diff --git a/.claude/rules/data-sync-storage.md b/.claude/rules/data-sync-storage.md index 953cf0908a..d6809a939f 100644 --- a/.claude/rules/data-sync-storage.md +++ b/.claude/rules/data-sync-storage.md @@ -142,7 +142,7 @@ paths: This path holds user data. A silent wrong answer here looks exactly like an empty database, so prove late-completion and cancellation behavior with a test. - **A synced CKRecord type or field reaches CloudKit Production before anything writes it.** Both apps pin the container to Production, where nothing is auto-created, and CloudKit rejects a whole record that carries an undeclared field. Write fields only through `record.fields(SomeSyncField.self)`, never `record["..."]`. To ship one: add it in Development, deploy to Production in the Console, run `apple/scripts/export-cloudkit-schema.sh`, commit `apple/CloudKit/production-schema.ckdb`, then add it to the matching `verifiedInProduction`. `ProductionSchemaParityTests` checks code against the snapshot in both directions, not against the server, so never edit the snapshot by hand: `apple/scripts/export-cloudkit-schema.sh --check` compares it with Production. A tag's `app` job runs `--check-deployed` instead, which fails only on a record type, field, field type, index, role or grant the tag's snapshot has and Production lacks. -- **A storage format stays readable by the previous stable.** Sparkle never downgrades, and a beta user can turn betas off (`apple/RELEASING.md`, Storage check). +- **A storage format stays readable by the previous stable.** Sparkle never downgrades, and a beta user can turn betas off (`apple/RELEASING.md`, Storage check). The one exception is a value the previous stable has no field for, such as a theme color in a slot added after it: that file and its synced record are written at the oldest schema that holds it, and everything else keeps the old format (`.claude/rules/theme.md`, Model and Sync). - **Persist, then notify.** `SyncChangeTracker.markDeleted()` runs after the storage write, because its notification can start a sync that re-uploads the record from the stale file. - **Concurrent schema loads await the one in-flight `loadTask`** in `SQLSchemaProvider`; never guard with a flag that returns without data. - **Every metadata read goes through `DatabaseManager.withMetadataDriver`**, never `MetadataConnectionPool.shared` directly, so an embedded engine with `supportsConnectionPooling = false` (DuckDB, PGlite) stays on its session driver; a second connection to it is a different, empty database. @@ -159,6 +159,7 @@ This path holds user data. A silent wrong answer here looks exactly like an empt | Light and dark theme ids (iCloud-synced, category `appearance`) | UserDefaults | `AppearanceSettings` through `AppSettingsManager` | | User themes (iCloud-synced, one `AppSettings` record `theme.` each) | JSON, one file per theme | `ThemeCatalog` (sync rules: `.claude/rules/theme.md`, Sync) | | Query history | SQLite FTS5 | `QueryHistoryStorage` | +| Query tab edit history: up to 50 versions per tab, kept 30 days or the shorter Query History window, after the tab closes too; purged with the connection and by Clear History | SQLite `query_tab_versions.db`, device-local, never synced | `QueryTabVersionStorage` | | Tab state | JSON | `TabPersistenceCoordinator`, `TabDiskActor` | | Filter defaults and presets | UserDefaults | `FilterSettingsStorage`, `FilterPresetStorage` | | Per-table filters | JSON, one file per table | `FilterSettingsStorage` | @@ -181,7 +182,11 @@ This path holds user data. A silent wrong answer here looks exactly like an empt | Snowflake SSO and MFA tokens, per host and user. A bare account's `NAME.USER` item stays for 0.83.1, which reads only that key | Keychain, device-local | `SnowflakeTokenCache` | | Whether `connections.json` was ever given an integrity tag, so a deleted tag reads as modified | Keychain `com.TablePro.connectionStoreStamped.`, device-local | `ConnectionStoreIntegrity` | | Whether Local only connections' secrets were copied to device-only items | UserDefaults `com.TablePro.localOnlySecretsKeptOnDevice`, device-local, never synced | `ConnectionStorage` | +| iCloud Keychain copies of Local only secrets the keychain refused to remove, retried at launch | UserDefaults `com.TablePro.localOnlySyncedCopiesPendingRemoval`, device-local, never synced | `ConnectionStorage` | +| Whether plugin secure fields moved out of `connections.json` into the Keychain, set only when every field moved and the save succeeded | UserDefaults `com.TablePro.pluginSecureFieldsMovedToKeychain`, device-local, never synced | `ConnectionStorage` | +| Whether a saved-queries store replaced a damaged file and waits for a full fetch, so push treats it as unreadable | Table `sync_refill` in the saved-queries SQLite store, dropped once a full fetch applied | `SQLFavoriteStorage` | | Deletions the server reported for a sync category that is off, applied when the category is next read | UserDefaults, device-local, never synced | `SyncMetadataStorage.deferredDeletions` | +| The iCloud account this device last saved the CloudKit change subscription for; cleared when sync turns on, on Upload Again, when the data is found removed, and when an activation brings a change no notice announced | UserDefaults `com.TablePro.sync.subscribedAccountId`, device-local, never synced | `SyncMetadataStorage.subscribedAccountId` through `SyncPushChannel` | | Where a connection came from (`origin`): this Mac, imported, iCloud | `connections.json`, never synced or exported | `ConnectionStorage` | | Table load timings (no identifying data, never uploaded) | JSON lines | `TableLoadHistoryStore` | | Reopen Last Session list (`LastSession.json`, plus `LastOpenConnections.json` holding only the ids that reconnect, which older builds read) | JSON | `LastOpenConnectionsStorage` | diff --git a/.claude/rules/plugin-system.md b/.claude/rules/plugin-system.md index e61ec10314..dc90c8595e 100644 --- a/.claude/rules/plugin-system.md +++ b/.claude/rules/plugin-system.md @@ -37,7 +37,7 @@ TableProPluginKit is built with Library Evolution, so an already-built plugin ke ## One PluginKit per process - R1 (the rule is in `apple/AGENTS.md`, Invariants that apply everywhere): a plugin that linked a package product depending on the `TableProPluginKit` package would carry its own copy of PluginKit's types, and the host's casts across that boundary fail. Products a plugin and the iOS app share stay PluginKit-free for that reason. -- The R1 check runs per binary (each plugin, each plugin test bundle, the app) and must print nothing: +- The R1 check runs per binary (each plugin, each plugin test bundle, the app) and must print nothing. A Release binary keeps no symbols, so run it on a Debug build, or on the dSYM's `Contents/Resources/DWARF/` for a Release one: ```bash nm -m "$BIN" | grep -v undefined | grep -E '_\$s17TableProPluginKit[0-9A-Za-z_]*Mn$' \ diff --git a/.claude/rules/tests.md b/.claude/rules/tests.md index 749bd65552..cd1648a222 100644 --- a/.claude/rules/tests.md +++ b/.claude/rules/tests.md @@ -73,7 +73,7 @@ UI flows: `.claude/rules/ui-tests.md`. Running tests and reading verdicts: `.cla Use one before writing a local helper. - `TableProTestSupport` (TableProCore): `waitUntil`, `AsyncGate`, `ManualClock` (production code takes `clock: any Clock = ContinuousClock()`), `BoundedCall`, `HeldOpenWriter`, `BackgroundPipeWriter`, `BackgroundPipeReader`. -- `apple/TableProTests/Helpers/`: `GlobalStateSuites`, `EphemeralDefaults`, `.appSettings { }`, `.recordingClipboard`, `CoordinatorFixture`, `OfflineRegistry`, `InMemoryColumnLayoutStore`, `InMemoryKeychain`, `PluginDriverStub`, `TestFixtures`, `AlwaysAllowGate`, `AlwaysDenyGate`. +- `apple/TableProTests/Helpers/`: `GlobalStateSuites`, `EphemeralDefaults`, `.appSettings { }`, `.recordingClipboard`, `CoordinatorFixture`, `OfflineRegistry`, `InMemoryColumnLayoutStore`, `InMemoryKeychain`, `PluginDriverStub`, `TestFixtures`, `AlwaysAllowGate`, `AlwaysDenyGate`, `CountingOpener`, `MCPInMemoryApprovalStore`, `MCPFixedApprovalReader`, `ArrayRowProvider`, `ScopeRecordingKeychain`. - `TableProMCPTestSupport` (TableProMacCore): `MCPTestClock`, `MockHttpServer`, `RecordingBridgeLogger`. `TableProTLSTestFixtures` (TableProCore), also for the iOS tests. ## Known failures and flakes diff --git a/.claude/rules/theme.md b/.claude/rules/theme.md index ca4cddb049..48ad7ede79 100644 --- a/.claude/rules/theme.md +++ b/.claude/rules/theme.md @@ -22,7 +22,7 @@ paths: - `TableProTheme` is Foundation only, so the iOS app can adopt it; AppKit and SwiftUI code stays in `apple/TablePro/Theme`. - **`ThemeSlot` is the one registry.** A slot exists only while a view reads it: removing its last reader removes the slot. - `isOverridden(_:)` means the slot draws a theme color, set or derived: ask it before swapping a native drawing path (a code table's selection, the caret). Only the theme editor asks `hasExplicitColor(_:)`. -- **Adding a slot bumps `ThemeSchema.current`.** `ThemeFileFormat` reads only the current schema: an older or newer one, or an unknown key, is refused rather than loaded without colors its author wrote, so a bump stops every older file from loading. The bump renames the sync content kind (Sync). +- **Adding a slot bumps `ThemeSchema.current`, and the slot records the schema that introduced it.** `ThemeFileFormat` reads every schema from 3 to the current one. A newer schema, an unknown key, or a key newer than the file's schema is refused rather than loaded without colors its author wrote. It writes the oldest schema that holds every key the theme sets, so a theme that sets no new slot stays readable by the previous stable (`.claude/rules/data-sync-storage.md`). The bump renames the sync content kind (Sync). - **An imported file is untrusted.** `ThemeFileImporter` checks the size before it reads, mints a fresh id every time, and never follows a VS Code `include` out of the file's folder. ## Reading colors @@ -55,7 +55,8 @@ paths: ## Sync -- **Each user theme syncs as the `AppSettings` record `theme.`** holding `ThemeFileFormat.encode(_:)` bytes. A user edit marks it dirty, a trash marks it deleted once the trash lands, and a load or migration marks nothing. Built-ins never sync. +- **Each user theme syncs as the `AppSettings` record `theme.`** holding `ThemeFileFormat.encode(_:)` bytes, at the oldest schema that holds its keys. A user edit marks it dirty, a trash marks it deleted once the trash lands, and a load or migration marks nothing. Built-ins never sync. +- **A record at a newer schema is skipped by an older Mac**, which keeps it as the merge base, so that Mac shows its default theme and its next edit of the theme drops the newer keys. A theme that sets no newer key stays at the old schema and syncs with it. This lasts only while an older build shares the zone, and it is named in the release that bumps the schema. - **A synced theme is untrusted.** `ThemeCatalog.applyRemote(category:data:)` checks the category id (never a built-in), a 64 KB cap, nesting depth, a full decode and the file id, and never writes over bytes it has not read. A validation failure is a logged skip; only a disk failure returns `.failed`, which withholds the change token so the pull replays. A remote delete trashes the file through `removeWithoutSync(ids:)`. - The light and dark theme ids sync. The appearance mode and fonts are device-local, and a received `appearance` record never changes them. - `SyncCoordinator.themeContentKind` names `ThemeSchema.current`, so a Mac upgraded past a bump refetches the zone once. After the first full pull, only local themes the server lacks are marked dirty, once. diff --git a/.claude/skills/fix-issue/references/platform-windows.md b/.claude/skills/fix-issue/references/platform-windows.md index e7baaae3d1..d823df2775 100644 --- a/.claude/skills/fix-issue/references/platform-windows.md +++ b/.claude/skills/fix-issue/references/platform-windows.md @@ -43,8 +43,8 @@ Pass `--platform windows`, which a Windows shell (Git Bash, MSYS2, Cygwin) picks | `app` | Windows only: anything under `windows/src/TablePro.App` changed, before `ui`; the Native AOT publish CI tests | | `ui [filter]` | Windows only: the change is visible in the app, or a flow the UI tests cover changed | -- Pick tests by what you changed: `src/` has its tests in `tests/.Tests` where that folder exists. `TablePro.Graphics` is reached through `TablePro.DataGrid.Tests`, and `TablePro.Editor` and `TablePro.App` through the UI tests. A core crate has its tests in its own folder. -- On macOS or Linux, `build`, `test`, `core` and `bindings` run. `app`, `ui` and `TablePro.DataGrid.Tests` report INCONCLUSIVE there: the pull request's `windows / Windows (x64)` and `windows / Windows (Arm64)` jobs run them. +- Pick tests by what you changed: `src/` has its tests in `tests/.Tests` where that folder exists. `TablePro.Graphics` is reached through `TablePro.DataGrid.Tests` and `TablePro.Editor.Tests`, `TablePro.Editor` through `TablePro.Editor.Tests` and the UI tests, and `TablePro.App` through the UI tests. A core crate has its tests in its own folder. +- On macOS or Linux, `build`, `test`, `core` and `bindings` run. `app`, `ui`, `TablePro.DataGrid.Tests` and `TablePro.Editor.Tests` report INCONCLUSIVE there: the pull request's `windows / Windows (x64)` and `windows / Windows (Arm64)` jobs run them. - There is no lint step: `windows.yml` runs none. `build` is the lint, since `windows/Directory.Build.props` turns on the latest recommended analyzers and the `.editorconfig` style with warnings as errors. - Extra review pass: none beyond the security pass. - Traps and remedies: `references/verification-windows.md`. diff --git a/.claude/skills/fix-issue/references/verification-windows.md b/.claude/skills/fix-issue/references/verification-windows.md index 77cdcafa0a..e849cbf8f0 100644 --- a/.claude/skills/fix-issue/references/verification-windows.md +++ b/.claude/skills/fix-issue/references/verification-windows.md @@ -24,7 +24,7 @@ Read this when a `verify.sh` verdict needs interpreting for the Windows app, bef - **A filter that matches nothing exits 8**, "Zero tests ran". The wrapper reports it as INCONCLUSIVE; check the executed count anyway. - **A bare filter word matches part of a test's full name**, namespace and class included, through `--filter-method "**"`. xunit's own options go after `--`: `--filter-class`, `--filter-trait "Needs=Core"`, `--filter-query`. -- **`TablePro.DataGrid.Tests` runs on Windows only.** Its csproj turns `IsTestProject` off elsewhere, so the solution's run on a Mac compiles it and runs the other four projects: fewer cases than CI counts on Windows. +- **`TablePro.DataGrid.Tests` and `TablePro.Editor.Tests` run on Windows only.** Their csproj files turn `IsTestProject` off elsewhere, so the solution's run on a Mac compiles them and runs the other four projects: fewer cases than CI counts on Windows. - **The `Needs=Core` tests load the real core** that the build copied beside the test assembly. They run on every runner. - **Cargo exits 101 for a compile error and for a failed test.** The `core` step tells them apart by whether a test binary ran. A filter that matches nothing exits 0 with "0 passed", which the wrapper reports as INCONCLUSIVE. - **`core` passes `--target `**, as the csproj and `core.yml` do, so it reuses the build instead of compiling the core again into `target/debug`. diff --git a/.claude/skills/fix-issue/scripts/lib/platform-apple.sh b/.claude/skills/fix-issue/scripts/lib/platform-apple.sh index a1fb1c2acd..00c039a90d 100644 --- a/.claude/skills/fix-issue/scripts/lib/platform-apple.sh +++ b/.claude/skills/fix-issue/scripts/lib/platform-apple.sh @@ -633,10 +633,13 @@ platform_step() { code=0 failed="" unbuilt="" - for target in mac ios; do - echo "== localization.py plugins --target $target" >> "$log" - (cd "$APPLE_ROOT" && python3 scripts/localization.py plugins --target "$target") >> "$log" 2>&1 \ - || { code=1; failed="$failed plugins:$target"; } + # The iOS widget extension has its own catalog and links no package. + for target in mac ios widget; do + if [ "$target" != widget ]; then + echo "== localization.py plugins --target $target" >> "$log" + (cd "$APPLE_ROOT" && python3 scripts/localization.py plugins --target "$target") >> "$log" 2>&1 \ + || { code=1; failed="$failed plugins:$target"; } + fi echo "== localization.py extracted --target $target" >> "$log" (cd "$APPLE_ROOT" && python3 scripts/localization.py extracted --target "$target") >> "$log" 2>&1 case $? in diff --git a/.claude/skills/fix-issue/scripts/lib/platform-windows.sh b/.claude/skills/fix-issue/scripts/lib/platform-windows.sh index 76edf690e7..e53998f7cf 100644 --- a/.claude/skills/fix-issue/scripts/lib/platform-windows.sh +++ b/.claude/skills/fix-issue/scripts/lib/platform-windows.sh @@ -368,7 +368,7 @@ platform_step() { else report_dotnet_tests "$log" "$filtered" "$code" if [ -z "$project" ] && ! on_windows; then - note "TablePro.DataGrid.Tests compiled but did not run: it runs on Windows" + note "TablePro.DataGrid.Tests and TablePro.Editor.Tests compiled but did not run: they run on Windows" fi fi diagnose_environment "$log" && STATUS=INCONCLUSIVE diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index ed47f97fc2..3c3180c49c 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -343,7 +343,7 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1 + - uses: taiki-e/install-action@c509219cf42fba86913069efd0ade2458c59337b # v2.90.1 with: tool: cargo-deny@${{ env.CARGO_DENY_VERSION }},cargo-shear@${{ env.CARGO_SHEAR_VERSION }},cargo-about@${{ env.CARGO_ABOUT_VERSION }} - name: Bans, licenses and sources @@ -543,7 +543,7 @@ jobs: add-job-id-key: false save-if: ${{ github.ref == 'refs/heads/main' }} - &docker-nextest - uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1 + uses: taiki-e/install-action@c509219cf42fba86913069efd0ade2458c59337b # v2.90.1 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - &docker-available @@ -697,7 +697,7 @@ jobs: workspaces: . key: tablepro-session save-if: ${{ github.ref == 'refs/heads/main' }} - - uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1 + - uses: taiki-e/install-action@c509219cf42fba86913069efd0ade2458c59337b # v2.90.1 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -772,7 +772,7 @@ jobs: # As in the fast job: the cache restores DuckDB's sources without files. - name: Drop the emptied DuckDB sources run: rm -rf ../target/duckdb-sources - - uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1 + - uses: taiki-e/install-action@c509219cf42fba86913069efd0ade2458c59337b # v2.90.1 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index e742097778..407080ccf9 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -51,7 +51,7 @@ jobs: tar -xzf "$archive" -C "$RUNNER_TEMP/actionlint" actionlint echo "$RUNNER_TEMP/actionlint" >> "$GITHUB_PATH" - - uses: taiki-e/install-action@68a389c788b2cd04016ce8c80ee23b80d2ab3d68 # v2.88.1 + - uses: taiki-e/install-action@c509219cf42fba86913069efd0ade2458c59337b # v2.90.1 with: tool: zizmor@${{ env.ZIZMOR_VERSION }} @@ -208,6 +208,7 @@ jobs: go mod verify go mod tidy -diff GOOS=darwin go vet ./... + go test ./... # go tool would build staticcheck for GOOS too, and Linux cannot run a darwin binary. go build -modfile=tools.mod -o "$RUNNER_TEMP/staticcheck" honnef.co/go/tools/cmd/staticcheck GOOS=darwin "$RUNNER_TEMP/staticcheck" ./... diff --git a/.github/workflows/ios-tests.yml b/.github/workflows/ios-tests.yml index 2eee442713..988ae229f0 100644 --- a/.github/workflows/ios-tests.yml +++ b/.github/workflows/ios-tests.yml @@ -79,9 +79,13 @@ jobs: key: ${{ steps.spm.outputs.cache-primary-key }} # Only Xcode adds a new string to the catalog. A command-line build extracts it and moves on. - - name: Check the app's strings reach its catalog + # The widget extension has its own catalog. + - name: Check the app's strings reach its catalogs if: ${{ !cancelled() }} - run: python3 scripts/localization.py extracted --target ios + run: | + set -euo pipefail + python3 scripts/localization.py extracted --target ios + python3 scripts/localization.py extracted --target widget # --parallel is what makes swift test write the XCTest xunit file. Without it only the Swift # Testing one appears, and TableProOracle's cases are all XCTest. diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index 74c5d61aff..edb06fc6b8 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -146,7 +146,7 @@ jobs: toolchain: "1.98" targets: ${{ matrix.target }} # windows/global.json takes the newest 10.0 SDK. - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + - uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 with: dotnet-version: 10.0.x # As Microsoft's desktop app workflows do: the committed manifest keeps the development identity. @@ -298,7 +298,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + - uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 with: dotnet-version: 10.0.x - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 diff --git a/.github/workflows/windows-grid-gate.yml b/.github/workflows/windows-grid-gate.yml index 12c54681c7..964b6b9bf8 100644 --- a/.github/workflows/windows-grid-gate.yml +++ b/.github/workflows/windows-grid-gate.yml @@ -51,7 +51,7 @@ jobs: run: rm -rf "target/$TARGET/duckdb-sources" target/duckdb-sources - name: Build the core with the release profile run: cargo build --locked --release -p tablepro-ffi --target $env:TARGET - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + - uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 with: dotnet-version: 10.0.x - name: Run the gate diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 73e44364b9..e298071914 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -66,7 +66,7 @@ jobs: - name: Drop the emptied DuckDB sources run: rm -rf "target/${{ matrix.target }}/duckdb-sources" target/duckdb-sources # windows/global.json takes the newest 10.0 SDK. - - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 + - uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 with: dotnet-version: 10.0.x - name: Start Windows' OpenSSH Server diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md index 4a121eeca6..0e2b962dbc 100644 --- a/.oss-scanner/threat_model.md +++ b/.oss-scanner/threat_model.md @@ -13,7 +13,7 @@ The user is trusted: the SQL they type, the connections they create, the plugins 1. **The database server and the network to it.** A malicious or compromised server, or anyone on the path when TLS is not verified, controls every byte of the protocol, every result set and every error message. This includes the HTTP and REST engines and the SSH server of a tunnel. 2. **Database contents.** Cell values (text, BLOBs, JSON, images, PHP-serialized data), table, column and schema names, comments and object definitions can be written by another user of a shared database. -3. **Files the user opens or imports.** `.sql`, CSV/TSV, JSON/JSONL, XLSX, Parquet, SQLite and DuckDB files, `.tablepro` connection bundles, connection files from other apps (TablePlus, DBeaver, DataGrip, Navicat `.ncx`, Sequel Ace, Beekeeper), project folders scanned for credentials (`.env`, docker-compose, framework configs), `~/.pgpass`, `~/.ssh/config`. +3. **Files the user opens or imports.** `.sql`, CSV/TSV, JSON/JSONL, XLSX, Parquet, SQLite and DuckDB files, `.tablepro` connection bundles, connection files from other apps (TablePlus, DBeaver, DataGrip, Navicat `.ncx`, Sequel Ace, Beekeeper, DBX), project folders scanned for credentials (`.env`, docker-compose, framework configs), `~/.pgpass`, `~/.ssh/config`. 4. **Links.** `tablepro://` deep links and database URLs (`postgresql://`, `mysql://`, `mongodb+srv://` and others in `apple/TablePro/Info.plist`) opened from a browser or another app. 5. **Local MCP clients.** The Mac app runs an MCP server on loopback. A process without a valid token, or a client acting beyond its token's scopes and connection allowlist, is untrusted. So are web pages reaching that port (DNS rebinding) and the remote MCP servers TablePro connects to as a client. 6. **AI model output.** Tool calls from a model, which database contents can steer by prompt injection. diff --git a/Cargo.lock b/Cargo.lock index e1b9b55dd5..d6ef499938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2274,7 +2274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2812,7 +2812,7 @@ dependencies = [ "gobject-sys", "libc", "system-deps 9.0.0", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5404,8 +5404,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +source = "git+https://github.com/TableProApp/rust-postgres?rev=f8bdf78d12df410a1c1b485445a9d0683361a173#f8bdf78d12df410a1c1b485445a9d0683361a173" dependencies = [ "base64 0.22.1", "byteorder", @@ -5422,8 +5421,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +source = "git+https://github.com/TableProApp/rust-postgres?rev=f8bdf78d12df410a1c1b485445a9d0683361a173#f8bdf78d12df410a1c1b485445a9d0683361a173" dependencies = [ "bytes", "chrono", @@ -5726,7 +5724,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6315,15 +6313,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6381,7 +6379,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7888,6 +7886,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "deadpool", "deadpool-postgres", "fallible-iterator 0.2.0", "futures-util", @@ -8055,6 +8054,7 @@ dependencies = [ "thiserror 2.0.20", "tokio", "tokio-util", + "toml_edit", "tracing", "uuid", ] @@ -8238,7 +8238,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8495,8 +8495,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +source = "git+https://github.com/TableProApp/rust-postgres?rev=f8bdf78d12df410a1c1b485445a9d0683361a173#f8bdf78d12df410a1c1b485445a9d0683361a173" dependencies = [ "async-trait", "byteorder", @@ -8626,6 +8625,7 @@ dependencies = [ "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", + "toml_writer", "winnow 1.0.4", ] @@ -9433,7 +9433,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 33abb65cd8..98d254b261 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,8 @@ async-trait = "0.1" thiserror = "2" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["raw_value"] } +# Edits a TOML config file and keeps its comments and layout. +toml_edit = "0.25.15" sqlparser = { version = "0.63", default-features = false, features = ["std"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "process"] } tokio-util = { version = "0.7", default-features = false, features = ["codec", "rt"] } @@ -195,6 +197,16 @@ testcontainers = { version = "0.27", default-features = false, features = ["aws- testcontainers-modules = { version = "0.15", default-features = false, features = ["aws-lc-rs"] } oo7 = { version = "0.6", default-features = false, features = ["tokio", "native_crypto", "unstable"] } +# TablePro's fork of rust-postgres, pinned by revision: the tokio-postgres-v0.7.18 tag plus the +# copy_in fix in rust-postgres/rust-postgres#1402, without which a COPY the server refuses before +# copy-in mode leaves the connection out of step. All three crates come from the fork, so the +# driver's postgres-types and postgres-protocol stay the ones tokio-postgres uses. Remove this +# once a tokio-postgres release has the fix. +[patch.crates-io] +tokio-postgres = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" } +postgres-types = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" } +postgres-protocol = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" } + [workspace.lints.rust] unsafe_op_in_unsafe_fn = "deny" diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000000..55a4831458 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,682 @@ +# Roadmap + +The work that makes the Linux and Windows apps complete daily-use clients, the core work under them, and the sync service and features every platform still lacks. The bar is the macOS app plus what the market expects from a daily-use client. + +Written 2026-11-11 against main `8581bd6c7`, from code inventories of the macOS, Linux and Windows apps and the Rust core, and a survey of 20 other clients ([Sources](#sources)). The maintainer's answers of 2026-11-13 are under [Decisions](#decisions). What the Linux app does today is in [linux/docs/features.md](linux/docs/features.md). + +## How to track + +- Tick a box when its PR merges and write the PR after it: `- [x] Windows: #4321`. +- A partial PR leaves the box open and adds `partly #4321: `. +- IDs never change. A new item goes at the end of its section with the next free number. +- Tier, from the survey: **CORE** every daily user expects it, **STD** most serious clients have it, **ADV** a pro-tier differentiator. +- `macOS: yes` means the macOS app has it; its code and docs are the reference. `paid` means it is a paid feature there. +- `needs C-07` names a core item that lands first. +- A `server` box is work in tablepro-cloud, outside this repository. +- Recount the progress table: + +```sh +awk '/^## /{p=$0} /^ *- \[[ x]\]/{t[p]++} /^ *- \[x\]/{d[p]++} END{for(k in t) print d[k]+0"/"t[k]" "k}' ROADMAP.md +``` + +## Order + +1. Linux 0.1.0: the Linux items of Phase 0. +2. Phase 1, then Windows: the Windows items of Phase 0 and every CORE item of Phase 2, then the first public Windows release (D-04). +3. Phases 3 to 8, by tier and demand. + +## Progress + +| Section | Boxes | Done | +|---|---|---| +| Phase 0: Release blockers | 16 | 0 | +| Phase 1: Core enablers | 23 | 0 | +| Phase 2: Windows daily use | 62 | 0 | +| Phase 3: Standard features both apps lack | 116 | 0 | +| Phase 4: Advanced tier | 60 | 0 | +| Phase 5: Engines | 27 | 0 | +| Phase 6: Quality bars | 8 | 0 | +| Phase 7: Sync and teams | 15 | 0 | +| Phase 8: Beyond the market | 40 | 0 | +| Total | 367 | 0 | + +## Where each app stands + +**macOS**, the reference: 48 engine types (5 bundled driver plugins, 31 in the registry), and every survey tier up to ADV. Compare & Sync, Server Dashboard, Table Health, Query Insights, Data Rewind, EXPLAIN diagrams, charts, map mode, routine debugger, users and roles, backup and restore, an agent-mode AI with 15 providers, an MCP server with 47 tools, iCloud sync and Team features. + +**Linux**, 0.1.0-beta.3: 9 engines from the core. Covers most of the CORE tier: connections with SSH, TLS and sockets, filters, editing with undo, completion, formatter, snippets, Vim, history, saved queries, structure editing, ER diagram, 7 export formats, dump and SQL import, AI chat, MCP with 11 tools, a CLI. Missing: all of admin and monitoring, query parameters, drawn plans, charts, schema compare, data transfer, backup through native tools, SOCKS, cloud IAM, users and roles, view and routine editing, the 10 paid features. 0.1.0 waits on [release criteria](linux/docs/releasing.md#release-criteria) 8 and 10. + +**Windows**, unreleased: 9 engines through the FFI. Has the connection library (groups, tags, favourites), in-process SSH, TLS with remedies, paging and sort, table-row editing with SQL review, a Direct2D editor with highlighting and multiple cursors, run statement, selection or script with transactions, Safe Mode (3 levels), tab tear-out, session restore, an MSIX build. Missing most of the CORE tier past that: filters, value viewers, row details, FK navigation, copy formats, completion, formatter, history, saved queries, .sql files, export, import, structure, sidebar actions, undo of edits. + +**Core**: the logic mostly exists, the FFI does not. `ddl`, `export`, `import`, `dump`, `filter`, history, saved queries, `ai`, `mcp`, `cli`, `account` and `foreign-import` are core code the FFI does not expose. The formatter, completion scope, script runner, key-based change tracker, copy renderer, highlight rules, Open Quickly and ER layout sit in `linux/crates/app`, GTK-free. Phase 1 moves both down so Windows builds on the code Linux already runs. + +## What the market expects + +- Top reasons users pick or leave a client: startup speed and memory, price model, engine breadth, native feel and dark mode, stability on big tables, OS availability, editor smarts, abandoned tools, trust (no forced AI, no required login), guards against production mistakes. +- Native on all three OSes is rare: most cross-platform clients are JVM (DataGrip, DBeaver, DbVisualizer) or Electron (Beekeeper, DbGate, Antares). +- Safe Mode is a stated reason to pick TablePlus. TablePro has it on every platform. +- Moving into STD during 2025-2026: an MCP server, AI with per-action consent, cloud IAM sign-in, a command palette, visual EXPLAIN, schema compare. +- Azure Data Studio retired on 2026-02-28. SQL notebooks now live only in the VS Code mssql extension. + +## Principles + +- Core first: logic both apps need lands in a core crate and the FFI before a Windows UI uses it. A Linux move keeps Linux's behavior and switches Linux to the core version in the same PR. +- Native UI per platform (`AGENTS.md`), no shared UI layer. +- Linux and Windows link their drivers at compile time for good ([core ADR 0025](core/docs/decisions/0025-linux-and-windows-link-their-drivers.md)). Oracle is blocked on a client library. +- One license and one feature gate on every platform: what is paid on macOS is paid on Linux and Windows (D-02). +- A new question that would reverse a decision goes under [Decisions](#decisions) as the next D item, and waits for the maintainer. + +## Phase 0: Release blockers + +### Windows first public release + +- [ ] **R-01** RUSTSEC-2026-0459 (bundled SQLite before 3.54.0): `deny.toml` blocks a Windows release until sqlx allows libsqlite3-sys 0.39 and the pin is raised +- [ ] **R-02** Maintainer setup, `windows/RELEASING.md` steps 1-6: tag rulesets, Artifact Signing with a Public Trust profile, the OIDC app registration, the `windows-release` secrets, `WINDOWS_SIGNING_*` and `WINDOWS_PUBLISHER` (permanent after the first release), the `windows-winget` environment +- [ ] **R-03** Dry run of `release-windows.yml` from main: the x64 package, the bundle, install-and-start tests and winget validation, all unverified (ADR 0005) +- [ ] **R-04** Real hardware: Windows 10 build 17763, Native AOT, Mica on a GPU, TabView in the title bar on Windows 10, loading from a signed MSIX (ADR 0001, 0004) +- [ ] **R-05** Drop the Windows App SDK metapackage once Community Toolkit 8.3 is stable (41.8 MB of unused ONNX and DirectML per architecture) +- [ ] **R-06** Updates: an App Installer file or the Store, an in-app update check, stable and beta channels +- [ ] **R-07** Live-server Windows tests for the 7 engines no Windows suite connects to: MySQL, SQL Server, DuckDB, ClickHouse, Cassandra, MongoDB, Redis +- [ ] **R-08** Windows user guide on docs.tablepro.app and the download page +- [ ] **R-09** First `windows-v0.1.0` tag, once the rest of Phase 0 and every CORE item in Phase 2 are done; no public build before it (D-04) + +### Linux 0.1.0 + +- [ ] **R-10** [Release criterion](linux/docs/releasing.md#release-criteria) 8: the manual pass with Orca, the keyboard alone, High Contrast, 1.5x text and a right-to-left locale +- [ ] **R-11** Release criterion 10: a dated week of daily use +- [ ] **R-12** Bring [linux/docs/features.md](linux/docs/features.md) up to date: about 40 features from #4108 to #4159, #4145 and #4147 are missing, and the Saved queries, Sidebar, Preferences, Connections import, MCP Tools and Copy rows are stale +- [ ] **R-13** The display-scale pass on a real GPU and a HiDPI screen, and a hand install on Ubuntu 26.04 +- [ ] **R-14** #4099 (KDE Wallet prompt opens behind the window) and #4101 (`flatpak list` shows "Usage data") +- [ ] **R-15** After 0.1.0: the Flathub manifest and exception request, by the maintainer, under [Flathub's requirements](linux/docs/decisions/0041-beta-repositories-and-release-packages.md#context) +- [ ] **R-16** After 0.1.0: an AUR PKGBUILD, kept by the community + +## Phase 1: Core enablers + +Each item ends in an FFI surface Windows needs. Linux paths are under `linux/crates/app/src/`. + +- [ ] **C-01** Filters through the FFI: `table_query` takes a `FilterSet` (always `FilterSet::default()` in `core/crates/ffi/src/session.rs` today) and `operators_for` +- [ ] **C-02** Export through the FFI: `core::export` encoders (CSV, TSV, JSON, JSON Lines, HTML, Markdown, SQL INSERT, XML, XLSX) as a streaming job with progress and Stop; the sinks and ledger in `ui/export/` move down +- [ ] **C-03** CSV import and pasted rows through the FFI: `core::import::{csv_import, plan_import, PastedRows}` +- [ ] **C-04** SQL dump and SQL file import through the FFI: `core::dump`, `SqlFileReader`, with the orchestration in `ui/export/dump_job.rs` and `services/import_file.rs` moved down +- [ ] **C-05** Structure through the FFI: the table definition (columns, indexes, keys, comments), CREATE statements, `core::ddl` drafts, `diff_table`, plans and catalog operations (core ADR 0013); the logic in `services/pending_structure.rs` moves down +- [ ] **C-06** Storage through the FFI: `QueryHistory` (FTS5, retention, export), `SavedQueries` with folders and keywords, `Favourites`, `DraftStore` (deferred by core ADR 0017) +- [ ] **C-07** SQL formatter into core: `ui/editor/format_plan.rs` and `significant_tokens.rs` +- [ ] **C-08** Completion context into core: `completion/completion_scope.rs`, `catalog_read.rs` and the fuzzy ranking; proposal objects stay per app +- [ ] **C-09** Script runner into core: `script_runner.rs` and `run_log.rs` with `BatchErrorPolicy` (the FFI's `QueryEditor::run` takes one statement or one `GO` batch) +- [ ] **C-10** Pending row edits keyed by primary key into core: `services/change_tracker.rs` and `undo_op.rs`, so edits survive sort, filter and paging and undo is shared; replaces the position-keyed FFI `TableEditor` and Windows `UndoHistory.cs` +- [ ] **C-11** Copy and display text into core: `ui/clipboard/copy_renderer.rs`, `services/grid/display_text.rs`, `date_display.rs`, `hex_dump.rs`, `ui/format/thousands.rs` +- [ ] **C-12** Grid helpers into core: highlight rules (`services/grid/highlight.rs` and `persistence/highlight_rules_store.rs`) and find in grid (`find_matches.rs`) +- [ ] **C-13** Open Quickly matching and index into core: `ui/quick_open/{matching,search_index}.rs` +- [ ] **C-14** ER diagram model, layout and scene into core: `ui/diagram/{layout,model,scene}.rs`; drawing and export stay per app +- [ ] **C-15** Snippets in a neutral format: `builtin_snippets` is written in GtkSourceView syntax today; core keeps a neutral form and each app converts +- [ ] **C-16** Foreign import through the FFI: a Windows `ForeignHost` and `ForeignAppDirs`, passwords into Credential Manager, DPAPI-held passwords read (core ADR 0008) +- [ ] **C-17** AI through the FFI: providers, prompts, schema context, streamed replies +- [ ] **C-18** MCP server through the FFI, with a Windows host +- [ ] **C-19** Account through the FFI: the Windows `AppClient` once registered with tablepro-cloud, and the entitlement gate that every paid feature on Windows needs (core ADR 0016, D-02) +- [ ] **C-20** CLI on Windows: a `Host`, MCP dispatch, peer verification on the pipe, PowerShell completion (core ADR 0018) +- [ ] **C-21** Query-result editing through the FFI (core ADR 0015, after ADR 0019) +- [ ] **C-22** Connection library through the FFI: export to JSON, duplicate with secrets +- [ ] **C-23** uniffi 0.32 once uniffi-bindgen-cs supports it (issue #183, core ADR 0002) + +## Phase 2: Windows daily use + +The first public Windows release waits for every CORE item here and the Windows items of Phase 0 (D-04). + +### Connections + +- [ ] **W-01** Duplicate a connection with its secrets · CORE · needs C-22 +- [ ] **W-02** Export the connections file (no secrets); import of TablePro's file exists · STD · needs C-22 +- [ ] **W-03** Paste a URL on the Connections page to start a connection (today only in the engine chooser) · STD +- [ ] **W-04** Jump hosts in the SSH section; the core and the FFI record carry them · STD +- [ ] **W-05** Connect timeout field · STD +- [ ] **W-06** Group colour (the record has `Color`) · STD +- [ ] **W-07** Import from TablePlus, DBeaver, DataGrip, Beekeeper, Navicat and DBX · STD · needs C-16 +- [ ] **W-08** Production cue: the connection colour on the tab strip, title bar and toolbar, and an environment label · CORE + +### Sidebar + +- [ ] **W-09** Object context menu: Open, Open in New Tab, Copy Name, Show CREATE, Truncate, Drop, through Safe Mode · CORE · needs C-05 +- [ ] **W-10** Create and drop databases and schemas · CORE · needs C-05 +- [ ] **W-11** Database switcher per connection and per query tab · CORE +- [ ] **W-12** Open a routine, trigger or sequence: its definition read-only, with Copy and Open in Editor · STD +- [ ] **W-13** Columns, indexes and keys under a table node · STD · needs C-05 +- [ ] **W-14** Table info: rows, size, comment · STD + +### Grid + +- [ ] **W-15** Filters: per-column operators, raw WHERE, match any or all, kept per table · CORE · needs C-01 +- [ ] **W-16** Filter by a cell's value, or exclude it · CORE · needs C-01 +- [ ] **W-17** Value viewers for long text, JSON, binary (hex) and images; a tooltip on a cut cell · CORE +- [ ] **W-18** Row details pane with editing · STD +- [ ] **W-19** Follow a foreign key to the referenced row, and show referencing rows · CORE +- [ ] **W-20** Copy as CSV or TSV with headers, JSON, Markdown, INSERT, IN clause · CORE · needs C-11 +- [ ] **W-21** Reorder, hide and freeze columns, kept per table (the control has frozen columns) · STD +- [ ] **W-22** Context menu on query-result grids · CORE +- [ ] **W-23** Selection summary: count, sum, average, min, max · STD +- [ ] **W-24** Find in the grid · STD · needs C-12 +- [ ] **W-25** Grid display settings: NULL text, date format, grid font, row height, stripes · STD · needs C-11 +- [ ] **W-26** Highlight rules · ADV · needs C-12 + +### Editing + +- [ ] **W-27** Undo and redo of pending edits · CORE · needs C-10 +- [ ] **W-28** Paste rows from the clipboard · STD · needs C-03, C-10 +- [ ] **W-29** Edit query results that read one keyed table · STD · needs C-21 +- [ ] **W-30** Type editors: date and time picker, enum menu, JSON check, boolean toggle · STD + +### SQL editor + +- [ ] **W-31** Completion: tables, columns, aliases, schemas, functions, keywords · CORE · needs C-08 +- [ ] **W-32** Formatter · CORE · needs C-07 +- [ ] **W-33** Replace, and regular expressions in Find · CORE +- [ ] **W-34** Open and save .sql files, drop a file on the editor, recent files · CORE +- [ ] **W-35** Query history with search, pins and retention · CORE · needs C-06 +- [ ] **W-36** Saved queries in folders · CORE · needs C-06 +- [ ] **W-37** Snippets with tab stops · STD · needs C-15 +- [ ] **W-38** Run a whole script with an error policy, one entry per result · CORE · needs C-09 +- [ ] **W-39** Word wrap, bracket matching, the current-statement band, Go to Error · STD +- [ ] **W-40** Vim mode · STD +- [ ] **W-41** Code folding · STD + +### Structure + +- [ ] **W-42** Structure view: columns, indexes, foreign keys, comments, the CREATE statement · CORE · needs C-05 +- [ ] **W-43** Table designer: create and alter columns, indexes and foreign keys, with the SQL reviewed · CORE · needs C-05 +- [ ] **W-44** ER diagram · STD · needs C-14 + +### Import and export + +- [ ] **W-45** Export a table or a result to CSV, JSON, XLSX, SQL INSERT, Markdown, HTML and XML, with progress and Stop · CORE · needs C-02 +- [ ] **W-46** CSV import into a table with a column mapping · CORE · needs C-03 +- [ ] **W-47** Import SQL File and Export Database · CORE · needs C-04 + +### NoSQL + +- [ ] **W-48** Redis key tab: the value by type, length, TTL, Load More (opening a key does nothing today) · CORE +- [ ] **W-49** MongoDB: documents as JSON with a tree view · STD +- [ ] **W-50** Cassandra paging past the first page (the FFI refuses offsets; page by paging state) · CORE + +### Workspace + +- [ ] **W-51** Open Quickly: objects, saved and past queries, commands · STD · needs C-13 +- [ ] **W-52** Settings: editor font family, grid font, NULL display, history, AI and MCP pages · CORE +- [ ] **W-53** A toast when a long query finishes in a background tab · STD +- [ ] **W-54** Every string localizable, the editor's menu and find bar included; Vietnamese first · STD +- [ ] **W-55** Editor colour schemes · STD +- [ ] **W-56** Grid accessibility: row header peers, and a workaround for the empty `ColumnHeaders` in Windows App SDK 2.6.4 · CORE + +### AI, MCP, account, integration + +- [ ] **W-57** AI assistant: providers, chat with schema context, Explain, Review, Optimize, Fix, AI policy per connection · STD · needs C-17 +- [ ] **W-58** MCP server with an External Clients level per connection · ADV · needs C-18 +- [ ] **W-59** Account page, license ask, and the paid-feature gate on the macOS license (D-02) · STD · needs C-19 +- [ ] **W-60** `tablepro://` and database URL links · STD +- [ ] **W-61** File associations for .sql, SQLite and DuckDB files · STD +- [ ] **W-62** Launch arguments and the `tablepro` CLI (the execution alias ignores arguments today) · STD · needs C-20 + +## Phase 3: Standard features both apps lack + +The Windows box assumes its Phase 2 base. Linux-only items have no Windows box. + +### Connections + +- **S-01** SOCKS5 and HTTP proxy with remote DNS · STD · macOS: yes + - [ ] core: a `TransportKind` beside DirectTcp, SshForward and LocalSocket + - [ ] Linux + - [ ] Windows +- **S-02** Cloud IAM sign-in: AWS RDS and Aurora tokens, Google Cloud SQL IAM · STD · macOS: yes + - [ ] core: an `AuthMode` beside Password and Kerberos + - [ ] Linux + - [ ] Windows +- **S-03** Microsoft Entra ID for SQL Server (core ADR 0005) · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-04** Startup SQL per connection · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-05** Password from a command, an environment variable or a file; 1Password, Vault and AWS Secrets Manager after · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-06** Reusable SSH profiles · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-07** Encrypted connection export with secrets · STD · macOS: paid + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-08** Kerberos for PostgreSQL and MySQL; an SPN override for SQL Server outside the client's realm (a tiberius change) · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-09** Pre-connect and post-disconnect scripts; a tunnel command (kubectl port-forward, AWS SSM) · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows + +### Grid and editing + +- **S-10** Saved filter presets and a default filter per table · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-11** Foreign key value picker in the cell editor · STD · macOS: yes + - [ ] Linux + - [ ] Windows +- **S-12** Edit binary values from and to a file; array and spatial values · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-13** A clock in the grid's cell editor (Row Details has one) · STD + - [ ] Linux +- **S-14** Display As: UUID forms, Unix timestamps, JSON inside text; MessagePack, Protobuf and PHP after · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-15** Search a value across every table · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows + +### SQL editor + +- **S-16** Query parameters (`:name`, `?`) with a prompt, LIKE and IN expansion · STD · macOS: yes + - [ ] core: parameter scan in `sql_syntax`, binding per driver + - [ ] Linux + - [ ] Windows +- **S-17** EXPLAIN as a tree and a diagram, with EXPLAIN ANALYZE where the engine has it · STD · macOS: yes + - [ ] core: a plan model parsed per engine + - [ ] Linux + - [ ] Windows +- **S-18** Result tabs: keep earlier results, pin one, compare two · STD · macOS: yes + - [ ] Linux + - [ ] Windows +- **S-19** Split editor or tab groups · STD · macOS: no + - [ ] Linux + - [ ] Windows +- **S-20** Inline diagnostics: syntax errors, unknown tables and columns · ADV · macOS: partly + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-21** Shortcut customization with conflicts and reset · STD · macOS: yes + - [ ] Linux + - [ ] Windows +- **S-22** Commands in Open Quickly (Linux finds objects and queries only) · STD · macOS: partly + - [ ] Linux + +### Structure and objects + +- **S-23** Create and edit views, routines and triggers; refresh materialized views · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-24** Check constraints and partitions in the structure tab · STD · macOS: partly + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-25** Drag to reorder columns, with the ALTER previewed · STD + - [ ] Linux + - [ ] Windows +- **S-26** Users, roles and privileges for MySQL, MariaDB, PostgreSQL and SQL Server · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-27** PostgreSQL enum, composite and domain types; extensions · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-28** Rename and drop views, routines and triggers from the sidebar; Copy Name · CORE + - [ ] Linux + +### Admin + +- **S-29** Process list with cancel and terminate · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-30** Server variables and status · STD · macOS: partly + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-31** Maintenance: VACUUM, ANALYZE, REINDEX, OPTIMIZE, CHECK, REPAIR, SQLite integrity check · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows + +### Import, export, transfer + +- **S-32** Import JSON, JSON Lines and XLSX; CSV into a new table; lift Linux's 64 MiB CSV limit · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-33** Export Parquet and JSON Lines files · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-34** Copy rows to another connection with a column mapping · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-35** Backup and restore through the engines' tools (pg_dump and pg_restore, mysqldump, mongodump). Flatpak needs `flatpak-spawn --host` or bundled tools; MSIX needs them on PATH · STD · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-36** Reusable export presets · STD · macOS: no + - [ ] core + - [ ] Linux + - [ ] Windows + +### NoSQL + +- **S-37** Redis: edit values and TTL, create, rename and delete keys, a namespace tree · STD · macOS: yes + - [ ] core: `write_batch` returns `Unsupported(Tables)` today + - [ ] Linux + - [ ] Windows +- **S-38** Redis Cluster and Sentinel; Valkey tested · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows +- **S-39** MongoDB: index management, a document tree editor, validation rules · STD · macOS: partly + - [ ] core + - [ ] Linux + - [ ] Windows + +### Visualization + +- **S-40** Charts from a result: bar, line, area, scatter · STD · macOS: paid + - [ ] Linux + - [ ] Windows +- **S-41** Map view for geometry columns · ADV · macOS: yes + - [ ] core + - [ ] Linux + - [ ] Windows + +### Workspace + +- **S-42** An app theme override and custom themes (Linux follows the system; Windows has Light, Dark, System) · STD · macOS: yes + - [ ] Linux + - [ ] Windows +- **S-43** French, Korean, Turkish, Simplified and Traditional Chinese, which macOS ships · STD + - [ ] Linux + - [ ] Windows +- **S-44** GNOME Shell search provider for connections · STD + - [ ] Linux + +## Phase 4: Advanced tier + +macOS has every item here, most as paid features, and the same license gates them on Linux and Windows (D-02). Linux and Windows have none; Linux's account page lists all 10 paid features as absent. + +- **A-01** Compare & Sync: schema diff and row diff between connections, a sync script, saved comparisons · paid + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-02** Server Dashboard: sessions, metrics, slow queries, locks and blocking chains, replication status, saved-query tiles + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-03** Table Health: bloat, unused and redundant indexes, missing keys, XID age, stale statistics, a fix script + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-04** Query Insights: most run, slowest, got slower, failures · paid + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-05** Data Rewind: restore values from before a save, kept 7 days · paid + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-06** AI agent mode: tool calls with approval, `@` context, slash commands, saved conversations, AI rules per connection + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-07** More AI providers: Gemini, xAI, OpenRouter, Copilot, Bedrock, llama.cpp (Linux has Anthropic, OpenAI and OpenAI-compatible) + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-08** MCP client: the assistant calls outside MCP servers + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-09** MCP server parity: 47 tools, resources, prompts, subscriptions, scoped tokens, pairing, an activity log, client setup snippets (Linux has 11 tools and one token) + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-10** Linked SQL folders with Git status, commit, pull and push · paid + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-11** Environment variables in connection fields · paid + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-12** Masking rules across the grid, copies, exports, AI and MCP + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-13** Generate test rows: up to 1M, foreign-key aware, seeded + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-14** Join builder from foreign keys + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-15** ER diagram editing (add keys and columns) and Mermaid, DOT and PDF export + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-16** Data dictionary export to Markdown and HTML + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-17** Copy tables and databases across connections and engines, with type translation + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-18** Routine debugger for PL/pgSQL + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-19** Live messages: PostgreSQL LISTEN/NOTIFY, Redis channels + - [ ] core + - [ ] Linux + - [ ] Windows +- **A-20** Data Files: open CSV, JSON and Excel without importing, through the DuckDB driver + - [ ] core + - [ ] Linux + - [ ] Windows + +## Phase 5: Engines + +The core has 9 drivers; macOS has 48 engine types. Linux and Windows link drivers at compile time ([core ADR 0025](core/docs/decisions/0025-linux-and-windows-link-their-drivers.md)), so each engine is a core crate on a client library that meets [adding-drivers.md, The client library](linux/docs/adding-drivers.md#1-the-client-library): maintained on crates.io, TLS through aws-lc-rs, a stream or Unix socket transport for SSH, and a way to stop a statement. + +Microsoft's mssql-tds is checked again each release as a SQL Server client. Today its TLS on Linux goes through OpenSSL and it takes no outside pull requests, so tiberius stays. + +### On the existing drivers + +- **E-01** MariaDB as its own entry on Windows + - [ ] Windows +- **E-02** Test and list Redshift, CockroachDB, YugabyteDB, TiDB, OceanBase, Databend and Valkey (untested today) + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-03** MongoDB sharded clusters and Amazon Keyspaces tested + - [ ] core + +### New drivers, by survey demand + +- **E-04** Oracle · STD · blocked: the oracle crate has one maintainer, no release since 2025-01-02, and needs the proprietary Oracle client; checked again each release + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-05** Snowflake, over the SQL REST API · STD + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-06** BigQuery, over REST · STD + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-07** Trino and Presto, over HTTP · STD + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-08** libSQL/Turso and Cloudflare D1, over HTTP · ADV + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-09** DynamoDB · ADV + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-10** Elasticsearch and OpenSearch · ADV + - [ ] core + - [ ] Linux + - [ ] Windows +- **E-11** Kafka, Neo4j, InfluxDB, etcd, Typesense and the rest of macOS's registry, on user demand + - [ ] core +- **E-12** Dropped: engines added after install on Windows. Linux and Windows link drivers for good (D-03, [core ADR 0025](core/docs/decisions/0025-linux-and-windows-link-their-drivers.md)) + +## Phase 6: Quality bars + +Linux already holds each bar; these bring Windows to it, plus one shared item. + +- [ ] **Q-01** Large-catalog case on Windows: 100 schemas of 500 tables and one database of 50,000; the sidebar, its search and completion within time bounds +- [ ] **Q-02** 1,000,000-row paging and export on Windows, with bounds on time and resident memory +- [ ] **Q-03** The value corpus on Windows: dates at their extremes, decimals, JSON, UUIDs, Unicode and binary read back correctly per engine +- [ ] **Q-04** Kill-during-save recovery test on Windows +- [ ] **Q-05** Windows manual pass: Narrator, the keyboard alone, high contrast, 200% text +- [ ] **Q-06** Cold start and memory budgets in CI on Linux and Windows (survey reason 1); the first Windows launch took 16.7 s with JIT +- [ ] **Q-07** On Windows, offer the crash report on the next launch, as Linux does +- [ ] **Q-08** Security review of the Windows surfaces in `.oss-scanner/threat_model.md` before the first release, as Linux had before beta.1 + +## Phase 7: Sync and teams + +A TablePro sync service on every platform (D-01). macOS keeps iCloud beside it, and the user picks one. It syncs connections and groups, secrets encrypted end to end, saved queries and settings, and Team Library and Team Catalog; iCloud Sync, Team Library and Team Catalog are 3 of the 10 paid features. + +- **T-01** Design ADR: the data model, conflict rules (field by field, as core ADR 0010 merges CloudKit records), end-to-end encryption of secrets and key recovery, whether query history syncs, and iCloud beside the service on macOS + - [ ] core +- **T-02** The service: storage per account and team, the sync API, the entitlement check (core ADR 0016) + - [ ] server +- **T-03** Sync client in the core: pull, push, merge, an offline queue, and the device list + - [ ] core +- **T-04** Connections, groups, tags, colours and SSH profiles, without their secrets + - [ ] Linux + - [ ] Windows +- **T-05** Secrets encrypted on the device before they leave it, never readable on the server + - [ ] core + - [ ] Linux + - [ ] Windows +- **T-06** Saved queries, snippets, favourites and settings + - [ ] Linux + - [ ] Windows +- **T-07** Team Library and Team Catalog: shared connections and queries with roles, as on macOS + - [ ] core + - [ ] server + - [ ] Linux + - [ ] Windows +- **T-08** macOS on the service: everything T-04 to T-07 sync, beside iCloud, a setting to pick one, and a move from iCloud to the service + - [ ] macOS + +## Phase 8: Beyond the market + +Survey features no client covers on every platform, and the macOS app lacks them too, so each has a macOS box. All but G-06 and G-12 were taken (D-05). + +- **G-01** SQL notebooks: SQL and Markdown cells with results (only VS Code mssql since Azure Data Studio retired) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-02** Scheduled jobs: export, backup or run SQL on a schedule, with a notification (Navicat, DBeaver PRO, TablePlus 27.2) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-03** Go to definition, find usages and rename in SQL (DataGrip, Navicat 18, dbForge) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-04** Object dependencies: uses and used by (DataGrip, DBeaver, SSMS, pgAdmin) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-05** Schema in version control: compare a database against SQL files or migrations (SSMS, VS Code mssql, dbForge, DBeaver PRO) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-06** Not taken (D-05): a visual MongoDB aggregation pipeline builder (Navicat) +- **G-07** Column profiling: nulls, distinct values, distribution (Navicat 17) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-08** Pivot tables on a result (dbForge, Navicat) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-09** Edit tables with no primary key through `ctid` or `rowid`, with a warning (DBeaver, DataGrip) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-10** Server log viewer and live trace such as Extended Events (SSMS, Querious, MySQL Workbench) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-11** Run a query every N seconds and chart it (DbVisualizer) + - [ ] core + - [ ] Linux + - [ ] Windows + - [ ] macOS +- **G-12** Not taken (D-05): a browser-hosted edition (CloudBeaver, DbGate, pgAdmin server mode) + +## Decisions + +Answered by the maintainer on 2026-11-13. + +| ID | Question | Decision | +|---|---|---| +| D-01 | Sync and team features on Linux and Windows | A TablePro sync service on every platform, with iCloud kept beside it on macOS ([Phase 7](#phase-7-sync-and-teams)). It reverses the Linux roadmap's earlier "no cloud sync through a TablePro service" | +| D-02 | Paid features on Linux and Windows | One license and one feature gate on all three platforms | +| D-03 | Engines added after install on Windows | No. Linux and Windows link drivers at compile time for good ([core ADR 0025](core/docs/decisions/0025-linux-and-windows-link-their-drivers.md)) | +| D-04 | The bar for the first public Windows release | Phase 0 plus every CORE item in Phase 2, with no public build before | +| D-05 | Which survey gaps to take | All but G-06 and G-12 ([Phase 8](#phase-8-beyond-the-market)) | +| D-06 | Which comes first | Linux 0.1.0, then Phase 1 and Windows ([Order](#order)) | +| D-07 | An Android app | Not now | + +## Out of scope + +- Drivers added after install on Linux and Windows ([core ADR 0025](core/docs/decisions/0025-linux-and-windows-link-their-drivers.md)) +- Embedded scripting: JavaScript, Python, Lua. SQL is enough, and a scripting engine widens the attack surface +- AppImage and Snap ([Linux ADR 0018](linux/docs/decisions/0018-ppa-copr-and-flathub-later.md), [0041](linux/docs/decisions/0041-beta-repositories-and-release-packages.md)) +- KDE-native styling: the Linux app runs with the Adwaita style everywhere +- An Android app, for now (D-07) +- G-06 and G-12 (D-05) + +## Sources + +- Code inventories of `apple/`, `linux/`, `windows/` and `core/` at `8581bd6c7`, and the docs site clone (last commit 2026-10-28, behind the app: Neo4j, InfluxDB, Qdrant, Memcached, StarRocks, masking, Table Health, Live Messages, result tabs and account sign-in have no page yet). +- Survey, from official feature pages and 2025-2026 release notes: + - DataGrip: https://www.jetbrains.com/help/datagrip/2026.3/release-notes-datagrip.html + - DBeaver: https://dbeaver.com/edition/, https://dbeaver.io/news/ + - TablePlus: https://tableplus.com/osx/changelog + - Navicat: https://www.navicat.com/en/navicat-18-highlights + - Beekeeper Studio: https://www.beekeeperstudio.io/features + - DbVisualizer: https://www.dbvis.com/whatsnew + - pgAdmin: https://www.pgadmin.org/news/ + - SSMS 22: https://learn.microsoft.com/ssms/release-notes-22 + - VS Code mssql: https://learn.microsoft.com/en-us/sql/tools/visual-studio-code-extensions/mssql/mssql-extension-visual-studio-code + - DbGate: https://www.dbgate.io/features + - Postico 2: https://releases.eggerapps.at/postico2/changelog + - Sequel Ace: https://github.com/Sequel-Ace/Sequel-Ace/releases + - Also HeidiSQL, dbForge (search summaries only, devart.com refuses fetches), MySQL Workbench, Antares, Querious, Chat2DB. +- Reasons users switch: about 20 Hacker News threads and GitHub trackers, for example https://news.ycombinator.com/item?id=39660592, https://news.ycombinator.com/item?id=46078571, https://github.com/TablePlus/TablePlus/issues/2669. diff --git a/apple/.swiftlint.yml b/apple/.swiftlint.yml index 6647ff85a7..18f3dbed90 100644 --- a/apple/.swiftlint.yml +++ b/apple/.swiftlint.yml @@ -485,6 +485,8 @@ custom_rules: - '.*/[^/]*Tests/' - '.*Tests\.swift$' - '.*/(WebPageOpener|DataLinkPolicy|MCPPairingService|NotificationsSettingsView|SyncSettingsView|ConnectionRecoveryPerformer|OpenConnectionIntent)\.swift$' + # An MCP client's install link, built from the bundled catalog, opened only when its scheme has a handler app. + - '.*/MCPClientWorkspace\.swift$' # Open TablePro's own links, and the user's files and URLs, in the TablePro bundle the command ships in. - '.*/(Handshake|CLILiveEnvironment)\.swift$' - '.*/(AppDelegate\+MainMenuActions|WhatsNewWindowController|SupportAskCoordinator|MainContentCoordinator\+Rewind|GoogleSignInService|ChatGPTCodexService)\.swift$' diff --git a/apple/CONTRIBUTING.md b/apple/CONTRIBUTING.md index cc18b17b8f..c33139ff66 100644 --- a/apple/CONTRIBUTING.md +++ b/apple/CONTRIBUTING.md @@ -150,7 +150,7 @@ xcodebuild -importLocalizations -project TablePro.xcodeproj -localizationPath Lo scripts/localization.py status # what is translated, per language ``` -The export builds the app first, so run `scripts/download-libs.sh` and `scripts/generate-project.sh` once before it. Pass an existing language code to fill gaps, or a new one to start a language. Commit the catalog, not the export: `Localization/` is ignored. The iPhone and iPad app has its own catalog: run the same commands on `TableProMobile/TableProMobile.xcodeproj` with the `TableProMobile` scheme. Plugin and package strings, and strings a command-line build added: [`AGENTS.md`](AGENTS.md), Every change 3. +The export builds the app first, so run `scripts/download-libs.sh` and `scripts/generate-project.sh` once before it. Pass an existing language code to fill gaps, or a new one to start a language. Commit the catalog, not the export: `Localization/` is ignored. The iPhone and iPad app has its own catalogs, one for the app and one for its widget: run the same commands on `TableProMobile/TableProMobile.xcodeproj` with the `TableProMobile` scheme, and `status` with `--target ios` or `--target widget`. Plugin and package strings, and strings a command-line build added: [`AGENTS.md`](AGENTS.md), Every change 3. ## Troubleshooting diff --git a/apple/Configs/NoExportedSymbols.exp b/apple/Configs/NoExportedSymbols.exp new file mode 100644 index 0000000000..058f29f449 --- /dev/null +++ b/apple/Configs/NoExportedSymbols.exp @@ -0,0 +1,2 @@ +# Exports nothing. dyld finds main through LC_MAIN and a plugin's principal class through the Objective-C runtime, +# and nothing binds to these binaries' symbols, so every public package symbol would only be a dead-strip root. diff --git a/apple/Configs/PluginKit.xcconfig b/apple/Configs/PluginKit.xcconfig index fef1b5c3ce..207ba58516 100644 --- a/apple/Configs/PluginKit.xcconfig +++ b/apple/Configs/PluginKit.xcconfig @@ -1,3 +1,3 @@ // When to raise each, and how the build stamps them: .claude/rules/plugin-system.md, ABI. -TABLEPRO_PLUGINKIT_VERSION = 43 +TABLEPRO_PLUGINKIT_VERSION = 44 TABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19 diff --git a/apple/Native/HanaBridge/internal/hana/bridge.go b/apple/Native/HanaBridge/internal/hana/bridge.go index 64ba0bbf4d..43a6002574 100644 --- a/apple/Native/HanaBridge/internal/hana/bridge.go +++ b/apple/Native/HanaBridge/internal/hana/bridge.go @@ -28,6 +28,21 @@ func (b *Bridge) Explain(sessionID uint64, operationID uint64, requestJSON []byt return encoded, encodedFailure(bridgeFailure) } +func (b *Bridge) OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) { + encoded, bridgeFailure := b.openCursorOnSession(sessionID, operationID, requestJSON) + return encoded, encodedFailure(bridgeFailure) +} + +func (b *Bridge) Fetch(sessionID uint64, operationID uint64) (result []byte, failure []byte) { + encoded, bridgeFailure := b.fetchOnSession(sessionID, operationID) + return encoded, encodedFailure(bridgeFailure) +} + +func (b *Bridge) CloseCursor(sessionID uint64, operationID uint64) (result []byte, failure []byte) { + encoded, bridgeFailure := b.closeCursorOnSession(sessionID, operationID) + return encoded, encodedFailure(bridgeFailure) +} + func (b *Bridge) Ping(sessionID uint64, operationID uint64) (failure []byte) { return encodedFailure(b.pingSession(sessionID, operationID)) } @@ -101,6 +116,34 @@ func (b *Bridge) explainOnSession(sessionID uint64, operationID uint64, requestJ return entry.explain(operationID, request) } +func (b *Bridge) openCursorOnSession(sessionID uint64, operationID uint64, requestJSON []byte) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + request, failure := decodeRequest[cursorRequest](requestJSON) + if failure != nil { + return nil, failure + } + return entry.openCursor(operationID, request) +} + +func (b *Bridge) fetchOnSession(sessionID uint64, operationID uint64) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + return entry.fetch(operationID) +} + +func (b *Bridge) closeCursorOnSession(sessionID uint64, operationID uint64) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + return entry.closeCursor(operationID) +} + func (b *Bridge) pingSession(sessionID uint64, operationID uint64) *bridgeError { entry, failure := b.sessions.lookup(sessionID) if failure != nil { diff --git a/apple/Native/HanaBridge/internal/hana/cursor.go b/apple/Native/HanaBridge/internal/hana/cursor.go new file mode 100644 index 0000000000..949f2b9667 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/cursor.go @@ -0,0 +1,191 @@ +package hana + +import ( + "context" + "database/sql" + "sync/atomic" +) + +const ( + defaultCursorBatchRows = 1_000 + maxCursorBatchRows = 100_000 + cursorBatchBytes = 4 << 20 + noCursorMessage = "no cursor is open for this operation" + cursorParametersMessage = "a cursor takes no parameters" +) + +type cursorRequest struct { + executeRequest + BatchRows int `json:"batchRows"` +} + +func (r cursorRequest) batchRows() int { + if r.BatchRows <= 0 { + return defaultCursorBatchRows + } + return min(r.BatchRows, maxCursorBatchRows) +} + +type cursor struct { + op *operation + rows *sql.Rows + reader *resultReader + maxRows int + maxBytes int +} + +func (c *cursor) nextBatch() (*rowBatch, error) { + batch := &rowBatch{maxRows: c.maxRows, maxBytes: c.maxBytes} + ended, err := c.reader.read(batch) + batch.done = ended + return batch, err +} + +type openedStatement struct { + header *resultEnvelope + rows *sql.Rows + columns []columnInfo +} + +func (s *session) openCursor(operationID uint64, request cursorRequest) ([]byte, *bridgeError) { + if request.hasParameters() { + return nil, internalError(cursorParametersMessage) + } + // Once the result exists a stop sends no CANCEL SESSION: it would mostly reach an idle session, and a refused + // cancel severs the session. The row loop polls the stop flag, and the forced sever covers an unanswered fetch. + var resultReady atomic.Bool + op, conn, failure := s.beginOperation(operationID, func(reason stopReason) { + if !resultReady.Load() { + s.interruptStatement(reason) + } + }) + if failure != nil { + return nil, failure + } + op.stopAfter(secondsDuration(request.TimeoutSeconds)) + opened, err := openStatement(conn, op, request.SQL) + if err != nil { + return nil, s.endOperation(op, err) + } + if opened.rows == nil { + s.slot.finish(op) + opened.header.sessionLost = s.currentState() == sessionLost + return appendCursorOpenedJSON(nil, opened.header, &rowBatch{done: true}), nil + } + resultReady.Store(true) + // The query timeout covers producing the result. Moving its rows can take far longer. + op.disarmDeadline() + current := &cursor{ + op: op, + rows: opened.rows, + reader: &resultReader{rows: opened.rows, columns: opened.columns, op: op}, + maxRows: request.batchRows(), + maxBytes: cursorBatchBytes, + } + batch, failure := s.advance(current) + if failure != nil { + return nil, failure + } + opened.header.sessionLost = s.currentState() == sessionLost + return appendCursorOpenedJSON(nil, opened.header, batch), nil +} + +func (s *session) fetch(operationID uint64) ([]byte, *bridgeError) { + current, failure := s.takeCursor(operationID) + if failure != nil { + return nil, failure + } + batch, failure := s.advance(current) + if failure != nil { + return nil, failure + } + return appendCursorBatchJSON(nil, batch, s.currentState() == sessionLost), nil +} + +func (s *session) closeCursor(operationID uint64) ([]byte, *bridgeError) { + current, failure := s.takeCursor(operationID) + if failure != nil { + return nil, failure + } + _, err := runUnderWatchdog(cancelDeadline, s.loseConnection, current.rows.Close) + if isConnectionFailure(err) { + s.loseConnection() + } + s.slot.finish(current.op) + return appendCursorClosedJSON(nil, s.currentState() == sessionLost), nil +} + +func (s *session) advance(current *cursor) (*rowBatch, *bridgeError) { + batch, err := current.nextBatch() + if err == nil && !batch.done { + if s.park(current) { + return batch, nil + } + err = errSlotClosed + } + err = joinCleanupFailure(err, current.rows.Close()) + if failure := s.endOperation(current.op, err); failure != nil { + return nil, failure + } + return batch, nil +} + +func (s *session) endOperation(op *operation, err error) *bridgeError { + op.settle() + var failure *bridgeError + if err != nil { + failure = s.failure(op, err) + } + s.slot.finish(op) + return failure +} + +func (s *session) takeCursor(operationID uint64) (*cursor, *bridgeError) { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == sessionClosed { + return nil, closedError() + } + current := s.parkedCursor + if current == nil || current.op.id != operationID { + return nil, internalError(noCursorMessage) + } + s.parkedCursor = nil + return current, nil +} + +func (s *session) park(current *cursor) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == sessionClosed { + return false + } + s.parkedCursor = current + return true +} + +func openStatement(conn *sql.Conn, op *operation, statement string) (openedStatement, error) { + if op.stopped() { + return openedStatement{}, errOperationStopped + } + if routesToExec(statement) { + envelope, err := runCountingStatement(conn, statement) + return openedStatement{header: envelope}, err + } + rows, err := conn.QueryContext(context.Background(), statement) + if err != nil { + return openedStatement{}, err + } + found, err := moveToFirstResultSet(rows) + var columns []columnInfo + if err == nil && found { + columns, err = describeColumns(rows) + } + if err != nil || !found { + if err = joinCleanupFailure(err, rows.Close()); err != nil { + return openedStatement{}, err + } + return openedStatement{header: affectedRowsEnvelope(0)}, nil + } + return openedStatement{header: tabularEnvelope(columns), rows: rows, columns: columns}, nil +} diff --git a/apple/Native/HanaBridge/internal/hana/cursor_test.go b/apple/Native/HanaBridge/internal/hana/cursor_test.go new file mode 100644 index 0000000000..caea056889 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/cursor_test.go @@ -0,0 +1,412 @@ +package hana + +import ( + "encoding/json" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +const selectAll = "SELECT ID, NAME FROM T" + +type cursorReply struct { + Columns []string `json:"columns"` + ColumnTypeNames []string `json:"columnTypeNames"` + RowsAffected int64 `json:"rowsAffected"` + HasResultSet bool `json:"hasResultSet"` + Rows [][]*string `json:"rows"` + TruncatedLobCount int `json:"truncatedLobCount"` + Done bool `json:"done"` + SessionLost bool `json:"sessionLost"` + IsTruncated bool `json:"isTruncated"` +} + +func decodeReply(t *testing.T, encoded []byte, failure *bridgeError) cursorReply { + t.Helper() + if failure != nil { + t.Fatalf("the helper answered with a failure: %v", failure) + } + var reply cursorReply + if err := json.Unmarshal(encoded, &reply); err != nil { + t.Fatalf("the reply is not JSON: %v\n%s", err, encoded) + } + return reply +} + +func requireRefused(t *testing.T, encoded []byte, failure *bridgeError, kind string) *bridgeError { + t.Helper() + if failure == nil { + t.Fatalf("the helper answered %s, want a %s failure", encoded, kind) + } + if failure.Kind != kind { + t.Fatalf("the failure is %v, want kind %s", failure, kind) + } + return failure +} + +func cursorOver(sql string, batchRows int) cursorRequest { + return cursorRequest{executeRequest: executeRequest{SQL: sql}, BatchRows: batchRows} +} + +func mustOpen(t *testing.T, entry *session, operationID uint64, request cursorRequest) cursorReply { + t.Helper() + encoded, failure := entry.openCursor(operationID, request) + return decodeReply(t, encoded, failure) +} + +func mustFetch(t *testing.T, entry *session, operationID uint64) cursorReply { + t.Helper() + encoded, failure := entry.fetch(operationID) + return decodeReply(t, encoded, failure) +} + +func mustExecute(t *testing.T, entry *session, operationID uint64, request executeRequest) cursorReply { + t.Helper() + encoded, failure := entry.execute(operationID, request) + return decodeReply(t, encoded, failure) +} + +func requireFetchRefused(t *testing.T, entry *session, operationID uint64) *bridgeError { + t.Helper() + encoded, failure := entry.fetch(operationID) + return requireRefused(t, encoded, failure, kindInternal) +} + +func parkedOperation(entry *session) *operation { + entry.mu.Lock() + defer entry.mu.Unlock() + return entry.parkedCursor.op +} + +func firstColumn(rows [][]*string) []string { + values := make([]string, 0, len(rows)) + for _, row := range rows { + if row[0] == nil { + values = append(values, "NULL") + continue + } + values = append(values, *row[0]) + } + return values +} + +func numbered(count int) []string { + values := make([]string, count) + for index := range values { + values[index] = strconv.Itoa(index) + } + return values +} + +func drain(t *testing.T, entry *session, operationID uint64, opened cursorReply) (batchSizes []int, ids []string) { + t.Helper() + reply := opened + batchSizes = append(batchSizes, len(reply.Rows)) + ids = append(ids, firstColumn(reply.Rows)...) + for !reply.Done { + reply = mustFetch(t, entry, operationID) + batchSizes = append(batchSizes, len(reply.Rows)) + ids = append(ids, firstColumn(reply.Rows)...) + } + return batchSizes, ids +} + +func TestCursorReadsEveryRowInBatchesAndMarksTheLastDone(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + + if !slices.Equal(opened.Columns, []string{"ID", "NAME"}) { + t.Fatalf("columns are %v", opened.Columns) + } + if !slices.Equal(opened.ColumnTypeNames, []string{"INTEGER", "NVARCHAR"}) { + t.Fatalf("column types are %v", opened.ColumnTypeNames) + } + if !opened.HasResultSet || opened.Done { + t.Fatalf("the opened cursor reads hasResultSet %v, done %v", opened.HasResultSet, opened.Done) + } + sizes, ids := drain(t, entry, 1, opened) + if !slices.Equal(sizes, []int{3, 3, 3, 1}) { + t.Fatalf("batch sizes are %v, want [3 3 3 1]", sizes) + } + if !slices.Equal(ids, numbered(10)) { + t.Fatalf("rows arrived as %v", ids) + } + if !server.lastRows().closed.Load() { + t.Fatal("the rows stayed open after the last batch") + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestCursorMarksAnExactMultipleDoneOnItsLastBatch(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(6))) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + if opened.Done { + t.Fatal("the first of two full batches reads done") + } + last := mustFetch(t, entry, 1) + + if len(last.Rows) != 3 || !last.Done { + t.Fatalf("the last batch has %d rows and done %v, want 3 and true", len(last.Rows), last.Done) + } + requireSlotFree(t, entry, 2) +} + +func TestCursorOverNoRowsSendsTheHeaderAndDone(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(0))) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + + if len(opened.Rows) != 0 || !opened.Done || !opened.HasResultSet { + t.Fatalf("an empty result opened with %d rows, done %v, hasResultSet %v", len(opened.Rows), opened.Done, opened.HasResultSet) + } + if !slices.Equal(opened.Columns, []string{"ID", "NAME"}) { + t.Fatalf("columns are %v", opened.Columns) + } + requireSlotFree(t, entry, 2) +} + +func TestCursorEndsABatchAtItsByteBudget(t *testing.T) { + wide := table(10) + wide.name = strings.Repeat("x", 1<<20) + entry := connectedSession(t, newFakeServer(wide)) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 1_000)) + sizes, ids := drain(t, entry, 1, opened) + + if !slices.Equal(sizes, []int{4, 4, 2}) { + t.Fatalf("batch sizes are %v, want [4 4 2] for 1 MiB rows under a 4 MiB budget", sizes) + } + if !slices.Equal(ids, numbered(10)) { + t.Fatalf("rows arrived as %v", ids) + } +} + +func TestStatementWithoutAResultSetOpensNoCursor(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + opened := mustOpen(t, entry, 1, cursorOver("INSERT INTO T VALUES (1, 'a')", 3)) + + if opened.HasResultSet || !opened.Done || opened.RowsAffected != 1 || len(opened.Rows) != 0 { + t.Fatalf("an insert opened as %+v", opened) + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestClosingACursorEarlyClosesItsRowsAndFreesTheSlot(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + encoded, failure := entry.closeCursor(1) + closed := decodeReply(t, encoded, failure) + + if closed.SessionLost { + t.Fatal("closing a cursor lost the session") + } + if !rows.closed.Load() { + t.Fatal("closing the cursor left its rows open") + } + if served := rows.served.Load(); served != 4 { + t.Fatalf("the server handed out %d rows, want the 3 sent and the 1 read ahead", served) + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestStopBetweenFetchesFailsTheNextFetchWithoutCancellingOnTheServer(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + op := parkedOperation(entry) + + entry.cancel(1) + op.inFlight.Wait() + encoded, failure := entry.fetch(1) + + requireRefused(t, encoded, failure, kindCancelled) + if !rows.closed.Load() { + t.Fatal("a stopped cursor left its rows open") + } + for _, statement := range server.executed() { + if strings.HasPrefix(statement, "ALTER SYSTEM CANCEL SESSION") { + t.Fatalf("a stop between fetches sent %q, though no statement was running", statement) + } + } + requireSlotFree(t, entry, 2) +} + +func TestStopDuringAFetchEndsItAtTheNextRowWithoutCancellingOnTheServer(t *testing.T) { + held := table(10) + held.hold = holdRow(4) + server := newFakeServer(held) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + op := parkedOperation(entry) + type outcome struct { + encoded []byte + failure *bridgeError + } + fetched := make(chan outcome, 1) + go func() { + encoded, failure := entry.fetch(1) + fetched <- outcome{encoded: encoded, failure: failure} + }() + select { + case <-held.hold.reached: + case <-time.After(5 * time.Second): + t.Fatal("the fetch never reached the held row") + } + + entry.cancel(1) + op.inFlight.Wait() + close(held.hold.released) + + var result outcome + select { + case result = <-fetched: + case <-time.After(5 * time.Second): + t.Fatal("the stopped fetch never answered") + } + requireRefused(t, result.encoded, result.failure, kindCancelled) + if served := rows.served.Load(); served != 5 { + t.Fatalf("the server handed out %d rows, want the fetch to stop at the row it was reading", served) + } + if !rows.closed.Load() { + t.Fatal("a fetch stopped in flight left the rows open") + } + for _, statement := range server.executed() { + if strings.HasPrefix(statement, "ALTER SYSTEM CANCEL SESSION") { + t.Fatalf("a stop after the result existed sent %q", statement) + } + } + requireSlotFree(t, entry, 2) +} + +func TestFetchThatFailsClosesTheCursor(t *testing.T) { + failing := table(10) + failing.failAt = 5 + server := newFakeServer(failing) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + encoded, failure := entry.fetch(1) + + refused := requireRefused(t, encoded, failure, kindInternal) + if !strings.Contains(refused.Message, errFakeRead.Error()) { + t.Fatalf("the failure reads %q, want the server's read error", refused.Message) + } + if !rows.closed.Load() { + t.Fatal("a failed fetch left the rows open") + } + requireFetchRefused(t, entry, 1) + // The next statement reads a table that no longer fails, so only a held slot can fail it. + server.mu.Lock() + server.table.failAt = -1 + server.mu.Unlock() + requireSlotFree(t, entry, 2) +} + +func TestFetchWithNoOpenCursorIsRefused(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + refused := requireFetchRefused(t, entry, 1) + if refused.Message != noCursorMessage { + t.Fatalf("the refusal reads %q", refused.Message) + } + encoded, failure := entry.closeCursor(1) + requireRefused(t, encoded, failure, kindInternal) + requireSlotFree(t, entry, 2) +} + +func TestFetchForAnotherOperationLeavesTheCursorOpen(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + mustOpen(t, entry, 5, cursorOver(selectAll, 3)) + + requireFetchRefused(t, entry, 6) + + next := mustFetch(t, entry, 5) + if !slices.Equal(firstColumn(next.Rows), []string{"3", "4", "5"}) { + t.Fatalf("the cursor resumed at %v", firstColumn(next.Rows)) + } +} + +func TestCursorRefusesParameters(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + request := cursorOver(selectAll+" WHERE ID = ?", 3) + request.Parameters = &[]cell{textCell("1")} + + encoded, failure := entry.openCursor(1, request) + + requireRefused(t, encoded, failure, kindInternal) + requireSlotFree(t, entry, 2) +} + +func TestQueryTimeoutStopsCoveringTheCursorOnceItIsOpen(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + request := cursorOver(selectAll, 3) + request.TimeoutSeconds = 0.2 + + mustOpen(t, entry, 1, request) + + op := parkedOperation(entry) + op.mu.Lock() + armed := op.deadline != nil + op.mu.Unlock() + if armed { + t.Fatal("the query timeout is still armed after the cursor opened") + } + // Twice the timeout: a deadline left armed would have stopped the cursor by now. + time.Sleep(400 * time.Millisecond) + if op.stopped() { + t.Fatal("the query timeout stopped an open cursor") + } + if next := mustFetch(t, entry, 1); len(next.Rows) != 3 { + t.Fatalf("a fetch past the timeout read %d rows", len(next.Rows)) + } +} + +func TestClosingTheSessionClosesAParkedCursor(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + entry.close() + + deadline := time.Now().Add(5 * time.Second) + for !rows.closed.Load() { + if time.Now().After(deadline) { + t.Fatal("closing the session left the parked cursor's rows open") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestCollectedExecuteStillCutsAtTheRowCap(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + cut := mustExecute(t, entry, 1, executeRequest{SQL: selectAll, RowCap: 4}) + whole := mustExecute(t, entry, 2, executeRequest{SQL: selectAll, RowCap: 10}) + uncapped := mustExecute(t, entry, 3, executeRequest{SQL: selectAll}) + + if !slices.Equal(firstColumn(cut.Rows), numbered(4)) || !cut.IsTruncated { + t.Fatalf("a cap of 4 read %v, truncated %v", firstColumn(cut.Rows), cut.IsTruncated) + } + if len(whole.Rows) != 10 || whole.IsTruncated { + t.Fatalf("a cap equal to the row count read %d rows, truncated %v", len(whole.Rows), whole.IsTruncated) + } + if len(uncapped.Rows) != 10 || uncapped.IsTruncated { + t.Fatalf("no cap read %d rows, truncated %v", len(uncapped.Rows), uncapped.IsTruncated) + } +} diff --git a/apple/Native/HanaBridge/internal/hana/envelope.go b/apple/Native/HanaBridge/internal/hana/envelope.go index 5b5d3a743b..39210549fd 100644 --- a/apple/Native/HanaBridge/internal/hana/envelope.go +++ b/apple/Native/HanaBridge/internal/hana/envelope.go @@ -154,12 +154,8 @@ func tabularEnvelope(columns []columnInfo) *resultEnvelope { } func (e *resultEnvelope) appendJSON(buffer []byte) []byte { - buffer = append(buffer, `{"columns":`...) - buffer = appendStringArray(buffer, e.columns) - buffer = append(buffer, `,"columnTypeNames":`...) - buffer = appendStringArray(buffer, e.columnTypeNames) - buffer = append(buffer, `,"columnClassifications":`...) - buffer = appendOptionalStringArray(buffer, e.columnClassifications) + buffer = append(buffer, '{') + buffer = e.appendColumnFields(buffer) buffer = append(buffer, `,"rows":[`...) for index, row := range e.rows { if index > 0 { @@ -182,6 +178,72 @@ func (e *resultEnvelope) appendJSON(buffer []byte) []byte { return append(buffer, '}') } +func (e *resultEnvelope) appendColumnFields(buffer []byte) []byte { + buffer = append(buffer, `"columns":`...) + buffer = appendStringArray(buffer, e.columns) + buffer = append(buffer, `,"columnTypeNames":`...) + buffer = appendStringArray(buffer, e.columnTypeNames) + buffer = append(buffer, `,"columnClassifications":`...) + return appendOptionalStringArray(buffer, e.columnClassifications) +} + +type rowBatch struct { + maxRows int + maxBytes int + encodedRows []byte + count int + truncatedLobs int + done bool +} + +func (b *rowBatch) full() bool { + return b.count >= b.maxRows || len(b.encodedRows) >= b.maxBytes +} + +func (b *rowBatch) add(row []cell, truncatedLobs int) { + if b.count > 0 { + b.encodedRows = append(b.encodedRows, ',') + } + b.encodedRows = appendRow(b.encodedRows, row) + b.count++ + b.truncatedLobs += truncatedLobs +} + +func appendCursorOpenedJSON(buffer []byte, header *resultEnvelope, batch *rowBatch) []byte { + buffer = append(buffer, '{') + buffer = header.appendColumnFields(buffer) + buffer = append(buffer, `,"rowsAffected":`...) + buffer = strconv.AppendInt(buffer, header.rowsAffected, 10) + buffer = append(buffer, `,"hasResultSet":`...) + buffer = strconv.AppendBool(buffer, header.hasResultSet) + buffer = append(buffer, ',') + buffer = appendBatchFields(buffer, batch, header.sessionLost) + return append(buffer, '}') +} + +func appendCursorBatchJSON(buffer []byte, batch *rowBatch, sessionLost bool) []byte { + buffer = append(buffer, '{') + buffer = appendBatchFields(buffer, batch, sessionLost) + return append(buffer, '}') +} + +func appendBatchFields(buffer []byte, batch *rowBatch, sessionLost bool) []byte { + buffer = append(buffer, `"rows":[`...) + buffer = append(buffer, batch.encodedRows...) + buffer = append(buffer, `],"truncatedLobCount":`...) + buffer = strconv.AppendInt(buffer, int64(batch.truncatedLobs), 10) + buffer = append(buffer, `,"done":`...) + buffer = strconv.AppendBool(buffer, batch.done) + buffer = append(buffer, `,"sessionLost":`...) + return strconv.AppendBool(buffer, sessionLost) +} + +func appendCursorClosedJSON(buffer []byte, sessionLost bool) []byte { + buffer = append(buffer, `{"sessionLost":`...) + buffer = strconv.AppendBool(buffer, sessionLost) + return append(buffer, '}') +} + func appendRow(buffer []byte, row []cell) []byte { buffer = append(buffer, '[') for index, value := range row { diff --git a/apple/Native/HanaBridge/internal/hana/fakedriver_test.go b/apple/Native/HanaBridge/internal/hana/fakedriver_test.go new file mode 100644 index 0000000000..3ce08ef0e0 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/fakedriver_test.go @@ -0,0 +1,212 @@ +package hana + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "io" + "reflect" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +var errFakeRead = errors.New("the fake server failed to read a row") + +type fakeTable struct { + rowCount int + // The row whose read fails, or -1. + failAt int + name string + hold *rowHold +} + +// Holds the read of one row until the test releases it, like a fetch waiting on the server. +type rowHold struct { + at int + reached chan struct{} + released chan struct{} +} + +func holdRow(at int) *rowHold { + return &rowHold{at: at, reached: make(chan struct{}, 1), released: make(chan struct{})} +} + +func table(rowCount int) fakeTable { + return fakeTable{rowCount: rowCount, failAt: -1} +} + +func (t fakeTable) nameOf(index int) string { + if t.name != "" { + return t.name + } + return "row " + strconv.Itoa(index) +} + +type fakeServer struct { + table fakeTable + + mu sync.Mutex + opened []*fakeRows + execs []string +} + +func newFakeServer(table fakeTable) *fakeServer { + return &fakeServer{table: table} +} + +func (f *fakeServer) Connect(context.Context) (driver.Conn, error) { + return &fakeConn{server: f}, nil +} + +func (f *fakeServer) Driver() driver.Driver { + return fakeDriver{} +} + +func (f *fakeServer) open() *fakeRows { + f.mu.Lock() + defer f.mu.Unlock() + rows := &fakeRows{table: f.table} + f.opened = append(f.opened, rows) + return rows +} + +func (f *fakeServer) lastRows() *fakeRows { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.opened) == 0 { + return nil + } + return f.opened[len(f.opened)-1] +} + +func (f *fakeServer) recordExec(statement string) { + f.mu.Lock() + defer f.mu.Unlock() + f.execs = append(f.execs, statement) +} + +func (f *fakeServer) executed() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.execs...) +} + +type fakeDriver struct{} + +func (fakeDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("the fake server opens connections through its connector") +} + +type fakeConn struct { + server *fakeServer +} + +func (c *fakeConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("the fake server prepares no statements") +} + +func (c *fakeConn) Close() error { + return nil +} + +func (c *fakeConn) Begin() (driver.Tx, error) { + return nil, errors.New("the fake server has no transactions") +} + +func (c *fakeConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return c.server.open(), nil +} + +func (c *fakeConn) ExecContext(_ context.Context, statement string, _ []driver.NamedValue) (driver.Result, error) { + c.server.recordExec(statement) + return driver.RowsAffected(1), nil +} + +type fakeRows struct { + table fakeTable + served atomic.Int64 + closed atomic.Bool +} + +func (r *fakeRows) Columns() []string { + return []string{"ID", "NAME"} +} + +func (r *fakeRows) Close() error { + r.closed.Store(true) + return nil +} + +func (r *fakeRows) Next(dest []driver.Value) error { + index := int(r.served.Load()) + if hold := r.table.hold; hold != nil && index == hold.at { + hold.reached <- struct{}{} + <-hold.released + } + if index == r.table.failAt { + return errFakeRead + } + if index >= r.table.rowCount { + return io.EOF + } + dest[0] = int64(index) + dest[1] = r.table.nameOf(index) + r.served.Add(1) + return nil +} + +func (r *fakeRows) ColumnTypeDatabaseTypeName(index int) string { + if index == 0 { + return "INTEGER" + } + return "NVARCHAR" +} + +func (r *fakeRows) ColumnTypeScanType(index int) reflect.Type { + if index == 0 { + return reflect.TypeFor[int64]() + } + return reflect.TypeFor[string]() +} + +func connectedSession(t *testing.T, server *fakeServer) *session { + t.Helper() + db := sql.OpenDB(server) + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("the fake server refused a connection: %v", err) + } + entry := &session{ + dialer: newTrackingDialer(nil, nil), + db: db, + controlDB: sql.OpenDB(server), + state: sessionConnected, + conn: conn, + connectionID: 1, + } + entry.slot = newOperationSlot(ForcedSeverGrace, entry.loseConnection) + t.Cleanup(entry.close) + return entry +} + +// A slot the cursor still held would block this statement forever. +func requireSlotFree(t *testing.T, entry *session, operationID uint64) { + t.Helper() + finished := make(chan *bridgeError, 1) + go func() { + _, failure := entry.execute(operationID, executeRequest{SQL: "SELECT ID, NAME FROM T"}) + finished <- failure + }() + select { + case failure := <-finished: + if failure != nil { + t.Fatalf("a statement after the cursor failed: %v", failure) + } + case <-time.After(5 * time.Second): + t.Fatal("the cursor still holds the operation slot") + } +} diff --git a/apple/Native/HanaBridge/internal/hana/opslot.go b/apple/Native/HanaBridge/internal/hana/opslot.go index 5afce97c6a..174fbfee15 100644 --- a/apple/Native/HanaBridge/internal/hana/opslot.go +++ b/apple/Native/HanaBridge/internal/hana/opslot.go @@ -154,6 +154,16 @@ func (o *operation) stopAfter(timeout time.Duration) { o.deadline = time.AfterFunc(timeout, func() { o.requestStop(stopTimedOut) }) } +// Unlike settle, the operation stays stoppable: a cursor still answers Stop after its timeout is gone. +func (o *operation) disarmDeadline() { + o.mu.Lock() + defer o.mu.Unlock() + if o.deadline != nil { + o.deadline.Stop() + o.deadline = nil + } +} + func (o *operation) settle() { o.mu.Lock() o.settled = true diff --git a/apple/Native/HanaBridge/internal/hana/session.go b/apple/Native/HanaBridge/internal/hana/session.go index 4e900bff2b..4e66ff3639 100644 --- a/apple/Native/HanaBridge/internal/hana/session.go +++ b/apple/Native/HanaBridge/internal/hana/session.go @@ -64,6 +64,8 @@ type session struct { state sessionState conn *sql.Conn connectionID int64 + // Set only between fetches; a fetch takes it out while it reads. + parkedCursor *cursor controlMu sync.Mutex control *sql.Conn @@ -267,7 +269,11 @@ func (s *session) adopt(conn *sql.Conn, connectionID int64) *bridgeError { } func (s *session) beginStatement(operationID uint64) (*operation, *sql.Conn, *bridgeError) { - op, err := s.slot.begin(operationID, s.interruptStatement) + return s.beginOperation(operationID, s.interruptStatement) +} + +func (s *session) beginOperation(operationID uint64, interrupt interruptFunc) (*operation, *sql.Conn, *bridgeError) { + op, err := s.slot.begin(operationID, interrupt) if err != nil { return nil, nil, s.failure(nil, err) } @@ -380,16 +386,23 @@ func (s *session) close() { s.state = sessionClosed conn := s.conn s.conn = nil + parked := s.parkedCursor + s.parkedCursor = nil s.mu.Unlock() s.slot.close() s.dialer.sever() - go s.release(conn) + go s.release(conn, parked) } -func (s *session) release(conn *sql.Conn) { +func (s *session) release(conn *sql.Conn, parked *cursor) { defer func() { _ = recover() }() + // Open rows hold the connection, so it cannot close before them. + if parked != nil { + _ = parked.rows.Close() + s.slot.finish(parked.op) + } if conn != nil { _ = conn.Close() } diff --git a/apple/Native/HanaBridge/internal/hana/statement.go b/apple/Native/HanaBridge/internal/hana/statement.go index a43e53deeb..5265d9176f 100644 --- a/apple/Native/HanaBridge/internal/hana/statement.go +++ b/apple/Native/HanaBridge/internal/hana/statement.go @@ -152,19 +152,27 @@ func rowsAffected(result sql.Result) int64 { } func readFirstResultSet(rows *sql.Rows, op *operation, rowLimit int) (*resultEnvelope, error) { + found, err := moveToFirstResultSet(rows) + if err != nil { + return nil, err + } + if !found { + return affectedRowsEnvelope(0), nil + } + return readResultSet(rows, op, rowLimit) +} + +func moveToFirstResultSet(rows *sql.Rows) (bool, error) { for { columns, err := rows.Columns() if err != nil { - return nil, err + return false, err } if len(columns) > 0 { - return readResultSet(rows, op, rowLimit) + return true, nil } if !rows.NextResultSet() { - if err := rows.Err(); err != nil { - return nil, err - } - return affectedRowsEnvelope(0), nil + return false, rows.Err() } } } @@ -175,25 +183,68 @@ func readResultSet(rows *sql.Rows, op *operation, rowLimit int) (*resultEnvelope return nil, err } envelope := tabularEnvelope(columns) - for rows.Next() { - if op.stopped() { - return nil, errOperationStopped + reader := &resultReader{rows: rows, columns: columns, op: op} + collected := &collectedRows{limit: rowLimit} + ended, err := reader.read(collected) + if err != nil { + return nil, err + } + envelope.rows = collected.rows + envelope.isTruncated = !ended + envelope.truncatedLobCount = collected.truncatedLobs + return envelope, nil +} + +type rowSink interface { + full() bool + add(row []cell, truncatedLobs int) +} + +type resultReader struct { + rows *sql.Rows + columns []columnInfo + op *operation + // Next has moved to a row that no sink has taken yet, so the next read starts with it. + pending bool +} + +// The bool is true once the result set has no rows left. +func (r *resultReader) read(sink rowSink) (bool, error) { + for { + if !r.pending { + if !r.rows.Next() { + return true, r.rows.Err() + } + r.pending = true } - if rowLimit > 0 && len(envelope.rows) >= rowLimit { - envelope.isTruncated = true - break + if r.op.stopped() { + return false, errOperationStopped } - row, truncatedLobs, err := scanRow(rows, columns, op) + if sink.full() { + return false, nil + } + row, truncatedLobs, err := scanRow(r.rows, r.columns, r.op) if err != nil { - return nil, err + return false, err } - envelope.rows = append(envelope.rows, row) - envelope.truncatedLobCount += truncatedLobs - } - if err := rows.Err(); err != nil { - return nil, err + r.pending = false + sink.add(row, truncatedLobs) } - return envelope, nil +} + +type collectedRows struct { + limit int + rows [][]cell + truncatedLobs int +} + +func (c *collectedRows) full() bool { + return c.limit > 0 && len(c.rows) >= c.limit +} + +func (c *collectedRows) add(row []cell, truncatedLobs int) { + c.rows = append(c.rows, row) + c.truncatedLobs += truncatedLobs } func describeColumns(rows *sql.Rows) (columns []columnInfo, err error) { diff --git a/apple/Native/HanaBridge/server.go b/apple/Native/HanaBridge/server.go index 511a302c9c..0645971207 100644 --- a/apple/Native/HanaBridge/server.go +++ b/apple/Native/HanaBridge/server.go @@ -13,16 +13,19 @@ import ( "github.com/TableProApp/TablePro/Native/HanaBridge/internal/hana" ) -const protocolVersion = 1 +const protocolVersion = 2 const ( - opcodeOpen byte = 1 - opcodeConnect byte = 2 - opcodeExecute byte = 3 - opcodeExplain byte = 4 - opcodePing byte = 5 - opcodeCancel byte = 6 - opcodeClose byte = 7 + opcodeOpen byte = 1 + opcodeConnect byte = 2 + opcodeExecute byte = 3 + opcodeExplain byte = 4 + opcodePing byte = 5 + opcodeCancel byte = 6 + opcodeClose byte = 7 + opcodeOpenCursor byte = 8 + opcodeFetch byte = 9 + opcodeCloseCursor byte = 10 ) const ( @@ -46,6 +49,9 @@ type backend interface { Connect(sessionID uint64, operationID uint64) (result []byte, failure []byte) Execute(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) Explain(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) + OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) + Fetch(sessionID uint64, operationID uint64) (result []byte, failure []byte) + CloseCursor(sessionID uint64, operationID uint64) (result []byte, failure []byte) Ping(sessionID uint64, operationID uint64) (failure []byte) Cancel(sessionID uint64, operationID uint64) Close(sessionID uint64) @@ -119,6 +125,12 @@ func (s *server) dispatch(request frame.Frame) { s.cancelOperation(request) case opcodeClose: s.closeSession(request) + case opcodeOpenCursor: + s.start(request, s.openCursor) + case opcodeFetch: + s.start(request, s.fetch) + case opcodeCloseCursor: + s.start(request, s.closeCursor) default: s.reply(request.ID, nil, hana.InternalFailure(fmt.Sprintf("unknown opcode %d", request.Code))) } @@ -157,6 +169,18 @@ func (s *server) explain(body requestBody) ([]byte, []byte) { return s.backend.Explain(body.Session, body.Operation, body.Request) } +func (s *server) openCursor(body requestBody) ([]byte, []byte) { + return s.backend.OpenCursor(body.Session, body.Operation, body.Request) +} + +func (s *server) fetch(body requestBody) ([]byte, []byte) { + return s.backend.Fetch(body.Session, body.Operation) +} + +func (s *server) closeCursor(body requestBody) ([]byte, []byte) { + return s.backend.CloseCursor(body.Session, body.Operation) +} + func (s *server) ping(body requestBody) ([]byte, []byte) { if failure := s.backend.Ping(body.Session, body.Operation); failure != nil { return nil, failure diff --git a/apple/Native/HanaBridge/server_test.go b/apple/Native/HanaBridge/server_test.go new file mode 100644 index 0000000000..4eab00f27c --- /dev/null +++ b/apple/Native/HanaBridge/server_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "encoding/json" + "io" + "slices" + "sync" + "testing" + + "github.com/TableProApp/TablePro/Native/HanaBridge/internal/frame" +) + +type recordedCall struct { + name string + session uint64 + operation uint64 + request string +} + +type recordingBackend struct { + mu sync.Mutex + calls []recordedCall +} + +func (b *recordingBackend) record(call recordedCall) { + b.mu.Lock() + defer b.mu.Unlock() + b.calls = append(b.calls, call) +} + +func (b *recordingBackend) recorded() []recordedCall { + b.mu.Lock() + defer b.mu.Unlock() + return append([]recordedCall(nil), b.calls...) +} + +func (b *recordingBackend) Open([]byte) (uint64, []byte) { + return 1, nil +} + +func (b *recordingBackend) Connect(uint64, uint64) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) Execute(uint64, uint64, []byte) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) Explain(uint64, uint64, []byte) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) ([]byte, []byte) { + b.record(recordedCall{name: "openCursor", session: sessionID, operation: operationID, request: string(requestJSON)}) + return []byte(`{"done":false}`), nil +} + +func (b *recordingBackend) Fetch(sessionID uint64, operationID uint64) ([]byte, []byte) { + b.record(recordedCall{name: "fetch", session: sessionID, operation: operationID}) + return []byte(`{"done":true}`), nil +} + +func (b *recordingBackend) CloseCursor(sessionID uint64, operationID uint64) ([]byte, []byte) { + b.record(recordedCall{name: "closeCursor", session: sessionID, operation: operationID}) + return nil, []byte(`{"kind":"internal","message":"no cursor"}`) +} + +func (b *recordingBackend) Ping(uint64, uint64) []byte { + return nil +} + +func (b *recordingBackend) Cancel(uint64, uint64) {} + +func (b *recordingBackend) Close(uint64) {} + +func (b *recordingBackend) CloseAll() {} + +func readReply(t *testing.T, replies io.Reader) frame.Frame { + t.Helper() + reply, err := frame.Read(replies, frame.MaxBodyLength) + if err != nil { + t.Fatalf("reading a reply failed: %v", err) + } + return reply +} + +func exchange(t *testing.T, requests io.Writer, replies io.Reader, request frame.Frame) frame.Frame { + t.Helper() + if err := frame.Write(requests, request); err != nil { + t.Fatalf("writing frame %d failed: %v", request.ID, err) + } + reply := readReply(t, replies) + if reply.ID != request.ID { + t.Fatalf("the reply answers frame %d, want %d", reply.ID, request.ID) + } + return reply +} + +func TestServerRoutesCursorOperationsAndAnnouncesProtocolTwo(t *testing.T) { + requestReader, requests := io.Pipe() + replies, replyWriter := io.Pipe() + backend := &recordingBackend{} + exited := make(chan int, 1) + go func() { + exited <- run(requestReader, replyWriter, backend) + _ = replyWriter.Close() + }() + + var greeting struct { + Protocol int `json:"protocol"` + } + if err := json.Unmarshal(readReply(t, replies).Body, &greeting); err != nil || greeting.Protocol != 2 { + t.Fatalf("the greeting names protocol %d (%v), want 2", greeting.Protocol, err) + } + + opened := exchange(t, requests, replies, frame.Frame{ID: 1, Code: opcodeOpenCursor, Body: []byte( + `{"session":4,"operation":9,"request":{"sql":"SELECT 1","batchRows":5}}`, + )}) + fetched := exchange(t, requests, replies, frame.Frame{ID: 2, Code: opcodeFetch, Body: []byte(`{"session":4,"operation":9}`)}) + closed := exchange(t, requests, replies, frame.Frame{ID: 3, Code: opcodeCloseCursor, Body: []byte(`{"session":4,"operation":9}`)}) + _ = requests.Close() + + if opened.Code != statusOK || string(opened.Body) != `{"done":false}` { + t.Fatalf("open answered %d %s", opened.Code, opened.Body) + } + if fetched.Code != statusOK || string(fetched.Body) != `{"done":true}` { + t.Fatalf("fetch answered %d %s", fetched.Code, fetched.Body) + } + if closed.Code != statusError { + t.Fatalf("a refused close answered status %d", closed.Code) + } + if code := <-exited; code != exitInputClosed { + t.Fatalf("the server exited with %d", code) + } + want := []recordedCall{ + {name: "openCursor", session: 4, operation: 9, request: `{"sql":"SELECT 1","batchRows":5}`}, + {name: "fetch", session: 4, operation: 9}, + {name: "closeCursor", session: 4, operation: 9}, + } + if got := backend.recorded(); !slices.Equal(got, want) { + t.Fatalf("the backend saw %+v, want %+v", got, want) + } +} diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 7b85a5049e..e32362d417 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -36,10 +36,13 @@ let package = Package( .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), + .library(name: "TableProWeaviateClient", targets: ["TableProWeaviateClient"]), .library(name: "TableProQdrantCore", targets: ["TableProQdrantCore"]), + .library(name: "TableProQdrantClient", targets: ["TableProQdrantClient"]), .library(name: "TableProInfluxDBCore", targets: ["TableProInfluxDBCore"]), .library(name: "TableProNeo4jCore", targets: ["TableProNeo4jCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), + .library(name: "TableProFirebaseClient", targets: ["TableProFirebaseClient"]), .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), .library(name: "TableProDocumentPath", targets: ["TableProDocumentPath"]), @@ -205,12 +208,25 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + // Only the plugin links this; the app tiers statements with TableProWeaviateCore alone. + .target( + name: "TableProWeaviateClient", + dependencies: ["TableProWeaviateCore"], + path: "Sources/TableProWeaviateClient" + ), .target( name: "TableProQdrantCore", dependencies: [], path: "Sources/TableProQdrantCore", swiftSettings: pluginCoreSettings ), + // Only the plugin links this; the app tiers statements with TableProQdrantCore alone. + .target( + name: "TableProQdrantClient", + dependencies: ["TableProQdrantCore"], + path: "Sources/TableProQdrantClient", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProInfluxDBCore", dependencies: [], @@ -228,9 +244,15 @@ let package = Package( ), .target( name: "TableProFirebaseCore", - dependencies: ["TableProGoogleCloud"], + dependencies: [], path: "Sources/TableProFirebaseCore" ), + // Only the plugin links this; the app tiers statements with TableProFirebaseCore alone. + .target( + name: "TableProFirebaseClient", + dependencies: ["TableProFirebaseCore", "TableProGoogleCloud"], + path: "Sources/TableProFirebaseClient" + ), .target( name: "TableProR2SQLCore", dependencies: ["TableProSQLQuoting"], @@ -252,7 +274,8 @@ let package = Package( dependencies: [], path: "Sources/TableProSQLQuoting" ), - // Starts every child process outside PluginKit. Foundation only, so it builds on Linux too. + // Starts every child process outside PluginKit and reads and writes its descriptors. Foundation only, with the + // descriptor code behind canImport(Darwin), so it builds on Linux too. .target( name: "TableProProcess", dependencies: [], @@ -388,13 +411,13 @@ let package = Package( path: "Tests/TableProNeo4jCoreTests" ), .testTarget( - name: "TableProWeaviateCoreTests", - dependencies: ["TableProWeaviateCore", "TableProTLSTestFixtures"], - path: "Tests/TableProWeaviateCoreTests" + name: "TableProWeaviateClientTests", + dependencies: ["TableProWeaviateClient", "TableProWeaviateCore", "TableProTLSTestFixtures"], + path: "Tests/TableProWeaviateClientTests" ), .testTarget( name: "TableProFirebaseCoreTests", - dependencies: ["TableProFirebaseCore", "TableProGoogleCloud"], + dependencies: ["TableProFirebaseCore", "TableProFirebaseClient", "TableProGoogleCloud"], path: "Tests/TableProFirebaseCoreTests" ), .testTarget( @@ -409,7 +432,7 @@ let package = Package( ), .testTarget( name: "TableProQdrantCoreTests", - dependencies: ["TableProQdrantCore"], + dependencies: ["TableProQdrantCore", "TableProQdrantClient"], path: "Tests/TableProQdrantCoreTests", swiftSettings: pluginCoreSettings ), @@ -514,7 +537,7 @@ let package = Package( ), .testTarget( name: "TableProProcessTests", - dependencies: ["TableProProcess"], + dependencies: ["TableProProcess", "TableProTestSupport"], path: "Tests/TableProProcessTests" ), .testTarget( @@ -541,7 +564,7 @@ let package = Package( ), .testTarget( name: "TableProTLSClientIdentityTests", - dependencies: ["TableProTLSClientIdentity", "TableProTLSTestFixtures"], + dependencies: ["TableProTLSClientIdentity", "TableProTLSTestFixtures", "TableProTestSupport"], path: "Tests/TableProTLSClientIdentityTests" ), .testTarget( diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift index eb0a50ead9..2f6b38273a 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift @@ -446,6 +446,135 @@ struct DerivedTableParser { return target.reference } + // MARK: - Join scope + + struct JoinSource: Equatable { + let table: String + let schema: String? + let alias: String? + // Whether the name the statement exposes it by, the alias or else the table, was quoted. + let isQuoted: Bool + } + + private struct FromItemNames { + let sources: [JoinSource] + // One stored table named directly, not a join group, a derived table or a table function. + let isTable: Bool + let end: Int + } + + // Walking back from an ON, a clause word means it belongs to no FROM list, as in CREATE INDEX, GRANT or a trigger. + private static let joinScopeBreakers: Set = [ + "SELECT", "WHERE", "GROUP", "HAVING", "ORDER", "LIMIT", "OFFSET", "FETCH", "SET", "VALUES", "RETURNING", "INTO", + "UNION", "INTERSECT", "EXCEPT", "MINUS", "WINDOW", "QUALIFY", "INSERT", "DELETE", "MERGE", "CREATE", "ALTER", + "DROP", "GRANT", "REVOKE" + ] + + // Set only when the ON is the token right before `cursor` and follows a table joined by JOIN. The earlier tables + // are the ones before it since the last comma of its FROM list or join group, which is all such an ON may name. + func joinScope(in stream: SQLTokenStream, cursor: Int) -> (joined: JoinSource, earlier: [JoinSource])? { + guard let on = stream.tokens.lastIndex(where: { $0.range.location < cursor }), stream.isWord(on, "ON"), + stream.kind(on - 1) != .comma, !stream.isWord(on - 1, in: Self.joinWords), + let start = joinListStart(before: on, cursor: cursor, in: stream) else { return nil } + var earlier: [JoinSource] = [] + var isJoined = false + var index = start + while index < on { + let item = fromItemNames(at: index, to: on, in: stream) + let next = nextFromItem(after: item.end, to: on, in: stream).index + guard next < on else { + guard isJoined, item.isTable, let joined = item.sources.first, + firstTopLevel(["ON"], from: item.end, to: on, in: stream) == nil else { return nil } + return (joined, Array(earlier.reversed())) + } + if stream.kind(next - 1) == .comma { + earlier = [] + isJoined = false + } else { + earlier += item.sources + isJoined = stream.isWord(next - 1, "JOIN") || stream.isWord(next - 1, "STRAIGHT_JOIN") + } + index = next + } + return nil + } + + // The first item of the FROM list holding `on` at the cursor's query level, or of the join group around it. + private func joinListStart(before on: Int, cursor: Int, in stream: SQLTokenStream) -> Int? { + var floor = 0 + if let open = stream.enclosingParens(before: cursor).last { + guard isStatementBody(open, in: stream) else { + return opensFromItem(open, in: stream) ? open + 1 : nil + } + floor = open + 1 + } + var index = on - 1 + while index >= floor { + let token = stream.tokens[index] + if token.kind == .closeParen, token.partner >= floor { + index = token.partner - 1 + continue + } + // After a dot a word is a name, as in `a.set`. + if token.kind == .word, stream.kind(index - 1) != .dot { + switch token.upper { + case "FROM" where !stream.isWord(index - 1, "DISTINCT"), "UPDATE": + return index + 1 + case "USING" where stream.kind(index + 1) != .openParen: + return index + 1 + case let word where Self.joinScopeBreakers.contains(word): + return nil + default: + break + } + } + index -= 1 + } + return nil + } + + private func fromItemNames(at start: Int, to end: Int, in stream: SQLTokenStream) -> FromItemNames { + var index = start + while stream.isWord(index, "LATERAL") || stream.isWord(index, "ONLY") { index += 1 } + guard index < end else { return FromItemNames(sources: [], isTable: false, end: end) } + if stream.kind(index) == .openParen { + let close = stream.after(index) - 1 + guard !isQueryBody(index, in: stream) else { + return FromItemNames(sources: [], isTable: false, end: alias(after: close, in: stream)?.end ?? close + 1) + } + return FromItemNames(sources: groupNames(from: index + 1, to: min(close, end), in: stream), isTable: false, end: close + 1) + } + guard let path = path(at: index, to: end, in: stream) else { + return FromItemNames(sources: [], isTable: false, end: index + 1) + } + index = path.end + if stream.kind(index) == .openParen { + let close = stream.after(index) - 1 + return FromItemNames(sources: [], isTable: false, end: alias(after: close, in: stream)?.end ?? close + 1) + } + if stream.kind(index) == .star { index += 1 } + var source = JoinSource(table: path.table, schema: path.schema, alias: nil, isQuoted: path.isQuoted) + if let alias = alias(after: index - 1, in: stream) { + let aliasIndex = stream.isWord(index, "AS") ? index + 1 : index + source = JoinSource( + table: path.table, schema: path.schema, alias: alias.name, isQuoted: stream.kind(aliasIndex) == .quoted + ) + index = alias.end + } + return FromItemNames(sources: [source], isTable: true, end: index) + } + + private func groupNames(from start: Int, to end: Int, in stream: SQLTokenStream) -> [JoinSource] { + var names: [JoinSource] = [] + var index = start + while index < end { + let item = fromItemNames(at: index, to: end, in: stream) + names += item.sources + index = nextFromItem(after: item.end, to: end, in: stream).index + } + return names + } + // MARK: - Definitions // One query level. A CTE of its WITH list is in scope from the next definition on, and only a level that holds the diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift index 0cdf7d7599..e2a213f2df 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift @@ -14,10 +14,18 @@ public protocol SQLCompletionCatalog: Sendable { func allowedValues(forColumn column: String, in references: [TableReference]) async -> [String] func allColumnsInScope(for references: [TableReference]) async -> [SQLCompletionItem] func allColumnsFromCachedTables() async -> [SQLCompletionItem] + // The table's own keys, none pointing outside its schema. + func foreignKeys(of tableName: String, schema: String?) async -> [SQLCompletionForeignKey] + // For a surface that spells nothing after ranking; nil leaves the name as stored and written. + func writtenText(of identifier: SQLCompletionIdentifier) -> String? } public extension SQLCompletionCatalog { func tableCompletionItems(inSchema schema: String, isQuoted: Bool) async -> [SQLCompletionItem] { await tableCompletionItems(inSchema: schema) } + + func foreignKeys(of tableName: String, schema: String?) async -> [SQLCompletionForeignKey] { [] } + + func writtenText(of identifier: SQLCompletionIdentifier) -> String? { nil } } diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift index b5892646b0..9df5182b68 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift @@ -10,6 +10,7 @@ public enum SQLCompletionKind: String, CaseIterable, Sendable { case alias case `operator` case favorite // Saved SQL favorite (keyword expansion) + case joinCondition public var iconName: String { switch self { @@ -22,12 +23,14 @@ public enum SQLCompletionKind: String, CaseIterable, Sendable { case .alias: return "a.circle.fill" case .operator: return "equal.circle.fill" case .favorite: return "star.circle.fill" + case .joinCondition: return "link.circle.fill" } } /// Lower sorts first within the same context. public var basePriority: Int { switch self { + case .joinCondition: return 0 case .favorite: return 50 case .column: return 100 case .table: return 200 @@ -57,6 +60,8 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { public let caseFolding: SQLCompletionCaseFolding // Set on a name the catalog stores, so the connection's identifier rules can spell it. public let identifier: SQLCompletionIdentifier? + // The `a.x = b.y` pairs of a join condition, so each name is spelled like any other. + public let columnPairs: [SQLCompletionColumnPair] public var matchedRanges: [Range] = [] public var fuzzyPenalty: Int = 0 // `o.total` names a table and a column; a column literally named `order.total` is one name. @@ -71,7 +76,8 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { sortPriority: Int? = nil, filterText: String? = nil, caseFolding: SQLCompletionCaseFolding = .fixed, - identifier: SQLCompletionIdentifier? = nil + identifier: SQLCompletionIdentifier? = nil, + columnPairs: [SQLCompletionColumnPair] = [] ) { self.id = UUID() self.label = label @@ -83,6 +89,7 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { self.filterText = filterText ?? label.lowercased() self.caseFolding = caseFolding self.identifier = identifier + self.columnPairs = columnPairs } private init(recasing item: SQLCompletionItem, label: String, insertText: String) { @@ -96,6 +103,7 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { self.filterText = item.filterText self.caseFolding = item.caseFolding self.identifier = item.identifier + self.columnPairs = item.columnPairs self.matchedRanges = item.matchedRanges self.fuzzyPenalty = item.fuzzyPenalty self.isQualifiedReference = item.isQualifiedReference diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift index 4085547445..6c839f856a 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift @@ -258,27 +258,9 @@ public final class SQLCompletionProvider { } case .on: + items += await joinConditionItems(for: context.joinScope) items += await allowedValueItems(for: context) - items += await columnItems(for: context.tableReferences) - for ref in context.tableReferences { - let qualifier = ref.alias ?? ref.tableName - let cols = await catalog?.columnCompletionItems(for: ref.tableName, schema: ref.schema) ?? [] - for col in cols { - var item = SQLCompletionItem( - label: "\(qualifier).\(col.label)", - kind: .column, - insertText: "\(qualifier).\(col.label)", - detail: col.detail, - documentation: "Column from \(ref.tableName)", - sortPriority: 80, - identifier: col.identifier.map { - SQLCompletionIdentifier([Self.qualifierPart(for: ref)] + $0.parts) - } - ) - item.isQualifiedReference = true - items.append(item) - } - } + items += await qualifiedColumnItems(for: context.tableReferences) items += SQLKeywords.operatorItems() items += dialectOperatorItems() items += filterKeywords([ @@ -578,6 +560,85 @@ public final class SQLCompletionProvider { return await catalog?.allColumnsInScope(for: resolved) ?? [] } + // A catalog that already qualifies a column once two tables are in scope keeps its own item, so the name is + // offered once and with the catalog's spelling. + private func qualifiedColumnItems(for references: [TableReference]) async -> [SQLCompletionItem] { + var items = await columnItems(for: references) + var indexByLabel: [String: Int] = [:] + for (index, item) in items.enumerated() where item.kind == .column && indexByLabel[item.label] == nil { + indexByLabel[item.label] = index + } + for ref in references { + let cols = await catalog?.columnCompletionItems(for: ref.tableName, schema: ref.schema) ?? [] + for col in cols { + let label = "\(ref.identifier).\(col.label)" + if let index = indexByLabel[label] { + items[index].sortPriority = min(items[index].sortPriority, Self.qualifiedColumnPriority) + continue + } + let parts = [Self.qualifierPart(for: ref)] + (col.identifier?.parts ?? [.catalog(col.label)]) + var item = SQLCompletionItem( + label: label, + kind: .column, + insertText: catalog?.writtenText(of: SQLCompletionIdentifier(parts)) ?? "\(ref.identifier).\(col.insertText)", + detail: col.detail, + documentation: "Column from \(ref.tableName)", + sortPriority: Self.qualifiedColumnPriority, + identifier: col.identifier.map { SQLCompletionIdentifier([Self.qualifierPart(for: ref)] + $0.parts) } + ) + item.isQualifiedReference = true + indexByLabel[label] = items.count + items.append(item) + } + } + return items + } + + private static let qualifiedColumnPriority = 80 + + private func joinConditionItems(for scope: SQLJoinScope?) async -> [SQLCompletionItem] { + guard let scope, let catalog else { return [] } + let joined = await joinSide(scope.joined, in: catalog) + var earlier: [SQLJoinConditions.Side] = [] + for reference in scope.earlier { + earlier.append(await joinSide(reference, in: catalog)) + } + var seen = Set() + return SQLJoinConditions.conditions(joined: joined, earlier: earlier).compactMap { condition in + let item = joinConditionItem(condition, joined: scope.joined, catalog: catalog) + return seen.insert(item.label).inserted ? item : nil + } + } + + private func joinSide(_ reference: TableReference, in catalog: any SQLCompletionCatalog) async -> SQLJoinConditions.Side { + SQLJoinConditions.Side( + reference: reference, + keys: await catalog.foreignKeys(of: reference.tableName, schema: reference.schema), + columns: await catalog.columnCompletionItems(for: reference.tableName, schema: reference.schema).map(\.label) + ) + } + + private func joinConditionItem( + _ condition: SQLJoinConditions.Condition, + joined: TableReference, + catalog: any SQLCompletionCatalog + ) -> SQLCompletionItem { + let pairs = condition.columns.map { columns in + SQLCompletionColumnPair( + left: SQLCompletionIdentifier([Self.qualifierPart(for: condition.earlier), .catalog(columns.earlier)]), + right: SQLCompletionIdentifier([Self.qualifierPart(for: joined), .catalog(columns.joined)]) + ) + } + let written = { (name: SQLCompletionIdentifier) in catalog.writtenText(of: name) ?? name.plainText } + return SQLCompletionItem( + label: SQLCompletionColumnPair.condition(pairs.map { (left: $0.left.plainText, right: $0.right.plainText) }), + kind: .joinCondition, + insertText: SQLCompletionColumnPair.condition(pairs.map { (left: written($0.left), right: written($0.right)) }), + detail: condition.constraintName.isEmpty ? nil : condition.constraintName, + columnPairs: pairs + ) + } + // ORDER BY sorts by an output alias before a source column of the same name, so the alias leads. private func outputAliasItems( _ spellings: [String], besides columns: [SQLCompletionItem], insideQuotes: Bool @@ -843,7 +904,7 @@ public final class SQLCompletionProvider { } case .select, .where_, .and, .on, .having, .groupBy, .orderBy, .returning, .using, .window: - if item.kind == .column { + if item.kind == .column || item.kind == .joinCondition { score -= 200 } case .set, .insertColumns: diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift index 1266a5c3fd..e47d27274d 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift @@ -130,6 +130,7 @@ public struct SQLContext: Sendable { public private(set) var quotedName: SQLQuotedName? public let outputAliases: [String] // SELECT-list aliases an ORDER BY at the cursor may name, as written public let targetTable: TableReference? // The table the INSERT, UPDATE or MERGE at the cursor writes + public let joinScope: SQLJoinScope? // Right after the ON of a join public init( clauseType: SQLClauseType, @@ -148,7 +149,8 @@ public struct SQLContext: Sendable { comparisonColumn: String? = nil, quotedName: SQLQuotedName? = nil, outputAliases: [String] = [], - targetTable: TableReference? = nil + targetTable: TableReference? = nil, + joinScope: SQLJoinScope? = nil ) { self.clauseType = clauseType self.prefix = prefix @@ -167,6 +169,7 @@ public struct SQLContext: Sendable { self.quotedName = quotedName self.outputAliases = outputAliases self.targetTable = targetTable + self.joinScope = joinScope } public func replacingTableReferences(_ references: [TableReference]) -> SQLContext { @@ -390,6 +393,11 @@ public final class SQLContextAnalyzer { writtenTable($0, among: tableReferences) } : nil + let joinScope = resolution.clause == .on && dotPrefix == nil && quotedName == nil && !isCastTarget + ? Self.derivedTableParser.joinScope(in: tokens, cursor: clauseEnd).flatMap { + SQLJoinScope($0, references: tableReferences, cteNames: cteNames) + } + : nil return SQLContext( clauseType: isCastTarget ? .castTarget : resolution.clause, @@ -408,7 +416,8 @@ public final class SQLContextAnalyzer { comparisonColumn: comparisonColumn, quotedName: quotedName, outputAliases: outputAliases, - targetTable: targetTable + targetTable: targetTable, + joinScope: joinScope ) } diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift index 172e2763a5..2e394ac414 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift @@ -53,6 +53,31 @@ public struct SQLCompletionIdentifier: Sendable, Hashable { public static func catalog(_ name: String) -> SQLCompletionIdentifier { SQLCompletionIdentifier([.catalog(name)]) } + + // Each part as stored or as written, with no quotes. + public var plainText: String { + parts.map { part in + switch part { + case .catalog(let name): return name + case .statement(let text, _): return text + } + }.joined(separator: ".") + } +} + +public struct SQLCompletionColumnPair: Sendable, Hashable { + public let left: SQLCompletionIdentifier + public let right: SQLCompletionIdentifier + + public init(left: SQLCompletionIdentifier, right: SQLCompletionIdentifier) { + self.left = left + self.right = right + } + + // One equality per pair, in key order. + public static func condition(_ sides: [(left: String, right: String)]) -> String { + sides.map { "\($0.left) = \($0.right)" }.joined(separator: " AND ") + } } // Whether the token being completed already opened a quoted name, and whether that quote is closed. @@ -127,6 +152,17 @@ public struct SQLIdentifierSpelling: Sendable { typedPrefix: String, quoting: SQLIdentifierTokenQuoting ) -> SQLCompletionItem { + if !item.columnPairs.isEmpty { + // The typed text is a qualifier, which says nothing about how the columns are cased. + guard quoting == .none else { return item } + let sides = item.columnPairs.map { pair in + (left: spelled(pair.left.parts, typedPrefix: ""), right: spelled(pair.right.parts, typedPrefix: "")) + } + return item.recased( + label: SQLCompletionColumnPair.condition(sides.map { (left: $0.left.label, right: $0.right.label) }), + insertText: SQLCompletionColumnPair.condition(sides.map { (left: $0.left.text, right: $0.right.text) }) + ) + } guard let identifier = item.identifier else { return item } if quoting != .none, identifier.parts.count == 1, case .catalog(let name) = identifier.parts[0] { // The user's own quote asks for the exact name: inside a closed pair it goes in without the pair. @@ -136,9 +172,14 @@ public struct SQLIdentifierSpelling: Sendable { // A qualified reference replaces the user's quotes, so it is spelled as if none were typed. guard quoting != .enclosed || item.isQualifiedReference else { return item } + let name = spelled(identifier.parts, typedPrefix: typedPrefix) + return item.recased(label: name.label, insertText: name.text) + } + + private func spelled(_ parts: [SQLCompletionIdentifier.Part], typedPrefix: String) -> (label: String, text: String) { var labels: [String] = [] var texts: [String] = [] - for part in identifier.parts { + for part in parts { switch part { case .catalog(let name): switch form(of: name, typedPrefix: typedPrefix) { @@ -154,7 +195,7 @@ public struct SQLIdentifierSpelling: Sendable { texts.append(isQuoted ? quoted(text) : text) } } - return item.recased(label: labels.joined(separator: "."), insertText: texts.joined(separator: ".")) + return (labels.joined(separator: "."), texts.joined(separator: ".")) } private func canStayBare(_ name: String) -> Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLJoinConditions.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLJoinConditions.swift new file mode 100644 index 0000000000..47a49e09af --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLJoinConditions.swift @@ -0,0 +1,169 @@ +import Foundation + +// One foreign key, its column pairs in key order. +public struct SQLCompletionForeignKey: Sendable, Hashable { + public let name: String + public let columns: [String] + public let referencedTable: String + public let referencedSchema: String? + public let referencedColumns: [String] + + public init( + name: String, + columns: [String], + referencedTable: String, + referencedSchema: String? = nil, + referencedColumns: [String] + ) { + self.name = name + self.columns = columns + self.referencedTable = referencedTable + self.referencedSchema = referencedSchema + self.referencedColumns = referencedColumns + } + + // A driver reports one row per column pair. + public struct Row: Sendable { + public let name: String + public let column: String + public let referencedTable: String + public let referencedSchema: String? + public let referencedColumn: String + + public init(name: String, column: String, referencedTable: String, referencedSchema: String?, referencedColumn: String) { + self.name = name + self.column = column + self.referencedTable = referencedTable + self.referencedSchema = referencedSchema + self.referencedColumn = referencedColumn + } + } + + // A composite key's rows share its name and target. Keys keep the order the driver read them in. + public static func grouping(_ rows: [Row]) -> [SQLCompletionForeignKey] { + struct Identity: Hashable { + let name: String + let table: String + let schema: String? + } + var order: [Identity] = [] + var pairs: [Identity: [(column: String, referenced: String)]] = [:] + for row in rows { + let identity = Identity(name: row.name, table: row.referencedTable, schema: row.referencedSchema) + if pairs[identity] == nil { order.append(identity) } + let pair = (column: row.column, referenced: row.referencedColumn) + guard pairs[identity]?.contains(where: { $0 == pair }) != true else { continue } + pairs[identity, default: []].append(pair) + } + return order.map { identity in + let keyPairs = pairs[identity] ?? [] + return SQLCompletionForeignKey( + name: identity.name, + columns: keyPairs.map { $0.column }, + referencedTable: identity.table, + referencedSchema: identity.schema, + referencedColumns: keyPairs.map { $0.referenced } + ) + } + } +} + +// The table whose ON the cursor follows, and the tables that ON may name, nearest first. +public struct SQLJoinScope: Sendable, Equatable { + public let joined: TableReference + public let earlier: [TableReference] + + public init(joined: TableReference, earlier: [TableReference]) { + self.joined = joined + self.earlier = earlier + } + + // Only a stored table has foreign keys, so a CTE or a derived table on either side is left out. + init?( + _ found: (joined: DerivedTableParser.JoinSource, earlier: [DerivedTableParser.JoinSource]), + references: [TableReference], + cteNames: [String] + ) { + func stored(_ source: DerivedTableParser.JoinSource) -> TableReference? { + if source.schema == nil, cteNames.contains(where: { $0.caseInsensitiveCompare(source.table) == .orderedSame }) { + return nil + } + let reference = references.first { + $0.tableName == source.table && $0.alias == source.alias && $0.schema == source.schema + } ?? TableReference( + tableName: source.table, alias: source.alias, schema: source.schema, isIdentifierQuoted: source.isQuoted + ) + return reference.isDerived ? nil : reference + } + guard let joined = stored(found.joined) else { return nil } + let earlier = found.earlier.compactMap(stored) + guard !earlier.isEmpty else { return nil } + self.init(joined: joined, earlier: earlier) + } +} + +enum SQLJoinConditions { + struct Side { + let reference: TableReference + let keys: [SQLCompletionForeignKey] + let columns: [String] + } + + struct ColumnPair: Equatable { + let earlier: String + let joined: String + } + + struct Condition: Equatable { + let earlier: TableReference + let constraintName: String + let columns: [ColumnPair] + } + + // Per earlier table, nearest first: its keys to the joined table, then the joined table's keys to it. The earlier + // table's column goes first either way, and a self-join yields both orientations. + static func conditions(joined: Side, earlier: [Side]) -> [Condition] { + earlier.flatMap { side -> [Condition] in + let outgoing = side.keys.filter { targets($0, from: side.reference, joined.reference) }.map { key in + Condition(earlier: side.reference, constraintName: key.name, columns: zip(key.columns, key.referencedColumns).map { + ColumnPair(earlier: spelling($0.0, in: side.columns), joined: spelling($0.1, in: joined.columns)) + }) + } + let incoming = joined.keys.filter { targets($0, from: joined.reference, side.reference) }.map { key in + Condition(earlier: side.reference, constraintName: key.name, columns: zip(key.columns, key.referencedColumns).map { + ColumnPair(earlier: spelling($0.1, in: side.columns), joined: spelling($0.0, in: joined.columns)) + }) + } + return outgoing + incoming + } + } + + // A key with no target schema points into its own table's namespace. The catalog keeps only keys that stay in the + // namespace they were read in, so with the owner unqualified a key reaches the unqualified tables, and with the + // owner qualified it reaches only tables qualified the same way. + private static func targets( + _ key: SQLCompletionForeignKey, + from owner: TableReference, + _ reference: TableReference + ) -> Bool { + guard key.referencedTable.caseInsensitiveCompare(reference.tableName) == .orderedSame, + !key.columns.isEmpty, key.columns.count == key.referencedColumns.count else { return false } + switch (key.referencedSchema ?? owner.schema, reference.schema) { + case (nil, nil): + return true + case let (keySchema?, schema?): + return keySchema.caseInsensitiveCompare(schema) == .orderedSame + case (.some, nil): + return owner.schema == nil + case (nil, .some): + return false + } + } + + // SQLite reports a key's columns as its REFERENCES clause spells them, which may differ in case from the table's. + private static func spelling(_ column: String, in columns: [String]) -> String { + guard !columns.contains(column) else { return column } + let matches = columns.filter { $0.caseInsensitiveCompare(column) == .orderedSame } + return matches.count == 1 ? matches[0] : column + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift index 6aef52ba87..4d706359dd 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift @@ -38,6 +38,15 @@ public enum BigtableRowWrites { BigtableRequest(verb: .delete, table: table, key: key) } + // The locator is the base64 of the key the row was read with; the _key text is only the fallback. + public static func originalKey(locator: String?, keyCell: BigtableRowSet.Cell?) throws -> Data? { + guard let locator else { return try keyCell.flatMap { try key($0) } } + guard let decoded = Data(base64Encoded: locator), !decoded.isEmpty else { + throw BigtableError.malformedResponse(String(localized: "This row cannot be found again from what was read. Reload the table.")) + } + return decoded + } + public static func key(_ cell: BigtableRowSet.Cell) throws -> Data? { switch cell { case .absent: return nil diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift index 63240c11ed..17e221ff48 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift @@ -15,12 +15,18 @@ public enum BigtableBytes { /// The bytes as text when `display` would show them as themselves. public static func printableText(_ data: Data) -> String? { - guard let text = String(data: data, encoding: .utf8), isPrintable(text), !text.hasPrefix(base64Prefix) else { + guard let text = exactText(data), isPrintable(text), !text.hasPrefix(base64Prefix) else { return nil } return text } + // String(data:encoding:) drops a leading byte order mark, which names a different row or column. + static func exactText(_ data: Data) -> String? { + let text = String(decoding: data, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(data) ? text : nil + } + public static func parse(_ text: String) throws -> Data { guard text.hasPrefix(base64Prefix) else { return Data(text.utf8) } guard let data = Data(base64Encoded: String(text.dropFirst(base64Prefix.count))) else { diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellLifetime.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellLifetime.swift new file mode 100644 index 0000000000..a742825116 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellLifetime.swift @@ -0,0 +1,78 @@ +// +// CassandraCellLifetime.swift +// TableProCassandraCore +// + +import Foundation + +// A write resets the TTL of every cell it writes to its own USING TTL, or to the table's default without one, so an +// edit has to carry what each cell had left when it was read. +public enum CassandraCellLifetime: Hashable, Sendable { + case expiring(seconds: Int) + case permanent + + // Nil for a cell read as null, whose edit is written without a TTL as before. + public init?(valueIsNull: Bool, timeToLive: String?) { + guard !valueIsNull else { return nil } + guard let timeToLive else { + self = .permanent + return + } + guard let seconds = Int(timeToLive), seconds > 0 else { return nil } + self = .expiring(seconds: seconds) + } + + public static func remaining(seconds: Int, after elapsed: Duration) -> Int { + max(1, seconds - Int(max(elapsed.components.seconds, 0))) + } +} + +// The lifetimes of one row's cells as a browse read them, with what the table says about naming the row. +public struct CassandraRowLifetimes: Sendable, Equatable { + public let columns: [String] + public let lifetimes: [CassandraCellLifetime?] + public let readAt: ContinuousClock.Instant + public let layout: CassandraTableLayout + + // Nil when every cell would be written the way a write without USING writes it, so nothing is kept for the row. + public init?( + columns: [String], + lifetimes: [CassandraCellLifetime?], + readAt: ContinuousClock.Instant, + layout: CassandraTableLayout + ) { + guard columns.count == lifetimes.count else { return nil } + let changesAWrite = lifetimes.contains { lifetime in + switch lifetime { + case .expiring: return true + case .permanent: return layout.defaultTimeToLive != 0 + case nil: return false + } + } + guard changesAWrite else { return nil } + self.columns = columns + self.lifetimes = lifetimes + self.readAt = readAt + self.layout = layout + } + + // A cell whose TTL ran out since the read is gone from the server, so an edit of it would live for one second. + public func hasExpired(_ column: String, at now: ContinuousClock.Instant) -> Bool { + guard let index = columns.firstIndex(of: column), case .expiring(let seconds) = lifetimes[index] else { + return false + } + return readAt.duration(to: now) >= .seconds(seconds) + } + + // The USING TTL a write of `column` keeps the cell's lifetime with, or nil for none. A cell that never expired + // is written with 0 unless the table's default TTL is 0, where a write without USING keeps it the same. + public func timeToLive(of column: String, at now: ContinuousClock.Instant) -> Int? { + guard let index = columns.firstIndex(of: column), let lifetime = lifetimes[index] else { return nil } + switch lifetime { + case .expiring(let seconds): + return CassandraCellLifetime.remaining(seconds: seconds, after: readAt.duration(to: now)) + case .permanent: + return layout.defaultTimeToLive == 0 ? nil : 0 + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift new file mode 100644 index 0000000000..771217b554 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift @@ -0,0 +1,83 @@ +// +// CassandraConditionalWrite.swift +// TableProCassandraCore +// + +import Foundation +import TableProSQLGrammar + +// A grid write names its row by key and is conditioned on it: an INSERT is an upsert, and an UPDATE writes its cells +// whether or not the row is there, so either would otherwise replace or recreate a row the grid never showed. +// A condition that fails answers one row whose `[applied]` is false, which the server reports as a success. +public enum CassandraConditionalWrite { + public static let appliedColumn = "[applied]" + public static let insertCondition = "IF NOT EXISTS" + public static let updateCondition = "IF EXISTS" + + private static let grammar = SQLLexicalReadings.resolve(databaseTypeId: "Cassandra", declared: nil, session: nil) + .execution + + public enum Refusal: Sendable, Equatable { + case keyTaken + case rowMissing + } + + public static func insertStatement(target: String, names: [String], values: [String]) -> String { + "INSERT INTO \(target) (\(names.joined(separator: ", "))) VALUES (\(values.joined(separator: ", "))) " + + insertCondition + } + + public static func updateStatement( + target: String, + assignments: [String], + keyCondition: String, + timeToLive: Int? = nil + ) -> String { + "UPDATE \(target)\(usingClause(timeToLive: timeToLive)) SET \(assignments.joined(separator: ", ")) " + + "WHERE \(keyCondition) \(updateCondition)" + } + + public static func usingClause(timeToLive: Int?) -> String { + timeToLive.map { " USING TTL \($0)" } ?? "" + } + + public static func batch(_ statements: [String]) -> String { + "BEGIN BATCH " + statements.map { $0 + "; " }.joined() + "APPLY BATCH" + } + + public static func refusal(statement: String, firstColumn: String?, firstValue: String?) -> Refusal? { + guard firstColumn == appliedColumn, firstValue?.lowercased() == "false" else { return nil } + if let statements = CQLBatch.statements(in: statement, grammar: grammar) { + return batchRefusal(statements) + } + let normalized = statement.uppercased() + if normalized.hasPrefix("INSERT"), normalized.hasSuffix(" " + insertCondition) { + return .keyTaken + } + if normalized.hasPrefix("UPDATE"), normalized.hasSuffix(" " + updateCondition) { + return .rowMissing + } + return nil + } + + // An edit whose cells keep different TTLs is a batch of UPDATEs on the row, each IF EXISTS or, for static + // cells, unconditioned. A batch with any other statement or condition is left to its result. + private static func batchRefusal(_ statements: [String]) -> Refusal? { + var holdsRow = false + for statement in statements { + let words = codeWords(statement) + guard words.first == "UPDATE" else { return nil } + guard let condition = words.firstIndex(of: "IF") else { continue } + guard words[condition...].elementsEqual(["IF", "EXISTS"]) else { return nil } + holdsRow = true + } + return holdsRow ? .rowMissing : nil + } + + private static func codeWords(_ statement: String) -> [String] { + SQLCodeProjection.code(of: statement, grammar: grammar) + .uppercased() + .split { !($0.isLetter || $0.isNumber || $0 == "_") } + .map(String.init) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeRead.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeRead.swift new file mode 100644 index 0000000000..d7d9646b92 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeRead.swift @@ -0,0 +1,95 @@ +// +// CassandraLifetimeRead.swift +// TableProCassandraCore +// + +import Foundation + +// The browse the plugin sends with a TTL() selector after the shown columns for each one that can carry its own +// TTL. `SELECT *` takes no other selector, so it is spelled out in the order the server returns it. +public enum CassandraLifetimeRead { + public struct Plan: Equatable, Sendable { + public let cql: String + public let shownColumns: [String] + // The column of each TTL() selector, in the order the selectors follow the shown columns. + public let lifetimeColumns: [String] + public let lifetimeValueIndices: [Int] + public let keyIndices: [String: Int] + } + + // Nil unless `cql` is a browse of `target` as the renderer writes it, every key column is shown, and at least one + // shown column takes a TTL() selector. + public static func plan(cql: String, target: String, layout: CassandraTableLayout) -> Plan? { + guard let select = select(of: cql, target: target) else { return nil } + let shown = select.columns ?? layout.selectOrder + var lifetimeColumns: [String] = [] + var lifetimeValueIndices: [Int] = [] + for (index, column) in shown.enumerated() + where layout.acceptsTimeToLive(column) && !lifetimeColumns.contains(column) { + lifetimeColumns.append(column) + lifetimeValueIndices.append(index) + } + guard !lifetimeColumns.isEmpty else { return nil } + var keyIndices: [String: Int] = [:] + for key in layout.primaryKey { + guard let index = shown.firstIndex(of: key) else { return nil } + keyIndices[key] = index + } + let selectors = shown.map(CassandraObjectQueries.quote) + + lifetimeColumns.map { "TTL(\(CassandraObjectQueries.quote($0)))" } + return Plan( + cql: "SELECT \(selectors.joined(separator: ", ")) FROM \(target)\(select.tail)", + shownColumns: shown, + lifetimeColumns: lifetimeColumns, + lifetimeValueIndices: lifetimeValueIndices, + keyIndices: keyIndices + ) + } + + private struct Select { + // Nil for `*`. + let columns: [String]? + let tail: String + } + + private static func select(of cql: String, target: String) -> Select? { + var rest = Substring(cql).unicodeScalars + guard consume("SELECT ", from: &rest) else { return nil } + var columns: [String]? + if !consume("* ", from: &rest) { + var names: [String] = [] + repeat { + guard let name = quotedName(from: &rest) else { return nil } + names.append(name) + } while consume(", ", from: &rest) + guard consume(" ", from: &rest) else { return nil } + columns = names + } + guard consume("FROM " + target, from: &rest) else { return nil } + let tail = String(Substring(rest)) + guard tail.isEmpty || tail.hasPrefix(" WHERE ") else { return nil } + return Select(columns: columns, tail: tail) + } + + private static func consume(_ text: String, from rest: inout Substring.UnicodeScalarView) -> Bool { + guard rest.starts(with: text.unicodeScalars) else { return false } + rest = rest.dropFirst(text.unicodeScalars.count) + return true + } + + // A double-quoted identifier, `""` standing for one quote, as SQLQuoting.identifier writes it. + private static func quotedName(from rest: inout Substring.UnicodeScalarView) -> String? { + guard consume("\"", from: &rest) else { return nil } + var name = String.UnicodeScalarView() + while let scalar = rest.first { + rest = rest.dropFirst() + guard scalar == "\"" else { + name.append(scalar) + continue + } + guard consume("\"", from: &rest) else { return String(name) } + name.append(scalar) + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeWrite.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeWrite.swift new file mode 100644 index 0000000000..f2b3319274 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeWrite.swift @@ -0,0 +1,181 @@ +// +// CassandraLifetimeWrite.swift +// TableProCassandraCore +// + +import Foundation + +// A CQL write carries one TTL for all the cells it writes, so cells that keep different TTLs go in statements of +// their own, joined in one batch. The batch names one partition, so Cassandra and ScyllaDB apply it atomically. +// A cell with no TTL to keep is written without USING, as before. +public enum CassandraLifetimeWrite { + public struct KeyCondition { + public let cql: String + public let values: [Value] + + public init(cql: String, values: [Value]) { + self.cql = cql + self.values = values + } + } + + public struct RowKeys { + public let row: KeyCondition + public let partition: KeyCondition + + public init(row: KeyCondition, partition: KeyCondition) { + self.row = row + self.partition = partition + } + } + + public struct Assignment { + public let cql: String + public let parameters: [Value] + public let timeToLive: Int? + public let isStatic: Bool + + public init(cql: String, parameters: [Value], timeToLive: Int?, isStatic: Bool) { + self.cql = cql + self.parameters = parameters + self.timeToLive = timeToLive + self.isStatic = isStatic + } + } + + public struct RestoredCell { + public let column: String + public let value: Value + public let isKey: Bool + public let isNull: Bool + public let isStatic: Bool + public let timeToLive: Int? + + public init(column: String, value: Value, isKey: Bool, isNull: Bool, isStatic: Bool, timeToLive: Int?) { + self.column = column + self.value = value + self.isKey = isKey + self.isNull = isNull + self.isStatic = isStatic + self.timeToLive = timeToLive + } + } + + public struct Statement { + public let cql: String + public let parameters: [Value] + } + + // Every statement of the batch is held to the row with IF EXISTS, except one that writes only static cells + // beside others that do not: CQL refuses clustering columns in an UPDATE of static columns alone, so that one + // names the partition and the batch's other conditions hold it. + public static func update( + target: String, + assignments: [Assignment], + row: KeyCondition, + partition: KeyCondition? + ) -> Statement { + let writesClusteringRow = assignments.contains { !$0.isStatic } + let statements = grouped(assignments, by: \.timeToLive).map { group -> Statement in + let set = group.items.map(\.cql) + let parameters = group.items.flatMap(\.parameters) + if let partition, writesClusteringRow, group.items.allSatisfy(\.isStatic) { + return Statement( + cql: plainUpdate(target: target, timeToLive: group.timeToLive, set: set, key: partition.cql), + parameters: parameters + partition.values + ) + } + return Statement( + cql: CassandraConditionalWrite.updateStatement( + target: target, assignments: set, keyCondition: row.cql, timeToLive: group.timeToLive + ), + parameters: parameters + row.values + ) + } + return batched(statements) + } + + // The INSERT carries the key and also writes the row marker, so it takes the cells that live longest: those + // that never expire, then those left to the table's default, then the longest TTL. The rest are UPDATEs. + public static func restore( + target: String, + cells: [RestoredCell], + keys: RowKeys? + ) -> Statement { + let valued = cells.filter { !$0.isKey && !$0.isNull } + guard let keys, valued.contains(where: { $0.timeToLive != nil }) else { + return insert(target: target, cells: cells, timeToLive: nil) + } + let groups = grouped(valued, by: \.timeToLive) + let inserted = longestLiving(groups.map { $0.timeToLive }) + let insertion = insert( + target: target, + cells: cells.filter { $0.isKey || $0.isNull || $0.timeToLive == inserted }, + timeToLive: inserted + ) + let updates = groups.filter { $0.timeToLive != inserted }.map { group -> Statement in + let key = group.items.allSatisfy(\.isStatic) ? keys.partition : keys.row + let set = group.items.map { "\(CassandraObjectQueries.quote($0.column)) = ?" } + return Statement( + cql: plainUpdate(target: target, timeToLive: group.timeToLive, set: set, key: key.cql), + parameters: group.items.map(\.value) + key.values + ) + } + return batched([insertion] + updates) + } + + static func longestLiving(_ timesToLive: [Int?]) -> Int? { + if timesToLive.contains(0) { return 0 } + if timesToLive.contains(where: { $0 == nil }) { return nil } + return timesToLive.compactMap { $0 }.max() + } + + private static func grouped( + _ items: [Item], + by timeToLive: (Item) -> Int? + ) -> [(timeToLive: Int?, items: [Item])] { + var groups: [(timeToLive: Int?, items: [Item])] = [] + for item in items { + let key = timeToLive(item) + if let index = groups.firstIndex(where: { $0.timeToLive == key }) { + groups[index].items.append(item) + } else { + groups.append((key, [item])) + } + } + return groups + } + + private static func insert( + target: String, + cells: [RestoredCell], + timeToLive: Int? + ) -> Statement { + let names = cells.map { CassandraObjectQueries.quote($0.column) }.joined(separator: ", ") + let markers = Array(repeating: "?", count: cells.count).joined(separator: ", ") + return Statement( + cql: "INSERT INTO \(target) (\(names)) VALUES (\(markers))" + + CassandraConditionalWrite.usingClause(timeToLive: timeToLive), + parameters: cells.map(\.value) + ) + } + + private static func plainUpdate(target: String, timeToLive: Int?, set: [String], key: String) -> String { + "UPDATE \(target)\(CassandraConditionalWrite.usingClause(timeToLive: timeToLive)) SET " + + "\(set.joined(separator: ", ")) WHERE \(key)" + } + + private static func batched(_ statements: [Statement]) -> Statement { + if statements.count == 1, let only = statements.first { return only } + return Statement( + cql: CassandraConditionalWrite.batch(statements.map(\.cql)), + parameters: statements.flatMap(\.parameters) + ) + } +} + +extension CassandraLifetimeWrite.KeyCondition: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.RowKeys: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.Assignment: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.RestoredCell: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.Statement: Sendable where Value: Sendable {} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraRecentEntries.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraRecentEntries.swift new file mode 100644 index 0000000000..dfdbb46820 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraRecentEntries.swift @@ -0,0 +1,47 @@ +// +// CassandraRecentEntries.swift +// TableProCassandraCore +// + +import Foundation + +// At most `capacity` entries in two halves. When the newer half is full the older half is dropped whole, so the +// entries written longest ago go first and no write costs more than a dictionary insert. +public struct CassandraRecentEntries { + public let capacity: Int + private var newer: [Key: Value] = [:] + private var older: [Key: Value] = [:] + + public init(capacity: Int) { + self.capacity = max(capacity, 2) + } + + public subscript(key: Key) -> Value? { + newer[key] ?? older[key] + } + + public var count: Int { + newer.count + older.count + } + + public mutating func set(_ value: Value, for key: Key) { + older.removeValue(forKey: key) + if newer[key] == nil, newer.count >= capacity / 2 { + older = newer + newer = [:] + } + newer[key] = value + } + + public mutating func removeValue(for key: Key) { + newer.removeValue(forKey: key) + older.removeValue(forKey: key) + } + + public mutating func removeAll() { + newer = [:] + older = [:] + } +} + +extension CassandraRecentEntries: Sendable where Key: Sendable, Value: Sendable {} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraTableLayout.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraTableLayout.swift new file mode 100644 index 0000000000..59bf99ab45 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraTableLayout.swift @@ -0,0 +1,107 @@ +// +// CassandraTableLayout.swift +// TableProCassandraCore +// + +import Foundation + +// What system_schema says about a table's columns: which ones a browse can read a TTL() for, and how an edit names +// the row or, for static columns, the partition. +public struct CassandraTableLayout: Sendable, Equatable { + public enum Kind: String, Sendable { + case partitionKey = "partition_key" + case clustering + case staticColumn = "static" + case regular + } + + public struct Column: Sendable, Equatable { + public let name: String + public let type: String + public let kind: Kind + public let position: Int + + public init(name: String, type: String, kind: Kind, position: Int) { + self.name = name + self.type = type + self.kind = kind + self.position = position + } + } + + public let partitionKey: [String] + public let clustering: [String] + public let staticColumns: Set + public let selectOrder: [String] + // Nil when the table's options could not be read. + public let defaultTimeToLive: Int? + private let lifetimeColumns: Set + + // CQL3Type.Native without counter and empty: each holds one cell. + private static let singleCellTypes: Set = [ + "ascii", "bigint", "blob", "boolean", "date", "decimal", "double", "duration", "float", "inet", "int", + "smallint", "text", "time", "timestamp", "timeuuid", "tinyint", "uuid", "varchar", "varint", + ] + private static let singleCellPrefixes = ["frozen<", "tuple<", "vector<"] + + // Nil for a table system_schema lists no columns for, such as a virtual table, and for a COMPACT STORAGE table, + // whose `empty` value column `SELECT *` leaves out. + public init?(columns: [Column], defaultTimeToLive: Int?) { + guard !columns.isEmpty, !columns.contains(where: { Self.normalized($0.type) == "empty" }) else { return nil } + let byPosition = { (kind: Kind) in + columns.filter { $0.kind == kind }.sorted { $0.position < $1.position }.map(\.name) + } + let byName = { (kind: Kind) in + columns.filter { $0.kind == kind }.map(\.name).sorted { $0.utf8.lexicographicallyPrecedes($1.utf8) } + } + partitionKey = byPosition(.partitionKey) + clustering = byPosition(.clustering) + let statics = byName(.staticColumn) + staticColumns = Set(statics) + // Cassandra's allColumnsInSelectOrder: the key in order, then static and regular columns by name. + selectOrder = partitionKey + clustering + statics + byName(.regular) + self.defaultTimeToLive = defaultTimeToLive + lifetimeColumns = Set(columns.filter { column in + (column.kind == .regular || column.kind == .staticColumn) && Self.holdsOneCell(column.type) + }.map(\.name)) + } + + public var primaryKey: [String] { partitionKey + clustering } + + // TTL() refuses a key column, and before Cassandra 5.0 a non-frozen collection or user type. A counter has no + // TTL, and an UPDATE of one refuses USING TTL. + public func acceptsTimeToLive(_ column: String) -> Bool { + lifetimeColumns.contains(column) + } + + public static func columnsQuery(keyspace: String, table: String) -> String { + """ + SELECT column_name, type, kind, position FROM system_schema.columns + WHERE keyspace_name = '\(CassandraObjectQueries.escapeLiteral(keyspace))' + AND table_name = '\(CassandraObjectQueries.escapeLiteral(table))' + """ + } + + public static func defaultTimeToLiveQuery(keyspace: String, table: String) -> String { + """ + SELECT default_time_to_live FROM system_schema.tables + WHERE keyspace_name = '\(CassandraObjectQueries.escapeLiteral(keyspace))' + AND table_name = '\(CassandraObjectQueries.escapeLiteral(table))' + """ + } + + // One row of `columnsQuery`, nil when a part is missing or the kind is not one CQL has. + public static func column(name: String?, type: String?, kind: String?, position: String?) -> Column? { + guard let name, let type, let kind = kind.flatMap(Kind.init(rawValue:)) else { return nil } + return Column(name: name, type: type, kind: kind, position: position.flatMap { Int($0) } ?? 0) + } + + private static func holdsOneCell(_ type: String) -> Bool { + let type = normalized(type) + return singleCellTypes.contains(type) || singleCellPrefixes.contains { type.hasPrefix($0) } + } + + private static func normalized(_ type: String) -> String { + type.trimmingCharacters(in: .whitespaces).lowercased() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraWireText.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraWireText.swift new file mode 100644 index 0000000000..f38c31f1f2 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraWireText.swift @@ -0,0 +1,19 @@ +// +// CassandraWireText.swift +// TableProCassandraCore +// + +import Foundation + +public enum CassandraWireText { + // Foundation's UTF-8 initializers drop a leading byte order mark, and a key that starts with one names another row. + public static func text(_ bytes: UnsafeRawBufferPointer) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } + + public static func text(_ bytes: Data) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionDestination.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionDestination.swift index fe424e678d..a9ef52212e 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionDestination.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionDestination.swift @@ -20,9 +20,9 @@ public struct ConnectionDestination: Equatable, Sendable { case verifyIdentity } + // A linked SSH profile is named by the servers it dials, so moving the profile moves the destination. public enum Tunnel: Equatable, Sendable { case none - case profile(UUID) case ssh(server: Endpoint, jumps: [Endpoint]) } @@ -100,8 +100,6 @@ public struct ConnectionDestination: Equatable, Sendable { switch tunnel { case .none: return "none" - case .profile(let id): - return "profile:" + id.uuidString.lowercased() case .ssh(let server, let jumps): return ([server] + jumps) .map { Self.normalizedHost($0.host) + ":" + String($0.port) } diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionFieldTrust.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionFieldTrust.swift index b99565d3d9..dbb106f145 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionFieldTrust.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/ConnectionFieldTrust.swift @@ -130,6 +130,7 @@ public enum ConnectionFieldCatalog { "bqMaxBytesBilled": .setting, "cassandraConsistency": .setting, "cassandraSerialConsistency": .setting, + "commitMode": .setting, "completionIdentifierCase": .setting, "connectionOptions": .setting, "connectTimeoutSeconds": .setting, diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift index 317be54e12..905b25e6cf 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift @@ -45,13 +45,13 @@ public final class ConnectionManager: @unchecked Sendable { // A teardown wait that gives up leaves the teardown running and counting as suspension-blocking. public func connect( - _ connection: DatabaseConnection, + _ saved: DatabaseConnection, prompter: (any ConnectionPrompter)? = nil ) async throws -> ConnectionSession { // Ahead of beginAttempt, so a refusal neither cancels an attempt in flight nor tears down the open session. - if let refusal = driverFactory.refusal(for: connection) { throw refusal } - let generation = beginAttempt(for: connection.id) - switch await awaitTeardown(of: connection.id, bound: .after(teardownWaitLimit)) { + if let refusal = driverFactory.refusal(for: saved) { throw refusal } + let generation = beginAttempt(for: saved.id) + switch await awaitTeardown(of: saved.id, bound: .after(teardownWaitLimit)) { case .cleared: break case .cancelled: @@ -59,6 +59,9 @@ public final class ConnectionManager: @unchecked Sendable { case .stillClosing: throw ConnectionError.previousSessionStillClosing } + guard isCurrentAttempt(generation, for: saved.id) else { throw CancellationError() } + // Everything after this mints for, tunnels to and dials the endpoint this names. + let connection = try await driverFactory.resolvingEndpointAlias(for: saved) guard isCurrentAttempt(generation, for: connection.id) else { throw CancellationError() } let stored = try secureStore.retrieve(forKey: Self.passwordKey(for: connection.id)) let password = try await driverFactory.resolvePassword(for: connection, stored: stored, secureStore: secureStore) diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift index f1d92e1622..5fe26dfd00 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift @@ -27,6 +27,17 @@ public protocol DatabaseDriver: AnyObject, Sendable { func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] func fetchDatabases() async throws -> [String] + // The table in `database`, for an engine that names the database beside the schema (SQL Server's + // `[db].[schema].[table]`), so the read never follows a `USE` the session ran. Any other engine reads as above. + func fetchColumns(table: String, schema: String?, database: String?) async throws -> [ColumnInfo] + func fetchIndexes(table: String, schema: String?, database: String?) async throws -> [IndexInfo] + func fetchForeignKeys(table: String, schema: String?, database: String?) async throws -> [ForeignKeyInfo] + + // Runs a statement sent with `StreamOptions.enforcesReadOnly` in the engine's read-only mode, or refuses it. + var enforcesReadOnlyMode: Bool { get } + // Whether the mode can run now: its own transaction would end a transaction or a table lock the session holds. + func entersReadOnlyMode() async -> Bool + func switchDatabase(to name: String) async throws var supportsSchemas: Bool { get } func switchSchema(to name: String) async throws @@ -58,6 +69,25 @@ public extension DatabaseDriver { func sessionLexicalFacts() async -> SQLSessionLexicalFacts? { nil } + func fetchColumns(table: String, schema: String?, database: String?) async throws -> [ColumnInfo] { + try await fetchColumns(table: table, schema: schema) + } + + func fetchIndexes(table: String, schema: String?, database: String?) async throws -> [IndexInfo] { + try await fetchIndexes(table: table, schema: schema) + } + + func fetchForeignKeys(table: String, schema: String?, database: String?) async throws -> [ForeignKeyInfo] { + try await fetchForeignKeys(table: table, schema: schema) + } + + var enforcesReadOnlyMode: Bool { false } + + func entersReadOnlyMode() async -> Bool { + guard enforcesReadOnlyMode else { return false } + return await sessionTransactionState() == .idle + } + func beginTransaction(mode: PluginTransactionAccessMode) async throws { try await beginTransaction() } diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/Protocols/DriverFactory.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/Protocols/DriverFactory.swift index e2ef856f52..5f2e49d1f4 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/Protocols/DriverFactory.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/Protocols/DriverFactory.swift @@ -18,7 +18,10 @@ public protocol DriverFactory: Sendable { preTunnelHost: String?, refreshPassword: (@Sendable () async throws -> String?)? ) throws -> any DatabaseDriver - // Runs before the tunnel, on the connection as saved, so a sign-in failure comes before any SSH prompt. + // Runs first, on the connection as saved. The connect mints for, tunnels to and dials what it returns, which can + // name another host, such as the RDS endpoint a CNAME stands for. It is never saved. + func resolvingEndpointAlias(for connection: DatabaseConnection) async throws -> DatabaseConnection + // Runs before the tunnel, so a sign-in failure comes before any SSH prompt. func resolvePassword( for connection: DatabaseConnection, stored: String?, @@ -49,4 +52,8 @@ public extension DriverFactory { func dialsSSHTunnel(for connection: DatabaseConnection) -> Bool { connection.sshEnabled } + + func resolvingEndpointAlias(for connection: DatabaseConnection) async throws -> DatabaseConnection { + connection + } } diff --git a/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdRangeScan.swift b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdRangeScan.swift new file mode 100644 index 0000000000..b7fa51787f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdRangeScan.swift @@ -0,0 +1,157 @@ +import Foundation + +public extension EtcdKeyRange { + init(key: Data, rangeEnd: Data?) { + self.key = key + self.rangeEnd = rangeEnd + } +} + +public struct EtcdRangeBatch { + public let items: [Item] + public let more: Bool + public let revision: Int64? + + public init(items: [Item], more: Bool, revision: Int64?) { + self.items = items + self.more = more + self.revision = revision + } +} + +public struct EtcdPageStart: Equatable, Sendable { + public let range: EtcdKeyRange + public let revision: Int64? + + public init(range: EtcdKeyRange, revision: Int64?) { + self.range = range + self.revision = revision + } +} + +// Reads a key range a bounded batch per request instead of in one response. A batch sends no sort order, so etcd +// applies the limit as it reads, in ascending key order; a descending sort makes it read the whole range first. Every +// request after the first reads at the first one's revision, so the batches are one snapshot, as a single request is. +public enum EtcdRangeScan { + public static func keyAfter(_ key: Data) -> Data { + key + Data([0]) + } + + // The rest of `range` after `key`, in ascending order. + public static func range(after key: Data, in range: EtcdKeyRange) -> EtcdKeyRange { + EtcdKeyRange(key: keyAfter(key), rangeEnd: range.rangeEnd) + } + + // The part of `range` below `key`, which is where a descending read goes on. + public static func range(before key: Data, in range: EtcdKeyRange) -> EtcdKeyRange { + EtcdKeyRange(key: range.key, rangeEnd: key) + } + + // Hands each batch to `body` until it returns false, the range ends or `maxItems` have been read. + public static func scan( + _ range: EtcdKeyRange, + batchSize: Int, + maxItems: Int, + key: (Item) -> Data, + fetch: (_ range: EtcdKeyRange, _ limit: Int64, _ revision: Int64?) async throws -> EtcdRangeBatch, + body: ([Item]) throws -> Bool + ) async throws { + var current = range + var revision: Int64? + var read = 0 + while read < maxItems { + try Task.checkCancellation() + let batch = try await fetch(current, Int64(min(max(1, batchSize), maxItems - read)), revision) + revision = revision ?? batch.revision + read += batch.items.count + guard try body(batch.items), batch.more, let last = batch.items.last else { return } + current = Self.range(after: key(last), in: range) + } + } + + // Where the page at `offset` begins, found by reading keys alone and never past the offset. Nil when the range + // has no key at that offset. + public static func pageStart( + of range: EtcdKeyRange, + skipping offset: Int, + batchSize: Int, + fetchKeys: (_ range: EtcdKeyRange, _ limit: Int64, _ revision: Int64?) async throws -> EtcdRangeBatch + ) async throws -> EtcdPageStart? { + guard offset > 0 else { return EtcdPageStart(range: range, revision: nil) } + var current = range + var revision: Int64? + var remaining = offset + while remaining > 0 { + try Task.checkCancellation() + let batch = try await fetchKeys(current, Int64(min(remaining, max(1, batchSize))), revision) + revision = revision ?? batch.revision + guard batch.more, let last = batch.items.last else { return nil } + remaining -= batch.items.count + current = Self.range(after: last, in: range) + } + return EtcdPageStart(range: current, revision: revision) + } +} + +public enum EtcdCompaction { + // etcd's ErrCompacted, sent as gRPC OutOfRange. + public static func isCompacted(grpcCode: Int?, message: String) -> Bool { + grpcCode == 11 && message.contains("required revision has been compacted") + } + + // A read pinned to its first batch's revision fails once the store compacts past it. It starts over once at the + // current revision, which is what a single request would have read. + public static func restartingOnce( + when isCompacted: (Error) -> Bool, + _ read: () async throws -> Value + ) async throws -> Value { + do { + return try await read() + } catch let error where isCompacted(error) { + return try await read() + } + } +} + +// The matching rows of one page, gathered from rows read in ascending key order. A descending page is the tail of the +// ascending matches, so it keeps only the last `offset + limit` of them rather than every row in the range. +public struct EtcdPageCollector { + private let offset: Int + private let limit: Int + private let ascending: Bool + private var skipped = 0 + private var kept: [Item] = [] + + public init(offset: Int, limit: Int, ascending: Bool) { + self.offset = max(0, offset) + self.limit = max(0, limit) + self.ascending = ascending + } + + // False once an ascending page is complete, so the read can stop. + public mutating func add(_ item: Item) -> Bool { + if ascending { + guard kept.count < limit else { return false } + if skipped < offset { + skipped += 1 + } else { + kept.append(item) + } + return kept.count < limit + } + let capacity = offset + limit + guard capacity > 0 else { return true } + kept.append(item) + if kept.count >= 2 * capacity { + kept.removeFirst(kept.count - capacity) + } + return true + } + + public var page: [Item] { + guard !ascending else { return kept } + return Array(kept.suffix(offset + limit).dropLast(offset).reversed()) + } + + var heldCount: Int { kept.count } +} diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Config/FirebaseConnectionSettings.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Config/FirebaseConnectionSettings.swift index b76179a1c6..bfa9318339 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Config/FirebaseConnectionSettings.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore public enum FirebaseFieldID { public static let authMethod = "fbAuthMethod" diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreCellCodec.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreCellCodec.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreCellCodec.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreCellCodec.swift index 7a911cec8f..668f0cbc0f 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreCellCodec.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreCellCodec.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore /// The grid only holds text, so an edit is read by the type the field already had: a string field /// stays a string even when the new text looks like a number. diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreClient.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreClient.swift index 94014f42c6..cfeb8c381c 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreClient.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore import TableProGoogleCloud public final class FirestoreClient: @unchecked Sendable { diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreDocument.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreDocument.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreDocument.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreDocument.swift index f810394b4c..60f3877559 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreDocument.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreDocument.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore public struct FirestoreDocument: Sendable, Equatable { /// Relative to `.../documents`, such as `users/alice`. diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreDocumentJSON.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreDocumentJSON.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreDocumentJSON.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreDocumentJSON.swift index 4a6ef31fe0..ba6c0f1f38 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreDocumentJSON.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreDocumentJSON.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore /// Types JSON has no word for are single-key objects such as `{"$timestamp": "..."}`. `$double` /// marks a double with no fraction, and `$map` wraps a real map whose only key is one of the tags. diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreQuery.swift similarity index 84% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreQuery.swift index be24f39373..49d3149337 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreQuery.swift @@ -1,76 +1,5 @@ import Foundation - -public struct FirebaseFilterSpec: Codable, Sendable, Equatable { - public let column: String - public let op: String - public let value: String - public let secondValue: String? - - public init(column: String, op: String, value: String, secondValue: String? = nil) { - self.column = column - self.op = op - self.value = value - self.secondValue = secondValue - } - - /// The host sends BETWEEN as `value: "lower,upper"` with the upper bound again in `secondValue`, - /// so the lower bound is `value` without that suffix, whatever commas the bounds hold. - public var betweenBounds: (lower: String, upper: String)? { - guard let upper = secondValue else { - let parts = FirebaseScalars.split(value, separator: ",", omittingEmpty: false) - return parts.count == 2 ? (parts[0], parts[1]) : nil - } - let suffix = Array(("," + upper).unicodeScalars) - let scalars = Array(value.unicodeScalars) - guard scalars.count >= suffix.count, Array(scalars.suffix(suffix.count)) == suffix else { - return (value, upper) - } - var lower = String.UnicodeScalarView() - lower.append(contentsOf: scalars.dropLast(suffix.count)) - return (String(lower), upper) - } -} - -public struct FirebaseSortSpec: Codable, Sendable, Equatable { - public let column: String - public let ascending: Bool - - public init(column: String, ascending: Bool) { - self.column = column - self.ascending = ascending - } -} - -/// Turned into a query only when it runs, because filter values are typed by the fields the -/// collection was last seen holding. -public struct FirebaseBrowseSpec: Codable, Sendable, Equatable { - public let table: String - public let filters: [FirebaseFilterSpec] - public let logicMode: String - public let sorts: [FirebaseSortSpec] - public let limit: Int - public let offset: Int - - public init( - table: String, - filters: [FirebaseFilterSpec] = [], - logicMode: String = "AND", - sorts: [FirebaseSortSpec] = [], - limit: Int, - offset: Int - ) { - self.table = table - self.filters = filters - self.logicMode = logicMode - self.sorts = sorts - self.limit = limit - self.offset = offset - } - - public var usesOr: Bool { - logicMode.uppercased() == "OR" - } -} +import TableProFirebaseCore /// A table in the sidebar is either a collection path (`users`, `users/alice/orders`) or a /// collection group, `*/orders`, which is every `orders` collection at any depth. diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreValue.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreValue.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreValue.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreValue.swift index a79f4d27b8..929c75bac9 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreValue.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Firestore/FirestoreValue.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore public indirect enum FirestoreValue: Sendable, Hashable { case null diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseClient.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseClient.swift index dd77ec8e1e..3d7d9acfbc 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseClient.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore import TableProGoogleCloud public final class RealtimeDatabaseClient: @unchecked Sendable { diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseQuery.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseQuery.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseQuery.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseQuery.swift index 5af5a6c63b..ea7c9d1dd4 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseQuery.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore /// The REST API has no offset, so a page asks the server for the first `offset + limit` children in /// its order and drops the leading `offset` here. diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseRows.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseRows.swift similarity index 77% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseRows.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseRows.swift index 6fcc4b9fc7..91935022c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabaseRows.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/RealtimeDatabase/RealtimeDatabaseRows.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore public struct RealtimeDatabaseRowSet: Sendable, Equatable { public let columns: [String] @@ -99,35 +100,3 @@ public enum RealtimeDatabaseCell { } } } - -public enum RealtimeDatabasePath { - /// Characters the Realtime Database refuses in a key; checked before a write so the error - /// names the key instead of coming back as a bare 400. - static let forbidden = CharacterSet(charactersIn: ".#$[]/").union(.controlCharacters) - - public static func normalized(_ path: String) -> String { - FirebaseScalars.split(path, separator: "/").joined(separator: "/") - } - - /// An empty key stands for the node itself, which is how a node holding a plain value shows. - public static func join(_ parent: String, _ key: String) -> String { - let base = normalized(parent) - guard !key.isEmpty else { return base } - return base.isEmpty ? key : base + "/" + key - } - - /// The `$key` of the row a node holding a plain value shows as. A key cannot hold `/`, so this - /// can never name a child. - public static func nodeKey(_ path: String) -> String { - "/" + normalized(path) - } - - public static func validateKey(_ key: String) throws { - guard !key.isEmpty, key.rangeOfCharacter(from: forbidden) == nil, key.utf8.count <= 768 else { - throw FirebaseError.configuration(String( - format: String(localized: "\"%@\" is not a valid key. Keys cannot be empty or contain . # $ [ ] or /."), - key - )) - } - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Wire/FirebaseTransport.swift similarity index 98% rename from apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift rename to apple/Packages/TableProCore/Sources/TableProFirebaseClient/Wire/FirebaseTransport.swift index c60b7ab6b9..d85a55b7a3 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseClient/Wire/FirebaseTransport.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseCore import TableProGoogleCloud public extension GoogleRESTResponse { diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabasePath.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabasePath.swift new file mode 100644 index 0000000000..9ec0d0dc72 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/RealtimeDatabase/RealtimeDatabasePath.swift @@ -0,0 +1,33 @@ +import Foundation + +public enum RealtimeDatabasePath { + /// Characters the Realtime Database refuses in a key; checked before a write so the error + /// names the key instead of coming back as a bare 400. + static let forbidden = CharacterSet(charactersIn: ".#$[]/").union(.controlCharacters) + + public static func normalized(_ path: String) -> String { + FirebaseScalars.split(path, separator: "/").joined(separator: "/") + } + + /// An empty key stands for the node itself, which is how a node holding a plain value shows. + public static func join(_ parent: String, _ key: String) -> String { + let base = normalized(parent) + guard !key.isEmpty else { return base } + return base.isEmpty ? key : base + "/" + key + } + + /// The `$key` of the row a node holding a plain value shows as. A key cannot hold `/`, so this + /// can never name a child. + public static func nodeKey(_ path: String) -> String { + "/" + normalized(path) + } + + public static func validateKey(_ key: String) throws { + guard !key.isEmpty, key.rangeOfCharacter(from: forbidden) == nil, key.utf8.count <= 768 else { + throw FirebaseError.configuration(String( + format: String(localized: "\"%@\" is not a valid key. Keys cannot be empty or contain . # $ [ ] or /."), + key + )) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseBrowseSpec.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseBrowseSpec.swift new file mode 100644 index 0000000000..48f6178ba0 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseBrowseSpec.swift @@ -0,0 +1,73 @@ +import Foundation + +public struct FirebaseFilterSpec: Codable, Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } + + /// The host sends BETWEEN as `value: "lower,upper"` with the upper bound again in `secondValue`, + /// so the lower bound is `value` without that suffix, whatever commas the bounds hold. + public var betweenBounds: (lower: String, upper: String)? { + guard let upper = secondValue else { + let parts = FirebaseScalars.split(value, separator: ",", omittingEmpty: false) + return parts.count == 2 ? (parts[0], parts[1]) : nil + } + let suffix = Array(("," + upper).unicodeScalars) + let scalars = Array(value.unicodeScalars) + guard scalars.count >= suffix.count, Array(scalars.suffix(suffix.count)) == suffix else { + return (value, upper) + } + var lower = String.UnicodeScalarView() + lower.append(contentsOf: scalars.dropLast(suffix.count)) + return (String(lower), upper) + } +} + +public struct FirebaseSortSpec: Codable, Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +/// Turned into a query only when it runs, because filter values are typed by the fields the +/// collection was last seen holding. +public struct FirebaseBrowseSpec: Codable, Sendable, Equatable { + public let table: String + public let filters: [FirebaseFilterSpec] + public let logicMode: String + public let sorts: [FirebaseSortSpec] + public let limit: Int + public let offset: Int + + public init( + table: String, + filters: [FirebaseFilterSpec] = [], + logicMode: String = "AND", + sorts: [FirebaseSortSpec] = [], + limit: Int, + offset: Int + ) { + self.table = table + self.filters = filters + self.logicMode = logicMode + self.sorts = sorts + self.limit = limit + self.offset = offset + } + + public var usesOr: Bool { + logicMode.uppercased() == "OR" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseJSON.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseJSON.swift index 3e551da1a8..4644c210b6 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseJSON.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseJSON.swift @@ -144,7 +144,7 @@ public indirect enum FirebaseJSON: Sendable, Hashable { /// JSON has no NaN or infinity, so those are written as strings; the caller that needs them /// back as numbers wraps them in a tag of its own. - static func doubleText(_ value: Double) -> String { + package static func doubleText(_ value: Double) -> String { guard value.isFinite else { return value.isNaN ? "\"NaN\"" : (value > 0 ? "\"Infinity\"" : "\"-Infinity\"") } diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseStatement.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseStatement.swift index a1f3b9df80..361cb0b725 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseStatement.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Wire/FirebaseStatement.swift @@ -196,14 +196,14 @@ public struct FirebaseRequest: Sendable, Equatable { // Paths and request lines are written per Unicode scalar, so they are read back per scalar too. A Character // joins a combining mark to the separator before it: `/` + U+0301 no longer splits, and " " + U+FE0F drops the mark. -enum FirebaseScalars { - static func split(_ text: String, separator: Unicode.Scalar, maxSplits: Int = .max, omittingEmpty: Bool = true) -> [String] { +package enum FirebaseScalars { + package static func split(_ text: String, separator: Unicode.Scalar, maxSplits: Int = .max, omittingEmpty: Bool = true) -> [String] { text.unicodeScalars .split(separator: separator, maxSplits: maxSplits, omittingEmptySubsequences: omittingEmpty) .map(String.init) } - static func removingPrefix(_ prefix: String, from text: String) -> String? { + package static func removingPrefix(_ prefix: String, from text: String) -> String? { guard text.unicodeScalars.starts(with: prefix.unicodeScalars) else { return nil } return String(text.unicodeScalars.dropFirst(prefix.unicodeScalars.count)) } diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift index 19a4dcad91..d25cea91e8 100644 --- a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift @@ -41,7 +41,8 @@ public enum GzipFile { let output = try FileHandle(forWritingTo: destination) defer { try? output.close() } let inflater = try GzipInflater(budget: budget) - while let block = try input.read(upToCount: readSize), !block.isEmpty { + // Each read returns an autoreleased buffer, and this loop never suspends, so nothing drains them until it ends. + while let block = try autoreleasepool(invoking: { try input.read(upToCount: readSize) }), !block.isEmpty { try Task.checkCancellation() try block.withUnsafeBytes { raw in try inflater.inflate(raw) { try output.write(contentsOf: $0) } diff --git a/apple/Packages/TableProCore/Sources/TableProHTTP/LoopbackRedirectListener.swift b/apple/Packages/TableProCore/Sources/TableProHTTP/LoopbackRedirectListener.swift new file mode 100644 index 0000000000..fbf9c30d86 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProHTTP/LoopbackRedirectListener.swift @@ -0,0 +1,499 @@ +import Darwin +import Foundation +import Network +import os + +public enum LoopbackRedirectListenerError: Error, Equatable, Sendable { + case portInUse + case timedOut +} + +public enum LoopbackRedirectAnswer: Sendable { + case ignore(Data) + case settle(Data, Result) +} + +// Waits on loopback for one browser redirect (RFC 8252 section 7.3). `answer` sees each complete request head; +// a request it ignores gets its reply and changes nothing, so a page probing the port cannot end the wait. +public final class LoopbackRedirectListener: @unchecked Sendable { + public typealias Answer = @Sendable (_ requestHead: Data) -> LoopbackRedirectAnswer + + private struct Admission { + let connection: NWConnection + let order: UInt64 + var awaitingHead: Bool + } + + private struct Teardown { + let listener: NWListener? + let connections: [NWConnection] + let ipv6Hold: Int32 + let readyContinuation: CheckedContinuation? + let readyTimeout: DispatchWorkItem? + let outcomeContinuation: CheckedContinuation? + let outcomeTimeout: DispatchWorkItem? + } + + private let port: UInt16 + private let headTimeout: TimeInterval + private let answer: Answer + private let holdIPv6: @Sendable (UInt16) -> LoopbackPortHold + private let queue = DispatchQueue(label: "com.TablePro.LoopbackRedirectListener") + private let lock = NSLock() + private var listener: NWListener? + private var ipv6Hold: Int32 = -1 + private var portAttempts = 1 + private var connections: [ObjectIdentifier: Admission] = [:] + private var admissions: UInt64 = 0 + private var readyContinuation: CheckedContinuation? + private var readyTimeout: DispatchWorkItem? + private var outcomeContinuation: CheckedContinuation? + private var outcomeTimeout: DispatchWorkItem? + private var settled: Result? + + // Port 0 lets the kernel pick one. + public convenience init(port: UInt16, headTimeout: TimeInterval = 5, answer: @escaping Answer) { + self.init(port: port, headTimeout: headTimeout, holdIPv6: { LoopbackPortHold.ipv6(port: $0) }, answer: answer) + } + + init( + port: UInt16, + headTimeout: TimeInterval, + holdIPv6: @escaping @Sendable (UInt16) -> LoopbackPortHold, + answer: @escaping Answer + ) { + self.port = port + self.headTimeout = headTimeout + self.holdIPv6 = holdIPv6 + self.answer = answer + } + + @discardableResult + public func start(readyTimeout: TimeInterval = 10) async throws -> UInt16 { + let listener: NWListener + do { + listener = try makeListener() + } catch { + LoopbackRedirectLimits.logger.error("Could not create the loopback redirect listener") + throw LoopbackRedirectListenerError.portInUse + } + return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let refusal = lock.withLock { () -> (any Error)? in + if let settled { + guard case .failure(let error) = settled else { return CancellationError() } + return error + } + self.listener = listener + readyContinuation = continuation + let item = DispatchWorkItem { [weak self] in + self?.finish(.failure(LoopbackRedirectListenerError.timedOut)) + } + self.readyTimeout = item + queue.asyncAfter(deadline: .now() + readyTimeout, execute: item) + return nil + } + guard let refusal else { + listener.start(queue: queue) + return + } + listener.cancel() + continuation.resume(throwing: refusal) + } + } + + public func outcome(timeout: TimeInterval, timeoutError: any Error) async throws -> Outcome { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + let early = lock.withLock { () -> Result? in + if let settled { return settled } + outcomeContinuation = continuation + let item = DispatchWorkItem { [weak self] in + self?.finish(.failure(timeoutError)) + } + outcomeTimeout = item + queue.asyncAfter(deadline: .now() + timeout, execute: item) + return nil + } + if let early { + continuation.resume(with: early) + } + } + } onCancel: { + finish(.failure(CancellationError())) + } + } + + public func stop() { + finish(.failure(CancellationError())) + } + + private func makeListener() throws -> NWListener { + guard let endpointPort = NWEndpoint.Port(rawValue: port) else { + throw LoopbackRedirectListenerError.portInUse + } + let parameters = NWParameters.tcp + parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(.loopback), port: endpointPort) + parameters.allowLocalEndpointReuse = true + let listener = try NWListener(using: parameters) + listener.stateUpdateHandler = { [weak self, weak listener] state in + guard let listener else { return } + self?.listenerChanged(state, of: listener) + } + listener.newConnectionHandler = { [weak self] connection in + guard let self else { + connection.cancel() + return + } + self.accept(connection) + } + return listener + } + + private func listenerChanged(_ state: NWListener.State, of source: NWListener) { + // A listener given up for another kernel port reports its own cancellation; that ends nothing. + let isCurrent = lock.withLock { listener === source } + guard isCurrent else { return } + switch state { + case .ready: + let awaitingReady = lock.withLock { readyContinuation != nil } + guard awaitingReady else { return } + guard let port = source.port?.rawValue, port > 0 else { + finish(.failure(LoopbackRedirectListenerError.portInUse)) + return + } + holdIPv6Loopback(port: port) + case .failed(let error), .waiting(let error): + LoopbackRedirectLimits.logger.error( + "The loopback redirect listener failed (\(LoopbackRedirectLimits.code(of: error), privacy: .public))" + ) + finish(.failure(LoopbackRedirectListenerError.portInUse)) + default: + break + } + } + + // A redirect to localhost may come in over ::1, so no other process may take that port (RFC 8252 section 8.3). + private func holdIPv6Loopback(port bound: UInt16) { + switch holdIPv6(bound) { + case .held(let descriptor): + let kept = lock.withLock { () -> Bool in + guard settled == nil else { return false } + ipv6Hold = descriptor + return true + } + guard kept else { + close(descriptor) + return + } + resumeReady(bound) + case .unavailable: + resumeReady(bound) + case .taken(let code): + // A port the kernel picked is only a bad draw; a fixed redirect port has no alternative. + if port == 0, listenOnAnotherKernelPort() { return } + LoopbackRedirectLimits.logger.error("Another process holds the IPv6 loopback port (errno \(code, privacy: .public))") + finish(.failure(LoopbackRedirectListenerError.portInUse)) + } + } + + private func listenOnAnotherKernelPort() -> Bool { + guard let replacement = try? makeListener() else { return false } + let (swapped, previous) = lock.withLock { () -> (Bool, NWListener?) in + guard settled == nil, readyContinuation != nil, + portAttempts < LoopbackRedirectLimits.maximumKernelPortAttempts + else { return (false, nil) } + portAttempts += 1 + let previous = listener + listener = replacement + return (true, previous) + } + guard swapped else { + replacement.cancel() + return false + } + previous?.cancel() + replacement.start(queue: queue) + return true + } + + private func resumeReady(_ port: UInt16) { + let (continuation, item) = lock.withLock { () -> (CheckedContinuation?, DispatchWorkItem?) in + let pending = (readyContinuation, readyTimeout) + readyContinuation = nil + readyTimeout = nil + return pending + } + item?.cancel() + continuation?.resume(returning: port) + } + + // When every slot is taken, the oldest connection that never sent its request makes room for the new one. + private func accept(_ connection: NWConnection) { + let (admitted, evicted) = lock.withLock { () -> (Bool, NWConnection?) in + guard settled == nil else { return (false, nil) } + var evicted: NWConnection? + if connections.count >= LoopbackRedirectLimits.maximumConnections { + let idle = connections.filter { $0.value.awaitingHead } + guard let oldest = idle.min(by: { $0.value.order < $1.value.order }) else { return (false, nil) } + connections[oldest.key] = nil + evicted = oldest.value.connection + } + admissions += 1 + connections[ObjectIdentifier(connection)] = Admission(connection: connection, order: admissions, awaitingHead: true) + return (true, evicted) + } + evicted?.cancel() + guard admitted else { + connection.cancel() + return + } + connection.stateUpdateHandler = { [weak self, weak connection] state in + guard let connection else { return } + switch state { + case .failed, .cancelled: + self?.release(connection) + default: + break + } + } + connection.start(queue: queue) + receive(on: connection, head: Data()) + queue.asyncAfter(deadline: .now() + headTimeout) { [weak self, weak connection] in + guard let self, let connection, self.dropIfAwaitingHead(connection) else { return } + connection.cancel() + } + } + + private func receive(on connection: NWConnection, head: Data) { + let remaining = LoopbackRedirectLimits.maximumRequestHeadBytes - head.count + connection.receive(minimumIncompleteLength: 1, maximumLength: remaining) { [weak self] content, _, isComplete, error in + guard let self else { + connection.cancel() + return + } + var received = head + if let content { + received.append(content) + } + if error != nil, received.isEmpty { + connection.cancel() + return + } + let hasFullHead = LoopbackRedirectRequest.isComplete(received) + if error == nil, !isComplete, !hasFullHead, received.count < LoopbackRedirectLimits.maximumRequestHeadBytes { + self.receive(on: connection, head: received) + return + } + self.respond(on: connection, head: received) + } + } + + private func respond(on connection: NWConnection, head: Data) { + switch answer(head) { + case .ignore(let response): + markAnswered(connection) + send(response, on: connection) + case .settle(let response, let result): + release(connection) + send(response, on: connection) + finish(result) + } + } + + private func send(_ response: Data, on connection: NWConnection) { + connection.send(content: response, completion: .contentProcessed { _ in + connection.cancel() + }) + } + + private func markAnswered(_ connection: NWConnection) { + lock.withLock { + connections[ObjectIdentifier(connection)]?.awaitingHead = false + return + } + } + + private func dropIfAwaitingHead(_ connection: NWConnection) -> Bool { + lock.withLock { () -> Bool in + let key = ObjectIdentifier(connection) + guard connections[key]?.awaitingHead == true else { return false } + connections[key] = nil + return true + } + } + + private func release(_ connection: NWConnection) { + lock.withLock { + connections[ObjectIdentifier(connection)] = nil + } + } + + private func finish(_ result: Result) { + let teardown = lock.withLock { () -> Teardown? in + guard settled == nil else { return nil } + settled = result + let teardown = Teardown( + listener: listener, + connections: connections.values.map { $0.connection }, + ipv6Hold: ipv6Hold, + readyContinuation: readyContinuation, + readyTimeout: readyTimeout, + outcomeContinuation: outcomeContinuation, + outcomeTimeout: outcomeTimeout + ) + listener = nil + connections = [:] + ipv6Hold = -1 + readyContinuation = nil + readyTimeout = nil + outcomeContinuation = nil + outcomeTimeout = nil + return teardown + } + guard let teardown else { return } + teardown.readyTimeout?.cancel() + teardown.outcomeTimeout?.cancel() + teardown.listener?.cancel() + teardown.connections.forEach { $0.cancel() } + if teardown.ipv6Hold >= 0 { + close(teardown.ipv6Hold) + } + if let ready = teardown.readyContinuation { + switch result { + case .failure(let error): + ready.resume(throwing: error) + case .success: + ready.resume(throwing: CancellationError()) + } + } + teardown.outcomeContinuation?.resume(with: result) + } +} + +public enum LoopbackRedirectRequest { + public static func isComplete(_ head: Data) -> Bool { + head.range(of: Data("\r\n\r\n".utf8)) != nil || head.range(of: Data("\n\n".utf8)) != nil + } + + public static func text(of head: Data) -> String { + String(bytes: head, encoding: .utf8) ?? String(bytes: head, encoding: .isoLatin1) ?? "" + } + + // The target of a `GET` in origin form, the only shape a browser sends for a redirect. + public static func getTarget(_ head: String) -> String? { + let scalars = head.unicodeScalars + let lineEnd = scalars.firstIndex(of: "\n") ?? scalars.endIndex + var requestLine = String(scalars[scalars.startIndex.. String? { + let matches = items.filter { $0.name == name } + guard matches.count == 1 else { return nil } + return matches[0].value + } + + // A redirect query is form encoded, so "+" stands for a space. + public static func formDecodedItems(in components: URLComponents) -> [URLQueryItem] { + (components.percentEncodedQueryItems ?? []).map { item in + URLQueryItem(name: formDecoded(item.name), value: item.value.map(formDecoded)) + } + } + + public static func constantTimeEquals(_ lhs: String, _ rhs: String) -> Bool { + let left = Array(lhs.utf8) + let right = Array(rhs.utf8) + guard left.count == right.count else { return false } + var difference: UInt8 = 0 + for index in left.indices { + difference |= left[index] ^ right[index] + } + return difference == 0 + } + + private static func formDecoded(_ value: String) -> String { + let spaced = value.replacingOccurrences(of: "+", with: " ") + return spaced.removingPercentEncoding ?? spaced + } +} + +public enum LoopbackRedirectPage { + public static func response(status: String, title: String, message: String) -> Data { + let body = Data(""" + TablePro + +

\(title)

\(message)

+ """.utf8) + let headers = [ + "HTTP/1.1 \(status)", + "Content-Type: text/html; charset=utf-8", + "Content-Length: \(body.count)", + "Cache-Control: no-store", + "X-Content-Type-Options: nosniff", + "Referrer-Policy: no-referrer", + "Content-Security-Policy: default-src 'none'; style-src 'unsafe-inline'", + "Connection: close" + ] + return Data((headers.joined(separator: "\r\n") + "\r\n\r\n").utf8) + body + } + + public static let badRequest = response(status: "400 Bad Request", title: "Bad Request", message: "") +} + +enum LoopbackRedirectLimits { + static let maximumRequestHeadBytes = 16_384 + static let maximumConnections = 16 + static let maximumKernelPortAttempts = 5 + static let logger = Logger(subsystem: "com.TablePro", category: "LoopbackRedirectListener") + + static func code(of error: NWError) -> Int32 { + if case .posix(let code) = error { return code.rawValue } + return -1 + } +} + +enum LoopbackPortHold: Sendable { + case held(Int32) + case unavailable + case taken(Int32) + + // Bound but never listening: a browser that tries ::1 first is not answered there and falls back to 127.0.0.1. + static func ipv6(port: UInt16) -> LoopbackPortHold { + let descriptor = socket(AF_INET6, SOCK_STREAM, 0) + guard descriptor >= 0 else { + let failure = errno + return isUnavailable(failure) ? .unavailable : .taken(failure) + } + _ = fcntl(descriptor, F_SETFD, FD_CLOEXEC) + var address = sockaddr_in6() + address.sin6_len = UInt8(MemoryLayout.size) + address.sin6_family = sa_family_t(AF_INET6) + address.sin6_port = port.bigEndian + guard inet_pton(AF_INET6, "::1", &address.sin6_addr) == 1 else { + close(descriptor) + return .unavailable + } + let result = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { generic in + bind(descriptor, generic, socklen_t(MemoryLayout.size)) + } + } + if result == 0 { + return .held(descriptor) + } + let failure = errno + close(descriptor) + return isUnavailable(failure) ? .unavailable : .taken(failure) + } + + private static func isUnavailable(_ code: Int32) -> Bool { + code == EADDRNOTAVAIL || code == EAFNOSUPPORT || code == EPROTONOSUPPORT + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift index 2d96cde84c..482f9d9b15 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift @@ -6,43 +6,55 @@ import Foundation enum InfluxCSV { static func records(_ data: Data) -> [[String]] { var reader = Reader() - data.withUnsafeBytes { reader.read($0) } + reader.read(data) return reader.finish() } - private struct Reader { - var records: [[String]] = [] - var record: [String] = [] - var field: [UInt8] = [] - var inQuotes = false - var lineHasContent = false + // Holds its state between reads, so a body can be split anywhere: inside a quoted field, between the two quotes of + // an escaped one, or between the CR and LF of one line end. + struct Reader { + private var records: [[String]] = [] + private var record: [String] = [] + private var field: [UInt8] = [] + private var inQuotes = false + // A quote right after the one that closed a quoted field is an escaped quote inside it. + private var closedQuote = false + // An LF right after the CR that ended a record is the same line end. + private var endedOnCarriageReturn = false + private var lineHasContent = false + + mutating func read(_ data: Data) { + data.withUnsafeBytes { read($0) } + } mutating func read(_ bytes: UnsafeRawBufferPointer) { - var index = 0 - while index < bytes.count { - let byte = bytes[index] - index += 1 + for byte in bytes { + if endedOnCarriageReturn { + endedOnCarriageReturn = false + if byte == Self.lineFeed { continue } + } if inQuotes { - if byte == Self.quote, index < bytes.count, bytes[index] == Self.quote { - field.append(Self.quote) - index += 1 - } else if byte == Self.quote { + if byte == Self.quote { inQuotes = false + closedQuote = true } else { field.append(byte) } continue } + let afterClosingQuote = closedQuote + closedQuote = false switch byte { case Self.quote: + if afterClosingQuote { field.append(Self.quote) } inQuotes = true lineHasContent = true case Self.comma: endField() lineHasContent = true case Self.carriageReturn: - if index < bytes.count, bytes[index] == Self.lineFeed { index += 1 } endRecord() + endedOnCarriageReturn = true case Self.lineFeed: endRecord() default: @@ -52,9 +64,16 @@ enum InfluxCSV { } } + // The records completed so far. One still open stays for the next read. + mutating func drain() -> [[String]] { + let completed = records + records = [] + return completed + } + mutating func finish() -> [[String]] { if lineHasContent || !record.isEmpty { endRecord() } - return records + return drain() } private mutating func endField() { diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift index cdfa81e9cd..d874a4634b 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift @@ -104,6 +104,13 @@ public struct InfluxGridQuery: Sendable, Equatable, Hashable { return marked ? InfluxStatement.marked(statement, language: language) : statement } + // A read that matched nothing comes back with no columns at all, so the grid still gets them from the schema. + public func emptyResult(schema: [InfluxColumn]) -> InfluxResultTable { + let names = projection.isEmpty ? schema.map(\.name) : projection + let types = Dictionary(schema.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) + return InfluxResultTable(columns: names, typeNames: names.map { types[$0] ?? InfluxTypeName.string }, rows: []) + } + // InfluxQL answers `COUNT(*)` with one `count_` per field, the point count only when there is one field. public static func rowCount(from table: InfluxResultTable) -> Int? { guard let row = table.rows.first else { return 0 } diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift new file mode 100644 index 0000000000..c03223406d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift @@ -0,0 +1,166 @@ +import Foundation + +// A grid page or export read as its body arrives. Each batch holds the rows the collected read gives for the same bytes, +// under one header that the measurement's schema types, as `typed(by:)` types the collected table. +public struct InfluxGridStream { + public enum Body: Sendable, Equatable { + // 3.x SQL and InfluxQL: CSV, with the header record on the first record batch only. + case csv + // 2.x InfluxQL with `chunked=true`: one JSON object per line, each in the v1 shape. + case influxQLChunks + } + + static let batchRows = 5_000 + // A wide header shrinks the batch, so one holds about a million cells at any width. + static let batchCells = 1 << 20 + + private let body: Body + private let schema: [InfluxColumn] + private let emptyResult: InfluxResultTable + private var csv = InfluxCSV.Reader() + private var lines = InfluxJSONLines() + private var layout: InfluxResponseDecoder.V3Layout? + private var header: InfluxResultTable? + private var pending: [[String?]] = [] + private var bytesRead = 0 + private var rowsRead = 0 + + public init(body: Body, grid: InfluxGridQuery, schema: [InfluxColumn]) { + self.body = body + self.schema = schema + emptyResult = grid.emptyResult(schema: schema) + } + + // Every batch carries the same columns and types, and the first one's are the stream's header. + public mutating func consume(_ chunk: Data) throws -> [InfluxResultTable] { + switch body { + case .csv: + bytesRead += chunk.count + csv.read(chunk) + return try csvBatches(csv.drain()) + case .influxQLChunks: + var batches: [InfluxResultTable] = [] + for line in lines.consume(chunk) { + if let batch = try influxQLBatch(line) { batches.append(batch) } + } + return batches + } + } + + // What is left once the body ends. A stream that has sent nothing gets its header here: from the CSV header record, + // or from the schema when the body named no columns. + public mutating func finish() throws -> [InfluxResultTable] { + var batches: [InfluxResultTable] = [] + switch body { + case .csv: + batches = try csvBatches(csv.finish()) + if let layout, !pending.isEmpty || header == nil { + batches.append(batch(pending, layout: layout)) + pending = [] + } + case .influxQLChunks: + if let line = lines.finish(), let batch = try influxQLBatch(line) { + batches.append(batch) + } + } + if header == nil { + batches.append(fixedHeader(emptyResult)) + } + return batches + } + + private mutating func csvBatches(_ records: [[String]]) throws -> [InfluxResultTable] { + var records = records[...] + if layout == nil, let columns = records.popFirst() { + layout = InfluxResponseDecoder.V3Layout(columns: columns) + } + guard let layout, !records.isEmpty else { return [] } + rowsRead += records.count + try InfluxResponseDecoder.CellBudget.perByte(bytes: bytesRead).check(rows: rowsRead, columns: layout.columns.count) + let size = max(1, min(Self.batchRows, Self.batchCells / max(layout.columns.count, 1))) + var batches: [InfluxResultTable] = [] + for record in records { + pending.append(layout.row(record)) + if pending.count >= size { + batches.append(batch(pending, layout: layout)) + pending.removeAll(keepingCapacity: true) + } + } + return batches + } + + // The first batch fixes the header. A column the schema lacks is typed from that batch's values, where the collected + // read infers it from every row. + private mutating func batch(_ rows: [[String?]], layout: InfluxResponseDecoder.V3Layout) -> InfluxResultTable { + let header = self.header ?? fixedHeader(layout.table(rows).typed(by: schema)) + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: rows) + } + + private mutating func influxQLBatch(_ line: Data) throws -> InfluxResultTable? { + let table = try InfluxResponseDecoder.decodeV1JSON(line) + guard !table.columns.isEmpty else { return nil } + guard let header else { + let typed = table.typed(by: schema) + _ = fixedHeader(typed) + return typed + } + if table.columns == header.columns { + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: table.rows) + } + let position = Dictionary(header.columns.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) + let targets = try table.columns.map { column -> Int in + guard let index = position[column] else { + throw InfluxError.malformedResponse( + String(format: String(localized: "column %@ first appears in a later chunk"), column) + ) + } + return index + } + let rows = table.rows.map { row -> [String?] in + var mapped = [String?](repeating: nil, count: header.columns.count) + for (index, target) in targets.enumerated() where index < row.count { + mapped[target] = row[index] + } + return mapped + } + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: rows) + } + + private mutating func fixedHeader(_ table: InfluxResultTable) -> InfluxResultTable { + if let header { return header } + let fixed = InfluxResultTable(columns: table.columns, typeNames: table.typeNames, rows: []) + header = fixed + return fixed + } +} + +// The lines of a body that arrives in pieces, each line whole however the pieces split it. +struct InfluxJSONLines { + private var partial = Data() + + mutating func consume(_ chunk: Data) -> [Data] { + var lines: [Data] = [] + var start = chunk.startIndex + while let end = chunk[start...].firstIndex(of: Self.lineFeed) { + var line = Data() + swap(&line, &partial) + line.append(chunk[start ..< end]) + if !Self.isBlank(line) { lines.append(line) } + start = chunk.index(after: end) + } + partial.append(chunk[start...]) + return lines + } + + mutating func finish() -> Data? { + var line = Data() + swap(&line, &partial) + return Self.isBlank(line) ? nil : line + } + + private static func isBlank(_ line: Data) -> Bool { + line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D || $0 == 0x0A } + } + + private static let lineFeed = UInt8(ascii: "\n") +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift index e70ee42d85..e254fb5ec0 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift @@ -76,6 +76,8 @@ public enum InfluxCredentials: Sendable, Equatable { public enum InfluxRequests { public static let bucketPageSize = 100 + // 2.x's own default for `chunked=true`. + public static let influxQLChunkSize = 10_000 // 3.x with auth on refuses an anonymous `/ping`. Before the version is known the token goes out as `Token`. public static func ping(credentials: InfluxCredentials, generation: InfluxGeneration? = nil) -> InfluxRequest { @@ -148,13 +150,16 @@ public enum InfluxRequests { InfluxRequest(method: "GET", path: "/api/v2/orgs", headers: authHeaders(credentials, generation: .v2)) } + // `streamed` asks 2.x InfluxQL for chunks: without it the server gathers the whole result in its memory before the + // first byte. 3.x writes CSV per record batch either way. public static func query( _ statement: InfluxStatement, database: String, retentionPolicy: String? = nil, organization: String, generation: InfluxGeneration, - credentials: InfluxCredentials + credentials: InfluxCredentials, + streamed: Bool = false ) throws -> InfluxRequest { var headers = authHeaders(credentials, generation: generation) switch (generation, statement.language) { @@ -190,6 +195,10 @@ public enum InfluxRequests { var form = [InfluxQueryItem("q", statement.text)] if !database.isEmpty { form.append(InfluxQueryItem("db", database)) } if let retentionPolicy, !retentionPolicy.isEmpty { form.append(InfluxQueryItem("rp", retentionPolicy)) } + if streamed { + form.append(InfluxQueryItem("chunked", "true")) + form.append(InfluxQueryItem("chunk_size", String(influxQLChunkSize))) + } return InfluxRequest( method: "POST", path: "/query", diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift index 4d1eb53d11..e9d6c7d518 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift @@ -6,24 +6,38 @@ public enum InfluxResponseDecoder { var records = InfluxCSV.records(data)[...] guard let columns = records.popFirst() else { return .empty } try CellBudget.perByte(of: data).check(rows: records.count, columns: columns.count) - let rows: [[String?]] = records.map { record in - columns.indices.map { index in + let layout = V3Layout(columns: columns) + return layout.table(records.map(layout.row)) + } + + // A 3.x CSV record as a row under its header. The stream maps records with this too, so a cell reads the same. + struct V3Layout { + let columns: [String] + // Only `time` is inferred as a timestamp, so it is the one column whose zone-less values get a `Z`. + private let timeColumns: [Int] + + init(columns: [String]) { + self.columns = columns + timeColumns = columns.indices.filter { columns[$0] == InfluxTypeName.timeColumn } + } + + func row(_ record: [String]) -> [String?] { + var row: [String?] = columns.indices.map { index in guard index < record.count, !record[index].isEmpty else { return nil } return record[index] } - } - let typeNames = columns.indices.map { index in - InfluxTypeName.inferred(column: columns[index], values: rows.map { $0[index] }) - } - let timestampColumns = typeNames.indices.filter { typeNames[$0] == InfluxTypeName.timestamp } - let normalized = timestampColumns.isEmpty ? rows : rows.map { row in - var row = row - for index in timestampColumns { + for index in timeColumns { row[index] = row[index].map(InfluxTypeName.normalizedTimestamp) } return row } - return InfluxResultTable(columns: columns, typeNames: typeNames, rows: normalized) + + func table(_ rows: [[String?]]) -> InfluxResultTable { + let typeNames = columns.indices.map { index in + InfluxTypeName.inferred(column: columns[index], values: rows.map { $0[index] }) + } + return InfluxResultTable(columns: columns, typeNames: typeNames, rows: rows) + } } // InfluxQL over `/query` (1.x and the 2.x compatibility API). Every series of every statement lands in one grid: a @@ -159,12 +173,16 @@ public enum InfluxResponseDecoder { } // Rows are as wide as every header together, so a short record costs more cells than bytes. - private struct CellBudget { + struct CellBudget { let limit: Int // Every v3 CSV row carries every column, a byte per cell at least: 16 cells per byte, or a million at any size. static func perByte(of data: Data) -> CellBudget { - let (scaled, overflow) = data.count.multipliedReportingOverflow(by: 16) + perByte(bytes: data.count) + } + + static func perByte(bytes: Int) -> CellBudget { + let (scaled, overflow) = bytes.multipliedReportingOverflow(by: 16) return CellBudget(limit: overflow ? .max : max(scaled, 1 << 20)) } diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift index cb7d49a806..91ea59b7e3 100644 --- a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift @@ -23,4 +23,69 @@ public enum KafkaOffsetMath { let (start, overflows) = end.subtractingReportingOverflow(step) return overflows ? floor : max(floor, start) } + + // A v2 batch header keeps the last offset it covers after compaction empties it or its records are aborted. + public static func offset(afterBatchAt baseOffset: Int64, lastOffsetDelta: Int32) -> Int64? { + let (last, overflows) = baseOffset.addingReportingOverflow(Int64(max(0, lastOffsetDelta))) + return overflows ? nil : offset(after: last) + } + + // As far back as the last range's density puts the missing records, a quarter more for an uneven spread, so a + // sparse stretch is crossed in a few reads and a dense one is not read past the page. At most `maximumGrowth` + // times the last range, which is also the step after a range that held nothing. + public static func nextWidth( + lastWidth: Int64, + lastRecords: Int, + needed: Int, + maximumGrowth: Int64 + ) -> Int64 { + let ceiling = max(1, lastWidth).multipliedReportingOverflow(by: max(2, maximumGrowth)) + let cap = ceiling.overflow ? Int64.max : ceiling.partialValue + guard lastRecords > 0 else { return cap } + let estimate = Double(max(0, needed)) * Double(max(1, lastWidth)) / Double(lastRecords) * 1.25 + guard estimate < Double(cap) else { return cap } + return max(1, Int64(estimate.rounded(.up))) + } +} + +// Offsets are not one per record: transaction markers, aborted records and compaction leave offsets with no +// record, so a tail window is widened by the records it holds, not by the offsets it spans. +public struct KafkaTailWalk { + public private(set) var records: [Record] = [] + // Nil once the walk holds `wanted` records or has read down to the floor. + public private(set) var nextRange: (start: Int64, end: Int64)? + public private(set) var start: Int64 + private let wanted: Int + private let floor: Int64 + private let maximumGrowth: Int64 + + public init(wanted: Int, end: Int64, firstStart: Int64, floor: Int64, maximumGrowth: Int64 = 4) { + self.wanted = wanted + self.floor = floor + self.maximumGrowth = maximumGrowth + start = firstStart + nextRange = wanted > 0 ? (firstStart, end) : nil + } + + // How many more records the walk takes; a read of `nextRange` keeps only its newest this many. + public var needed: Int { max(0, wanted - records.count) } + + // `earlier` holds the newest records of `nextRange` in offset order, all older than the ones already held. + public mutating func absorb(_ earlier: [Record]) { + guard let range = nextRange else { return } + start = range.start + records = Array((earlier + records).suffix(wanted)) + guard records.count < wanted, range.start > floor else { + nextRange = nil + return + } + let (span, overflows) = range.end.subtractingReportingOverflow(range.start) + let width = KafkaOffsetMath.nextWidth( + lastWidth: overflows ? Int64.max : span, + lastRecords: earlier.count, + needed: needed, + maximumGrowth: maximumGrowth + ) + nextRange = (KafkaOffsetMath.stepBack(from: range.start, by: width, floor: floor), range.start) + } } diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift index bb11e7e97d..8ba659cab5 100644 --- a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift @@ -73,17 +73,20 @@ public enum KafkaRecordBatchDecoder { // The blob is a sequence of batches cut at partition_max_bytes, so a trailing partial batch // is normal. Control batches hold transaction markers, never rows for the grid. // Every batch charges `budget`, so the records kept across the whole blob stay within it. + // `nextOffset` follows the last whole batch of any kind, nil when none was whole: a reply of only + // markers, aborted or compacted batches still moves the read on. public static func decode( blob: Data, partition: Int32, abortedTransactions: [KafkaAbortedTransaction] = [], skipAborted: Bool, budget: inout DecompressionBudget - ) throws -> (records: [KafkaRecord], truncated: Bool) { + ) throws -> (records: [KafkaRecord], truncated: Bool, nextOffset: Int64?) { let bytes = [UInt8](blob) var cursor = 0 var records: [KafkaRecord] = [] var truncated = false + var nextOffset: Int64? // A partition can carry several producers at once, so an aborted run belongs to its // producer. Keying this on the offset alone let one producer's commit marker end // another's aborted run, and its rolled-back records then reached the grid. @@ -120,6 +123,7 @@ public enum KafkaRecordBatchDecoder { ) } let attributes = Self.int16(bytes, at: cursor + 21) + let lastOffsetDelta = Self.int32(bytes, at: cursor + 23) let baseTimestamp = Self.int64(bytes, at: cursor + 27) let producerId = Self.int64(bytes, at: cursor + 43) let count = Int(Self.int32(bytes, at: cursor + 57)) @@ -140,6 +144,7 @@ public enum KafkaRecordBatchDecoder { // A control batch is its producer's commit or abort marker, and ends only that // producer's run. abortedProducers.remove(producerId) + nextOffset = KafkaOffsetMath.offset(afterBatchAt: baseOffset, lastOffsetDelta: lastOffsetDelta) cursor = end continue } @@ -157,11 +162,12 @@ public enum KafkaRecordBatchDecoder { partition: partition )) } + nextOffset = KafkaOffsetMath.offset(afterBatchAt: baseOffset, lastOffsetDelta: lastOffsetDelta) cursor = end } if cursor < bytes.count { truncated = true } - return (records, truncated) + return (records, truncated, nextOffset) } /// The records inside a batch use ZIG-ZAG signed varints, unlike the unsigned LEB128 the diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift index c9051e99ed..2a75a10868 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift @@ -19,6 +19,12 @@ public enum MSSQLDefinitionText { ["NATIONAL", "TEXT"], ["BINARY", "VARYING"], ] private static let longestMultiWordType = 3 + // ALTER COLUMN takes COLLATE on these and their ISO synonyms only, never on text, ntext or an alias type. + private static let characterTypes: Set<[String]> = [ + ["char"], ["varchar"], ["nchar"], ["nvarchar"], ["character"], ["char", "varying"], ["character", "varying"], + ["national", "char"], ["national", "character"], ["national", "char", "varying"], + ["national", "character", "varying"], + ] public static func isParenthesizedExpression(_ text: String) -> Bool { SQLExpressionFragment.isSingleExpression(text, databaseTypeId: databaseTypeId) @@ -54,6 +60,11 @@ public enum MSSQLDefinitionText { return SQLExpressionFragment.isSingleType(source.substring(from: index), databaseTypeId: databaseTypeId) } + public static func isCharacterType(_ text: String) -> Bool { + let name = text.prefix { $0 != "(" }.lowercased().filter { $0 != "[" && $0 != "]" } + return characterTypes.contains(name.split(whereSeparator: \.isWhitespace).map(String.init)) + } + public static func isCollationName(_ text: String) -> Bool { !text.isEmpty && text.unicodeScalars.allSatisfy { scalar in ("A"..."Z").contains(scalar) || ("a"..."z").contains(scalar) || ("0"..."9").contains(scalar) || scalar == "_" @@ -205,6 +216,14 @@ public struct MSSQLDefinitionRefusal: Error, LocalizedError, Sendable, Equatable )) } + public static func foreignKeyAction(_ constraint: String, _ action: String) -> MSSQLDefinitionRefusal { + MSSQLDefinitionRefusal(reason: String( + format: String(localized: "The referential action of foreign key %@ is not one SQL Server has: %@"), + constraint, + action + )) + } + public static func indexFilter(_ index: String) -> MSSQLDefinitionRefusal { MSSQLDefinitionRefusal(reason: String( format: String(localized: "The filter of index %@ is not one SQL expression, so it cannot be written."), diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift index 8984ccce8a..64913d299a 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift @@ -96,42 +96,171 @@ public enum MSSQLSchemaQueries { } } + // A column keeps COLLATE only where it differs from `databaseCollation`; nil writes every column's. public static func createTableStatement( schema: String, table: String, columns: [MSSQLColumnRow], - primaryKeyColumns: [String], - foreignKeys: [MSSQLForeignKeyRow] + keyConstraints: [MSSQLKeyConstraintRow] = [], + checkConstraints: [MSSQLCheckConstraintRow] = [], + foreignKeys: [MSSQLForeignKeyRow] = [], + databaseCollation: String? ) throws -> String { - var parts = try columns.map { column -> String in - let type = column.displayType.uppercased() - guard MSSQLDefinitionText.isColumnType(type) else { - throw MSSQLDefinitionRefusal.columnType(column.name, type) + var parts = try columns.map { try columnDefinition($0, databaseCollation: databaseCollation) } + parts += keyConstraintDefinitions(keyConstraints) + parts += try foreignKeyDefinitions(foreignKeys, schema: schema) + parts += try checkConstraintDefinitions(checkConstraints, table: table) + let body = parts.map { " " + $0 }.joined(separator: ",\n") + let created = "CREATE TABLE \(bracketed(schema: schema, table: table)) (\n\(body)\n);" + // CREATE TABLE has no way to declare a check disabled or untrusted, and rows the source let past one would + // fail the restore, so it is switched off before any row goes in. + let unchecked = checkConstraints.filter { $0.definition != nil && ($0.isDisabled || $0.isNotTrusted) } + .map { "ALTER TABLE \(bracketed(schema: schema, table: table)) NOCHECK CONSTRAINT \(SQLQuoting.identifier($0.name, quote: .bracket));" } + return ([created] + unchecked).joined(separator: "\n") + } + + // After the rows: a check the source enforced without having checked its rows is turned back on the same way. + public static func checkConstraintReenableStatements( + schema: String, + table: String, + checkConstraints: [MSSQLCheckConstraintRow] + ) -> [String] { + checkConstraints.filter { $0.definition != nil && $0.isNotTrusted && !$0.isDisabled }.map { + "ALTER TABLE \(bracketed(schema: schema, table: table)) WITH NOCHECK CHECK CONSTRAINT " + + "\(SQLQuoting.identifier($0.name, quote: .bracket));" + } + } + + private static func columnDefinition(_ column: MSSQLColumnRow, databaseCollation: String?) throws -> String { + let name = SQLQuoting.identifier(column.name, quote: .bracket) + if column.isComputed { + guard let expression = column.computedDefinition.flatMap(MSSQLDefinitionText.expression) else { + throw MSSQLDefinitionRefusal.computedColumn(column.name) } - var definition = " \(SQLQuoting.identifier(column.name, quote: .bracket)) \(type)" - if column.isIdentity { definition += " IDENTITY(1,1)" } - definition += column.isNullable ? " NULL" : " NOT NULL" - if let value = column.defaultValue { - guard let expression = MSSQLDefinitionText.expression(value) else { - throw MSSQLDefinitionRefusal.columnDefault(column.name) - } - definition += " DEFAULT \(expression)" + // NOT NULL is allowed on a computed column only with PERSISTED. + guard column.isPersisted else { return "\(name) AS \(expression)" } + return "\(name) AS \(expression) PERSISTED" + (column.isNullable ? "" : " NOT NULL") + } + let type = column.displayType.uppercased() + guard MSSQLDefinitionText.isColumnType(type) else { + throw MSSQLDefinitionRefusal.columnType(column.name, type) + } + var definition = "\(name) \(type)" + if let collation = column.collation, collation != databaseCollation { + guard MSSQLDefinitionText.isCollationName(collation) else { + throw MSSQLDefinitionRefusal.collation(column.name, collation) + } + definition += " COLLATE \(collation)" + } + if column.isIdentity { + let identity = "\(column.identitySeed ?? ""),\(column.identityIncrement ?? "")" + guard MSSQLDefinitionText.isIdentitySpecification(identity) else { + throw MSSQLDefinitionRefusal.identity(column.name, identity) + } + definition += " IDENTITY(\(identity))" + } + definition += column.isNullable ? " NULL" : " NOT NULL" + if let value = column.defaultValue { + guard let expression = MSSQLDefinitionText.expression(value) else { + throw MSSQLDefinitionRefusal.columnDefault(column.name) + } + definition += " DEFAULT \(expression)" + } + return definition + } + + private static func keyConstraintDefinitions(_ rows: [MSSQLKeyConstraintRow]) -> [String] { + let constraints = grouped(rows, by: { $0.name }) + let primaryFirst = constraints.filter { $0.first?.isPrimaryKey == true } + + constraints.filter { $0.first?.isPrimaryKey != true } + return primaryFirst.compactMap { keyRows -> String? in + guard let first = keyRows.first else { return nil } + var definition = "CONSTRAINT \(SQLQuoting.identifier(first.name, quote: .bracket))" + definition += first.isPrimaryKey ? " PRIMARY KEY" : " UNIQUE" + if let kind = MSSQLDefinitionText.indexKind(first.indexKind) { + definition += " \(kind)" + } + let keys = keyRows.map { key in + SQLQuoting.identifier(key.columnName, quote: .bracket) + (key.isDescending ? " DESC" : "") + } + return definition + " (\(keys.joined(separator: ", ")))" + } + } + + // A referenced table without a schema resolves against the login's default schema, so it is always qualified. + private static func foreignKeyDefinitions(_ rows: [MSSQLForeignKeyRow], schema: String) throws -> [String] { + try grouped(rows, by: { $0.constraintName }).compactMap { keyRows -> String? in + guard let first = keyRows.first else { return nil } + guard let onDelete = MSSQLDefinitionText.referentialAction(first.onDelete) else { + throw MSSQLDefinitionRefusal.foreignKeyAction(first.constraintName, first.onDelete) + } + guard let onUpdate = MSSQLDefinitionText.referentialAction(first.onUpdate) else { + throw MSSQLDefinitionRefusal.foreignKeyAction(first.constraintName, first.onUpdate) } + let columns = keyRows.map { SQLQuoting.identifier($0.columnName, quote: .bracket) } + let referencedColumns = keyRows.map { SQLQuoting.identifier($0.referencedColumn, quote: .bracket) } + let referencedSchema = first.referencedSchema.flatMap { $0.isEmpty ? nil : $0 } ?? schema + var definition: String = "CONSTRAINT \(SQLQuoting.identifier(first.constraintName, quote: .bracket))" + + " FOREIGN KEY (\(columns.joined(separator: ", ")))" + + " REFERENCES \(bracketed(schema: referencedSchema, table: first.referencedTable))" + + " (\(referencedColumns.joined(separator: ", ")))" + if onDelete != "NO ACTION" { definition += " ON DELETE \(onDelete)" } + if onUpdate != "NO ACTION" { definition += " ON UPDATE \(onUpdate)" } return definition } - if !primaryKeyColumns.isEmpty { - let keys = primaryKeyColumns.map { SQLQuoting.identifier($0, quote: .bracket) }.joined(separator: ", ") - parts.append(" CONSTRAINT \(SQLQuoting.identifier("PK_" + table, quote: .bracket)) PRIMARY KEY (\(keys))") + } + + // An unreadable definition is left out, as an unreadable default is. + private static func checkConstraintDefinitions(_ rows: [MSSQLCheckConstraintRow], table: String) throws -> [String] { + try rows.compactMap { row -> String? in + guard let definition = row.definition else { return nil } + guard let expression = MSSQLDefinitionText.expression(definition) else { + throw MSSQLDefinitionRefusal.checkConstraint(table) + } + return "CONSTRAINT \(SQLQuoting.identifier(row.name, quote: .bracket)) CHECK \(expression)" } - for key in foreignKeys { - parts.append( - " CONSTRAINT \(SQLQuoting.identifier(key.constraintName, quote: .bracket))" - + " FOREIGN KEY (\(SQLQuoting.identifier(key.columnName, quote: .bracket)))" - + " REFERENCES \(SQLQuoting.identifier(key.referencedTable, quote: .bracket))" - + " (\(SQLQuoting.identifier(key.referencedColumn, quote: .bracket)))" - ) + } + + private static func grouped(_ rows: [Row], by key: (Row) -> String) -> [[Row]] { + var order: [String] = [] + var groups: [String: [Row]] = [:] + for row in rows { + let name = key(row) + if groups[name] == nil { order.append(name) } + groups[name, default: []].append(row) } - return "CREATE TABLE \(bracketed(schema: schema, table: table)) (\n" + parts.joined(separator: ",\n") + "\n);" + return order.compactMap { groups[$0] } + } + + // SQL Server gives a column altered without COLLATE the database default, so a character column restates the + // collation the new definition carries, or else the one it had. + public static func alterColumnCollation(type: String, collation: String?, previous: String?) -> String? { + guard MSSQLDefinitionText.isCharacterType(type) else { return nil } + return collation.flatMap(nonEmpty) ?? previous.flatMap(nonEmpty) + } + + public static func alterColumnStatement( + qualifiedTable: String, + column: String, + type: String, + collation: String?, + isNullable: Bool + ) throws -> String { + guard MSSQLDefinitionText.isColumnType(type) else { + throw MSSQLDefinitionRefusal.columnType(column, type) + } + var statement = "ALTER TABLE \(qualifiedTable) ALTER COLUMN \(SQLQuoting.identifier(column, quote: .bracket)) \(type)" + if let collation { + guard MSSQLDefinitionText.isCollationName(collation) else { + throw MSSQLDefinitionRefusal.collation(column, collation) + } + statement += " COLLATE \(collation)" + } + return statement + (isNullable ? " NULL" : " NOT NULL") + } + + private static func nonEmpty(_ text: String) -> String? { + text.isEmpty ? nil : text } public static func qualifiedName(schema: String?, table: String) -> String { @@ -237,7 +366,7 @@ public enum MSSQLSchemaQueries { SELECT \(columnSelectList) FROM INFORMATION_SCHEMA.COLUMNS c - \(columnDefaultJoin) + \(columnCatalogJoin) LEFT JOIN ( SELECT kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc @@ -262,7 +391,7 @@ public enum MSSQLSchemaQueries { \(columnSelectList), c.TABLE_NAME FROM INFORMATION_SCHEMA.COLUMNS c - \(columnDefaultJoin) + \(columnCatalogJoin) LEFT JOIN ( SELECT kcu.TABLE_NAME, kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc @@ -277,11 +406,12 @@ public enum MSSQLSchemaQueries { """ } - public static let allColumnsTableNameIndex = 11 + public static let allColumnsTableNameIndex = 16 - /// A `rowversion` column reports its type as `timestamp`, and a system-versioned period or ledger column reports a - /// `GeneratedAlwaysType` above zero. Neither is identity or computed, and the server refuses a written value for - /// both. Before SQL Server 2016 the property does not exist and reads NULL, which is not above zero. + // A `rowversion` column reports its type as `timestamp`, and a system-versioned period or ledger column reports a + // `GeneratedAlwaysType` above zero. Neither is identity or computed, and the server refuses a written value for + // both. Before SQL Server 2016 the property does not exist and reads NULL, which is not above zero. + // `sys.identity_columns` keeps seed and increment as `sql_variant`, read here as text. private static let columnSelectList = """ c.COLUMN_NAME, c.DATA_TYPE, @@ -297,15 +427,27 @@ public enum MSSQLSchemaQueries { WHEN c.DATA_TYPE = 'timestamp' THEN 1 WHEN COLUMNPROPERTY(\(columnObjectID), c.COLUMN_NAME, 'GeneratedAlwaysType') > 0 THEN 1 ELSE 0 - END AS IS_SERVER_GENERATED + END AS IS_SERVER_GENERATED, + \(columnDefinitionSelectList) + """ + + // The parts a CREATE TABLE restates, after the flags every reader takes, read the same way for a `#name` table. + private static let columnDefinitionSelectList = """ + c.COLLATION_NAME, + cc.definition AS COMPUTED_DEFINITION, + cc.is_persisted AS IS_PERSISTED, + CONVERT(varchar(40), idc.seed_value) AS IDENTITY_SEED, + CONVERT(varchar(40), idc.increment_value) AS IDENTITY_INCREMENT """ private static let columnObjectID = "OBJECT_ID(QUOTENAME(c.TABLE_SCHEMA) + '.' + QUOTENAME(c.TABLE_NAME))" // `INFORMATION_SCHEMA.COLUMNS.COLUMN_DEFAULT` stops at 4,000 characters, which can leave a literal open. - private static let columnDefaultJoin = """ + private static let columnCatalogJoin = """ LEFT JOIN sys.columns sc ON sc.object_id = \(columnObjectID) AND sc.name = c.COLUMN_NAME LEFT JOIN sys.default_constraints dc ON dc.object_id = sc.default_object_id + LEFT JOIN sys.computed_columns cc ON cc.object_id = sc.object_id AND cc.column_id = sc.column_id + LEFT JOIN sys.identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id """ // A `#name` table lives in `tempdb` whatever database the session is on. @@ -363,10 +505,13 @@ public enum MSSQLSchemaQueries { WHEN c.DATA_TYPE = 'timestamp' THEN 1 WHEN sc.generated_always_type > 0 THEN 1 ELSE 0 - END AS IS_SERVER_GENERATED + END AS IS_SERVER_GENERATED, + \(columnDefinitionSelectList) FROM tempdb.INFORMATION_SCHEMA.COLUMNS c JOIN tempdb.sys.columns sc ON sc.object_id = \(object) AND sc.name = c.COLUMN_NAME LEFT JOIN tempdb.sys.default_constraints dc ON dc.object_id = sc.default_object_id + LEFT JOIN tempdb.sys.computed_columns cc ON cc.object_id = sc.object_id AND cc.column_id = sc.column_id + LEFT JOIN tempdb.sys.identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id WHERE c.TABLE_NAME = OBJECT_NAME(\(object), DB_ID(N'tempdb')) ORDER BY c.ORDINAL_POSITION """ @@ -401,6 +546,69 @@ public enum MSSQLSchemaQueries { """ } + // Rows are what `indexStatements` reads. A primary key's or UNIQUE constraint's index is left out: + // `createTableStatement` declares the constraint, which builds it. + public static func indexDefinitions(schema: String, table: String) -> String { + let object = MSSQLStringLiteral.quoted(bracketed(schema: schema, table: table)) + return """ + SELECT i.name, i.type_desc, i.is_unique, i.filter_definition, + c.name AS column_name, ic.is_included_column, ic.is_descending_key + FROM sys.indexes i + JOIN sys.index_columns ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id + JOIN sys.columns c + ON ic.object_id = c.object_id AND ic.column_id = c.column_id + WHERE i.object_id = OBJECT_ID(\(object)) + AND i.name IS NOT NULL + AND i.is_primary_key = 0 + AND i.is_unique_constraint = 0 + AND i.type IN (1, 2) + ORDER BY i.index_id, ic.is_included_column, ic.key_ordinal + """ + } + + // Rows are `name, type, type_desc, column_name, is_descending_key`, one per key column in key order. + public static func keyConstraints(schema: String, table: String) -> String { + let (catalog, object) = catalogObject(schema: schema, table: table) + return """ + SELECT kc.name, kc.type, i.type_desc, c.name AS column_name, ic.is_descending_key + FROM \(catalog)sys.key_constraints kc + JOIN \(catalog)sys.indexes i + ON i.object_id = kc.parent_object_id AND i.index_id = kc.unique_index_id + JOIN \(catalog)sys.index_columns ic + ON ic.object_id = i.object_id AND ic.index_id = i.index_id + JOIN \(catalog)sys.columns c + ON c.object_id = ic.object_id AND c.column_id = ic.column_id + WHERE kc.parent_object_id = \(object) + AND ic.key_ordinal > 0 + ORDER BY CASE kc.type WHEN 'PK' THEN 0 ELSE 1 END, kc.name, ic.key_ordinal + """ + } + + // Rows are `name, definition, is_disabled, is_not_trusted`. + public static func checkConstraints(schema: String, table: String) -> String { + let (catalog, object) = catalogObject(schema: schema, table: table) + return """ + SELECT cc.name, cc.definition, cc.is_disabled, cc.is_not_trusted + FROM \(catalog)sys.check_constraints cc + WHERE cc.parent_object_id = \(object) + ORDER BY cc.name + """ + } + + // A `#name` table's columns take tempdb's default, not the current database's. + public static func databaseCollation(forTable table: String) -> String { + let database = namesTemporaryTable(table) ? "N'tempdb'" : "DB_NAME()" + return "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(\(database), 'Collation'))" + } + + private static func catalogObject(schema: String, table: String) -> (catalog: String, object: String) { + if namesTemporaryTable(table) { + return ("tempdb.", temporaryObjectID(table)) + } + return ("", "OBJECT_ID(\(MSSQLStringLiteral.quoted(bracketed(schema: schema, table: table))))") + } + // One statement for a table's keys and a schema's, so the two reads cannot drift apart. public static func foreignKeys(schema: String, table: String?) -> String { let s = MSSQLStringLiteral.quoted(schema) @@ -455,6 +663,11 @@ public struct MSSQLColumnRow: Sendable, Equatable { public let isPrimaryKey: Bool public let isComputed: Bool public let isServerGenerated: Bool + public let collation: String? + public let computedDefinition: String? + public let isPersisted: Bool + public let identitySeed: String? + public let identityIncrement: String? public init( name: String, @@ -467,7 +680,12 @@ public struct MSSQLColumnRow: Sendable, Equatable { isIdentity: Bool, isPrimaryKey: Bool, isComputed: Bool = false, - isServerGenerated: Bool = false + isServerGenerated: Bool = false, + collation: String? = nil, + computedDefinition: String? = nil, + isPersisted: Bool = false, + identitySeed: String? = nil, + identityIncrement: String? = nil ) { self.name = name self.dataType = dataType @@ -480,6 +698,11 @@ public struct MSSQLColumnRow: Sendable, Equatable { self.isPrimaryKey = isPrimaryKey self.isComputed = isComputed self.isServerGenerated = isServerGenerated + self.collation = collation + self.computedDefinition = computedDefinition + self.isPersisted = isPersisted + self.identitySeed = identitySeed + self.identityIncrement = identityIncrement } /// Whether the server refuses any value written to the column, which is what the host treats as generated. @@ -523,6 +746,36 @@ public struct MSSQLIndexRow: Sendable, Equatable { } } +public struct MSSQLKeyConstraintRow: Sendable, Equatable { + public let name: String + public let isPrimaryKey: Bool + public let indexKind: String? + public let columnName: String + public let isDescending: Bool + + public init(name: String, isPrimaryKey: Bool, indexKind: String?, columnName: String, isDescending: Bool = false) { + self.name = name + self.isPrimaryKey = isPrimaryKey + self.indexKind = indexKind + self.columnName = columnName + self.isDescending = isDescending + } +} + +public struct MSSQLCheckConstraintRow: Sendable, Equatable { + public let name: String + public let definition: String? + public let isDisabled: Bool + public let isNotTrusted: Bool + + public init(name: String, definition: String?, isDisabled: Bool = false, isNotTrusted: Bool = false) { + self.name = name + self.definition = definition + self.isDisabled = isDisabled + self.isNotTrusted = isNotTrusted + } +} + public struct MSSQLForeignKeyRow: Sendable, Equatable { public let constraintName: String public let columnName: String @@ -574,7 +827,33 @@ public extension MSSQLSchemaQueries { isIdentity: (row[safe: 7] ?? nil) == "1", isPrimaryKey: (row[safe: 8] ?? nil) == "1", isComputed: (row[safe: 9] ?? nil) == "1", - isServerGenerated: (row[safe: 10] ?? nil) == "1" + isServerGenerated: (row[safe: 10] ?? nil) == "1", + collation: (row[safe: 11] ?? nil).flatMap { $0.isEmpty ? nil : $0 }, + computedDefinition: row[safe: 12] ?? nil, + isPersisted: (row[safe: 13] ?? nil) == "1", + identitySeed: row[safe: 14] ?? nil, + identityIncrement: row[safe: 15] ?? nil + ) + } + + static func parseKeyConstraintRow(_ row: [String?]) -> MSSQLKeyConstraintRow? { + guard let name = row[safe: 0] ?? nil, let column = row[safe: 3] ?? nil else { return nil } + return MSSQLKeyConstraintRow( + name: name, + isPrimaryKey: (row[safe: 1] ?? nil)?.trimmingCharacters(in: .whitespaces) == "PK", + indexKind: row[safe: 2] ?? nil, + columnName: column, + isDescending: (row[safe: 4] ?? nil) == "1" + ) + } + + static func parseCheckConstraintRow(_ row: [String?]) -> MSSQLCheckConstraintRow? { + guard let name = row[safe: 0] ?? nil else { return nil } + return MSSQLCheckConstraintRow( + name: name, + definition: row[safe: 1] ?? nil, + isDisabled: (row[safe: 2] ?? nil) == "1", + isNotTrusted: (row[safe: 3] ?? nil) == "1" ) } diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedItemEdit.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedItemEdit.swift new file mode 100644 index 0000000000..54b5bba162 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedItemEdit.swift @@ -0,0 +1,34 @@ +import Foundation + +// How an edit of an item the grid read is written. A write that keeps the loaded Value or Flags is a `cas` on the +// unique `gets` returned, so a change made since the read fails instead of being undone. +public enum MemcachedItemEdit: Equatable, Sendable { + case touch + case set + case compareAndSwap(unique: UInt64) + case needsRefresh + + // Nil when nothing the item stores was edited. + public static func write(editsValue: Bool, editsFlags: Bool, editsTTL: Bool, unique: UInt64?) -> MemcachedItemEdit? { + guard editsValue || editsFlags else { return editsTTL ? .touch : nil } + if editsValue, editsFlags { return .set } + guard let unique else { return .needsRefresh } + return .compareAndSwap(unique: unique) + } +} + +// protocol.txt, Storage commands: a `cas` answers `EXISTS` when the item was modified since it was fetched and +// `NOT_FOUND` when it does not exist. Either way nothing was stored. +public enum MemcachedCASRefusal: Equatable, Sendable { + case changed + case notFound + + public init?(replyTo command: MemcachedCommand, status: String) { + guard case .compareAndSwap = command else { return nil } + switch status { + case "EXISTS": self = .changed + case "NOT_FOUND": self = .notFound + default: return nil + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift index 21fb9221ce..b3502139a6 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -52,7 +52,10 @@ public enum MemcachedKeyListing { guard let equals = part.firstIndex(of: "=") else { continue } fields[part[.. String { + "cas \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(unique) \(MemcachedCommandParser.quoted(value))" + } + + public static func cas(key: String, flags: UInt32, exptime: Int64, unique: UInt64, bytes: Data) -> String { + "cas \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(unique) \(MemcachedCommandParser.quoted(bytes: bytes))" + } + public static func touch(key: String, exptime: Int64) -> String { "touch \(MemcachedCommandParser.argument(key)) \(exptime)" } diff --git a/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift b/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift index 2a9a28bd92..9f35ef576b 100644 --- a/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift +++ b/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift @@ -90,26 +90,68 @@ public extension Row { } } } + + // A one-line list never needs more than a preview, even of a value the cell holds whole. + func displayValues(maxBytes: Int = Cell.previewByteLimit) -> [String?] { + cells.map { cell -> String? in + switch cell { + case .null: + return nil + case .text(let value): + return Cell.preview(of: value, maxBytes: maxBytes) + case .truncatedText, .binary: + return cell.displayString + } + } + } } public extension Cell { + static let previewByteLimit = 4_096 + + // For text whose kind only the column's declared type tells, such as a row read back as strings. static func from(legacyValue value: String?, columnTypeName: String?, options: StreamOptions, ref: CellRef? = nil) -> Cell { guard let value else { return .null } - let bytes = value.utf8.count - let upper = (columnTypeName ?? "").uppercased() - let isBinary = upper.contains("BLOB") || upper.contains("BYTEA") || upper.contains("BINARY") || upper.contains("VARBINARY") || upper.contains("IMAGE") + if let columnTypeName, holdsBytes(typeName: columnTypeName) { + return .binary(byteCount: value.utf8.count, ref: ref) + } + return from(text: value, options: options, ref: ref) + } - if isBinary { - return .binary(byteCount: bytes, ref: ref) + // For a value the driver knows is text, whatever its column's type is called. + static func from(text value: String, options: StreamOptions, ref: CellRef? = nil) -> Cell { + let bytes = value.utf8.count + guard bytes > options.textTruncationBytes else { return .text(value) } + // A value with a ref can be read again by its key, so it never spends the budget. + if ref == nil, let budget = options.textBudget, budget.admit(bytes) { + return .text(value) } + return .truncatedText(prefix: preview(of: value, maxBytes: options.textTruncationBytes), totalBytes: bytes, ref: ref) + } - if bytes > options.textTruncationBytes { - let prefixSlice = value.prefix(options.textTruncationBytes) - return .truncatedText(prefix: String(prefixSlice), totalBytes: bytes, ref: ref) + // At most `maxBytes` of UTF-8, ending on a Character boundary. + static func preview(of value: String, maxBytes: Int) -> String { + guard value.utf8.count > maxBytes else { return value } + var end = value.startIndex + var used = 0 + while end < value.endIndex { + let next = value.index(after: end) + used += value.utf8.distance(from: end, to: next) + guard used <= maxBytes else { break } + end = next } + return String(value[.. Bool { + let base = typeName.prefix { $0 != "(" }.trimmingCharacters(in: .whitespaces).uppercased() + return byteTypeNames.contains(base) } + + private static let byteTypeNames: Set = [ + "BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "BYTEA", "BINARY", "VARBINARY", "IMAGE" + ] } private let byteCountFormatter = OSAllocatedUnfairLock(uncheckedState: { diff --git a/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift b/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift index fd619d01e1..1eab5e3482 100644 --- a/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift +++ b/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift @@ -1,4 +1,5 @@ import Foundation +import os public enum StreamElement: Sendable { case columns([ColumnInfo]) @@ -20,22 +21,60 @@ public struct StreamOptions: Sendable { public let inlineBinary: Bool public let maxRows: Int public let lazyContext: LazyContext? + // Long text with no ref stays whole while this admits it; nil cuts every long value. + public var textBudget: TextBudget? + // Safe Mode is Read-Only. A driver whose `enforcesReadOnlyMode` is true runs the statement in the engine's + // read-only mode or refuses it; any other driver runs it as it is. + public var enforcesReadOnly: Bool public init( - textTruncationBytes: Int = 4_096, + textTruncationBytes: Int = Cell.previewByteLimit, inlineBinary: Bool = false, maxRows: Int = 100_000, - lazyContext: LazyContext? = nil + lazyContext: LazyContext? = nil, + textBudget: TextBudget? = nil, + enforcesReadOnly: Bool = false ) { self.textTruncationBytes = textTruncationBytes self.inlineBinary = inlineBinary self.maxRows = maxRows self.lazyContext = lazyContext + self.textBudget = textBudget + self.enforcesReadOnly = enforcesReadOnly } public static let `default` = StreamOptions() } +// Shared by every cell of one request, and read from whatever thread the driver decodes on. +public final class TextBudget: Sendable { + private struct State: Sendable { + var remaining: Int + var refused = false + } + + private let state: OSAllocatedUnfairLock + + public init(bytes: Int) { + state = OSAllocatedUnfairLock(initialState: State(remaining: max(0, bytes))) + } + + public func admit(_ bytes: Int) -> Bool { + state.withLock { state in + guard bytes <= state.remaining else { + state.refused = true + return false + } + state.remaining -= bytes + return true + } + } + + public var hasRefused: Bool { + state.withLock { $0.refused } + } +} + public struct LazyContext: Sendable { public let table: String public let primaryKeyColumns: [String] diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift index 5932916173..b174813116 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift @@ -256,6 +256,23 @@ public enum MySQLObjectQueries { """ } + // The sidebar's read, apart from export's `eventList`, which keeps the four columns every server answers. + // A name narrows it to the one event a source tab reads again. + public static func eventDetails(schema: String, name: String? = nil, exactSchema: Bool) -> String { + let namePredicate = name.map { "\n AND EVENT_NAME = '\(escapeLiteral($0))'" } ?? "" + return """ + SELECT EVENT_NAME, EVENT_TYPE, STATUS, EVENT_SCHEMA, + EXECUTE_AT, INTERVAL_VALUE, INTERVAL_FIELD, STARTS, ENDS, + ON_COMPLETION, LAST_EXECUTED, DEFINER, EVENT_COMMENT + FROM information_schema.EVENTS + WHERE \(schemaMatch("EVENT_SCHEMA", schema, exact: exactSchema))\(namePredicate) + ORDER BY EVENT_NAME + """ + } + + // Its own statement, so a server without the variable loses that one attribute rather than the list. + public static let eventScheduler = "SELECT @@global.event_scheduler" + /// Qualified with the schema. Unqualified, the server resolves the name against the session /// database instead of the one being browsed, and returns a different routine's body or none. public static func routineDefinition(kind: String, schema: String?, name: String) -> String { diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift index f434425ace..aeada04b55 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift @@ -4,7 +4,7 @@ nonisolated public extension LibPQCopyDirection { var unsupportedMessage: String { switch self { case .copyIn: - return String(localized: "The query editor cannot send data to COPY FROM STDIN, so no rows were sent. Use Import to load a file into a table.") + return String(localized: "COPY FROM STDIN takes its rows from the lines after the line its ; ends, so no rows were sent.") case .copyOut: return String(localized: "The query editor cannot receive the output of COPY TO STDOUT, so it was discarded. Run a SELECT, or use Export to save the rows to a file.") case .copyBoth: diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCapabilities.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCapabilities.swift index 5e38c4ccb6..8fd987c88d 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCapabilities.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCapabilities.swift @@ -30,6 +30,8 @@ nonisolated public struct PostgreSQLCapabilities: Sendable, Equatable { /// Every range gained a companion multirange in 14, with a name the creator may choose. public var hasMultirangeTypes: Bool { serverVersion >= 140_000 } public var hasForeignTablesCatalog: Bool { serverVersion >= 90_100 } + // `pg_extension` and CREATE EXTENSION landed in 9.1. + public var hasExtensionsCatalog: Bool { serverVersion >= 90_100 } public var hasSequencesCatalog: Bool { serverVersion >= 100_000 } public var hasBypassRLS: Bool { serverVersion >= 90_500 } public var hasProcedureKind: Bool { serverVersion >= 110_000 } diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/Process/DescriptorRead.swift b/apple/Packages/TableProCore/Sources/TableProProcess/DescriptorRead.swift similarity index 98% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/Process/DescriptorRead.swift rename to apple/Packages/TableProCore/Sources/TableProProcess/DescriptorRead.swift index 65b00e75bf..0cd04b55da 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/Process/DescriptorRead.swift +++ b/apple/Packages/TableProCore/Sources/TableProProcess/DescriptorRead.swift @@ -1,3 +1,4 @@ +#if canImport(Darwin) import Darwin import Foundation @@ -62,3 +63,4 @@ public enum DescriptorRead { } } } +#endif diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/Process/DescriptorWrite.swift b/apple/Packages/TableProCore/Sources/TableProProcess/DescriptorWrite.swift similarity index 97% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/Process/DescriptorWrite.swift rename to apple/Packages/TableProCore/Sources/TableProProcess/DescriptorWrite.swift index 3925cfe0d4..7bf7b7e07e 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/Process/DescriptorWrite.swift +++ b/apple/Packages/TableProCore/Sources/TableProProcess/DescriptorWrite.swift @@ -1,3 +1,4 @@ +#if canImport(Darwin) import Darwin import Foundation @@ -35,3 +36,4 @@ public enum DescriptorWrite { } } } +#endif diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantConnectionSettings.swift similarity index 98% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantConnectionSettings.swift index 63e3d523e6..0f9b26885c 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantConnectionSettings.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore public enum QdrantFieldID { public static let apiKey = "qdApiKey" diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantFilterBuilder.swift similarity index 96% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantFilterBuilder.swift index 8b20a24cde..79c149baee 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantFilterBuilder.swift @@ -1,18 +1,5 @@ import Foundation - -public struct QdrantFilterSpec: Sendable, Equatable { - public let column: String - public let op: String - public let value: String - public let secondValue: String? - - public init(column: String, op: String, value: String, secondValue: String? = nil) { - self.column = column - self.op = op - self.value = value - self.secondValue = secondValue - } -} +import TableProQdrantCore public enum QdrantFilterError: Error, LocalizedError, Equatable, Sendable { case unsupportedOperator(String) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantPoint.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantPoint.swift index b725bd810c..a449315f06 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantPoint.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore public enum QdrantPointID { // Qdrant refuses an integer id spelled as a string and the other way round. diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantRowWrites.swift similarity index 87% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantRowWrites.swift index ce87b6ac90..52d03e4de0 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantRowWrites.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore public struct QdrantCell: Sendable, Equatable { public let column: String @@ -42,29 +43,11 @@ public struct QdrantRowWrite: Sendable, Equatable { public let landedCheck: String? } -public enum QdrantInsertCheck { - public static let tag = "QDRANT_INSERT_CHECK:" - - public static func encode(collection: String, id: Any, payload: [String: Any], vectors: [String: Any]) -> String? { - let spec: [String: Any] = ["collection": collection, "id": id, "payload": payload, "vectors": vectors] - guard let data = try? QdrantJSON.data(spec) else { return nil } - return tag + data.base64EncodedString() - } - - public static func decode(_ text: String) -> (collection: String, id: Any, payload: [String: Any], vectors: [String: Any])? { - guard text.hasPrefix(tag), - let data = Data(base64Encoded: String(text.dropFirst(tag.count))), - let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let collection = object["collection"] as? String, let id = object["id"], - let payload = object["payload"] as? [String: Any] - else { return nil } - return (collection, id, payload, object["vectors"] as? [String: Any] ?? [:]) - } - +public extension QdrantInsertCheck { // The payload and which vectors the point holds. Vector values are not compared: Qdrant sorts a // sparse vector's indices, stores float16 and uint8 vectors rounded and cosine vectors normalized, so // a vector written exactly reads back different. - public static func landed(_ point: QdrantPoint?, payload: [String: Any], vectors: [String: Any]) -> Bool { + static func landed(_ point: QdrantPoint?, payload: [String: Any], vectors: [String: Any]) -> Bool { guard let point, (try? QdrantJSON.text(point.payload)) == (try? QdrantJSON.text(payload)) else { return false } let names: Set switch point.vector { @@ -147,7 +130,7 @@ public enum QdrantRowWriter { let id = try pointID(change.id) return [try write("POST", "\(points)/delete?wait=true", ["points": [id]], change)] case .insert: - let cells = change.cells.filter { $0.text != nil && $0.text != defaultMarker } + let cells = change.cells.filter { $0.text != nil } let idText = cells.first { $0.column == QdrantSchema.idColumn }?.text.flatMap { $0.isEmpty ? nil : $0 } let id = try pointID(idText ?? newID()) let values = try typedValues(cells, typeName: typeName) @@ -181,9 +164,6 @@ public enum QdrantRowWriter { } } - // The host's marker for a cell the server fills; a duplicated row carries it in its key. - static let defaultMarker = "__DEFAULT__" - // An upsert updates only existing points this filter matches, and it matches none. private static var matchesNoPoint: [String: Any] { ["must": [["has_id": [Any]()]]] } diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantSchema.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantSchema.swift index 962faf11df..fa49b43695 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantSchema.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore // The first seven are Qdrant's payload index schemas; the rest are read off sampled values. public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantSortPlan.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantSortPlan.swift new file mode 100644 index 0000000000..04ec330955 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantSortPlan.swift @@ -0,0 +1,27 @@ +import Foundation +import TableProQdrantCore + +public enum QdrantSortPlan: Sendable, Equatable { + case byID + case byPayload(column: String, key: String, ascending: Bool) + + public static func resolve(_ sorts: [QdrantSortSpec]) throws -> QdrantSortPlan { + guard let sort = sorts.first else { return .byID } + guard sorts.count == 1 else { + throw QdrantError.unsupported(String(localized: "Qdrant sorts by one field at a time. Remove the sort from the other columns.")) + } + if sort.column == QdrantSchema.idColumn { + guard sort.ascending else { + throw QdrantError.unsupported(String(localized: "Qdrant lists points by id in ascending order only.")) + } + return .byID + } + guard !QdrantSchema.isVectorColumn(sort.column), sort.column != QdrantSchema.scoreColumn else { + throw QdrantError.unsupported(String(format: String(localized: "Qdrant cannot sort by %@."), sort.column)) + } + guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: sort.column)) else { + throw QdrantFilterError.keyNotAddressable(column: sort.column) + } + return .byPayload(column: sort.column, key: key, ascending: sort.ascending) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantValue.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantValue.swift index 3945423068..64da205308 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantValue.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/QdrantValue.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore public enum QdrantValueError: Error, LocalizedError, Equatable, Sendable { case notAnInteger(column: String, value: String) diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantClient.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantClient.swift index 98a69f3420..e586a5095e 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantClient.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore public struct QdrantScrollPage: @unchecked Sendable { public let points: [QdrantPoint] diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift similarity index 67% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift index 28c0f493e8..32b5413c65 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantScrollPager.swift @@ -53,36 +53,17 @@ public extension QdrantClient { ).points } - // Without vectors this reads payloads only, which is how an export learns every key before its header. + // Without vectors this reads payloads only, which is how an export learns every key before its header. A page is + // read only when the reader asks for the next one. func pointPages( collection: String, pageSize: Int, withVector: Bool = true ) -> AsyncThrowingStream<[QdrantPoint], Error> { - AsyncThrowingStream { continuation in - let task = Task { - do { - var cursor: Any? - repeat { - try Task.checkCancellation() - let page = try await self.scroll( - collection: collection, - body: Self.scrollBody( - filter: nil, limit: max(pageSize, 1), cursor: cursor, withData: true, withVector: withVector - ) - ) - if !page.points.isEmpty { - continuation.yield(page.points) - } - cursor = page.nextCursor - } while cursor != nil - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - continuation.onTermination = { @Sendable _ in task.cancel() } - } + let pager = QdrantPointPager( + client: self, collection: collection, pageSize: max(pageSize, 1), withVector: withVector + ) + return AsyncThrowingStream(unfolding: { try await pager.next() }) } static func scrollBody( @@ -106,3 +87,38 @@ public extension QdrantClient { return body } } + +// One reader pulls the pages, so one call at a time reaches the cursor. +private final class QdrantPointPager: @unchecked Sendable { + private let client: QdrantClient + private let collection: String + private let pageSize: Int + private let withVector: Bool + private var cursor: Any? + private var isDone = false + + init(client: QdrantClient, collection: String, pageSize: Int, withVector: Bool) { + self.client = client + self.collection = collection + self.pageSize = pageSize + self.withVector = withVector + } + + func next() async throws -> [QdrantPoint]? { + while !isDone { + try Task.checkCancellation() + let page = try await client.scroll( + collection: collection, + body: QdrantClient.scrollBody( + filter: nil, limit: pageSize, cursor: cursor, withData: true, withVector: withVector + ) + ) + cursor = page.nextCursor + isDone = cursor == nil + if !page.points.isEmpty { + return page.points + } + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantSortedPager.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantSortedPager.swift index 302080324e..7181a37299 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantSortedPager.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantSortedPager.swift @@ -1,4 +1,5 @@ import Foundation +import TableProQdrantCore // `order_by` walks Qdrant's range index, which holds only points with a value in it and holds a point // once per value. So the grid's rows are the ordered points first, then the points without a value in diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantTransport.swift similarity index 85% rename from apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift rename to apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantTransport.swift index b92ec486fa..fdad9ba712 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantClient/Wire/QdrantTransport.swift @@ -34,8 +34,6 @@ public struct QdrantHTTPResponse: Sendable, Equatable { } } -// The URLSession transport lives in the plugin: the app links this module for the query classifier, -// and an Objective-C class compiled into both would be registered twice in one process. public protocol QdrantTransport: Sendable { func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse func cancelAll() diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift index c2c5ddefab..aeaed2038e 100644 --- a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantBrowse.swift @@ -88,28 +88,3 @@ public enum QdrantBrowseQuery { ) } } - -public enum QdrantSortPlan: Sendable, Equatable { - case byID - case byPayload(column: String, key: String, ascending: Bool) - - public static func resolve(_ sorts: [QdrantSortSpec]) throws -> QdrantSortPlan { - guard let sort = sorts.first else { return .byID } - guard sorts.count == 1 else { - throw QdrantError.unsupported(String(localized: "Qdrant sorts by one field at a time. Remove the sort from the other columns.")) - } - if sort.column == QdrantSchema.idColumn { - guard sort.ascending else { - throw QdrantError.unsupported(String(localized: "Qdrant lists points by id in ascending order only.")) - } - return .byID - } - guard !QdrantSchema.isVectorColumn(sort.column), sort.column != QdrantSchema.scoreColumn else { - throw QdrantError.unsupported(String(format: String(localized: "Qdrant cannot sort by %@."), sort.column)) - } - guard let key = QdrantKeyPath.quoted(QdrantSchema.payloadKey(forColumn: sort.column)) else { - throw QdrantFilterError.keyNotAddressable(column: sort.column) - } - return .byPayload(column: sort.column, key: key, ascending: sort.ascending) - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterSpec.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterSpec.swift new file mode 100644 index 0000000000..5d5cf77165 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterSpec.swift @@ -0,0 +1,15 @@ +import Foundation + +public struct QdrantFilterSpec: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantInsertCheck.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantInsertCheck.swift new file mode 100644 index 0000000000..8afb20be51 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantInsertCheck.swift @@ -0,0 +1,21 @@ +import Foundation + +public enum QdrantInsertCheck { + public static let tag = "QDRANT_INSERT_CHECK:" + + public static func encode(collection: String, id: Any, payload: [String: Any], vectors: [String: Any]) -> String? { + let spec: [String: Any] = ["collection": collection, "id": id, "payload": payload, "vectors": vectors] + guard let data = try? QdrantJSON.data(spec) else { return nil } + return tag + data.base64EncodedString() + } + + public static func decode(_ text: String) -> (collection: String, id: Any, payload: [String: Any], vectors: [String: Any])? { + guard text.hasPrefix(tag), + let data = Data(base64Encoded: String(text.dropFirst(tag.count))), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = object["collection"] as? String, let id = object["id"], + let payload = object["payload"] as? [String: Any] + else { return nil } + return (collection, id, payload, object["vectors"] as? [String: Any] ?? [:]) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift b/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift index 7c760fe1e3..94c3c23d55 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift @@ -33,13 +33,13 @@ public struct QueryScript: Sendable, Equatable { /// How the session read literals when the text was split. public let session: SQLSessionLexicalFacts? - /// The text that was split, where each unit ends in it, and its engine, so the rest can be split again. + /// The text that was split, the statement each unit runs, and its engine, so the rest can be split again. private let source: String? - private let unitEnds: [Int] + private let statements: [SQLStatementScanner.ExecutableStatement] private let databaseType: DatabaseType? public init(units: [Unit], unitKind: UnitKind) { - self.init(units: units, unitKind: unitKind, session: nil, source: nil, unitEnds: [], databaseType: nil) + self.init(units: units, unitKind: unitKind, session: nil, source: nil, statements: [], databaseType: nil) } private init( @@ -47,14 +47,14 @@ public struct QueryScript: Sendable, Equatable { unitKind: UnitKind, session: SQLSessionLexicalFacts?, source: String?, - unitEnds: [Int], + statements: [SQLStatementScanner.ExecutableStatement], databaseType: DatabaseType? ) { self.units = units self.unitKind = unitKind self.session = session self.source = source - self.unitEnds = unitEnds + self.statements = statements self.databaseType = databaseType } @@ -93,7 +93,7 @@ public struct QueryScript: Sendable, Equatable { unitKind: .statement, session: session, source: text, - unitEnds: statements.map { $0.range.location + $0.range.length }, + statements: statements, databaseType: databaseType ) } @@ -107,7 +107,7 @@ public struct QueryScript: Sendable, Equatable { unitKind: .batch, session: session, source: nil, - unitEnds: [], + statements: [], databaseType: databaseType ) } @@ -136,12 +136,27 @@ public struct QueryScript: Sendable, Equatable { /// The text after unit `index`, split again under `session`, for a statement that changed how the session reads /// literals: after `SET sql_mode = 'NO_BACKSLASH_ESCAPES'` a backslash no longer escapes a quote, so the split /// made before it ran no longer matches what the server will read. Nil when the script cannot be split again. + /// + /// The rest starts where the scan went on after the unit, so a `COPY`'s rows stay rows and a client delimiter + /// stays in force. public func replanned(after index: Int, session: SQLSessionLexicalFacts?) -> QueryScript? { - guard unitKind == .statement, let source, let databaseType, unitEnds.indices.contains(index) else { + guard unitKind == .statement, let source, let databaseType, statements.indices.contains(index) else { return nil } - let rest = (source as NSString).substring(from: unitEnds[index]) - return QueryScript.plan(rest, databaseType: databaseType, sendsBatchesWhole: false, session: session) + let rest = SQLStatementScanner.executableStatements( + in: source, + after: statements[index], + through: (source as NSString).length, + grammar: Self.executionGrammar(for: databaseType, session: session) + ) + return QueryScript( + units: rest.map { Unit(sql: $0.sql) }, + unitKind: .statement, + session: session, + source: source, + statements: rest, + databaseType: databaseType + ) } } diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift index d849a7244e..5c6e02855c 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift @@ -47,7 +47,7 @@ public enum RedisArgumentCodec { } public static func quote(_ bytes: Data) -> String { - if let text = String(data: bytes, encoding: .utf8) { + if let text = RedisWireText.decode(bytes) { return isBare(text) ? text : quotedText(text) } return quotedBytes(bytes) diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift index 8d42794c8d..d92e7bc4bc 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift @@ -32,6 +32,17 @@ public enum RedisCommandSplitter { return nil } + // Every command with the last caret offset `command(at:in:)` gives it, for a caller that keeps them between + // caret moves. + public static func commandsWithCaretReach(in text: String) -> [(command: Command, reach: Int)] { + var lexer = RedisCommandLexer(text, endsCommands: true, decodesArguments: false) + var commands: [(command: Command, reach: Int)] = [] + while let found = lexer.next() { + commands.append((command: command(found, in: text), reach: found.reach)) + } + return commands + } + private static func command(_ found: RedisCommandLexer.Command, in text: String) -> Command { let range = NSRange(location: found.start, length: found.end - found.start) return Command(text: (text as NSString).substring(with: range), range: range) diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift index 32ac2c4114..1eb803d10b 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift @@ -34,10 +34,11 @@ nonisolated public enum RedisKeySummary { } } - public static func previewCommand(for kind: RedisKeyKind, key: String) -> [String] { + public static func previewCommand(for kind: RedisKeyKind, key: String, stringByteLimit: Int? = nil) -> [String] { switch kind { case .string: - return ["GET", key] + guard let stringByteLimit, stringByteLimit > 0 else { return ["GET", key] } + return ["GETRANGE", key, "0", String(stringByteLimit - 1)] case .hash: return ["HSCAN", key, "0", "COUNT", String(collectionPreviewLimit)] case .list: @@ -99,6 +100,6 @@ nonisolated public enum RedisKeySummary { ) else { return nil } - return String(data: data, encoding: .utf8) + return RedisWireText.decode(data) } } diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisReadCommands.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisReadCommands.swift index 83ec3f7cc0..80538d1b93 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisReadCommands.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisReadCommands.swift @@ -1,14 +1,15 @@ import Foundation // A command reads only when its verb is on this list, so a verb the list has never heard of counts as a write. +// Checked by name against Redis 8.10.3 `COMMAND DOCS`: a module command flagged readonly can still change state. public enum RedisReadCommands { public static let verbs: Set = [ - "GET", "MGET", "STRLEN", "GETRANGE", "SUBSTR", "EXISTS", "TYPE", "TTL", "PTTL", + "GET", "MGET", "STRLEN", "GETRANGE", "SUBSTR", "LCS", "DIGEST", "EXISTS", "TYPE", "TTL", "PTTL", "EXPIRETIME", "PEXPIRETIME", "KEYS", "SCAN", "RANDOMKEY", "DBSIZE", "DUMP", "HGET", "HMGET", "HGETALL", "HKEYS", "HVALS", "HLEN", "HEXISTS", "HRANDFIELD", - "HSCAN", "HSTRLEN", "LRANGE", "LLEN", "LINDEX", "LPOS", + "HSCAN", "HSTRLEN", "HTTL", "HPTTL", "HEXPIRETIME", "HPEXPIRETIME", "LRANGE", "LLEN", "LINDEX", "LPOS", "SMEMBERS", "SISMEMBER", "SMISMEMBER", "SCARD", "SRANDMEMBER", "SSCAN", - "SDIFF", "SINTER", "SUNION", "SINTERCARD", + "SDIFF", "SINTER", "SUNION", "SINTERCARD", "SDIFFCARD", "SUNIONCARD", "ZRANGE", "ZRANGEBYSCORE", "ZRANGEBYLEX", "ZREVRANGE", "ZREVRANGEBYSCORE", "ZREVRANGEBYLEX", "ZRANK", "ZREVRANK", "ZSCORE", "ZMSCORE", "ZCARD", "ZCOUNT", "ZLEXCOUNT", "ZSCAN", "ZRANDMEMBER", "ZDIFF", "ZINTER", "ZUNION", "ZINTERCARD", @@ -16,8 +17,20 @@ public enum RedisReadCommands { "PFCOUNT", "BITCOUNT", "BITPOS", "GETBIT", "BITFIELD_RO", "GEOPOS", "GEODIST", "GEOHASH", "GEOSEARCH", "GEORADIUS_RO", "GEORADIUSBYMEMBER_RO", "SORT_RO", "OBJECT", "COMMAND", "INFO", "TIME", "LASTSAVE", "PING", "ECHO", "LOLWUT", - "JSON.GET", "JSON.MGET", "JSON.TYPE", "JSON.OBJKEYS", "JSON.ARRLEN", "JSON.STRLEN", - "TS.RANGE", "TS.REVRANGE", "TS.GET", "TS.MGET", "TS.INFO", "FT.SEARCH", "FT.INFO" + "ARCOUNT", "ARGET", "ARGETRANGE", "ARGREP", "ARINFO", "ARLASTITEMS", "ARLEN", "ARMGET", "ARNEXT", "AROP", + "ARSCAN", + "JSON.GET", "JSON.MGET", "JSON.TYPE", "JSON.OBJKEYS", "JSON.ARRLEN", "JSON.STRLEN", "JSON.ARRINDEX", + "JSON.OBJLEN", "JSON.RESP", + "TS.RANGE", "TS.REVRANGE", "TS.GET", "TS.MGET", "TS.INFO", "TS.MRANGE", "TS.MREVRANGE", "TS.NRANGE", + "TS.NREVRANGE", "TS.QUERYINDEX", "TS.QUERYLABELS", "TS.READ", + "FT.SEARCH", "FT.INFO", "FT.AGGREGATE", "FT.HYBRID", "FT.PROFILE", "FT.EXPLAIN", "FT.EXPLAINCLI", "FT.GET", + "FT.MGET", "FT.SPELLCHECK", "FT.SUGGET", "FT.SUGLEN", "FT.SYNDUMP", "FT.TAGVALS", "FT.DICTDUMP", + "FT.ALIASLIST", + "BF.CARD", "BF.EXISTS", "BF.INFO", "BF.MEXISTS", "CF.COUNT", "CF.EXISTS", "CF.INFO", "CF.MEXISTS", + "CMS.INFO", "CMS.QUERY", "TOPK.COUNT", "TOPK.INFO", "TOPK.LIST", "TOPK.QUERY", + "TDIGEST.BYRANK", "TDIGEST.BYREVRANK", "TDIGEST.CDF", "TDIGEST.INFO", "TDIGEST.MAX", "TDIGEST.MIN", + "TDIGEST.QUANTILE", "TDIGEST.RANK", "TDIGEST.REVRANK", "TDIGEST.TRIMMED_MEAN", + "VCARD", "VDIM", "VEMB", "VGETATTR", "VINFO", "VISMEMBER", "VLINKS", "VRANDMEMBER", "VRANGE", "VSIM" ] // `CONFIG GET` reads, and every other `CONFIG` subcommand changes the server. diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisStringPreview.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisStringPreview.swift new file mode 100644 index 0000000000..c3398560a4 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisStringPreview.swift @@ -0,0 +1,46 @@ +import Foundation + +nonisolated public enum RedisStringPreview { + // Holds a grid page to its row count times 64 KiB, where reading each string whole had no bound. + public static let byteLimit = 65_536 + + // The mark JSONTruncation puts after a cut value. + public static let marker = "..." + + public enum Shown: Equatable, Sendable { + case text(String) + case bytes(Data) + } + + // STRLEN declined leaves only the read itself, and a read that filled the range may have been cut. + public static func isShortened(length: Int?, shownByteCount: Int, byteLimit: Int = RedisStringPreview.byteLimit) -> Bool { + guard let length else { return shownByteCount >= byteLimit } + return length > byteLimit + } + + public static func shown(_ bytes: Data, length: Int?, byteLimit: Int = RedisStringPreview.byteLimit) -> Shown { + guard isShortened(length: length, shownByteCount: bytes.count, byteLimit: byteLimit) else { + return RedisWireText.decode(bytes).map(Shown.text) ?? .bytes(bytes) + } + guard let text = RedisWireText.decode(wholeSequencePrefix(bytes)) else { return .bytes(bytes) } + return .text(text + marker) + } + + // Drops only a multi-byte sequence the range cut through, so a binary value never passes as text. + static func wholeSequencePrefix(_ bytes: Data) -> Data { + let tail = Array(bytes.suffix(3)) + for offset in stride(from: 1, through: tail.count, by: 1) { + let byte = tail[tail.count - offset] + if byte & 0xC0 == 0x80 { continue } + let sequenceLength: Int + switch byte { + case 0xC0...0xDF: sequenceLength = 2 + case 0xE0...0xEF: sequenceLength = 3 + case 0xF0...0xF7: sequenceLength = 4 + default: return bytes + } + return offset < sequenceLength ? Data(bytes.prefix(bytes.count - offset)) : bytes + } + return bytes + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisWireText.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisWireText.swift new file mode 100644 index 0000000000..3e00789e3a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisWireText.swift @@ -0,0 +1,9 @@ +import Foundation + +nonisolated public enum RedisWireText { + // String(data:encoding:) drops a leading byte order mark, which names a different key. + public static func decode(_ bytes: Data) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoCommandCatalog.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoCommandCatalog.swift index f851b5d9e2..a14bb38353 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoCommandCatalog.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoCommandCatalog.swift @@ -24,4 +24,17 @@ public enum MongoCommandCatalog { public static func preserves(_ name: String) -> Bool { name.unicodeScalars.allSatisfy(\.isASCII) && preservingCommands.contains(name.lowercased()) } + + // The commands among `preservingCommands` that read only, whatever their options: no `validate`, whose `repair` + // fixes the collection, no `explain`, `getMore` or `killCursors`. + public static let readCommands: Set = [ + "ping", "hello", "ismaster", "buildinfo", "hostinfo", "serverstatus", "connectionstatus", "whatsmyuri", + "listcommands", "getcmdlineopts", "getparameter", "getlog", "currentop", "top", "replsetgetstatus", + "replsetgetconfig", "connpoolstats", "listdatabases", "listcollections", "listindexes", "dbstats", + "collstats", "datasize", "usersinfo", "rolesinfo", "find", "count", "distinct", "aggregate", + ] + + public static func readsOnly(_ name: String) -> Bool { + name.unicodeScalars.allSatisfy(\.isASCII) && readCommands.contains(name.lowercased()) + } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementParser.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementParser.swift index f69d2a3e6c..0655ccfe0a 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementParser.swift @@ -1,25 +1,32 @@ import Foundation -/// What a statement the grammar accepted does: every call in order, and every key its literals spell. public struct MongoStatement: Equatable, Sendable { + public enum Receiver: Equatable, Sendable { + // `use` and `show`, which the driver rewrites into calls of the shell's own functions. + case shell + case database, collection, cursor, explainer + } + public struct Call: Equatable, Sendable { public let name: String public let isRead: Bool - /// The first key of a `runCommand` or `adminCommand` document, which names the command. + // The first key of a `runCommand` or `adminCommand` document, which names the command. public let commandName: String? + public let receiver: Receiver + // Named by `getSiblingDB`, `getDB`, `use` or a command's `$db`, `admin` for `adminCommand`, nil for the current one. + public let database: String? + // `db.`, `db["name"]`, `getCollection("name")`, or a command document's first value when it is a string. + public let collection: String? } public var calls: [Call] = [] public var objectKeys: Set = [] } -/// Accepts one mongosh statement whose arguments are literal data, and nothing else. -/// -/// Rejecting is always safe, since the caller then treats the statement as a write. Accepting has -/// to mean JavaScriptCore reads the text the same way, so the lexical rules for whitespace, -/// comments, strings, numbers and regular expression literals follow ECMAScript exactly, and -/// anything they would read differently is refused: escapes in identifiers, legacy octal, `?.`, -/// template literals, a second statement. +// Accepts one mongosh statement whose arguments are literal data, and nothing else. Rejecting is always safe, since +// the caller then treats the statement as a write. Accepting has to mean JavaScriptCore reads the text the same way, +// so whitespace, comments, strings, numbers and regular expressions follow ECMAScript exactly, and anything they +// would read differently is refused: escapes in identifiers, legacy octal, `?.`, template literals, a second statement. public struct MongoStatementParser { private struct Rejected: Error {} @@ -27,7 +34,15 @@ public struct MongoStatementParser { case database, client, collection, cursor, explainer, value } - /// Deep enough for any real filter, shallow enough that a crafted `[[[...]]]` cannot exhaust the stack. + private struct Literal { + var string: String? + var firstKey: String? + var firstValue: String? + // A command's own `$db`, which libmongoc 1.28.1 sends instead of the database it was called on (mongoc-cmd.c). + var database: String? + } + + // Deep enough for any real filter, shallow enough that a crafted `[[[...]]]` cannot exhaust the stack. private static let maximumDepth = 64 private static let databaseReads: Set = [ @@ -55,7 +70,7 @@ public struct MongoStatementParser { private static let commandMethods: Set = ["runCommand", "adminCommand"] - /// Pure value constructors from the shell prelude. None of them reaches the server. + // Pure value constructors from the shell prelude. None of them reaches the server. private static let constructors: Set = [ "ObjectId", "ISODate", "Date", "NumberLong", "NumberInt", "NumberDecimal", "Int32", "Long", "Decimal128", "Double", "Timestamp", "BinData", "HexData", "UUID", "LegacyJavaUUID", @@ -63,7 +78,7 @@ public struct MongoStatementParser { "BSONRegExp", "BSONSymbol", "DBRef" ] - /// Non-writable globals, so no earlier statement can rebind them. + // Non-writable globals, so no earlier statement can rebind them. private static let numericConstants: Set = ["NaN", "Infinity"] private static let boundaryKeys: Set = ["MinKey", "MaxKey"] @@ -76,6 +91,8 @@ public struct MongoStatementParser { private var index: String.UnicodeScalarView.Index private var depth = 0 private var statement = MongoStatement() + private var database: String? + private var collection: String? private let shadowsCollection: (String) -> Bool private init(_ text: String, shadowsCollection: @escaping (String) -> Bool = { _ in false }) { @@ -84,7 +101,7 @@ public struct MongoStatementParser { self.shadowsCollection = shadowsCollection } - /// Whitespace and comments only, as JavaScript reads them: a `--` line is a decrement, not a comment. + // As JavaScript reads it: a `--` line is a decrement, not a comment. public static func isTrivia(_ text: String) -> Bool { var parser = MongoStatementParser(text) guard (try? parser.skipTrivia()) != nil else { return false } @@ -94,23 +111,40 @@ public struct MongoStatementParser { // `shadowsCollection` names what `db.` reaches instead of a collection: PluginKit's // MongoCollectionAccessor.isShadowedByDatabaseMember, which this module cannot import. public static func parse(_ text: String, shadowsCollection: @escaping (String) -> Bool) -> MongoStatement? { - if isShellCommand(text) { return MongoStatement() } + if let command = shellCommand(text) { + let call = MongoStatement.Call( + name: command.keyword, + isRead: true, + commandName: nil, + receiver: .shell, + database: command.keyword == "use" ? command.argument : nil, + collection: nil + ) + return MongoStatement(calls: [call]) + } var parser = MongoStatementParser(text, shadowsCollection: shadowsCollection) return try? parser.statementText() } - /// `use orders` and `show collections`, in exactly the shape the driver rewrites into a call - /// (`MongoShellCommandLine.rewrite`): anything looser reaches JavaScriptCore as typed. public static func isShellCommand(_ text: String) -> Bool { + shellCommand(text) != nil + } + + // `use orders` and `show collections`, in exactly the shape the driver rewrites into a call + // (`MongoShellCommandLine.rewrite`): anything looser reaches JavaScriptCore as typed. + private static func shellCommand(_ text: String) -> (keyword: String, argument: String)? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) let lowered = trimmed.lowercased() for keyword in ["use", "show"] where lowered.hasPrefix(keyword + " ") { let remainder = trimmed.dropFirst(keyword.count).trimmingCharacters(in: .whitespaces) let word = remainder.hasSuffix(";") ? String(remainder.dropLast()) : remainder - guard keyword == "use" else { return shownTopics.contains(word.lowercased()) } - return !word.isEmpty && word.unicodeScalars.allSatisfy(isDatabaseNameScalar) + guard keyword == "use" else { + return shownTopics.contains(word.lowercased()) ? (keyword: keyword, argument: word) : nil + } + guard !word.isEmpty, word.unicodeScalars.allSatisfy(isDatabaseNameScalar) else { return nil } + return (keyword: keyword, argument: word) } - return false + return nil } private static func isDatabaseNameScalar(_ scalar: Unicode.Scalar) -> Bool { @@ -135,7 +169,7 @@ public struct MongoStatementParser { return statement } - /// `.name` or `["name"]`, with trivia allowed around the dot as JavaScript allows it. + // Trivia is allowed around the dot, as JavaScript allows it. private mutating func memberName() throws -> String? { try skipTrivia() switch peek() { @@ -162,77 +196,96 @@ public struct MongoStatementParser { case .database: guard isCall else { guard !shadowsCollection(member) else { throw Rejected() } + collection = member return .collection } switch member { case "getSiblingDB": - try stringArgument() + database = try stringArgument() return .database case "getCollection": - try stringArgument() + collection = try stringArgument() return .collection case "getMongo": - try call(member, isRead: true) + try call(member, on: .database, isRead: true) return .client default: - try call(member, isRead: Self.databaseReads.contains(member)) + try call(member, on: .database, isRead: Self.databaseReads.contains(member)) return .value } case .client: guard isCall, member == "getDB" else { throw Rejected() } - try stringArgument() + database = try stringArgument() return .database case .collection: guard isCall else { throw Rejected() } switch member { case "find", "aggregate": - try call(member, isRead: true) + try call(member, on: .collection, isRead: true) return .cursor case "explain": - try call(member, isRead: true) + try call(member, on: .collection, isRead: true) return .explainer default: - try call(member, isRead: Self.collectionReads.contains(member)) + try call(member, on: .collection, isRead: Self.collectionReads.contains(member)) return .value } case .cursor: guard isCall else { throw Rejected() } let modifies = Self.cursorModifiers.contains(member) - try call(member, isRead: modifies || Self.cursorReads.contains(member)) + try call(member, on: .cursor, isRead: modifies || Self.cursorReads.contains(member)) return modifies ? .cursor : .value case .explainer: guard isCall else { throw Rejected() } - try call(member, isRead: Self.explainedReads.contains(member)) + try call(member, on: .explainer, isRead: Self.explainedReads.contains(member)) return .value case .value: throw Rejected() } } - private mutating func call(_ name: String, isRead: Bool) throws { - let firstKey = try arguments() - let commandName = Self.commandMethods.contains(name) ? firstKey : nil - statement.calls.append(MongoStatement.Call(name: name, isRead: isRead, commandName: commandName)) + private mutating func call(_ name: String, on receiver: MongoStatement.Receiver, isRead: Bool) throws { + let first = try arguments() + guard Self.commandMethods.contains(name) else { + statement.calls.append(MongoStatement.Call( + name: name, + isRead: isRead, + commandName: nil, + receiver: receiver, + database: database, + collection: receiver == .database ? nil : collection + )) + return + } + statement.calls.append(MongoStatement.Call( + name: name, + isRead: isRead, + commandName: first.firstKey, + receiver: receiver, + database: first.database ?? (name == "adminCommand" ? "admin" : database), + collection: first.firstValue + )) } - private mutating func stringArgument() throws { + private mutating func stringArgument() throws -> String { try expect("(") try skipTrivia() - _ = try string() + let argument = try string() try skipTrivia() try expect(")") + return argument } - /// The literal arguments of a call. Returns the first key of the first argument when it is a document. - private mutating func arguments() throws -> String? { + // Only the first argument's literal: a command is its first argument. + private mutating func arguments() throws -> Literal { try expect("(") try skipTrivia() - var firstKey: String? + var first = Literal() var isFirst = true while peek() != ")" { - let key = try value() + let literal = try value() if isFirst { - firstKey = key + first = literal isFirst = false } try skipTrivia() @@ -241,13 +294,12 @@ public struct MongoStatementParser { try skipTrivia() } try expect(")") - return firstKey + return first } // MARK: - Literals - /// One literal value. Returns its first key when it is a document. - private mutating func value() throws -> String? { + private mutating func value() throws -> Literal { depth += 1 defer { depth -= 1 } guard depth <= Self.maximumDepth else { throw Rejected() } @@ -259,7 +311,7 @@ public struct MongoStatementParser { case "[": try array() case "\"", "'": - _ = try string() + return Literal(string: try string()) case "/": try regularExpression() case "-", ".", "0"..."9": @@ -267,29 +319,32 @@ public struct MongoStatementParser { default: try word() } - return nil + return Literal() } - private mutating func document() throws -> String? { + private mutating func document() throws -> Literal { try expect("{") try skipTrivia() - var firstKey: String? + var literal = Literal() while peek() != "}" { let key = try propertyKey() // A literal's `__proto__` key sets its prototype rather than a field. guard key != "__proto__" else { throw Rejected() } statement.objectKeys.insert(key) - if firstKey == nil { firstKey = key } + if literal.firstKey == nil { literal.firstKey = key } try skipTrivia() try expect(":") - _ = try value() + let entry = try value() + // A key written twice keeps its first place and takes its last value. + if key == literal.firstKey { literal.firstValue = entry.string } + if key == "$db" { literal.database = entry.string } try skipTrivia() guard peek() == "," else { break } advance() try skipTrivia() } try expect("}") - return firstKey + return literal } private mutating func propertyKey() throws -> String { @@ -318,7 +373,7 @@ public struct MongoStatementParser { try expect("]") } - /// `true`, `false`, `null`, a prelude constructor called on literals, or `MinKey`/`MaxKey`. + // `true`, `false`, `null`, a prelude constructor called on literals, or `MinKey`/`MaxKey`. private mutating func word() throws { var name = try identifier() if Self.numericConstants.contains(name) { return } @@ -431,7 +486,7 @@ public struct MongoStatementParser { return scalar } - /// A decimal literal, optionally negated. Hex, octal, binary, BigInt and separators are refused. + // Decimal only: hex, octal, binary, BigInt and separators are refused. private mutating func number() throws { if peek() == "-" { advance() @@ -466,7 +521,7 @@ public struct MongoStatementParser { return count } - /// The lexical grammar of a regular expression literal: a `/` inside a class does not end it. + // A `/` inside a class does not end the literal. private mutating func regularExpression() throws { try expect("/") var inClass = false @@ -501,8 +556,8 @@ public struct MongoStatementParser { // MARK: - Lexing - /// An ASCII identifier. One followed by `\` or by a non-ASCII character JavaScript might read as - /// part of the name is refused, since the name the parser saw would not be the name that runs. + // ASCII only. One followed by `\` or by a non-ASCII character JavaScript might read as part of the name is + // refused, since the name the parser saw would not be the name that runs. private mutating func identifier() throws -> String { guard let first = peek(), isIdentifierStart(first) else { throw Rejected() } var name = String.UnicodeScalarView() @@ -522,7 +577,7 @@ public struct MongoStatementParser { isIdentifierStart(scalar) || ("0"..."9").contains(scalar) } - /// ECMAScript WhiteSpace and LineTerminator, and comments. An unterminated block comment is refused. + // ECMAScript WhiteSpace and LineTerminator, and comments. An unterminated block comment is refused. private mutating func skipTrivia() throws { while let scalar = peek() { if isTrivia(scalar) { diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls+Parser.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls+Parser.swift new file mode 100644 index 0000000000..4c129ee6bc --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls+Parser.swift @@ -0,0 +1,871 @@ +import Foundation + +extension SQLBareNameCalls { + // One query's blocks: each SELECT, the tables it reads and the names it holds. A shape it cannot read, such as a + // `WITH` that is not a list of named queries, counts as a call. + struct Parser { + typealias Block = SQLBareNameBlock + + private struct CommonTable { + var block: Int? + var columns: [String]? + } + + private enum Clause { + case from + case orderBy + case other + } + + private enum FromState { + case expectTable + case afterTable + case afterAlias + } + + private let code: StatementCode + private let grammar: SQLLexicalGrammar + private let reserved: Set + private let tokens: [TSQLTokens.Token] + private var closing: [Int: Int] = [:] + private(set) var blocks: [Block] = [] + private(set) var callsOutright = false + + init(_ code: StatementCode, grammar: SQLLexicalGrammar, reserved: Set) { + self.code = code + self.grammar = grammar + self.reserved = reserved + tokens = code.tokens + var open: [Int] = [] + for (index, token) in tokens.enumerated() { + if token.kind == .open { + open.append(index) + } else if token.kind == .close, let start = open.popLast() { + closing[start] = index + } + } + } + + // A query tab hands the gate its statements joined by line breaks, which Oracle does not split on, so a SELECT + // standing where a clause would is read as the next query. + // A write is refused before this is asked, so only queries and EXPLAIN PLAN reach it. + mutating func parse() { + let end = (tokens.lastIndex { $0.kind != .terminator }).map { $0 + 1 } ?? 0 + var index = 0 + while index < end { + let token = tokens[index] + guard token.depth == 0 else { + index += 1 + continue + } + if token.word == "EXPLAIN" { + // EXPLAIN PLAN plans its statement without running it, so the statement is read and dropped. + guard let plan = (index.., + parent: Int?, + commonTables: [String: CommonTable] + ) -> (first: Int?, end: Int) { + var index = range.lowerBound + let end = range.upperBound + var scope = commonTables + if word(at: index) == "WITH" { + guard let next = commonTableList(from: index + 1, end: end, scope: &scope) else { + callsOutright = true + return (nil, end) + } + index = next + } + var first: Int? + var last: Int? + var lastIsSelect = false + var compound = false + while index < end { + if tokens[index].kind == .open, let close = closing[index], close < end { + let block = queryExpression((index + 1).. Int { + let depth = tokens[start].depth + var index = start + while index < end { + if tokens[index].depth == depth, startsNextQuery(at: index, end: end) { return index } + index += 1 + } + return end + } + + private func startsNextQuery(at index: Int, end: Int) -> Bool { + switch word(at: index) { + case "SELECT": + return true + case "WITH": + guard identifier(at: index + 1) != nil, index + 2 < end else { return false } + return word(at: index + 2) == "AS" || tokens[index + 2].kind == .open + default: + return false + } + } + + // `name [(columns)] AS (query) [SEARCH ... | CYCLE ...], ...`. Nil when it is not that shape, as with an inline + // PL/SQL `WITH FUNCTION`. + private mutating func commonTableList( + from start: Int, + end: Int, + scope: inout [String: CommonTable] + ) -> Int? { + var index = start + while index < end { + guard let name = identifier(at: index) else { return nil } + index += 1 + var columns: [String]? + if index < end, tokens[index].kind == .open, let close = closing[index] { + columns = identifierList((index + 1)..= depth, tokens[index].kind != .comma, + tokens[index].kind != .open, word(at: index) != "SELECT" { + index += 1 + } + guard index < end, tokens[index].kind == .comma else { return index } + index += 1 + } + return nil + } + + private func branchEnd(from start: Int, end: Int) -> Int { + let depth = tokens[start].depth + var index = start + 1 + while index < end { + if tokens[index].depth == depth { + if let word = tokens[index].word, Self.setOperators.contains(word) { return index } + if startsNextQuery(at: index, end: end) { return index } + } + index += 1 + } + return end + } + + private mutating func selectBlock( + _ range: Range, + parent: Int?, + commonTables: [String: CommonTable] + ) -> Int { + let block = newBlock(parent: parent) + let depth = tokens[range.lowerBound].depth + var marks: [(index: Int, clause: Clause)] = [] + for index in (range.lowerBound + 1).. Clause? { + guard let word = word(at: index) else { return nil } + let next = index + 1 < end ? self.word(at: index + 1) : nil + switch word { + case "FROM": + return afterFrom ? nil : .from + case "WHERE", "HAVING", "INTO": + return .other + case "GROUP": + return next == "BY" && self.word(at: index - 1) != "WITHIN" ? .other : nil + case "ORDER": + return next == "BY" || next == "SIBLINGS" ? .orderBy : nil + case "CONNECT": + return next == "BY" ? .other : nil + case "START": + return next == "WITH" ? .other : nil + case "FOR": + return next == "UPDATE" ? .other : nil + case "MODEL", "WINDOW", "OFFSET": + return afterFrom ? .other : nil + case "FETCH": + return afterFrom && (next == "FIRST" || next == "NEXT") ? .other : nil + default: + return nil + } + } + + // MARK: - Select list + + private mutating func selectList(_ range: Range, block: Int, _ commonTables: [String: CommonTable]) { + var start = range.lowerBound + while start < range.upperBound, let word = word(at: start), ["DISTINCT", "UNIQUE", "ALL"].contains(word) { + start += 1 + } + // Dameng's `TOP n [PERCENT] [WITH TIES]`, whose count would otherwise read as an operand an alias follows. + if word(at: start) == "TOP", start + 1 < range.upperBound { + start += 1 + if tokens[start].kind == .open, let close = closing[start] { + expression(start..<(close + 1), block: block, commonTables: commonTables, ordersBy: false) + start = close + 1 + } else { + start += 1 + } + while start < range.upperBound, let word = word(at: start), ["PERCENT", "WITH", "TIES"].contains(word) { + start += 1 + } + } + guard start < range.upperBound else { return } + let depth = tokens[start].depth + var itemStart = start + for index in start...range.upperBound { + guard index == range.upperBound || (tokens[index].kind == .comma && tokens[index].depth == depth) else { + continue + } + if itemStart < index { selectItem(itemStart.., block: Int, _ commonTables: [String: CommonTable]) { + let last = item.upperBound - 1 + if isSymbol(Self.star, at: last) { + let qualifiers = chainNames(item.lowerBound..= 2, let alias = identifier(at: last) { + if word(at: last - 1) == "AS" { + expressionEnd = last - 1 + } else if endsOperand(at: last - 1) { + expressionEnd = last + } + if expressionEnd != item.upperBound { + blocks[block].aliases.insert(alias) + blocks[block].outputs.append(.name(alias)) + } + } + if expressionEnd == item.upperBound, let names = chainNames(item), let name = names.last { + blocks[block].outputs.append(.name(name)) + } + expression(item.lowerBound..) -> [String]? { + guard !range.isEmpty, range.count % 2 == 1 else { return nil } + var names: [String] = [] + for index in range { + if (index - range.lowerBound) % 2 == 0 { + guard let name = (index == range.lowerBound ? identifier(at: index) : chainPart(at: index)) else { + return nil + } + names.append(name) + } else if !isSymbol(Self.period, at: index) { + return nil + } + } + return names + } + + // MARK: - FROM + + private mutating func fromClause(_ range: Range, block: Int, commonTables: [String: CommonTable]) { + guard !range.isEmpty else { return } + let depth = tokens[range.lowerBound].depth + var state = FromState.expectTable + var lateral = false + var current: Int? + var index = range.lowerBound + let end = range.upperBound + while index < end { + let token = tokens[index] + let word = token.word + if token.kind == .comma, token.depth == depth { + state = .expectTable + current = nil + index += 1 + continue + } + if let word, Self.joinWords.contains(word) { + state = .expectTable + current = nil + index += 1 + continue + } + if let word, Self.joinPrefixes.contains(word) { + index += 1 + continue + } + switch word { + case "LATERAL": + lateral = true + index += 1 + continue + case "ON": + let stop = fromBoundary(after: index, end: end, depth: depth) + expression((index + 1).. (end: Int, source: Int?)? { + if tokens[index].kind == .open, let close = closing[index], close < end { + // Oracle 23ai's `(VALUES (...), (...)) t (a, b)`, which sees no table beside it, as an inline view does. + if let first = ((index + 1)..= 2 ? parts[parts.count - 2] : nil, name: name) + return (last + 1, append(.table(table, alias: nil), to: block)) + } + + // As Oracle 23ai runs MATCH_RECOGNIZE (measured): PARTITION BY, ORDER BY, each MEASURES expression and each DEFINE + // condition hold names. A MEASURES alias, a DEFINE variable, the row pattern, SUBSET and the clause's own words + // are not. A clause that does not read in Oracle's order is read whole, every name after AS included. + private mutating func rowPatternClause(_ range: Range, block: Int, commonTables: [String: CommonTable]) { + guard let first = range.first, + let clause = SQLRowPatternClause.read(tokens, body: range, text: code.code as NSString) + else { + expression(range, block: block, commonTables: commonTables, ordersBy: false, readsAfterAs: true) + return + } + let depth = tokens[first].depth + for list in [clause.partitionBy, clause.orderBy].compactMap({ $0 }) { + expression(list, block: block, commonTables: commonTables, ordersBy: false) + } + for item in clause.measures.map({ items($0, depth: depth) }) ?? [] { + let read = measureExpression(item, depth: depth) + expression(read, block: block, commonTables: commonTables, ordersBy: false, inRowPattern: true) + } + for item in items(clause.define, depth: depth) { + let read = definedCondition(item, depth: depth) + expression(read, block: block, commonTables: commonTables, ordersBy: false, inRowPattern: true) + } + } + + private func items(_ range: Range, depth: Int) -> [Range] { + var items: [Range] = [] + var start = range.lowerBound + for index in range.lowerBound...range.upperBound + where index == range.upperBound || (tokens[index].kind == .comma && tokens[index].depth == depth) { + if start < index { items.append(start.., depth: Int) -> Range { + if let alias = item.first(where: { tokens[$0].depth == depth && word(at: $0) == "AS" }) { + return item.lowerBound.. item.lowerBound, identifier(at: last) != nil, endsOperand(at: last - 1) else { return item } + return item.lowerBound.., depth: Int) -> Range { + guard let condition = item.first(where: { tokens[$0].depth == depth && word(at: $0) == "AS" }) else { + return item + } + return (condition + 1)..) -> [String] { + var names: [String] = [] + var index = range.lowerBound + while index < range.upperBound { + if word(at: index) == "PATTERN", index + 1 < range.upperBound, tokens[index + 1].kind == .open, + let close = closing[index + 1] { + names += ((index + 2).. Int { + blocks[block].sources.append(source) + return blocks[block].sources.count - 1 + } + + private func fromBoundary(after index: Int, end: Int, depth: Int) -> Int { + var next = index + 1 + while next < end { + let token = tokens[next] + if token.depth == depth { + if token.kind == .comma { return next } + if let word = token.word, Self.joinWords.contains(word) || Self.joinPrefixes.contains(word) { + return next + } + } + next += 1 + } + return end + } + + // MARK: - Expressions + + // `readsAfterAs`: a MATCH_RECOGNIZE clause read whole, where AS also leads a DEFINE condition. `inRowPattern`: a + // MATCH_RECOGNIZE measure or condition, where RUNNING and FINAL before a call are the clause's own words. + private mutating func expression( + _ range: Range, + block: Int, + commonTables: [String: CommonTable], + ordersBy: Bool, + readsAfterAs: Bool = false, + inRowPattern: Bool = false + ) { + var index = range.lowerBound + let end = range.upperBound + var afterAs = false + while index < end { + let token = tokens[index] + if token.kind == .open { + guard let close = closing[index], close < end else { return } + let inner = (index + 1).. 1 { + blocks[block].chains.append(parts) + } + index = last + 1 + } + } + + private mutating func resolveChains() { + for index in blocks.indices { + for parts in blocks[index].chains { + if visibleSource(answering: parts[0], from: index) { + blocks[index].candidates.append(.member(root: parts[0], member: parts[1])) + } else if parts.count >= 3, visibleTable(owner: parts[0], name: parts[1], from: index) { + continue + } else if parts.count <= 3, let last = parts.last, last == "NEXTVAL" || last == "CURRVAL" { + continue + } else { + callsOutright = true + } + } + } + } + + private func visibleSource(answering qualifier: String, from index: Int) -> Bool { + scope(from: index).contains { $0.answers(to: qualifier) } + } + + private func visibleTable(owner: String, name: String, from index: Int) -> Bool { + scope(from: index).contains { source in + guard case .table(let table, nil) = source else { return false } + return table.owner == owner && table.name == name + } + } + + private func scope(from index: Int) -> [Block.Source] { + var sources: [Block.Source] = [] + var current: Int? = index + var hops = 0 + while let block = current, hops <= blocks.count { + sources += blocks[block].sources + current = blocks[block].parent + hops += 1 + } + return sources + } + + // MARK: - Tokens + + private mutating func newBlock(parent: Int?) -> Int { + blocks.append(Block(parent: parent)) + return blocks.count - 1 + } + + private func startsQuery(_ range: Range) -> Bool { + guard let first = range.first(where: { tokens[$0].kind != .open }) else { return false } + return word(at: first) == "SELECT" || word(at: first) == "WITH" + } + + private func identifierList(_ range: Range) -> [String] { + guard !range.isEmpty else { return [] } + let depth = tokens[range.lowerBound].depth + return range.compactMap { tokens[$0].depth == depth ? identifier(at: $0) : nil } + } + + private func word(at index: Int) -> String? { + tokens.indices.contains(index) ? tokens[index].word : nil + } + + private func isSymbol(_ symbol: UInt16, at index: Int) -> Bool { + index >= 0 && code.isSymbol(symbol, at: index) + } + + // A name as the catalog stores it: unquoted in capitals, quoted as written. Nil for a reserved word, a literal or + // a symbol. + private func identifier(at index: Int) -> String? { + guard tokens.indices.contains(index) else { return nil } + if case .word(let word) = tokens[index].kind { + return reserved.contains(word) ? nil : word + } + return quotedOrWideName(at: index) + } + + // After a period any word names a member, a reserved one too. + private func chainPart(at index: Int) -> String? { + guard tokens.indices.contains(index) else { return nil } + if case .word(let word) = tokens[index].kind { return word } + return quotedOrWideName(at: index) + } + + private func quotedOrWideName(at index: Int) -> String? { + let token = tokens[index] + switch token.kind { + case .name: + var end = token.location + while end < code.text.length, SQLFunctionCallScanner.isIdentifierUnit(code.text.character(at: end)) { + end += 1 + } + return code.text.substring(with: NSRange(location: token.location, length: end - token.location)) + .uppercased() + case .quoted: + guard let span = SQLNonCodeSpan.span(at: token.location, in: code.text, grammar: grammar), + code.text.character(at: span.start) == Self.doubleQuote + else { return nil } + let inner = NSRange(location: span.start + 1, length: max(0, span.contentEnd - span.start - 1)) + return code.text.substring(with: inner).replacingOccurrences(of: "\"\"", with: "\"", options: .literal) + default: + return nil + } + } + + // A name, a literal, a number or a closing parenthesis, which an alias may follow. + private func endsOperand(at index: Int) -> Bool { + guard tokens.indices.contains(index) else { return false } + switch tokens[index].kind { + case .quoted, .name, .close: + return true + case .word(let word): + return word == "NULL" + || (!SQLBareNameCalls.oracleReservedWords.contains(word) && !Self.operatorWords.contains(word)) + case .other: + let unit = code.text.character(at: tokens[index].location) + return unit >= Self.digitZero && unit <= Self.digitZero + 9 + default: + return false + } + } + + private static let setOperators: Set = ["UNION", "INTERSECT", "MINUS", "EXCEPT"] + private static let joinWords: Set = ["JOIN", "APPLY"] + private static let joinPrefixes: Set = ["INNER", "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "NATURAL"] + + // Words a table can be followed by that are not its alias. + private static let fromKeywords: Set = joinWords.union(joinPrefixes).union([ + "USING", "PARTITION", "SUBPARTITION", "SAMPLE", "SEED", "PIVOT", "UNPIVOT", "MATCH_RECOGNIZE", "VERSIONS", + "LATERAL", "MODEL", "WINDOW", "OFFSET", "FETCH", "LIMIT", + ]) + + // Words after which an operand follows, so a name after one is never an alias. + private static let operatorWords: Set = [ + "CASE", "WHEN", "THEN", "ELSE", "LIKEC", "LIKE2", "LIKE4", "ESCAPE", "MEMBER", "SUBMULTISET", "MULTISET", + "EXCEPT", "SOME", "ANY", "ALL", "AT", "ZONE", "TIME", "LOCAL", "INTERVAL", "DATE", "TIMESTAMP", "NEW", + "CURSOR", "CONNECT_BY_ROOT", "COLLATE", "TO", "DAY", "MONTH", "YEAR", "HOUR", "MINUTE", "SECOND", "OVER", + "KEEP", "WITHIN", "TYPE", "JSON", "A", "EMPTY", "DANGLING", "PRESENT", "NAN", "INFINITE", "RETURNING", + "FORMAT", "DISTINCT", "PRIOR", "NOT", "AND", "OR", "IS", "IN", "LIKE", "BETWEEN", "EXISTS", "AS", "DIV", + "MOD", "XOR", "REGEXP", "RLIKE", "BINARY", + ] + + private static let period = UInt16(UnicodeScalar(".").value) + private static let star = UInt16(UnicodeScalar("*").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let at = UInt16(UnicodeScalar("@").value) + private static let doubleQuote = UInt16(UnicodeScalar("\"").value) + private static let digitZero = UInt16(UnicodeScalar("0").value) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls.swift new file mode 100644 index 0000000000..bfb07032f6 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls.swift @@ -0,0 +1,369 @@ +import Foundation + +// Oracle and Dameng call a function that takes no arguments when a query names it without parentheses, once the name +// is not a column of a table in scope. Measured on Oracle 23ai and DM8: a column wins; a correlated subquery sees the +// outer query's tables and an inline view does not; a select-list alias is seen by ORDER BY only; `pkg.f` and +// `owner.f` call; on Dameng `alias.f` calls package `alias` when its table has no column F. This reads every such name +// a query holds, and `SQLCallableCatalog` is what the session sees, so together they say whether the query calls one. +public enum SQLBareNameCalls { + public static let engines: Set = ["Oracle", "Dameng"] + + // Under every way the engine could lex the text. Only a query reads anything: a write is refused before this. + public static func read(_ sql: String, databaseTypeId: String, readings: SQLLexicalReadings) -> SQLBareNameReading { + let reserved = reservedWords(forDatabaseTypeId: databaseTypeId) + var reading = SQLBareNameReading() + reading.foldsFunctionNames = databaseTypeId == "Dameng" + for grammar in readings.all { + for statement in SQLStatementClassifier.statements(of: sql, grammar: grammar) { + var parser = Parser(StatementCode(statement, grammar: grammar), grammar: grammar, reserved: reserved) + parser.parse() + reading.callsOutright = reading.callsOutright || parser.callsOutright + if !parser.blocks.isEmpty { reading.queries.append(parser.blocks) } + } + } + return reading + } + + // V$RESERVED_WORDS with RESERVED = 'Y' on Oracle 23ai, and the ones of those DM8 reserves too. No function carries + // one of these names unquoted, so none is asked about. + static let oracleReservedWords: Set = [ + "ALL", "ALTER", "AND", "ANY", "AS", "ASC", "BETWEEN", "BY", "CHAR", "CHECK", "CLUSTER", "COMPRESS", "CONNECT", + "CREATE", "DATE", "DECIMAL", "DEFAULT", "DELETE", "DESC", "DISTINCT", "DROP", "ELSE", "EXCEPT", "EXCLUSIVE", + "EXISTS", "FLOAT", "FOR", "FROM", "GRANT", "GROUP", "HAVING", "IDENTIFIED", "IN", "INDEX", "INSERT", "INTEGER", + "INTERSECT", "INTO", "IS", "LIKE", "LOCK", "LONG", "MINUS", "MODE", "NOCOMPRESS", "NOT", "NOWAIT", "NULL", + "NUMBER", "OF", "ON", "OPTION", "OR", "ORDER", "PCTFREE", "PRIOR", "PUBLIC", "RAW", "RENAME", "RESOURCE", + "REVOKE", "SELECT", "SET", "SHARE", "SIZE", "SMALLINT", "START", "SYNONYM", "TABLE", "THEN", "TO", "TRIGGER", + "UNION", "UNIQUE", "UPDATE", "VALUES", "VARCHAR", "VARCHAR2", "VIEW", "WHERE", "WITH", + ] + + static let damengReservedWords: Set = oracleReservedWords.subtracting([ + "COMPRESS", "DATE", "EXCEPT", "EXCLUSIVE", "IDENTIFIED", "INTEGER", "LOCK", "LONG", "MODE", "NOCOMPRESS", + "NOWAIT", "NUMBER", "OPTION", "PCTFREE", "RAW", "RENAME", "RESOURCE", "SHARE", "SIZE", "SMALLINT", "THEN", + "VARCHAR", "VARCHAR2", + ]) + + static func reservedWords(forDatabaseTypeId typeId: String) -> Set { + typeId == "Dameng" ? damengReservedWords : oracleReservedWords + } +} + +public struct SQLTableName: Hashable, Sendable { + // Nil resolves as the session does: the current schema, its synonyms, then public synonyms. + public let owner: String? + public let name: String + + public init(owner: String?, name: String) { + self.owner = owner + self.name = name + } +} + +// What the session can call bare, read once per connection and schema with `query`. +public struct SQLCallableCatalog: Sendable, Equatable { + // Functions in the current schema and synonyms that reach a function, a synonym or a remote object. + public let functions: Set + // What a dotted call can start with: packages and synonyms to them, and schemas. + public let roots: Set + + public init(functions: Set, roots: Set) { + self.functions = functions + self.roots = roots + } + + // Rows of `query`: a kind, `F` or `R`, and a name. + public init(rows: [[String?]]) { + var functions: Set = [] + var roots: Set = [] + for row in rows where row.count >= 2 { + guard let kind = row[0], let name = row[1] else { continue } + if kind == "F" { functions.insert(name) } else if kind == "R" { roots.insert(name) } + } + self.init(functions: functions, roots: roots) + } + + // Owners are named, since a schema object called ALL_OBJECTS would otherwise answer instead. Oracle and DM8 both + // answer it. + public static let query = """ + SELECT 'F', o.OBJECT_NAME FROM SYS.ALL_OBJECTS o + WHERE o.OWNER = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') AND o.OBJECT_TYPE = 'FUNCTION' + UNION + SELECT 'F', s.SYNONYM_NAME FROM SYS.ALL_SYNONYMS s + WHERE s.OWNER IN (SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA'), 'PUBLIC') + AND (s.DB_LINK IS NOT NULL OR EXISTS (SELECT 1 FROM SYS.ALL_OBJECTS t WHERE t.OWNER = s.TABLE_OWNER + AND t.OBJECT_NAME = s.TABLE_NAME AND t.OBJECT_TYPE IN ('FUNCTION', 'SYNONYM'))) + UNION + SELECT 'R', o.OBJECT_NAME FROM SYS.ALL_OBJECTS o + WHERE o.OWNER = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') AND o.OBJECT_TYPE = 'PACKAGE' + UNION + SELECT 'R', s.SYNONYM_NAME FROM SYS.ALL_SYNONYMS s + WHERE s.OWNER IN (SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA'), 'PUBLIC') + AND (s.DB_LINK IS NOT NULL OR EXISTS (SELECT 1 FROM SYS.ALL_OBJECTS t WHERE t.OWNER = s.TABLE_OWNER + AND t.OBJECT_NAME = s.TABLE_NAME AND t.OBJECT_TYPE IN ('PACKAGE', 'SYNONYM'))) + UNION + SELECT 'R', u.USERNAME FROM SYS.ALL_USERS u + """ + + // The columns of each table as the session resolves its name: an object in the schema first, then a synonym there, + // then, for a name without an owner, a public synonym. Rows: the table's position, the rank, the database link and + // a column. Nil when a name cannot be written as a literal. + public static func columnsQuery(for tables: [SQLTableName]) -> String? { + var branches: [String] = [] + for (index, table) in tables.enumerated() { + guard let name = literal(table.name) else { return nil } + let owner: String + if let tableOwner = table.owner { + guard let quoted = literal(tableOwner) else { return nil } + owner = quoted + } else { + owner = "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" + } + branches.append(""" + SELECT \(index) I, 1 R, o.OWNER, o.OBJECT_NAME N, CAST(NULL AS VARCHAR2(128)) L FROM SYS.ALL_OBJECTS o \ + WHERE o.OWNER = \(owner) AND o.OBJECT_NAME = \(name) \ + AND o.OBJECT_TYPE IN ('TABLE', 'VIEW', 'MATERIALIZED VIEW') + """) + branches.append(""" + SELECT \(index), 2, s.TABLE_OWNER, s.TABLE_NAME, s.DB_LINK FROM SYS.ALL_SYNONYMS s \ + WHERE s.OWNER = \(owner) AND s.SYNONYM_NAME = \(name) + """) + if table.owner == nil { + branches.append(""" + SELECT \(index), 3, s.TABLE_OWNER, s.TABLE_NAME, s.DB_LINK FROM SYS.ALL_SYNONYMS s \ + WHERE s.OWNER = 'PUBLIC' AND s.SYNONYM_NAME = \(name) + """) + } + } + guard !branches.isEmpty else { return nil } + return """ + SELECT t.I, t.R, t.L, c.COLUMN_NAME FROM (\(branches.joined(separator: " UNION ALL "))) t \ + LEFT JOIN SYS.ALL_TAB_COLUMNS c ON c.OWNER = t.OWNER AND c.TABLE_NAME = t.N + """ + } + + // The first rank that resolves wins. A synonym to a remote object, or to another synonym, has no columns here. + public static func columns(fromRows rows: [[String?]], tables: [SQLTableName]) -> [SQLTableName: Set] { + var ranked: [Int: (rank: Int, remote: Bool, columns: Set)] = [:] + for row in rows where row.count >= 4 { + guard let index = row[0].flatMap({ Int($0) }), tables.indices.contains(index), + let rank = row[1].flatMap({ Int($0) }) + else { continue } + let remote = row[2].map { !$0.isEmpty } ?? false + if let current = ranked[index], current.rank < rank { continue } + var entry = ranked[index].flatMap { $0.rank == rank ? $0 : nil } + ?? (rank: rank, remote: remote, columns: Set()) + entry.remote = entry.remote || remote + if let column = row[3] { entry.columns.insert(column) } + ranked[index] = entry + } + var columns: [SQLTableName: Set] = [:] + for (index, entry) in ranked where !entry.remote { + columns[tables[index]] = entry.columns + } + return columns + } + + // A backslash escapes the quote on a DM8 server with BACKSLASH_ESCAPE set, so a name holding one is not written. + private static func literal(_ name: String) -> String? { + guard !name.isEmpty, !name.contains("\\"), + !name.unicodeScalars.contains(where: { $0.properties.generalCategory == .control }) + else { return nil } + return "'" + name.replacingOccurrences(of: "'", with: "''", options: .literal) + "'" + } +} + +// The names a query holds outside parentheses, block by block, before the catalog says which ones call. +public struct SQLBareNameReading: Sendable, Equatable { + // A dotted name that no table in scope starts, such as `pkg.f` or `owner.f`, or a name at a database link: a call, + // or an error the engine raises before it runs anything. + public internal(set) var callsOutright = false + var queries: [[SQLBareNameBlock]] = [] + // A DM8 instance set up with CASE_SENSITIVE=N keeps a name as written, so on Dameng a function or root matches in + // any case. Columns still match exactly, which can only count more names as calls. + var foldsFunctionNames = false + + // Nothing in the query could be a function's name. + public var namesNothing: Bool { + !callsOutright && queries.allSatisfy { blocks in blocks.allSatisfy(\.candidates.isEmpty) } + } + + // The tables whose columns decide, once the catalog holds a name the query uses. Empty when none does. + public func tablesToResolve(_ catalog: SQLCallableCatalog) -> Set { + let catalog = matching(catalog) + var tables: Set = [] + for blocks in queries where blocks.contains(where: { $0.namesSomething(in: catalog, folding: foldsFunctionNames) }) { + for block in blocks { + for case .table(let table, _) in block.sources { tables.insert(table) } + } + } + return tables + } + + // `columns` holds what `SQLCallableCatalog.columns(fromRows:tables:)` read. A table missing from it has columns + // nobody knows, so a name it might hold counts as a call. + public func callsFunction(_ catalog: SQLCallableCatalog, columns: [SQLTableName: Set]) -> Bool { + if callsOutright { return true } + let catalog = matching(catalog) + return queries.contains { blocks in + let resolver = SQLBareNameResolver(blocks: blocks, columns: columns, folding: foldsFunctionNames) + return blocks.indices.contains { resolver.blockCalls($0, catalog: catalog) } + } + } + + private func matching(_ catalog: SQLCallableCatalog) -> SQLCallableCatalog { + guard foldsFunctionNames else { return catalog } + return SQLCallableCatalog( + functions: Set(catalog.functions.map { $0.uppercased() }), + roots: Set(catalog.roots.map { $0.uppercased() }) + ) + } +} + +struct SQLBareNameBlock: Sendable, Equatable { + enum Source: Sendable, Equatable { + case table(SQLTableName, alias: String?) + // An inline view or a common table expression. Nil block: one that refers to itself, read by its column list. + case derived(block: Int?, columns: [String]?, alias: String?) + // `TABLE(...)`, `JSON_TABLE(...)`, a pipelined function, a remote table, a pivot: columns nobody knows. + case opaque(alias: String?) + + var alias: String? { + switch self { + case .table(_, let alias), .derived(_, _, let alias), .opaque(let alias): return alias + } + } + + func named(_ alias: String) -> Source { + switch self { + case .table(let table, _): return .table(table, alias: alias) + case .derived(let block, let columns, _): return .derived(block: block, columns: columns, alias: alias) + case .opaque: return .opaque(alias: alias) + } + } + + // The name a qualifier can use: the alias, or else the table's own name. + func answers(to qualifier: String) -> Bool { + if let alias { return alias == qualifier } + if case .table(let table, _) = self { return table.name == qualifier } + return false + } + } + + enum Output: Sendable, Equatable { + case name(String) + case all + case allOf(String) + } + + enum Candidate: Sendable, Equatable { + // `seesAliases`: an ORDER BY name, which resolves to a select-list alias first. + case name(String, seesAliases: Bool) + // `root.member` where a table in scope answers to `root`. + case member(root: String, member: String) + } + + var parent: Int? + var sources: [Source] = [] + var aliases: Set = [] + var outputs: [Output] = [] + var candidates: [Candidate] = [] + // A compound query's trailing ORDER BY sees only its first branch's names. + var orderByNamesOf: Int? + // Dotted names, resolved once every block's tables are read. + var chains: [[String]] = [] + + func namesSomething(in catalog: SQLCallableCatalog, folding: Bool) -> Bool { + candidates.contains { candidate in + switch candidate { + case .name(let name, _): return catalog.functions.contains(folding ? name.uppercased() : name) + case .member(let root, _): return catalog.roots.contains(folding ? root.uppercased() : root) + } + } + } +} + +private struct SQLBareNameResolver { + let blocks: [SQLBareNameBlock] + let columns: [SQLTableName: Set] + let folding: Bool + + func blockCalls(_ index: Int, catalog: SQLCallableCatalog) -> Bool { + let block = blocks[index] + return block.candidates.contains { candidate in + switch candidate { + case .name(let name, let seesAliases): + guard catalog.functions.contains(folding ? name.uppercased() : name) else { return false } + if seesAliases { + if let named = block.orderByNamesOf { + return outputNames(of: named, depth: 0)?.contains(name) != true + } + if block.aliases.contains(name) { return false } + } + return !isColumn(name, from: index) + case .member(let root, let member): + guard catalog.roots.contains(folding ? root.uppercased() : root) else { return false } + guard let source = source(answering: root, from: index) else { return true } + return columns(of: source, depth: 0)?.contains(member) != true + } + } + } + + private func isColumn(_ name: String, from index: Int) -> Bool { + var current: Int? = index + var hops = 0 + while let blockIndex = current, hops <= blocks.count { + if blocks[blockIndex].sources.contains(where: { columns(of: $0, depth: 0)?.contains(name) == true }) { + return true + } + current = blocks[blockIndex].parent + hops += 1 + } + return false + } + + private func source(answering qualifier: String, from index: Int) -> SQLBareNameBlock.Source? { + var current: Int? = index + var hops = 0 + while let blockIndex = current, hops <= blocks.count { + if let source = blocks[blockIndex].sources.first(where: { $0.answers(to: qualifier) }) { return source } + current = blocks[blockIndex].parent + hops += 1 + } + return nil + } + + private func columns(of source: SQLBareNameBlock.Source, depth: Int) -> Set? { + guard depth <= SQLStatementTiering.deepestNesting else { return nil } + switch source { + case .table(let table, _): + return columns[table] + case .derived(let block, let explicit, _): + if let explicit { return Set(explicit) } + return block.flatMap { outputNames(of: $0, depth: depth + 1) } + case .opaque: + return nil + } + } + + private func outputNames(of index: Int, depth: Int) -> Set? { + guard blocks.indices.contains(index), depth <= SQLStatementTiering.deepestNesting else { return nil } + let block = blocks[index] + var names: Set = [] + for output in block.outputs { + switch output { + case .name(let name): + names.insert(name) + case .all: + for source in block.sources { + guard let sourceColumns = columns(of: source, depth: depth + 1) else { return nil } + names.formUnion(sourceColumns) + } + case .allOf(let qualifier): + guard let source = block.sources.first(where: { $0.answers(to: qualifier) }), + let sourceColumns = columns(of: source, depth: depth + 1) + else { return nil } + names.formUnion(sourceColumns) + } + } + return names + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCallableCatalogCache.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCallableCatalogCache.swift new file mode 100644 index 0000000000..ea3a0f1e9d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCallableCatalogCache.swift @@ -0,0 +1,80 @@ +import Foundation + +// What Oracle and Dameng let a query call without parentheses, read from the session's own catalog and kept per +// connection and schema, so a query names its functions to Read-Only at the cost of one catalog read a minute. +public actor SQLCallableCatalogCache { + public typealias Reader = @Sendable (String) async throws -> [[String?]] + + // A function or a table another session creates is seen within this long. + private let lifetime: Duration + // Starts timing an entry's age on the clock. + private let stopwatch: @Sendable () -> @Sendable () -> Duration + + private struct Key: Hashable { + let connectionId: UUID + let schema: String? + } + + private struct Entry { + let value: Value + let age: @Sendable () -> Duration + } + + private var catalogs: [Key: Entry] = [:] + private var columns: [Key: [SQLTableName: Entry>]] = [:] + + public init>(lifetime: Duration = .seconds(60), clock: C = ContinuousClock()) { + self.lifetime = lifetime + stopwatch = { + let start = clock.now + return { start.duration(to: clock.now) } + } + } + + // Whether the query calls a function by a bare name. `read` runs one catalog query on the connection. + public func callsFunction( + _ reading: SQLBareNameReading, + connectionId: UUID, + schema: String?, + read: Reader + ) async throws -> Bool { + if reading.callsOutright { return true } + if reading.namesNothing { return false } + let key = Key(connectionId: connectionId, schema: schema) + let catalog = try await catalog(for: key, read: read) + let known = try await columns(of: reading.tablesToResolve(catalog), for: key, read: read) + return reading.callsFunction(catalog, columns: known) + } + + private func catalog(for key: Key, read: Reader) async throws -> SQLCallableCatalog { + if let cached = catalogs[key], cached.age() < lifetime { + return cached.value + } + let age = stopwatch() + let catalog = SQLCallableCatalog(rows: try await read(SQLCallableCatalog.query)) + catalogs[key] = Entry(value: catalog, age: age) + return catalog + } + + private func columns( + of tables: Set, + for key: Key, + read: Reader + ) async throws -> [SQLTableName: Set] { + guard !tables.isEmpty else { return [:] } + var known = (columns[key] ?? [:]).filter { $0.value.age() < lifetime } + let missing = tables.filter { known[$0] == nil }.sorted { ($0.owner ?? "", $0.name) < ($1.owner ?? "", $1.name) } + if !missing.isEmpty, let query = SQLCallableCatalog.columnsQuery(for: missing) { + let age = stopwatch() + let resolved = SQLCallableCatalog.columns(fromRows: try await read(query), tables: missing) + for table in missing { + // A table the catalog does not resolve, or one behind a database link, is kept with no columns. + known[table] = Entry(value: resolved[table] ?? [], age: age) + } + } + columns[key] = known + return tables.reduce(into: [:]) { result, table in + if let entry = known[table] { result[table] = entry.value } + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift index 2e44afde28..bb2f0f4ea6 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift @@ -13,8 +13,10 @@ public struct SQLCatalogObjectKinds: OptionSet, Sendable, Hashable { public static let types = SQLCatalogObjectKinds(rawValue: 1 << 3) public static let schemas = SQLCatalogObjectKinds(rawValue: 1 << 4) public static let databases = SQLCatalogObjectKinds(rawValue: 1 << 5) + // The kinds a driver declares for the sidebar, such as MySQL events and PostgreSQL extensions. + public static let serverObjects = SQLCatalogObjectKinds(rawValue: 1 << 6) - public static let objects: SQLCatalogObjectKinds = [.tables, .routines, .triggers, .types] + public static let objects: SQLCatalogObjectKinds = [.tables, .routines, .triggers, .types, .serverObjects] public static let everything: SQLCatalogObjectKinds = [.objects, .schemas, .databases] } @@ -71,7 +73,7 @@ public enum SQLCatalogEffectClassifier { return .opaque } let tokens = leadingTokens(of: trimmed, grammar: grammar) - let leading = leadingKeywordEffect(tokens: tokens, trimmed: trimmed) + let leading = leadingKeywordEffect(tokens: tokens, trimmed: trimmed, grammar: grammar) guard leading.kinds.isEmpty else { return leading } return leading.union(embeddedDefinitionEffect(tokens: tokens)) } @@ -87,9 +89,17 @@ public enum SQLCatalogEffectClassifier { return SQLCatalogEffect(kinds: definedKinds(after: tokens[(index + 1)...]), endsTransaction: false) } - private static func leadingKeywordEffect(tokens: [String], trimmed: String) -> SQLCatalogEffect { + private static func leadingKeywordEffect( + tokens: [String], + trimmed: String, + grammar: SQLLexicalGrammar + ) -> SQLCatalogEffect { guard let keyword = tokens.first else { return .none } + if grammar.contains(.transactionShortForms), tokens.count == 1 { + if keyword == "ET" { return SQLCatalogEffect(kinds: [], endsTransaction: true) } + if keyword == "BT" { return .none } + } if transactionEndKeywords.contains(keyword) { let rollsBackToSavepoint = keyword == "ROLLBACK" && tokens.dropFirst().prefix(2).contains("TO") return rollsBackToSavepoint ? .none : SQLCatalogEffect(kinds: [], endsTransaction: true) @@ -200,7 +210,7 @@ public enum SQLCatalogEffectClassifier { "SYNONYM": .tables, "FIELD": .tables, "FUNCTION": .routines, "PROCEDURE": .routines, "PROC": .routines, "ROUTINE": .routines, "AGGREGATE": .routines, "MACRO": .routines, "PACKAGE": .routines, "OPERATOR": .routines, - "TRIGGER": .triggers, "EVENT": .triggers, + "TRIGGER": .triggers, "EVENT": [.triggers, .serverObjects], "TYPE": .types, "DOMAIN": .types, "ENUM": .types, "SCHEMA": [.schemas, .objects], "NAMESPACE": [.schemas, .objects], "OWNED": [.schemas, .objects], "DATABASE": .everything, "KEYSPACE": .everything, "CATALOG": .everything, diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLDelimiterDirective.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLDelimiterDirective.swift new file mode 100644 index 0000000000..6cae802465 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLDelimiterDirective.swift @@ -0,0 +1,153 @@ +import Foundation + +/// A `DELIMITER //` line, which changes the text that ends a statement the way the mysql client reads a script. +/// +/// The client reads it as the first word of a line, in any case, followed by a space, a tab or the line's end, and only +/// where no statement text is pending. The line goes to no server. The argument is the first word after it, up to a +/// space, or a string quoted with `'`, `"` or a backtick; the rest of the line is ignored. An empty argument, or one +/// holding a backslash, is refused and the delimiter stays as it was. The delimiter then ends a statement wherever it +/// stands in code, inside a word too, matched case for case. +public struct SQLDelimiterDirective: Sendable, Equatable { + /// The line from its `DELIMITER` to the end of the line, the line feed excluded. + public let range: NSRange + + /// The delimiter the line sets, nil when the client refuses the argument and keeps the one before. + public let delimiter: String? + + public init(range: NSRange, delimiter: String?) { + self.range = range + self.delimiter = delimiter + } + + private static let word = Array("delimiter".utf16) + + /// The directive whose `DELIMITER` starts at `offset`, or nil when the line there is not one. `offset` has to be + /// code where no statement text is pending. + public static func line(at offset: Int, in text: NSString, length: Int) -> SQLDelimiterDirective? { + var before = offset - 1 + while before >= 0, isLineBlank(text.character(at: before)) { + before -= 1 + } + guard before < 0 || text.character(at: before) == SqlLexer.newline else { return nil } + return line(startingAt: offset, in: text, length: length) + } + + /// The directive whose `DELIMITER` starts at `offset`, for a reader that already knows only spaces and tabs stand + /// before it on its line. `text` has to hold the whole line unless the line ends the text. + public static func line(startingAt offset: Int, in text: NSString, length: Int) -> SQLDelimiterDirective? { + let wordEnd = offset + word.count + guard spellsWord(in: text, at: offset, length: length) else { return nil } + if wordEnd < length { + let next = text.character(at: wordEnd) + guard isLineBlank(next) || next == SqlLexer.newline || next == SqlLexer.carriageReturn else { return nil } + } + var lineEnd = wordEnd + while lineEnd < length, text.character(at: lineEnd) != SqlLexer.newline { + lineEnd += 1 + } + // The client reads a line without the carriage return of a CRLF break. + var argumentEnd = lineEnd + if argumentEnd > wordEnd, text.character(at: argumentEnd - 1) == SqlLexer.carriageReturn { + argumentEnd -= 1 + } + var argument: [UInt16] = [] + for index in wordEnd.. Bool { + let wordEnd = offset + word.count + return wordEnd < length && spellsWord(in: text, at: offset, length: length) + && isLineBlank(text.character(at: wordEnd)) + } + + // ASCII only: the client compares bytes, so `delımiter` is no command. + private static func spellsWord(in text: NSString, at offset: Int, length: Int) -> Bool { + guard offset + word.count <= length else { return false } + for (index, unit) in word.enumerated() where text.character(at: offset + index) | 0x20 != unit { + return false + } + return true + } + + // As mysql's `get_arg` reads it: a backslash keeps the next unit in an unquoted argument, and a quoted one ends at + // its first matching quote, or at the line's end when it never closes. + private static func delimiter(fromArgument argument: [UInt16]) -> String? { + var index = 0 + while index < argument.count, isArgumentSpace(argument[index]) { + index += 1 + } + guard index < argument.count else { return nil } + var units: [UInt16] = [] + let opener = argument[index] + if opener == SqlLexer.singleQuote || opener == SqlLexer.doubleQuote || opener == SqlLexer.backtick { + index += 1 + while index < argument.count, argument[index] != opener { + units.append(argument[index]) + index += 1 + } + } else { + while index < argument.count, argument[index] != SqlLexer.space { + if argument[index] == SqlLexer.backslash, index + 1 < argument.count { + index += 1 + } + units.append(argument[index]) + index += 1 + } + } + guard !units.isEmpty, !units.contains(SqlLexer.backslash) else { return nil } + return String(utf16CodeUnits: units, count: units.count) + } + + private static func isLineBlank(_ unit: UInt16) -> Bool { + unit == SqlLexer.space || unit == SqlLexer.tab + } + + // `my_isspace`, which skips the blanks before the argument. + private static func isArgumentSpace(_ unit: UInt16) -> Bool { + unit == SqlLexer.space || (unit >= 0x09 && unit <= 0x0D) + } +} + +/// The delimiter a `DELIMITER` line set, which ends a statement wherever it stands in code, matched unit for unit. +public struct SQLClientDelimiter: Sendable, Equatable { + public let units: [UInt16] + + /// Nil for `;`, which the statement tracker reads. + public init?(_ text: String) { + guard text != ";" else { return nil } + units = Array(text.utf16) + } + + public func starts(in text: NSString, at offset: Int, length: Int) -> Bool { + guard offset + units.count <= length else { return false } + for (index, unit) in units.enumerated() where text.character(at: offset + index) != unit { + return false + } + return true + } + + /// `word` as read at `start`, cut where the delimiter starts inside it: `END$$` is the word `END`. + public func cut( + _ word: (text: String, end: Int), + readAt start: Int, + in text: NSString, + length: Int + ) -> (text: String, end: Int) { + let wordStart = word.end - (word.text as NSString).length + var cursor = max(wordStart, start) + 1 + while cursor < word.end, !starts(in: text, at: cursor, length: length) { + cursor += 1 + } + guard cursor < word.end else { return word } + let raw = text.substring(with: NSRange(location: wordStart, length: cursor - wordStart)) + return (raw.utf16.allSatisfy { $0 < 0x80 } ? raw.uppercased() : raw, cursor) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLFileNamedAsTable.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLFileNamedAsTable.swift index 6029d61d64..98708c33b7 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLFileNamedAsTable.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLFileNamedAsTable.swift @@ -1,7 +1,7 @@ import Foundation // DuckDB reads a file named as a table. A quoted identifier, in `"` or a backtick, counts only with a `/`, a `.` or a -// `\`. +// `\`. So does a dotted name ending in a file extension, and a table function DuckDB does not keep inside the database. public enum SQLFileNamedAsTable { private static let tableIntroducers: Set = [ "FROM", "JOIN", "USING", "TABLE", "DESCRIBE", "DESC", "SHOW", "SUMMARIZE", "PIVOT", "UNPIVOT", @@ -16,6 +16,23 @@ public enum SQLFileNamedAsTable { // `extract(year FROM '2026-01-01'::DATE)` separates arguments with FROM. private static let functionsTakingFrom: Set = ["EXTRACT", "TRIM", "SUBSTRING", "OVERLAY"] + // Only a FROM or JOIN list holds table functions; PIVOT's and USING's parentheses hold expressions. + private static let callingIntroducers: Set = ["FROM", "JOIN"] + + // An allowlist: an extension DuckDB loads on its own brings table functions no list here knows. + static let tableFunctionsInsideDatabase: Set = [ + "RANGE", "GENERATE_SERIES", "UNNEST", "REPEAT", "JSON_EACH", "JSON_TREE", "DUCKDB_COLUMNS", + "DUCKDB_CONSTRAINTS", "DUCKDB_DATABASES", "DUCKDB_DEPENDENCIES", "DUCKDB_FUNCTIONS", "DUCKDB_INDEXES", + "DUCKDB_KEYWORDS", "DUCKDB_SCHEMAS", "DUCKDB_SEQUENCES", "DUCKDB_SETTINGS", "DUCKDB_TABLES", "DUCKDB_TYPES", + "DUCKDB_VARIABLES", "DUCKDB_VIEWS", "PRAGMA_DATABASE_SIZE", "PRAGMA_PLATFORM", "PRAGMA_SHOW", + "PRAGMA_STORAGE_INFO", "PRAGMA_TABLE_INFO", "PRAGMA_VERSION", + ] + + // DuckDB joins a dotted table name with `.` and reads it as a file in the working folder. + static let fileExtensions: Set = [ + "CSV", "TSV", "GZ", "ZST", "PARQUET", "JSON", "JSONL", "NDJSON", "GEOJSON", "GEOJSONL", "DB", "DDB", "DUCKDB", + ] + private static let nameContinuations = Set(".:&".utf16) private static let pathMarks = CharacterSet(charactersIn: "/.\\") @@ -31,6 +48,8 @@ public enum SQLFileNamedAsTable { var separatesArgumentsWithFrom = false var listsTables = false var expects = Expectation.nothing + var introducer = "" + var nameParts = 0 } public static func appears(in statement: String, grammar: SQLLexicalGrammar) -> Bool { @@ -46,32 +65,53 @@ public enum SQLFileNamedAsTable { break case .word(let word) where tableIntroducers.contains(word): levels[top].expects = .table + levels[top].introducer = word + levels[top].nameParts = 0 if word == "FROM" { levels[top].listsTables = true } case .word(let word) where tableListEnders.contains(word) && expected != .table: levels[top].listsTables = false - case .word("ONLY") where expected == .table: + case .word("ONLY") where expected == .table, .word("LATERAL") where expected == .table: levels[top].expects = .table - case .word, .name: + case .word(let word): + guard expected == .table else { break } + levels[top].nameParts += 1 + if levels[top].nameParts > 1, fileExtensions.contains(word) { return true } + levels[top].expects = .restOfName + case .name: if expected == .table { levels[top].expects = .restOfName } case .quoted: guard expected == .table else { break } if namesFile(at: token.location, in: source, grammar: grammar) { return true } + levels[top].nameParts += 1 + let quotedName = identifier(at: token.location, in: source, grammar: grammar)?.uppercased() + if levels[top].nameParts > 1, let quotedName, fileExtensions.contains(quotedName) { return true } levels[top].expects = .restOfName case .open: + if expected == .restOfName, callingIntroducers.contains(levels[top].introducer), + !callsTableFunctionInsideDatabase(before: index, in: tokens, source: source, grammar: grammar) { + return true + } // `FROM (notes JOIN 'payroll.csv' USING (id))` groups tables, and any other `(` starts afresh. levels.append( Parentheses( separatesArgumentsWithFrom: callsFunctionTakingFrom(before: index, in: tokens, source: source), - expects: expected == .table ? .table : .nothing + expects: expected == .table ? .table : .nothing, + introducer: expected == .table ? levels[top].introducer : "" ) ) case .close: if top > 0 { levels.removeLast() } case .comma: - if levels[top].listsTables { levels[top].expects = .table } + if levels[top].listsTables { + levels[top].expects = .table + levels[top].nameParts = 0 + } case .other: - let continuesName = nameContinuations.contains(source.character(at: token.location)) - if expected == .restOfName, continuesName { levels[top].expects = .table } + let mark = source.character(at: token.location) + if expected == .restOfName, nameContinuations.contains(mark) { + levels[top].expects = .table + if mark != period { levels[top].nameParts = 0 } + } case .terminator: break } @@ -106,6 +146,33 @@ public enum SQLFileNamedAsTable { return tableIntroducers.first(where: word.hasSuffix).map { .word($0) } ?? token.kind } + // A name the scanner cannot read counts as unknown. + private static func callsTableFunctionInsideDatabase( + before index: Int, + in tokens: [TSQLTokens.Token], + source: NSString, + grammar: SQLLexicalGrammar + ) -> Bool { + guard index >= 1 else { return false } + let callee = tokens[index - 1] + let name: String? = switch callee.kind { + case .word(let word): word + case .quoted: identifier(at: callee.location, in: source, grammar: grammar)?.uppercased() + default: nil + } + guard let name else { return false } + return name == "VALUES" || tableFunctionsInsideDatabase.contains(name) + } + + private static func identifier(at location: Int, in source: NSString, grammar: SQLLexicalGrammar) -> String? { + guard [SqlLexer.doubleQuote, SqlLexer.backtick].contains(source.character(at: location)), + let span = SQLNonCodeSpan.span(at: location, in: source, grammar: grammar) + else { + return nil + } + return source.substring(with: NSRange(location: location + 1, length: max(0, span.contentEnd - location - 1))) + } + private static func namesFile(at location: Int, in source: NSString, grammar: SQLLexicalGrammar) -> Bool { guard [SqlLexer.doubleQuote, SqlLexer.backtick].contains(source.character(at: location)), let span = SQLNonCodeSpan.span(at: location, in: source, grammar: grammar) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift index db77326018..ab874e2887 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift @@ -153,6 +153,26 @@ public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable { /// since no other fact here says that a hint's body is tokens or that a stage name keeps a quote. Curated only. public static let databendTokens = SQLLexicalGrammar(rawValue: 1 << 40) + /// A script reads as psql reads it: `COPY ... FROM STDIN` takes the lines after its own as rows, up to a line + /// holding only `\.`, and pg_dump's `\restrict` and `\unrestrict` lines run nothing, as no meta-command runs here. + /// See ``SQLPsqlScript``. Curated only. + public static let psqlScripts = SQLLexicalGrammar(rawValue: 1 << 41) + + /// The non-ASCII spaces ``SQLNonCodeSpan/spacesBeforeLexing`` lists end a name, as DuckDB turns them into spaces + /// before it lexes where its pre-lexing pass sees no quote, so `x` + U+200B + `E'...'` is a name and an escape + /// string. PostgreSQL reads every byte from 0x80 up as part of a name. Measured on DuckDB 1.5.4 and 2.0.0 and + /// PostgreSQL 17.11. That pass reads quotes its own way, so DuckDB's profile also doubts this fact. Curated only. + public static let nonASCIISpacesEndNames = SQLLexicalGrammar(rawValue: 1 << 42) + + // `BT` and `ET` begin and end a transaction, as Teradata reads them; anywhere else they are names. Curated only. + public static let transactionShortForms = SQLLexicalGrammar(rawValue: 1 << 43) + + /// A routine's body is one statement, as MySQL's `routine_body` reads it, and a compound one holds the `;`s inside + /// it: `IF ... END IF`, `WHILE ... END WHILE`, `LOOP ... END LOOP` and `REPEAT ... END REPEAT` as `BEGIN ... END` + /// does. A `SET` outside the return type starts the body, so a `begin` after it is a name. Measured on MariaDB + /// 13.0.2. Curated only. + public static let compoundStatementBodies = SQLLexicalGrammar(rawValue: 1 << 44) + /// Standard SQL: quotes close only on a doubled quote, `--` and flat `/* */` are the only comments. public static let ansi: SQLLexicalGrammar = [] diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index 73445d0e30..8ec9f9526a 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -68,9 +68,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// own; an engine that runs statements without one says so through its plugin. A `MERGE` that keeps its `;` is left /// out for the same reason: an engine that refuses one without it says so through its plugin. So is a CQL batch, /// which would read one statement where an unknown engine may run several. So are the two single body forms, which - /// would cut the `BEGIN ... END` body of an engine that has the usual one. So is a statement inside another, which - /// would make every `update` or `access` column an unknown engine names a write, and Snowflake's pipe operator, - /// which would cut a statement at the `->>` that is JSON's operator on an unknown engine. + /// would cut the `BEGIN ... END` body of an engine that has the usual one. So is MySQL's compound statement body, + /// which would hold a `LOOP` or a `WHILE ... DO` open past the `;` an unknown engine ends it at. So is a statement + /// inside another, which would make every `update` or `access` column an unknown engine names a write, and Snowflake's + /// pipe operator, which would cut a statement at the `->>` that is JSON's operator on an unknown engine. So are + /// psql's `COPY` rows, which would hide lines an unknown engine runs as statements. public static let everyKnownReading: [SQLLexicalGrammar] = { var seen: Set = [] var result: [SQLLexicalGrammar] = [] @@ -79,8 +81,10 @@ public struct SQLLexicalProfile: Sendable, Hashable { for reading in profile.readings { let unitless = reading.subtracting([ .plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements, - .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, - .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, + .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .compoundStatementBodies, + .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, .psqlScripts, + // An unknown engine reads `BT` and `ET` as names, not as transaction control. + .transactionShortForms, ]) if seen.insert(unitless).inserted { result.append(unitless) @@ -98,7 +102,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// are operators. From its grammar: a routine's body is a literal or `BEGIN ATOMIC ... END`. static let postgreSQL: SQLLexicalGrammar = [ .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .escapeStringContinuations, - .carriageReturnEndsLineComments, .atomicRoutineBodies, + .carriageReturnEndsLineComments, .atomicRoutineBodies, .psqlScripts, ] /// CockroachDB, from its scanner: PostgreSQL's grammar, and a `b'...'` byte literal read by the routine that reads @@ -109,10 +113,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// MySQL 8.4.11 and MariaDB 11.8.9, measured: a backslash escapes in `'` and `"` strings and is literal in a /// backtick identifier, block comments do not nest, `#` is a comment, `/*! */` runs, `--` needs whitespace after - /// it, and only a line feed ends a line comment. + /// it, and only a line feed ends a line comment. From the MySQL 8.4 manual, and by hand on MariaDB 13.0.2 (the probe + /// does not check it): a routine's body is one statement, which may be a compound one. static let mySQL: SQLLexicalGrammar = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, - .executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, + .executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, .compoundStatementBodies, ] /// SQLite 3.54, measured: brackets and backticks quote identifiers, `]]` does not escape, a backslash is literal, @@ -126,10 +131,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// quotes a name and a `'...'` after a line break is a string of its own, not part of an `E'...'`. 1.5.4 continues /// the `E'...'` as PostgreSQL does, a registry plugin built before 2.0 links 1.5, and `enable_legacy_parser` brings /// 1.5's parser back on 2.0, so a gate reads both. It has no `BEGIN ... END` body, so it takes PostgreSQL's - /// narrower fact. + /// narrower fact. Both versions turn the non-ASCII spaces into blanks before they lex, outside the quotes their + /// pre-lexing pass sees. static let duckDB: SQLLexicalGrammar = [ .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes, - .carriageReturnEndsLineComments, .atomicRoutineBodies, + .carriageReturnEndsLineComments, .atomicRoutineBodies, .nonASCIISpacesEndNames, ] /// Oracle 23.26, measured: a backslash is literal, block comments do not nest, `q'[...]'` is a literal, `$` and @@ -294,7 +300,13 @@ public struct SQLLexicalProfile: Sendable, Hashable { "libSQL": sqliteFamily, "Turso": sqliteFamily, "Cloudflare D1": sqliteFamily, - "DuckDB": SQLLexicalProfile(grammar: duckDB, undetermined: [.escapeStringContinuations]), + // DuckDB blanks a non-ASCII space only where its own pre-lexing pass is outside a quote, and that pass reads + // quotes its own way: after `a$$`, `E'\''` or `/* ' */` it keeps the space, a name character again + // (measured by hand on 1.5.4 and 2.0.0; the probe checks the execution reading only). So a gate reads both. + "DuckDB": SQLLexicalProfile( + grammar: duckDB, + undetermined: [.escapeStringContinuations, .nonASCIISpacesEndNames] + ), "Oracle": SQLLexicalProfile(grammar: oracle), "Dameng": SQLLexicalProfile( grammar: dameng, @@ -318,7 +330,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { "Trino": trinoFamily, "Presto": trinoFamily, "Teradata": SQLLexicalProfile( - grammar: .ansi, + grammar: [.transactionShortForms], undetermined: [.nestedBlockComments, .carriageReturnEndsLineComments] ), "SAP HANA": SQLLexicalProfile( diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift index 9fb7a0e504..8311a7f4b6 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift @@ -137,8 +137,8 @@ enum SQLLexicalRelevance { let length = text.length var facts: SQLLexicalGrammar = [ .plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements, - .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, - .pipeOperatorChains, .databendTokens, + .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .compoundStatementBodies, + .statementsAsValues, .pipeOperatorChains, .databendTokens, .psqlScripts, .transactionShortForms, ] var blockCommentOpeners = 0 var index = 0 @@ -196,6 +196,8 @@ enum SQLLexicalRelevance { if next == SqlLexer.singleQuote { facts.insert(.alternativeQuoting) } case capitalG, smallG: facts.insert(.batchSeparatorLines) + case let space where space >= 0xA0 && SQLNonCodeSpan.spacesBeforeLexing.contains(space): + facts.insert(.nonASCIISpacesEndNames) default: break } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift index 47e597afcc..52e67f17ff 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift @@ -119,8 +119,6 @@ public enum SQLNonCodeSpan { return span.end } - /// Whether `unit` continues a word, so a prefix like `E` or `q` glued to it is part of an identifier rather than - /// the start of a literal. public static func isWordUnit(_ unit: UInt16) -> Bool { if unit < 0x80 { return SqlDollarQuote.isIdentifierPart(unit) @@ -128,6 +126,29 @@ public enum SQLNonCodeSpan { return !SQLSeparatingCharacter.isSeparating(unit) } + /// Whether a literal prefix right after `unit` is part of the name `unit` belongs to, so `x$e'\'` is the name + /// `x$e` and a plain string. PostgreSQL 17.11 continues a name with `$` and every unit from 0x80 up, and DuckDB + /// 1.5.4 and 2.0.0 do too, less the spaces they blank (measured). CockroachDB's scanner reads bytes the same way, and + /// KingbaseES, a PostgreSQL fork, is taken to. Oracle and Dameng document `$` and `#` as name characters, and run + /// one statement per call. + public static func continuesName(beforePrefix unit: UInt16, grammar: SQLLexicalGrammar) -> Bool { + if unit >= 0x80 { + return !endsNameAsBlank(unit, grammar: grammar) + } + return SqlDollarQuote.isIdentifierPart(unit) || unit == SqlDollarQuote.dollar + || (unit == SqlLexer.hash && grammar.contains(.dollarAndHashInIdentifiers)) + } + + /// Whether a `$` right after `unit` is part of the name `unit` belongs to, as in `a$$b`, rather than a dollar + /// quote's opener. On DuckDB `SELECT` + U+200B + `$$...$$` opens one, measured on 2.0.0. + public static func continuesName(beforeDollar unit: UInt16, grammar: SQLLexicalGrammar) -> Bool { + SqlDollarQuote.isIdentifierContinuation(unit) && !endsNameAsBlank(unit, grammar: grammar) + } + + private static func endsNameAsBlank(_ unit: UInt16, grammar: SQLLexicalGrammar) -> Bool { + grammar.contains(.nonASCIISpacesEndNames) && spacesBeforeLexing.contains(unit) + } + // MARK: - Comments private static func commentSpan( @@ -272,8 +293,8 @@ public enum SQLNonCodeSpan { return closedSpan(.quoted, at: index, span, closerLength: 1) } - /// `E'...'`, `b'...'` and `q'[...]'`, which only start where a word could: `xE'` is an identifier followed by a - /// string. + /// `E'...'`, `b'...'` and `q'[...]'`, which only start where a name does not go on: `xE'` and `x$E'` are a name + /// followed by a string. private static func prefixedLiteralSpan( at index: Int, character: UInt16, @@ -281,7 +302,9 @@ public enum SQLNonCodeSpan { length: Int, grammar: SQLLexicalGrammar ) -> Span? { - guard index == 0 || !isWordUnit(text.character(at: index - 1)) else { return nil } + guard index == 0 || !continuesName(beforePrefix: text.character(at: index - 1), grammar: grammar) else { + return nil + } let startsEscapeString = grammar.contains(.escapeStringPrefix) && (character == capitalE || character == smallE) let startsEscapeBytes = grammar.contains(.escapeBytesPrefix) && character == smallB if startsEscapeString || startsEscapeBytes, @@ -296,7 +319,8 @@ public enum SQLNonCodeSpan { ) if grammar.contains(.escapeStringContinuations) { var newlines = span.newlines - while span.isClosed, let continued = continuingQuote(after: span.next, in: text, length: length) { + while span.isClosed, + let continued = continuingQuote(after: span.next, in: text, length: length, grammar: grammar) { span = SqlLexer.skipQuotedString( text, from: continued.quote, @@ -327,15 +351,21 @@ public enum SQLNonCodeSpan { private static let verticalTab: UInt16 = 0x0B - // DuckDB 1.5 turns these into spaces before it lexes (measured on 1.5.4). PostgreSQL and CockroachDB do not, and - // the text a join there leaves them does not parse, so they run none of it. - private static let spacesBeforeLexing: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) + // DuckDB turns these into spaces before it lexes, outside what its pre-lexing pass reads as a quote, and no other + // non-ASCII unit (measured on 1.5.4 and 2.0.0). PostgreSQL and CockroachDB read them as part of a name. + static let spacesBeforeLexing: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) .union(0x2000...0x200B) /// The quote that carries on the string closed just before `index`, as PostgreSQL's `quotecontinue` finds it: only /// blanks and `--` comments between, a line break among them. PostgreSQL 16 and later take a vertical tab for a - /// blank; an older server refuses the text. - private static func continuingQuote(after index: Int, in text: NSString, length: Int) -> (quote: Int, newlines: Int)? { + /// blank; an older server refuses the text. A non-ASCII space is a blank only where DuckDB 1.5 has blanked it + /// first: PostgreSQL 17.11 reads `E'a'` + U+00A0 + line break + `'b'` as a name between two strings. + private static func continuingQuote( + after index: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> (quote: Int, newlines: Int)? { var cursor = index var lineBroke = false var newlines = 0 @@ -350,7 +380,7 @@ public enum SQLNonCodeSpan { cursor += 1 case SqlLexer.space, SqlLexer.tab, verticalTab, SqlLexer.formFeed: cursor += 1 - case let unit where spacesBeforeLexing.contains(unit): + case let unit where endsNameAsBlank(unit, grammar: grammar): cursor += 1 case SqlLexer.dash where SqlLexer.startsLineComment(text, at: cursor, length: length): cursor = SqlLexer.endOfLineComment( @@ -376,7 +406,8 @@ public enum SQLNonCodeSpan { at: index, in: text, bufLen: length, - style: style + style: style, + continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) } ) else { return nil diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift new file mode 100644 index 0000000000..d5d1607083 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift @@ -0,0 +1,42 @@ +import Foundation + +// The scanner keeps the `;`s of a PostgreSQL rule's `DO ALSO (...; ...)` in its statement. A client that cuts at every +// `;` runs each piece, so a gate tiers them too: `DO ALSO (SELECT 1; DELETE FROM t)` by its `DELETE`. +enum SQLParenthesizedSemicolons { + static func pieces(in statement: String, grammar: SQLLexicalGrammar) -> [String] { + guard statement.utf16.contains(SqlLexer.semicolon) else { return [] } + let text = statement as NSString + let length = text.length + var cuts: [Int] = [] + var depth = 0 + var index = 0 + while index < length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + switch text.character(at: index) { + case SqlLexer.openParen: + depth += 1 + case SqlLexer.closeParen: + depth = max(0, depth - 1) + case SqlLexer.semicolon where depth > 0: + cuts.append(index) + default: + break + } + index += 1 + } + guard !cuts.isEmpty else { return [] } + var pieces: [String] = [] + var start = 0 + for cut in cuts + [length] { + let piece = StatementBlank.trimming(text.substring(with: NSRange(location: start, length: cut - start))) + if !piece.isEmpty { + pieces.append(piece) + } + start = cut + 1 + } + return pieces + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift new file mode 100644 index 0000000000..0860269016 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift @@ -0,0 +1,272 @@ +import Foundation + +/// The lines of a psql script that are not SQL, read the way psql 17 and 18 read them. +/// +/// `COPY ... FROM STDIN` takes its rows from the script itself: the lines after the line its `;` ends on, up to a line +/// holding only `\.`, or to the end of the script. psql skips them even when the `COPY` fails, since a row run as SQL +/// can be any statement. pg_dump writes a script that opens with `\restrict ` and closes with `\unrestrict `, +/// which only limit psql's own meta-commands; nothing here runs one, so both lines run nothing. +public enum SQLPsqlScript { + private static let backslash = SqlLexer.backslash + private static let period = UInt16(UnicodeScalar(".").value) + private static let restrict = Array("restrict".utf16) + private static let unrestrict = Array("unrestrict".utf16) + + /// Everything after the `;` at `semicolon` that belongs to a `COPY ... FROM STDIN`: the rest of the `;`'s line, then + /// each line up to and including the one holding only `\.`, its line break excluded, or up to the end of the text. + /// + /// psql runs code after the `;` on its own line once the rows are in. pg_dump never writes any, and here it never + /// runs, so a line break after it cannot decide whether it does. + public static func copyRows(afterSemicolonAt semicolon: Int, in text: NSString, length: Int) -> NSRange { + guard var lineStart = lineEnd(in: text, from: semicolon + 1, length: length).map({ $0 + 1 }) else { + return NSRange(location: semicolon + 1, length: length - semicolon - 1) + } + while lineStart < length { + if let markerEnd = endOfDataMarker(in: text, at: lineStart, length: length) { + return NSRange(location: semicolon + 1, length: markerEnd - semicolon - 1) + } + guard let next = lineEnd(in: text, from: lineStart, length: length) else { break } + lineStart = next + 1 + } + return NSRange(location: semicolon + 1, length: length - semicolon - 1) + } + + /// The rows of the `COPY` block `rows` spans, as ``copyRows(afterSemicolonAt:in:length:)`` found it: the lines + /// after the `;`'s line, each with its line break, up to the `\.` line or the end of the text. The `\.` line is + /// never part of them: psql 18 does not send it, and PostgreSQL 18 reads it as data in CSV. + public static func copyData(inRows rows: NSRange, of text: NSString) -> NSRange { + let end = NSMaxRange(rows) + guard let firstBreak = lineEnd(in: text, from: rows.location, length: end) else { + return NSRange(location: end, length: 0) + } + let start = firstBreak + 1 + var lastLineStart = start + if start < end { + let found = text.range( + of: "\n", + options: [.literal, .backwards], + range: NSRange(location: start, length: end - start) + ) + if found.location != NSNotFound { lastLineStart = found.location + 1 } + } + let dataEnd = endOfDataMarker(in: text, at: lastLineStart, length: text.length) == end ? lastLineStart : end + return NSRange(location: start, length: max(0, dataEnd - start)) + } + + /// Where the `\.` line that starts at `lineStart` ends, its line break excluded, or nil when the line is not one. + /// psql 18 matches `\.` and a LF or CRLF at a line's start only; a last line `\.` with no line break ends the rows + /// too. The unit after the marker is read to decide, so `text` has to hold it unless the marker ends the text. + public static func endOfDataMarker(in text: NSString, at lineStart: Int, length: Int) -> Int? { + guard lineStart + 1 < length, + text.character(at: lineStart) == backslash, + text.character(at: lineStart + 1) == period + else { + return nil + } + let after = lineStart + 2 + guard after < length else { return after } + switch text.character(at: after) { + case SqlLexer.newline: + return after + case SqlLexer.carriageReturn where after + 1 == length || text.character(at: after + 1) == SqlLexer.newline: + return after + default: + return nil + } + } + + /// How many units ``endOfDataMarker(in:at:length:)`` reads from a line's start: `\`, `.`, a carriage return and + /// the line feed after it. + public static let endOfDataMarkerLookahead = 4 + + /// The `\restrict` or `\unrestrict` line whose backslash is at `offset`, its line break excluded, when only blanks + /// stand before it on its line. psql reads both names case for case. + public static func restrictLine(at offset: Int, in text: NSString, length: Int) -> NSRange? { + var before = offset - 1 + while before >= 0, isLineBlank(text.character(at: before)) { + before -= 1 + } + guard before < 0 || text.character(at: before) == SqlLexer.newline, + let nameEnd = restrictNameEnd(in: text, at: offset, length: length) + else { + return nil + } + let end = lineEnd(in: text, from: nameEnd, length: length) ?? length + return NSRange(location: offset, length: end - offset) + } + + /// Whether a `\restrict` or `\unrestrict` line starts at `offset`, for a reader that already knows only blanks + /// stand before it on its line. `text` has to hold the unit after the name unless the name ends the text. + public static func startsRestrictLine(in text: NSString, at offset: Int, length: Int) -> Bool { + restrictNameEnd(in: text, at: offset, length: length) != nil + } + + /// How many units a reader needs from a backslash on to answer ``startsRestrictLine(in:at:length:)``. + public static let restrictLineLookahead = 1 + 10 + 1 + + private static func restrictNameEnd(in text: NSString, at offset: Int, length: Int) -> Int? { + guard offset < length, text.character(at: offset) == backslash, + let nameEnd = [restrict, unrestrict].lazy.compactMap({ name in + matches(name, in: text, at: offset + 1, length: length) ? offset + 1 + name.count : nil + }).first + else { + return nil + } + if nameEnd < length { + let next = text.character(at: nameEnd) + guard isLineBlank(next) || next == SqlLexer.newline || next == SqlLexer.carriageReturn else { return nil } + } + return nameEnd + } + + private static func lineEnd(in text: NSString, from offset: Int, length: Int) -> Int? { + guard offset < length else { return nil } + let found = text.range( + of: "\n", + options: .literal, + range: NSRange(location: offset, length: length - offset) + ) + return found.location == NSNotFound ? nil : found.location + } + + private static func matches(_ name: [UInt16], in text: NSString, at offset: Int, length: Int) -> Bool { + guard offset + name.count <= length else { return false } + for (index, unit) in name.enumerated() where text.character(at: offset + index) != unit { + return false + } + return true + } + + private static func isLineBlank(_ unit: UInt16) -> Bool { + unit == SqlLexer.space || unit == SqlLexer.tab + } +} + +/// psql's test for a statement that takes rows from the script: its first word is `COPY`, and among its first eight +/// words outside parentheses a `FROM` comes right before `STDIN` or `STDOUT`, which the server's grammar does not tell +/// apart. A quoted name is no word, and a dotted name is one word per part. +struct SQLCopyFromStandardInput { + private static let wordsRead = 8 + + private var words = 0 + private var parenDepth = 0 + private var followsFrom = false + private var settled = false + private(set) var readsRows = false + + var needsWords: Bool { + !settled + } + + mutating func observeWord(_ word: String) { + guard !settled, parenDepth == 0 else { return } + if words == 0, word != "COPY" { + settled = true + return + } + if followsFrom, word == "STDIN" || word == "STDOUT" { + readsRows = true + settled = true + return + } + followsFrom = word == "FROM" + words += 1 + if words == Self.wordsRead { + settled = true + } + } + + mutating func observeSymbol(_ symbol: UInt16) { + guard !settled else { return } + switch symbol { + case SqlLexer.openParen: + parenDepth += 1 + case SqlLexer.closeParen: + parenDepth = max(0, parenDepth - 1) + default: + break + } + if words == 0 { + settled = true + } + } + + mutating func observeOpaqueToken() { + if words == 0 { + settled = true + } + } +} + +/// A statement tracker that also tells a `COPY ... FROM STDIN`, for a grammar with ``SQLLexicalGrammar/psqlScripts``. +struct SQLPsqlStatementTracker: SQLStatementBoundaryTracking { + private var base: any SQLStatementBoundaryTracking + private var copy = SQLCopyFromStandardInput() + + init(base: any SQLStatementBoundaryTracking) { + self.base = base + } + + var needsWords: Bool { + base.needsWords || copy.needsWords + } + + var terminator: SQLStatementTerminator { + base.terminator + } + + var acceptsBindParameters: Bool { + base.acceptsBindParameters + } + + var holdsOpenBlock: Bool { + base.holdsOpenBlock + } + + var awaitsContinuation: Bool { + base.awaitsContinuation + } + + var copiesFromStandardInput: Bool { + copy.readsRows + } + + mutating func observeWord(_ word: String) { + copy.observeWord(word) + if base.needsWords { + base.observeWord(word) + } + } + + mutating func observeSymbol(_ symbol: UInt16) { + copy.observeSymbol(symbol) + base.observeSymbol(symbol) + } + + mutating func observeOpaqueToken() { + copy.observeOpaqueToken() + base.observeOpaqueToken() + } + + mutating func observeQuotedToken(opener: UInt16) { + copy.observeOpaqueToken() + base.observeQuotedToken(opener: opener) + } + + mutating func observeGap() { + base.observeGap() + } + + mutating func observeSemicolon() -> Bool { + base.observeSemicolon() + } + + mutating func continuesStatement(with word: String?) -> Bool { + base.continuesStatement(with: word) + } + + mutating func reset() { + base.reset() + copy = SQLCopyFromStandardInput() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLReadOnlyFunctionCalls.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLReadOnlyFunctionCalls.swift new file mode 100644 index 0000000000..4f8163d628 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLReadOnlyFunctionCalls.swift @@ -0,0 +1,457 @@ +import Foundation + +// The one place that says which calls Read-Only runs on an engine whose functions can write, when the driver cannot hold +// the statement to the engine's read-only mode. A built-in here reads only: nothing that writes, locks, sleeps, waits or +// reaches a file (GET_LOCK, SLEEP, LOAD_FILE, NEXTVAL, SETVAL, a DBMS_ package) is listed. A qualified or quoted name, a +// name the scanner cannot read, and anything else may be a function the user wrote. A call written without parentheses, +// which Oracle and Dameng accept for a function with no arguments, cannot be told from a column and is not seen. +public enum SQLReadOnlyFunctionCalls { + // An autonomous-transaction function on Oracle and Dameng, a stored function on MySQL, MariaDB and OceanBase, PL/pgSQL + // on the PostgreSQL family, a `fn::` body on SurrealDB. TiDB, StarRocks and Databend have no stored function or UDF that writes; SQL Server, + // Snowflake, BigQuery, Trino and Redshift refuse a write from a function a query calls; ClickHouse, the SQLite engines + // and Cassandra have none. + public static let queryFunctionsCanWrite: Set = [ + "Oracle", "Dameng", + "MySQL", "MariaDB", "OceanBase", + "PostgreSQL", "CockroachDB", "PGlite", "KingbaseES", + "SurrealDB", + ] + + static let commonBuiltins: Set = [ + "COUNT", "SUM", "AVG", "MIN", "MAX", "STDDEV", "STDDEV_POP", "STDDEV_SAMP", "VARIANCE", "VAR_POP", "VAR_SAMP", + "CORR", "COVAR_POP", "COVAR_SAMP", "GROUPING", + "ROW_NUMBER", "RANK", "DENSE_RANK", "PERCENT_RANK", "CUME_DIST", "NTILE", "LAG", "LEAD", "FIRST_VALUE", + "LAST_VALUE", "NTH_VALUE", + "COALESCE", "NULLIF", "GREATEST", "LEAST", "CAST", "EXTRACT", + "LOWER", "UPPER", "LENGTH", "TRIM", "LTRIM", "RTRIM", "SUBSTR", "SUBSTRING", "REPLACE", "CONCAT", "LPAD", "RPAD", + "REVERSE", "ASCII", "REGEXP_REPLACE", "REGEXP_SUBSTR", "REGEXP_INSTR", "REGEXP_LIKE", "POSITION", "TRANSLATE", + "ABS", "CEIL", "CEILING", "FLOOR", "ROUND", "MOD", "POWER", "SQRT", "EXP", "LN", "LOG", "SIGN", "SIN", "COS", + "TAN", "ASIN", "ACOS", "ATAN", "ATAN2", + "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", "LOCALTIMESTAMP", + "JSON_VALUE", "JSON_QUERY", "JSON_OBJECT", "JSON_ARRAY", "JSON_TABLE", + ] + + // Oracle and Dameng. + static let oracleBuiltins: Set = [ + "NVL", "NVL2", "DECODE", "LNNVL", "NANVL", "LISTAGG", "MEDIAN", "RATIO_TO_REPORT", + "TO_CHAR", "TO_DATE", "TO_NUMBER", "TO_TIMESTAMP", "TO_TIMESTAMP_TZ", "TO_DSINTERVAL", "TO_YMINTERVAL", + "NUMTODSINTERVAL", "NUMTOYMINTERVAL", "TO_NCHAR", "HEXTORAW", "RAWTOHEX", "ROWIDTOCHAR", "CONVERT", + "INSTR", "INSTRB", "LENGTHB", "SUBSTRB", "INITCAP", "CHR", "SOUNDEX", "REGEXP_COUNT", + "TRUNC", "BITAND", "WIDTH_BUCKET", "REMAINDER", + "ADD_MONTHS", "MONTHS_BETWEEN", "LAST_DAY", "NEXT_DAY", "NEW_TIME", "FROM_TZ", "TZ_OFFSET", "SYS_EXTRACT_UTC", + "SYS_GUID", "SYS_CONTEXT", "USERENV", "ORA_HASH", "STANDARD_HASH", "XMLTABLE", + ] + + // MySQL, MariaDB and the servers that answer as MySQL. + static let mysqlBuiltins: Set = [ + "IF", "IFNULL", "ISNULL", "GROUP_CONCAT", "JSON_ARRAYAGG", "JSON_OBJECTAGG", "BIT_AND", "BIT_OR", "BIT_XOR", + "ANY_VALUE", "CONVERT", + "CONCAT_WS", "CHAR_LENGTH", "CHARACTER_LENGTH", "OCTET_LENGTH", "BIT_LENGTH", "LCASE", "UCASE", "LEFT", "RIGHT", + "MID", "LOCATE", "INSTR", "FIND_IN_SET", "FIELD", "ELT", "STRCMP", "SUBSTRING_INDEX", "REPEAT", "SPACE", "CHAR", + "FORMAT", "QUOTE", "SOUNDEX", "HEX", "UNHEX", "BIN", "OCT", "CONV", + "TRUNCATE", "POW", "LOG2", "LOG10", "COT", "DEGREES", "RADIANS", "PI", "RAND", "CRC32", + "NOW", "SYSDATE", "CURDATE", "CURTIME", "LOCALTIME", "UTC_DATE", "UTC_TIME", "UTC_TIMESTAMP", "DATE", "TIME", + "TIMESTAMP", "YEAR", "QUARTER", "MONTH", "WEEK", "WEEKDAY", "WEEKOFYEAR", "YEARWEEK", "DAY", "DAYOFMONTH", + "DAYOFWEEK", "DAYOFYEAR", "DAYNAME", "MONTHNAME", "HOUR", "MINUTE", "SECOND", "MICROSECOND", "DATE_FORMAT", + "TIME_FORMAT", "STR_TO_DATE", "DATE_ADD", "DATE_SUB", "ADDDATE", "SUBDATE", "ADDTIME", "SUBTIME", "DATEDIFF", + "TIMEDIFF", "TIMESTAMPDIFF", "TIMESTAMPADD", "UNIX_TIMESTAMP", "FROM_UNIXTIME", "FROM_DAYS", "TO_DAYS", + "TO_SECONDS", "SEC_TO_TIME", "TIME_TO_SEC", "MAKEDATE", "MAKETIME", "PERIOD_ADD", "PERIOD_DIFF", "LAST_DAY", + "CONVERT_TZ", + "JSON_EXTRACT", "JSON_UNQUOTE", "JSON_CONTAINS", "JSON_CONTAINS_PATH", "JSON_KEYS", "JSON_LENGTH", "JSON_TYPE", + "JSON_VALID", "JSON_SEARCH", "JSON_DEPTH", + "MD5", "SHA", "SHA1", "SHA2", "UUID", "UUID_SHORT", + "DATABASE", "SCHEMA", "USER", "CURRENT_USER", "SESSION_USER", "SYSTEM_USER", "VERSION", "CONNECTION_ID", + ] + + // An older PostgreSQL-family driver without the mode. + static let postgresBuiltins: Set = [ + "NOW", "TO_CHAR", "TO_DATE", "TO_NUMBER", "TO_TIMESTAMP", "DATE_TRUNC", "DATE_PART", "AGE", "TRUNC", "LEFT", + "RIGHT", "CHAR_LENGTH", "CHARACTER_LENGTH", "OCTET_LENGTH", "SPLIT_PART", "STRING_AGG", "ARRAY_AGG", + "ARRAY_LENGTH", "INITCAP", "CHR", "MD5", "FORMAT", "CONCAT_WS", "TO_JSON", "TO_JSONB", "JSON_AGG", "JSONB_AGG", + "JSON_BUILD_OBJECT", "JSONB_BUILD_OBJECT", "JSON_BUILD_ARRAY", "JSONB_BUILD_ARRAY", "GENERATE_SERIES", + "BOOL_AND", "BOOL_OR", "EVERY", "POW", "LOG10", "PI", "RANDOM", + ] + + // The engines here that read MATCH_RECOGNIZE; DM8 answers it with a syntax error. Measured on Oracle 23ai: in its + // MEASURES and DEFINE these call the clause's own navigation, never a user function of that name, and + // `PATTERN (...)` is a row pattern. Elsewhere, a query nested in the clause included, each is a user's name. + static let rowPatternEngines: Set = ["Oracle"] + static let rowPatternFunctions: Set = ["PREV", "NEXT", "FIRST", "LAST", "CLASSIFIER", "MATCH_NUMBER"] + + private static let oracleFamily: Set = ["Oracle", "Dameng"] + private static let mysqlFamily: Set = ["MySQL", "MariaDB", "OceanBase", "TiDB", "StarRocks", "Databend"] + private static let postgresFamily: Set = ["PostgreSQL", "CockroachDB", "PGlite", "KingbaseES", "Redshift"] + + private static let oracleCalls = commonBuiltins.union(oracleBuiltins) + private static let mysqlCalls = commonBuiltins.union(mysqlBuiltins) + private static let postgresCalls = commonBuiltins.union(postgresBuiltins) + private static let anyCalls = commonBuiltins.union(oracleBuiltins).union(mysqlBuiltins).union(postgresBuiltins) + + public static func builtins(forDatabaseTypeId typeId: String) -> Set { + if oracleFamily.contains(typeId) { return oracleCalls } + if mysqlFamily.contains(typeId) { return mysqlCalls } + if postgresFamily.contains(typeId) { return postgresCalls } + return anyCalls + } + + // Reserved in Oracle, MySQL and PostgreSQL alike, so no function a query calls can carry the bare name. + static let reservedWords: Set = [ + "SELECT", "FROM", "WHERE", "AND", "OR", "NOT", "IN", "EXISTS", "AS", "ON", "WITH", "GROUP", "ORDER", "HAVING", + "UNION", "DISTINCT", "ELSE", "THEN", "ALL", "VALUES", "BETWEEN", "TABLE", + ] + + // A word another engine lets a user function carry stands before a parenthesis only in this syntax. + private enum Syntax { + // Right after a call's closing parenthesis: `SUM(x) OVER (`. + case afterCall + // After a call, opening with this word: `COUNT(*) FILTER (WHERE`, `MAX(x) KEEP (DENSE_RANK`. + case afterCallOpening(String) + // After one of these words: `GROUP BY (`, `GROUPING SETS (`, `USE INDEX (`. + case afterWord(Set) + // After a name, a literal or a closing parenthesis: `t PARTITION (`, `x LIKE (`, `) EXCEPT (`. + case afterOperand + // Opening with one of these words: `ARRAY(SELECT`, `LATERAL (SELECT`. + case opening(Set) + // After a comparison: `= ANY (`. + case afterComparison + // Its parenthesized part followed by one of these words: `MATCH (a) AGAINST`, `INTERVAL (n) DAY`. + case followedBy(Set) + } + + private static let intervalUnits: Set = [ + "MICROSECOND", "SECOND", "MINUTE", "HOUR", "DAY", "WEEK", "MONTH", "QUARTER", "YEAR", + ] + + private static let syntax: [String: [Syntax]] = [ + "BY": [.afterWord(["GROUP", "ORDER", "PARTITION", "CONNECT"])], + "OVER": [.afterCall], + "FILTER": [.afterCallOpening("WHERE")], + "KEEP": [.afterCallOpening("DENSE_RANK")], + "AGAINST": [.afterCall], + "MATCH": [.followedBy(["AGAINST"])], + "INTERVAL": [.followedBy(intervalUnits)], + "CASE": [.followedBy(["WHEN"])], + "WHEN": [.afterOperand, .afterWord(["CASE"])], + "PARTITION": [.afterOperand], + "PIVOT": [.afterOperand], + "UNPIVOT": [.afterOperand], + "CUBE": [.afterWord(["BY"])], + "ROLLUP": [.afterWord(["BY"])], + "SETS": [.afterWord(["GROUPING"])], + "ARRAY": [.opening(["SELECT", "WITH"])], + "LATERAL": [.opening(["SELECT", "WITH"])], + "ANY": [.afterComparison], + "SOME": [.afterComparison], + "JOIN": [.afterOperand, .afterWord(["INNER", "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "NATURAL", "FOR"])], + "USING": [.afterOperand], + "INDEX": [.afterWord(["USE", "FORCE", "IGNORE"])], + "KEY": [.afterWord(["USE", "FORCE", "IGNORE"])], + "LIKE": [.afterOperand], + "IS": [.afterOperand], + "INTERSECT": [.afterOperand], + "EXCEPT": [.afterOperand], + "MINUS": [.afterOperand], + "LIMIT": [.afterOperand], + "OFFSET": [.afterOperand], + // Word operators: `10 DIV (2)`, `a XOR (b)`. + "DIV": [.afterOperand], + "XOR": [.afterOperand], + "REGEXP": [.afterOperand], + "RLIKE": [.afterOperand], + "ILIKE": [.afterOperand], + "ESCAPE": [.afterOperand], + ] + + // After one of these a call can start, so a word that follows it is not standing after an operand. Word operators + // are among them: `10 DIV pivot(1)` calls `pivot`. + private static let introducers: Set = reservedWords.union(syntax.keys).union([ + "SET", "INTO", "RETURN", "RETURNING", "PRIOR", "CALL", "DO", "EXECUTE", "FOR", "CONNECT", + "MOD", "COLLATE", "BINARY", "UNIQUE", "SOUNDS", "MEMBER", "OF", "SIMILAR", "TO", "ASC", "DESC", "NULLS", + ]) + + private static let openParenthesis = UInt16(UnicodeScalar("(").value) + private static let closeParenthesis = UInt16(UnicodeScalar(")").value) + private static let period = UInt16(UnicodeScalar(".").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let quoteOpeners: Set = [0x22, 0x60, 0x5B] + private static let comparisons: Set = [0x3D, 0x3C, 0x3E, 0x21] + + // Under every way the engine could lex the text, as the classifier reads it. A type with a length, such as + // `DECIMAL(10, 2)` in a cast, is the engine's own type, not a call: `typeNames` are the engine's types. + public static func callsUnlistedFunction( + _ sql: String, + databaseTypeId: String, + readings: SQLLexicalReadings, + typeNames: Set + ) -> Bool { + let typeNames = Set(typeNames.map { $0.uppercased() }) + let known = builtins(forDatabaseTypeId: databaseTypeId) + let readsRowPatterns = rowPatternEngines.contains(databaseTypeId) + let tiering = SQLStatementTiering(databaseTypeId: databaseTypeId, readings: readings, request: nil) + return readings.all.contains { grammar in + SQLStatementClassifier.statements(of: sql, grammar: grammar).contains { statement in + // SurrealQL's built-ins are namespaced, `time::now()`; only a stored body can write. + if databaseTypeId == SQLStatementTiering.surrealDB { + return tiering.callsSurrealStoredCode(StatementCode(statement, grammar: grammar)) + } + return statementCallsUnlistedFunction( + statement, + grammar: grammar, + builtins: known, + typeNames: typeNames, + readsRowPatterns: readsRowPatterns + ) + } + } + } + + private static func statementCallsUnlistedFunction( + _ statement: String, + grammar: SQLLexicalGrammar, + builtins: Set, + typeNames: Set, + readsRowPatterns: Bool + ) -> Bool { + // A plain EXPLAIN plans its statement without running it. + if let explain = SQLExplainStatement.read(statement, grammar: grammar) { + switch explain.reading { + case .describesObject, .explains(_, false): + return false + case .explains(_, true), .unreadable: + break + } + } + let source = statement as NSString + let code = SQLCodeProjection.code(of: statement, grammar: grammar) as NSString + let calls = SQLFunctionCallScanner.calls(in: source, code: code) + var quoted: [Range]? + func site(of call: SQLFunctionCallScanner.Call, nameStart: Int) -> CallSite { + let spans = quoted ?? quotedSpans(in: source, grammar: grammar) + quoted = spans + return CallSite(code: code, quotedSpans: spans, nameStart: nameStart, parenthesis: call.parenthesis) + } + let rowPatterns = readsRowPatterns + ? RowPatternClauses(source: source, code: code, grammar: grammar, calls: calls) { + site(of: $0, nameStart: $1).matches(.afterOperand) + } + : nil + return calls.contains { call in + guard case .named(let name, let start) = call.callee, + !quoteOpeners.contains(source.character(at: start)) + else { return true } + let word = name.uppercased() + switch qualifier(before: start, in: code) { + case .member: + return true + case .cast: + return !typeNames.contains(word) + case .unqualified: + break + } + if builtins.contains(word) || typeNames.contains(word) || reservedWords.contains(word) { return false } + if rowPatterns?.ownsCall(named: word, at: start) == true { return false } + guard let rules = syntax[word] else { return true } + let callSite = site(of: call, nameStart: start) + return !rules.contains { callSite.matches($0) } + } + } + + // What each MATCH_RECOGNIZE clause holds: its row pattern, its MEASURES and DEFINE, where navigation is the clause's + // own, and the queries nested in it, which that navigation does not reach. A clause `SQLRowPatternClause` cannot read + // owns nothing. + private struct RowPatternClauses { + private enum Kind { + case clause + case navigation + case pattern + case query + } + + private var groups: [(range: Range, kind: Kind)] = [] + private var owned: Set = [] + + init( + source: NSString, + code: NSString, + grammar: SQLLexicalGrammar, + calls: [SQLFunctionCallScanner.Call], + opensClause: (SQLFunctionCallScanner.Call, Int) -> Bool + ) { + let openers = calls.filter { call in + guard case .named(let name, let start) = call.callee else { return false } + return name.uppercased() == "MATCH_RECOGNIZE" && opensClause(call, start) + } + guard !openers.isEmpty else { return } + let tokens = TSQLTokens.read(source, code: code, grammar: grammar) + for call in openers { + guard case .named(_, let start) = call.callee, + let open = tokens.firstIndex(where: { $0.kind == .open && $0.location == call.parenthesis }), + let close = Self.close(of: open, in: tokens), + let clause = SQLRowPatternClause.read(tokens, body: (open + 1).. Bool { + if owned.contains(start) { return true } + switch kind(at: start) { + case .pattern: + return true + case .navigation: + return SQLReadOnlyFunctionCalls.rowPatternFunctions.contains(word) + case .clause, .query, nil: + return false + } + } + + private func kind(at position: Int) -> Kind? { + groups.filter { $0.range.contains(position) }.max { $0.range.lowerBound < $1.range.lowerBound }?.kind + } + + private static func close(of open: Int, in tokens: [TSQLTokens.Token]) -> Int? { + ((open + 1).. [Range] { + var spans: [Range] = [] + var index = 0 + while index < source.length { + guard let span = SQLNonCodeSpan.span(at: index, in: source, grammar: grammar) else { + index += 1 + continue + } + if span.kind == .quoted { spans.append(span.start.. Qualifier { + var cursor = start - 1 + while cursor >= 0, SQLFunctionCallScanner.isBlank(code.character(at: cursor)) { + cursor -= 1 + } + guard cursor >= 0 else { return .unqualified } + let unit = code.character(at: cursor) + if unit == period { return .member } + guard unit == colon else { return .unqualified } + return cursor > 0 && code.character(at: cursor - 1) == colon ? .cast : .member + } + + private enum Neighbor: Equatable { + case start + case word(String) + case symbol(UInt16) + // A literal or a quoted name, which the projection blanked. + case quoted + } + + private struct CallSite { + let code: NSString + let quotedSpans: [Range] + let nameStart: Int + let parenthesis: Int + + func matches(_ rule: Syntax) -> Bool { + switch rule { + case .afterCall: + return before == .symbol(SQLReadOnlyFunctionCalls.closeParenthesis) + case .afterCallOpening(let word): + return before == .symbol(SQLReadOnlyFunctionCalls.closeParenthesis) && firstWord(after: parenthesis) == word + case .afterWord(let words): + guard case .word(let word) = before else { return false } + return words.contains(word) + case .afterOperand: + switch before { + case .quoted: + return true + case .symbol(let unit): + return unit == SQLReadOnlyFunctionCalls.closeParenthesis + case .word(let word): + return !SQLReadOnlyFunctionCalls.introducers.contains(word) + case .start: + return false + } + case .opening(let words): + return firstWord(after: parenthesis).map { words.contains($0) } ?? false + case .afterComparison: + guard case .symbol(let unit) = before else { return false } + return SQLReadOnlyFunctionCalls.comparisons.contains(unit) + case .followedBy(let words): + return wordAfterGroup().map { words.contains($0) } ?? false + } + } + + private var before: Neighbor { + var cursor = nameStart - 1 + while cursor >= 0, SQLFunctionCallScanner.isBlank(code.character(at: cursor)) { + if quotedSpans.contains(where: { $0.contains(cursor) }) { return .quoted } + cursor -= 1 + } + guard cursor >= 0 else { return .start } + let unit = code.character(at: cursor) + guard SQLFunctionCallScanner.isIdentifierUnit(unit) else { return .symbol(unit) } + var start = cursor + while start > 0, SQLFunctionCallScanner.isIdentifierUnit(code.character(at: start - 1)) { + start -= 1 + } + return .word(code.substring(with: NSRange(location: start, length: cursor - start + 1)).uppercased()) + } + + private func firstWord(after index: Int) -> String? { + var cursor = index + 1 + while cursor < code.length, SQLFunctionCallScanner.isBlank(code.character(at: cursor)) { + cursor += 1 + } + var end = cursor + while end < code.length, SQLFunctionCallScanner.isIdentifierUnit(code.character(at: end)) { + end += 1 + } + guard end > cursor else { return nil } + return code.substring(with: NSRange(location: cursor, length: end - cursor)).uppercased() + } + + private func wordAfterGroup() -> String? { + var depth = 0 + for index in parenthesis.. = ["THEN", "ELSE", "DO", "LOOP", "REPEAT"] + /// The words that open a routine body written as one statement, as in MySQL's + /// `CREATE PROCEDURE p() SELECT ... ORDER BY begin DESC`, where `begin` is a column. + private static let singleStatementBodyOpeners: Set = [ + "SELECT", "INSERT", "UPDATE", "DELETE", "REPLACE", "MERGE", + ] + + /// A trigger's event follows these, as in `BEFORE INSERT`, `INSTEAD OF DELETE` and `INSERT OR UPDATE`, so the + /// word after one is part of the header rather than the body. + private static let triggerEventLeaders: Set = ["BEFORE", "AFTER", "OF", "OR", "FOR"] + + private static let controlStatementOpeners: Set = ["IF", "WHILE", "LOOP", "REPEAT"] + + private static let handlerKinds: Set = ["CONTINUE", "EXIT", "UNDO"] + + /// The words before a `SET` that is part of MySQL's return type: the `SET(...)` type, `CHARACTER SET`, `CHAR SET`. + private static let wordsBeforeTypeSet: Set = ["RETURNS", "CHARACTER", "CHAR"] + private let grammar: SQLLexicalGrammar + private let readsBlocksEverywhere: Bool + private let readsCompoundBodies: Bool + private var openControlStatements: [ControlStatement] = [] + // An `IF` or `WHILE` where a statement can start, until the word that ends its condition at the same depth: + // `IF(a, 1, 2)` is a function wherever a value can stand. + private var conditionAwaitingBody: ControlStatement? + // The `CASE`s open here that are expressions, by the depth inside each: their `THEN` and `ELSE` start no statement. + private var caseExpressionDepths: [Int] = [] + // Inside PostgreSQL's `BEGIN ATOMIC` body, how far the statement being read is into `CREATE FUNCTION`: only such a + // statement has a body of its own, and elsewhere `begin atomic` is a column and its alias. + private var nestedDefinition = NestedDefinition.none + // A `REPEAT` block just opened, which a `(` right after it turns into the `REPEAT()` function. + private var repeatMayBeFunction = false + // A MySQL event's body follows its `DO`, and its schedule before that starts nothing. + private var awaitsEventBody = false + // T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE` + // or `ELSE`, or after a block the statement already opened. Anywhere else `begin` is in a name's place. + private var followsControlFlow = false + // The statement's own `IF`s still waiting for an `ELSE`, which may follow the `;` that ends the `IF`'s body. + private var conditionalsAwaitingElse = 0 + private var endedBeforeElse = false + private var pendingBeginMayOpen = false private var header = Header.opener private var definesRoutine = false private var openedBlockBody = false + private var routineBody = RoutineBody.unknown + private var previousWord: String? + // A trigger's events stand before its table, as in SQLite's `CREATE TRIGGER tr DELETE ON t`. + private var readsTriggerEvents = false + // A trigger's event list goes on after a word in `triggerEventLeaders`, or after a `,` that follows an event, as + // in `FOR INSERT, UPDATE`. + private var expectsTriggerEvent = false + private var followsTriggerEvent = false private var depth = 0 /// The open `CASE`s that have held no `;`, which only a `CASE` statement can. The next `END` closes one of them, /// whatever word follows it. private var openCaseExpressions = 0 private var parenDepth = 0 + private var ruleAction = RuleAction.none + private var ruleActionHeldSemicolon = false private var pendingBegin = false private var pendingEnd = false private var followsName = false @@ -45,17 +148,21 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private var atStatementPosition = false private var labelMayFollow = false private var inDeclaration = false - private var inHandlerDeclaration = false + private var handler = HandlerPart.none private var merge: SQLMergeStatementTracker? public init(grammar: SQLLexicalGrammar) { self.grammar = grammar + self.readsBlocksEverywhere = grammar.contains(.unterminatedStatements) + self.readsCompoundBodies = grammar.contains(.compoundStatementBodies) self.merge = grammar.contains(.terminatedMergeStatements) ? SQLMergeStatementTracker() : nil } public var needsWords: Bool { + // Inside parentheses no word opens or closes a block, and a `MERGE` there needs no `;`. + if readsBlocksEverywhere { return parenDepth == 0 } if case .read = header { - return definesRoutine || merge?.needsWords == true + return definesRoutine || ruleAction == .header || ruleAction == .afterDo || merge?.needsWords == true } return true } @@ -70,11 +177,17 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // An `END` still waiting for the word after it closes a block when the text ends. public var holdsOpenBlock: Bool { - depth > (pendingEnd ? 1 : 0) + depth > (pendingEnd ? 1 : 0) || ruleActionHeldSemicolon } public mutating func observeWord(_ word: String) { merge?.observeWord(word) + repeatMayBeFunction = false + let previous = previousWord + previousWord = word + let isTriggerEvent = expectsTriggerEvent && Self.singleStatementBodyOpeners.contains(word) + expectsTriggerEvent = Self.triggerEventLeaders.contains(word) + followsTriggerEvent = isTriggerEvent let isMember = followsDot followsName = true followsDot = false @@ -85,15 +198,60 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } let startsStatement = startsBodyStatement startsBodyStatement = false - let statementPosition = atStatementPosition - observeStatementPosition(after: word, wasAtStatementPosition: statementPosition) - guard !isMember, readHeader(word), definesRoutine, parenDepth == 0 else { return } + let statementPosition = atStatementPosition || readHandlerWord(word) + observeStatementPosition(after: word, previous: previous, wasAtStatementPosition: statementPosition) + guard !isMember else { return } + var isFirstWord = false + if case .opener = header { + isFirstWord = true + } + let pastHeader = readHeader(word) + if pastHeader, parenDepth == 0 { + observeRuleWord(word) + } + guard parenDepth == 0, readsBlocksEverywhere || (pastHeader && definesRoutine) else { return } + let startsBody = depth == 0 && routineBody == .unknown && !awaitsEventBody + if !readsBlocksEverywhere, depth == 0 { + observeBodyStart(word, after: previous, isTriggerEvent: isTriggerEvent) + } + let atStatementStart = depth == 0 ? startsBody : statementPosition + if readsCompoundBodies { + observeCompoundStatement(word, atStatementStart: atStatementStart) + } + if depth > 0, grammar.contains(.atomicRoutineBodies) { + observeNestedDefinition(word, startsStatement: startsStatement) + } switch word { - case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition): + case "BEGIN" where readsBlocksEverywhere: pendingBegin = true + pendingBeginMayOpen = opensNestedBlock( + atStatementPosition: statementPosition, + isFirstWord: isFirstWord, + after: previous + ) + case "BEGIN": + if opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord, after: previous) { + pendingBegin = true + pendingBeginMayOpen = true + } + case "IF" where readsBlocksEverywhere: + followsControlFlow = true + if depth == 0 { + conditionalsAwaitingElse += 1 + } + case "WHILE" where readsBlocksEverywhere: + followsControlFlow = true + case "ELSE" where readsBlocksEverywhere && openCaseExpressions == 0: + followsControlFlow = true + if depth == 0 { + conditionalsAwaitingElse = max(0, conditionalsAwaitingElse - 1) + } case "CASE" where !grammar.hasSingleBodyForm: depth += 1 openCaseExpressions += 1 + if !atStatementStart { + caseExpressionDepths.append(depth) + } case "END" where !grammar.hasSingleBodyForm || startsStatement: pendingEnd = true default: @@ -103,6 +261,20 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSymbol(_ symbol: UInt16) { merge?.observeSymbol(symbol) + if repeatMayBeFunction { + repeatMayBeFunction = false + if symbol == SqlLexer.openParen { + openControlStatements.removeLast() + depth -= 1 + } + } + previousWord = nil + readHandlerSymbol(symbol) + expectsTriggerEvent = symbol == Self.comma && followsTriggerEvent + followsTriggerEvent = false + if ruleAction == .afterDo, symbol != SqlLexer.openParen { + ruleAction = .none + } let opensBlock = symbol == SqlLexer.openParen && !grammar.contains(.atomicRoutineBodies) settlePendingBeforeNonWord(opensBlock: opensBlock) followsDot = followsName && symbol == Self.dot @@ -112,9 +284,16 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { switch symbol { case SqlLexer.openParen: parenDepth += 1 + if ruleAction == .afterDo { + ruleAction = .list(depth: parenDepth) + } endName() case SqlLexer.closeParen: parenDepth = max(0, parenDepth - 1) + if case .list(let listDepth) = ruleAction, parenDepth < listDepth { + ruleAction = .none + ruleActionHeldSemicolon = false + } default: break } @@ -122,6 +301,21 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeOpaqueToken() { merge?.observeOpaqueToken() + repeatMayBeFunction = false + switch handler { + case .condition, .sqlState: + handler = .afterCondition + case .afterCondition: + handler = .statement + default: + break + } + previousWord = nil + expectsTriggerEvent = false + followsTriggerEvent = false + if ruleAction == .afterDo { + ruleAction = .none + } settlePendingBeforeNonWord(opensBlock: !grammar.contains(.atomicRoutineBodies)) followsName = true followsDot = false @@ -136,24 +330,60 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSemicolon() -> Bool { pendingBegin = false + repeatMayBeFunction = false + conditionAwaitingBody = nil + // A `CASE` that holds a `;` is a statement. + caseExpressionDepths.removeAll() + nestedDefinition = .none + previousWord = nil + expectsTriggerEvent = false + followsTriggerEvent = false if pendingEnd { pendingEnd = false closeBlock() } + if readsBlocksEverywhere { + // A T-SQL `CASE` is only an expression, which no `;` can stand inside, so one still open was never closed. + depth = max(0, depth - openCaseExpressions) + } openCaseExpressions = 0 - parenDepth = 0 followsName = false followsDot = false - let endsStatement = depth == 0 - startsBodyStatement = !endsStatement - atStatementPosition = !endsStatement + let heldByRuleAction: Bool + if case .list(let listDepth) = ruleAction { + heldByRuleAction = depth == 0 && parenDepth >= listDepth + } else { + heldByRuleAction = false + } + if heldByRuleAction { + ruleActionHeldSemicolon = true + } else { + parenDepth = 0 + } + let endsStatement = depth == 0 && !heldByRuleAction + startsBodyStatement = depth > 0 + atStatementPosition = depth > 0 labelMayFollow = false inDeclaration = false - inHandlerDeclaration = false + handler = .none merge?.observeSemicolon(endsStatement: endsStatement) + endedBeforeElse = endsStatement && readsBlocksEverywhere && conditionalsAwaitingElse > 0 return endsStatement } + public var awaitsContinuation: Bool { + endedBeforeElse + } + + // T-SQL writes `IF c SELECT 1; ELSE SELECT 2;`, so the `;` ended the `IF`'s body rather than the `IF`. + public mutating func continuesStatement(with word: String?) -> Bool { + guard endedBeforeElse else { return false } + endedBeforeElse = false + guard word == "ELSE" else { return false } + merge?.observeSemicolon(endsStatement: false) + return true + } + public mutating func reset() { self = SQLRoutineBodyTracker(grammar: grammar) } @@ -175,8 +405,13 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } return false case .kind(let accountWords): + if word == "RULE", grammar.contains(.atomicRoutineBodies) { + ruleAction = .header + } if SqlBlockStructure.isRoutineKind(word, grammar: grammar) { definesRoutine = true + readsTriggerEvents = word == "TRIGGER" + awaitsEventBody = readsCompoundBodies && word == "EVENT" header = .name } else if word == "DEFINER" { header = .kind(accountWords: 2) @@ -195,21 +430,170 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } } - // A label, `lbl:`, is a word at a statement's start followed by `:`. A handler's body follows its condition. - private mutating func observeStatementPosition(after word: String, wasAtStatementPosition: Bool) { - atStatementPosition = depth > 0 && Self.statementIntroducers.contains(word) + // A label, `lbl:`, is a word at a statement's start followed by `:`. On MySQL a `DO` where a statement starts is + // the `DO` statement, an expression after it; any other ends a `WHILE` or `FOR` header. + private mutating func observeStatementPosition(after word: String, previous: String?, wasAtStatementPosition: Bool) { + atStatementPosition = depth > 0 && caseExpressionDepths.last != depth && Self.statementIntroducers.contains(word) + && !(readsCompoundBodies && word == "DO" && wasAtStatementPosition) labelMayFollow = wasAtStatementPosition if wasAtStatementPosition, word == "DECLARE" { inDeclaration = true } - if inDeclaration, word == "HANDLER" { - inHandlerDeclaration = true + // `handler` is not reserved, so `DECLARE handler CURSOR FOR ...` declares a cursor. + if inDeclaration, word == "HANDLER", let previous, Self.handlerKinds.contains(previous) { + handler = .header + } + } + + /// Returns whether `word` starts a handler's statement. + private mutating func readHandlerWord(_ word: String) -> Bool { + switch handler { + case .header where word == "FOR": + handler = .condition + case .condition where word == "SQLSTATE": + handler = .sqlState + case .condition where word == "NOT": + handler = .notFound + case .condition, .notFound: + handler = .afterCondition + case .afterCondition: + handler = .statement + return true + default: + break + } + return false + } + + // A MySQL error code arrives one digit at a time. + private mutating func readHandlerSymbol(_ symbol: UInt16) { + let isDigit = symbol >= Self.zero && symbol <= Self.nine + switch handler { + case .condition: + handler = .afterCondition + case .afterCondition where !isDigit: + handler = symbol == Self.comma ? .condition : .statement + default: + break + } + } + + private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool, after previous: String?) -> Bool { + if readsBlocksEverywhere { + return depth > 0 || definesRoutine || isFirstWord || followsControlFlow || openedBlockBody + } + if grammar.contains(.atomicRoutineBodies) { return depth == 0 || nestedDefinition == .routine } + if grammar.hasSingleBodyForm { return true } + switch routineBody { + case .statement: + return false + case .returnValue: + return depth == 0 && (previous == "AS" || previous == "IS") + case .unknown, .controlStatement: + // A MySQL body is the one control statement, so a `begin` beside it is a name. + guard depth > 0 else { return routineBody == .unknown || !readsCompoundBodies } + return atStatementPosition + } + } + + // Where a routine's body is one statement, the word that starts it. + private mutating func observeBodyStart(_ word: String, after previous: String?, isTriggerEvent: Bool) { + if word == "ON" { + readsTriggerEvents = false + } + if awaitsEventBody { + awaitsEventBody = word != "DO" + return + } + guard routineBody == .unknown else { return } + if Self.controlStatementOpeners.contains(word) { + routineBody = .controlStatement + } else if word == "RETURN", previous != "ON" { + // DB2's `COMMIT ON RETURN NO` is a header clause. + routineBody = .returnValue + } else if Self.singleStatementBodyOpeners.contains(word), !isTriggerEvent, !readsTriggerEvents { + routineBody = .statement + } else if readsCompoundBodies, word == "SET", !Self.wordsBeforeTypeSet.contains(previous ?? "") { + routineBody = .statement + } else if readsCompoundBodies, word == "DO" { + routineBody = .statement + } + } + + // A compound statement opens its block where a statement starts. An `IF` or a `WHILE` opens it only at the word + // that ends its condition, since a condition holds no statement and `IF(a, 1, 2)` is a function. + private mutating func observeCompoundStatement(_ word: String, atStatementStart: Bool) { + guard routineBody == .unknown || routineBody == .controlStatement else { return } + let awaited = conditionAwaitingBody.flatMap { $0.depth == depth ? $0.keyword : nil } + switch word { + case "IF", "WHILE": + if atStatementStart, conditionAwaitingBody == nil { + conditionAwaitingBody = ControlStatement(keyword: word, depth: depth) + } + case "THEN" where awaited == "IF": + conditionAwaitingBody = nil + openControlStatement("IF", startsStatement: true) + case "DO" where awaited == "WHILE": + conditionAwaitingBody = nil + openControlStatement("WHILE", startsStatement: true) + // MariaDB's Oracle mode writes `WHILE x LOOP ... END LOOP`. + case "LOOP" where awaited == "WHILE": + conditionAwaitingBody = nil + openControlStatement(word, startsStatement: true) + case "LOOP" where atStatementStart: + openControlStatement(word, startsStatement: true) + case "REPEAT" where atStatementStart: + openControlStatement(word, startsStatement: true) + repeatMayBeFunction = true + case "WHEN", "ELSE", "END": + if awaited != nil { + conditionAwaitingBody = nil + } + default: + break } } - private func opensNestedBlock(atStatementPosition: Bool) -> Bool { - guard depth > 0, !grammar.hasSingleBodyForm, !grammar.contains(.unterminatedStatements) else { return true } - return atStatementPosition || inHandlerDeclaration + private mutating func observeNestedDefinition(_ word: String, startsStatement: Bool) { + if startsStatement { + nestedDefinition = SqlBlockStructure.opensRoutineDefinition(word) ? .opener : .none + } else if nestedDefinition == .opener, !SqlBlockStructure.precedesRoutineKind(word) { + nestedDefinition = SqlBlockStructure.isRoutineKind(word, grammar: grammar) ? .routine : .none + } + } + + private mutating func openControlStatement(_ keyword: String, startsStatement: Bool) { + depth += 1 + openControlStatements.append(ControlStatement(keyword: keyword, depth: depth)) + atStatementPosition = startsStatement + } + + // `END IF` and its kin close the block their own keyword opened, and nothing where it opened none. + private mutating func closeControlStatement(_ keyword: String) { + guard let innermost = openControlStatements.last, innermost.keyword == keyword, innermost.depth == depth else { + return + } + openControlStatements.removeLast() + depth -= 1 + } + + private mutating func observeRuleWord(_ word: String) { + switch ruleAction { + case .header where word == "DO": + ruleAction = .afterDo + case .afterDo where word == "ALSO" || word == "INSTEAD": + break + case .afterDo: + ruleAction = .none + default: + break + } + } + + // `BEGIN TRY` and `BEGIN CATCH` are blocks wherever they stand, and nothing else follows a name that way. + private func beginOpensBlock(followedBy word: String) -> Bool { + if readsBlocksEverywhere, word == "TRY" || word == "CATCH" { return true } + return pendingBeginMayOpen && !SqlBlockStructure.beginStartsStatement(followedBy: word, grammar: grammar) } private mutating func endName() { @@ -227,7 +611,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { openBlockBody() return true } - if !SqlBlockStructure.beginStartsTransaction(followedBy: word) { + if beginOpensBlock(followedBy: word) { openBlockBody() } } @@ -237,8 +621,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { closeBlock() return false } - switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0) { + switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0, grammar: grammar) { case .closesControlFlow: + closeControlStatement(word) return true case .closesCaseStatement: closeBlock() @@ -254,7 +639,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private mutating func settlePendingBeforeNonWord(opensBlock: Bool) { if pendingBegin { pendingBegin = false - if opensBlock { + // A T-SQL `begin(` outside a routine is a name, as in `CREATE TABLE begin(a int)`. + let opensAfterName = !readsBlocksEverywhere || definesRoutine || depth > 0 + if opensBlock, pendingBeginMayOpen, opensAfterName { openBlockBody() } } @@ -268,14 +655,20 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // A trigger body holds no other; `BEGIN ATOMIC` is never a name, so a routine defined inside one nests. private mutating func openBlockBody() { guard !grammar.contains(.triggerBodies) || depth == 0 else { return } + if routineBody == .returnValue { + routineBody = .unknown + } depth += 1 openedBlockBody = true startsBodyStatement = true atStatementPosition = true + nestedDefinition = .none } private mutating func closeBlock() { depth = max(0, depth - 1) openCaseExpressions = max(0, openCaseExpressions - 1) + openControlStatements.removeAll { $0.depth > depth } + caseExpressionDepths.removeAll { $0 > depth } } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRowPatternClause.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRowPatternClause.swift new file mode 100644 index 0000000000..1cb3663180 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRowPatternClause.swift @@ -0,0 +1,136 @@ +import Foundation + +// MATCH_RECOGNIZE's parentheses, read in the order Oracle takes them: PARTITION BY, ORDER BY, MEASURES, the rows per match, +// AFTER MATCH SKIP, PATTERN, SUBSET, DEFINE. A section word counts only after an operand, never after `.`, `AS`, a comma +// or an operator, so `AS subset` or `x.define` opens nothing; one that still lands out of order makes the clause +// unreadable. Nil when the clause does not read in that order, and a reader then takes it whole as an expression. +struct SQLRowPatternClause { + // Token indices. + private(set) var partitionBy: Range? + private(set) var orderBy: Range? + private(set) var measures: Range? + let patternKeyword: Int + let pattern: Range + let define: Range + + private init(patternKeyword: Int, pattern: Range, define: Range) { + self.patternKeyword = patternKeyword + self.pattern = pattern + self.define = define + } + + private enum Section: Int { + case partitionBy + case orderBy + case measures + case rowsPerMatch + case afterMatch + case pattern + case subset + case define + } + + private struct Mark { + let section: Section + let keyword: Int + let content: Int + } + + // `body` is the tokens inside the clause's parentheses. + static func read(_ tokens: [TSQLTokens.Token], body: Range, text: NSString) -> SQLRowPatternClause? { + guard let first = body.first else { return nil } + let depth = tokens[first].depth + var marks: [Mark] = [] + var index = body.lowerBound + while index < body.upperBound { + guard tokens[index].depth == depth, let mark = mark(at: index, in: tokens, body: body, text: text) else { + index += 1 + continue + } + if let last = marks.last, mark.section.rawValue <= last.section.rawValue { return nil } + marks.append(mark) + if mark.section == .define { break } + index = mark.content + } + guard let patternAt = marks.firstIndex(where: { $0.section == .pattern }), + let define = marks.last, define.section == .define, + let close = close(of: marks[patternAt].content, in: tokens, before: body.upperBound), + marks[patternAt + 1].keyword == close + 1 + else { return nil } + let keyword = marks[patternAt].keyword + var clause = SQLRowPatternClause( + patternKeyword: keyword, + pattern: (marks[patternAt].content + 1).., + text: NSString + ) -> Mark? { + guard let word = tokens[index].word else { return nil } + let next = index + 1 < body.upperBound ? tokens[index + 1] : nil + let section: Section + switch word { + case "PARTITION" where next?.word == "BY": + section = .partitionBy + case "ORDER" where next?.word == "BY": + section = .orderBy + case "MEASURES": + section = .measures + case "ONE" where next?.word == "ROW", "ALL" where next?.word == "ROWS": + section = .rowsPerMatch + case "AFTER" where next?.word == "MATCH": + section = .afterMatch + // Its content starts at the parenthesis. + case "PATTERN" where next?.kind == .open: + section = .pattern + case "SUBSET": + section = .subset + case "DEFINE": + section = .define + default: + return nil + } + guard index == body.lowerBound || followsOperand(tokens[index - 1], text: text) else { return nil } + let length = section == .partitionBy || section == .orderBy ? 2 : 1 + return Mark(section: section, keyword: index, content: index + length) + } + + private static let operatorWords: Set = [ + "AS", "AND", "OR", "NOT", "IS", "IN", "LIKE", "BETWEEN", "CASE", "WHEN", "THEN", "ELSE", "BY", "MEASURES", + "SUBSET", "DEFINE", "PATTERN", "RUNNING", "FINAL", "PRIOR", "DISTINCT", "ALL", "ANY", "SOME", "EXISTS", "ESCAPE", + ] + + private static func followsOperand(_ token: TSQLTokens.Token, text: NSString) -> Bool { + switch token.kind { + case .close, .name, .quoted: + return true + case .word(let word): + return !operatorWords.contains(word) + case .other: + let unit = text.character(at: token.location) + return unit >= 0x30 && unit <= 0x39 + case .open, .comma, .terminator: + return false + } + } + + private static func close(of open: Int, in tokens: [TSQLTokens.Token], before end: Int) -> Int? { + guard open < end, tokens[open].kind == .open else { return nil } + return ((open + 1).. SQLSessionStateEffect { + SQLSessionStateEffect( + creates: creates.subtracting(next.clears).union(next.creates), + clears: clears.union(next.clears) + ) + } + + public func applied(to kinds: SQLSessionStateKinds) -> SQLSessionStateKinds { + kinds.subtracting(clears).union(creates) + } +} + +public enum SQLSessionDialect: Sendable, Hashable { + case postgres + case mySQL + case sqlServer + case oracle + case sqlite + case duckDB +} + +// No engine reports a session's temporary tables, settings or variables, and a dead connection answers nothing, so +// the text decides. A `CALL`, `DO`, `EXEC proc` or PL/SQL block is not read: what it sets up is out of sight. +public enum SQLSessionStateClassifier: Sendable { + // Past this a text is data, an INSERT's VALUES, and reading it all would cost a pass per run. Each text is cut before + // it is split, so a setup past this point in one long text, such as a T-SQL batch with no `;`, is not seen. + private static let classifiedPrefixLength = 65_536 + private static let modifierLookahead = 5 + + private static let period = UInt16(UnicodeScalar(".").value) + private static let atSign = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let equals = UInt16(UnicodeScalar("=").value) + + // What `skipping` names runs again on every new connection, such as a startup command, so it is never lost. + public static func effect( + ofStatements statements: [String], + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar, + skipping skipped: [String] = [] + ) -> SQLSessionStateEffect { + let skippedTexts = Set(skipped.map(comparableText)) + var effect = SQLSessionStateEffect.none + for text in statements { + let head = classifiedHead(of: text) + guard SQLSessionStateQuickCheck.mayChangeSession(head, dialect: dialect) else { continue } + for statement in SQLStatementClassifier.statements(of: head, grammar: grammar) + where SQLSessionStateQuickCheck.mayChangeSession(statement, dialect: dialect) + && !skippedTexts.contains(comparableText(statement)) { + effect = effect.followed(by: statementEffect(of: statement, dialect: dialect, grammar: grammar)) + } + } + return effect + } + + public static func effect(of sql: String, dialect: SQLSessionDialect, grammar: SQLLexicalGrammar) -> SQLSessionStateEffect { + effect(ofStatements: [sql], dialect: dialect, grammar: grammar) + } + + static func comparableText(_ statement: String) -> String { + var text = statement.trimmingCharacters(in: .whitespacesAndNewlines) + while text.hasSuffix(";") { + text = String(text.dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + } + return text + } + + // Every server reading of a `/*!` comment counts what it sets up, and only what all of them clear is cleared. + private static func statementEffect( + of statement: String, + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar + ) -> SQLSessionStateEffect { + // More versions than a reader takes one by one leaves the text as written, its executable comments unread. + let texts = SQLExecutableCommentReadings.texts(of: statement, grammar: grammar).map { Array($0) } ?? [statement] + var creates: SQLSessionStateKinds = [] + var clears: SQLSessionStateKinds? + for text in texts { + let reading = SQLStatementClassifier.runnableStatements(of: text, grammar: grammar) + .reduce(SQLSessionStateEffect.none) { partial, piece in + partial.followed(by: pieceEffect(of: piece, dialect: dialect, grammar: grammar)) + } + creates.formUnion(reading.creates) + clears = clears.map { $0.intersection(reading.clears) } ?? reading.clears + } + return SQLSessionStateEffect(creates: creates, clears: clears ?? []) + } + + private static func classifiedHead(of statement: String) -> String { + let text = statement as NSString + guard text.length > classifiedPrefixLength else { return statement } + let cut = text.rangeOfComposedCharacterSequence(at: classifiedPrefixLength).location + return text.substring(to: cut) + } + + private static func pieceEffect( + of piece: String, + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar + ) -> SQLSessionStateEffect { + guard !SQLProceduralBlock.runs(piece, grammar: grammar) else { return .none } + let code = StatementCode(piece, grammar: grammar) + guard let start = code.tokens.firstIndex(where: { $0.kind != .open }), code.tokens[start].word != nil else { + return .none + } + switch dialect { + case .postgres: + return postgresEffect(code, start: start) + case .mySQL: + return SQLSessionStateEffect(creates: mySQLKinds(code, start: start)) + case .sqlServer: + return SQLSessionStateEffect(creates: sqlServerKinds(code, start: start)) + case .oracle: + return SQLSessionStateEffect(creates: oracleKinds(code, start: start)) + case .sqlite: + return SQLSessionStateEffect(creates: sqliteKinds(code, start: start)) + case .duckDB: + return SQLSessionStateEffect(creates: duckDBKinds(code, start: start)) + } + } +} + +// MARK: - PostgreSQL + +extension SQLSessionStateClassifier { + private static let postgresObjectKinds: Set = ["TABLE", "VIEW", "SEQUENCE", "FUNCTION", "PROCEDURE", "TYPE"] + + // `SET LOCAL` lasts until the transaction ends, `SET TRANSACTION` covers one, and `SET CONSTRAINTS` the current one. + private static let postgresTransactionScopedSets: Set = ["LOCAL", "TRANSACTION", "CONSTRAINTS"] + + // KingbaseES ships each `pg_` function under a `sys_` name too. The `_xact_` forms end with the transaction. + private static let postgresSessionLockFunctions: Set = [ + "PG_ADVISORY_LOCK", "PG_ADVISORY_LOCK_SHARED", "PG_TRY_ADVISORY_LOCK", "PG_TRY_ADVISORY_LOCK_SHARED", + "SYS_ADVISORY_LOCK", "SYS_ADVISORY_LOCK_SHARED", "SYS_TRY_ADVISORY_LOCK", "SYS_TRY_ADVISORY_LOCK_SHARED", + ] + + private static func postgresEffect(_ code: StatementCode, start: Int) -> SQLSessionStateEffect { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + // A prepared statement runs what it holds only when executed. + var kinds: SQLSessionStateKinds = keyword == "PREPARE" ? [] : postgresCallKinds(code) + switch keyword { + case "CREATE": + let temporary = createsTemporary(code, after: start, kinds: postgresObjectKinds) + || createsInSchema(code, after: start, kinds: postgresObjectKinds) { $0.hasPrefix("PG_TEMP") } + // `ON COMMIT DROP` ends the table with the transaction. + if temporary, !code.tokens.indices.contains(where: { code.spells(["ON", "COMMIT", "DROP"], at: $0) }) { + kinds.insert(.temporaryObjects) + } + case "SELECT", "WITH": + if selectsIntoPostgresTemporary(code) { + kinds.insert(.temporaryObjects) + } + case "SET": + if let next, !postgresTransactionScopedSets.contains(next) { + kinds.insert(.sessionSettings) + } + case "PREPARE": + if next != nil, next != "TRANSACTION" { + kinds.insert(.preparedStatements) + } + case "DECLARE": + if declaresHeldCursor(code, after: start) { + kinds.insert(.heldCursors) + } + case "LISTEN": + kinds.insert(.listeners) + case "DISCARD": + switch next { + case "ALL": + // `RESET ALL` is no clear: `role` and `session_authorization` are `GUC_NO_RESET_ALL`, and only this + // resets them too. + return SQLSessionStateEffect(creates: kinds, clears: .all) + case "TEMP", "TEMPORARY": + return SQLSessionStateEffect(creates: kinds, clears: .temporaryObjects) + default: + break + } + case "DEALLOCATE": + let target = next == "PREPARE" ? code.word(at: start + 2) : next + if target == "ALL" { + return SQLSessionStateEffect(creates: kinds, clears: .preparedStatements) + } + case "UNLISTEN": + if code.isSymbol(SqlLexer.star, at: start + 1) { + return SQLSessionStateEffect(creates: kinds, clears: .listeners) + } + default: + break + } + return SQLSessionStateEffect(creates: kinds) + } + + private static func postgresCallKinds(_ code: StatementCode) -> SQLSessionStateKinds { + var kinds: SQLSessionStateKinds = [] + for index in code.tokens.indices where code.opensCall(at: index) { + guard let name = code.word(at: index) else { continue } + if postgresSessionLockFunctions.contains(name) { + kinds.insert(.sessionLocks) + } else if name == "SET_CONFIG", !isTransactionLocalSetConfig(code, opening: index + 1) { + kinds.insert(.sessionSettings) + } + } + return kinds + } + + // `set_config(name, value, true)` lasts until the transaction ends. + private static func isTransactionLocalSetConfig(_ code: StatementCode, opening open: Int) -> Bool { + let outside = code.tokens[open].depth + var argument = 0 + var third: [TSQLTokens.Token] = [] + var index = open + 1 + while index < code.tokens.count { + let token = code.tokens[index] + if token.kind == .close, token.depth == outside { break } + if token.kind == .comma, token.depth == outside + 1 { + argument += 1 + } else if argument == 2 { + third.append(token) + } + index += 1 + } + return third.count == 1 && third[0].word == "TRUE" + } + + private static func selectsIntoPostgresTemporary(_ code: StatementCode) -> Bool { + code.tokens.indices.contains { index in + guard code.word(at: index) == "INTO", let target = code.word(at: index + 1) else { return false } + return target == "TEMP" || target == "TEMPORARY" + || (target.hasPrefix("PG_TEMP") && code.isSymbol(period, at: index + 2)) + } + } + + // A cursor outlives its transaction only `WITH HOLD`. + private static func declaresHeldCursor(_ code: StatementCode, after start: Int) -> Bool { + let end = code.tokens[(start + 1)...].firstIndex { $0.word == "FOR" } ?? code.tokens.count + return ((start + 1).. = ["GLOBAL", "PERSIST", "PERSIST_ONLY"] + private static let mySQLSessionScopes: Set = ["SESSION", "LOCAL"] + + private static func mySQLKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + var kinds: SQLSessionStateKinds = [] + switch keyword { + case "CREATE": + if createsTemporary(code, after: start, kinds: ["TABLE"]) { + kinds.insert(.temporaryObjects) + } + case "SET": + kinds.formUnion(mySQLSetKinds(code, from: start + 1)) + case "PREPARE": + return next == nil ? [] : [.preparedStatements] + case "LOCK": + if next == "TABLE" || next == "TABLES" { + kinds.insert(.sessionLocks) + } + default: + break + } + if code.tokens.indices.contains(where: { code.opensCall(at: $0) && code.word(at: $0) == "GET_LOCK" }) { + kinds.insert(.sessionLocks) + } + if assignsMySQLUserVariable(code) { + kinds.insert(.userVariables) + } + return kinds + } + + // Plain `SET TRANSACTION` covers the next transaction only, `SET PASSWORD` and `SET DEFAULT ROLE` change the + // account, and MariaDB's `SET STATEMENT ... FOR` one statement. + private static func mySQLSetKinds(_ code: StatementCode, from first: Int) -> SQLSessionStateKinds { + guard code.tokens.indices.contains(first) else { return [] } + let next = code.word(at: first) + let after = code.word(at: first + 1) + if after == "TRANSACTION", let next { + return mySQLSessionScopes.contains(next) ? [.sessionSettings] : [] + } + switch next { + case "PASSWORD", "STATEMENT", "TRANSACTION", "RESOURCE": + return [] + case "DEFAULT" where after == "ROLE": + return [] + case "NAMES", "CHARACTER", "CHARSET", "ROLE": + return [.sessionSettings] + default: + break + } + return mySQLAssignmentKinds(code, from: first) + } + + // A scope word sets the scope of the assignments after it that name none, as the manual says. + private static func mySQLAssignmentKinds(_ code: StatementCode, from first: Int) -> SQLSessionStateKinds { + let depth = code.tokens[first].depth + var kinds: SQLSessionStateKinds = [] + var sessionScope = true + var startsAssignment = true + for index in first.. Bool { + code.tokens.indices.contains { index in + guard code.isSymbol(atSign, at: index), !code.isSymbol(atSign, at: index - 1), + !code.isSymbol(atSign, at: index + 1) + else { return false } + if code.word(at: index - 1) == "INTO" { return true } + return code.isSymbol(colon, at: index + 2) && code.isSymbol(equals, at: index + 3) + } + } +} + +// MARK: - SQL Server + +extension SQLSessionStateClassifier { + private static let sqlServerTemporaryKinds: Set = ["TABLE", "PROC", "PROCEDURE"] + + private static func sqlServerKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + switch keyword { + case "CREATE": + return createsSQLServerTemporary(code, after: start) ? [.temporaryObjects] : [] + case "SELECT", "WITH": + let intoTemporary = code.tokens.indices.contains { index in + code.word(at: index) == "INTO" && namesSQLServerTemporary(code, at: index + 1) + } + return intoTemporary ? [.temporaryObjects] : [] + case "SET": + // A `@variable` dies with its batch. + guard let next, !next.hasPrefix("@") else { return [] } + return [.sessionSettings] + case "EXEC", "EXECUTE": + // `EXECUTE AS` impersonates until a `REVERT`. + return next == "AS" ? [.sessionSettings] : sqlServerProcedureKinds(code, nameAt: start + 1) + case "DECLARE": + return declaresGlobalCursor(code, after: start) ? [.heldCursors] : [] + default: + // A batch that opens with a procedure's name runs it. + return sqlServerProcedureKinds(code, nameAt: start) + } + } + + private static func createsSQLServerTemporary(_ code: StatementCode, after start: Int) -> Bool { + let window = (start + 1).. Bool { + guard code.tokens.indices.contains(index) else { return false } + let token = code.tokens[index] + let first: Int + switch token.kind { + case .word, .name: + first = token.location + case .quoted: + first = token.location + 1 + default: + return false + } + return first < code.text.length && code.text.character(at: first) == SqlLexer.hash + } + + private static func sqlServerProcedureKinds(_ code: StatementCode, nameAt start: Int) -> SQLSessionStateKinds { + var index = start + if code.word(at: index)?.hasPrefix("@") == true, code.isSymbol(equals, at: index + 1) { + index += 2 + } + while code.isSymbol(period, at: index + 1), code.word(at: index + 2) != nil { + index += 2 + } + switch code.word(at: index) { + // An application role holds the session's security context until `sp_unsetapprole` or the end of the session. + case "SP_SET_SESSION_CONTEXT", "SP_SETAPPROLE": + return [.sessionSettings] + case "SP_GETAPPLOCK": + return [.sessionLocks] + default: + return [] + } + } + + // A cursor is global unless declared `LOCAL`; a cursor variable `DECLARE @c CURSOR` dies with its batch. + private static func declaresGlobalCursor(_ code: StatementCode, after start: Int) -> Bool { + guard let name = code.word(at: start + 1), !name.hasPrefix("@") else { return false } + let end = code.tokens[(start + 1)...].firstIndex { $0.word == "FOR" } ?? code.tokens.count + let words = code.tokens[(start + 1).. = ["SET", "ENABLE", "DISABLE", "FORCE"] + private static let sqliteObjectKinds: Set = ["TABLE", "VIEW", "TRIGGER", "INDEX"] + private static let duckDBObjectKinds: Set = ["TABLE", "VIEW", "SEQUENCE", "MACRO", "FUNCTION", "TYPE"] + + // Rows in a global temporary table are not counted: the table itself is permanent. + private static func oracleKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let next = code.word(at: start + 1) + switch code.word(at: start) { + case "ALTER": + let verb = code.word(at: start + 2) + return next == "SESSION" && verb.map(oracleSessionVerbs.contains) == true ? [.sessionSettings] : [] + case "SET": + return next == "ROLE" ? [.sessionSettings] : [] + case "CREATE": + // A private temporary table goes at commit unless it keeps its definition. + let isPrivate = code.spells(["PRIVATE", "TEMPORARY", "TABLE"], at: start + 1) + let preserved = code.tokens.indices.contains { code.spells(["PRESERVE", "DEFINITION"], at: $0) } + return isPrivate && preserved ? [.temporaryObjects] : [] + default: + return [] + } + } + + private static func sqliteKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + guard code.word(at: start) == "CREATE" else { return [] } + let temporary = createsTemporary(code, after: start, kinds: sqliteObjectKinds) + || createsInSchema(code, after: start, kinds: sqliteObjectKinds) { $0 == "TEMP" } + return temporary ? [.temporaryObjects] : [] + } + + private static func duckDBKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let next = code.word(at: start + 1) + switch code.word(at: start) { + case "CREATE": + let temporary = createsTemporary(code, after: start, kinds: duckDBObjectKinds) + || createsInSchema(code, after: start, kinds: duckDBObjectKinds) { $0 == "TEMP" } + return temporary ? [.temporaryObjects] : [] + case "SET": + switch next { + case nil, "GLOBAL", "LOCAL": + return [] + case "VARIABLE": + return [.userVariables] + default: + return [.sessionSettings] + } + default: + return [] + } + } +} + +// MARK: - Shared readings + +extension SQLSessionStateClassifier { + private static let temporaryMarkers: Set = ["TEMP", "TEMPORARY"] + + // `CREATE [OR REPLACE] [GLOBAL | LOCAL] TEMP TABLE`: the marker stands before the kind, a few words in. + private static func createsTemporary(_ code: StatementCode, after start: Int, kinds: Set) -> Bool { + for index in (start + 1).., + schema isTemporarySchema: (String) -> Bool + ) -> Bool { + let window = (start + 1).. String? { + tokens.indices.contains(index) ? tokens[index].word : nil + } + + func opensCall(at index: Int) -> Bool { + tokens.indices.contains(index + 1) && tokens[index].word != nil && tokens[index + 1].kind == .open + } +} + +// Whole words, read without the grammar: a hit inside a comment or a literal only costs the full read, and a data word +// such as `offset`, `block` or `template`, or an email address, is no hit. Every rule above needs one of these. +enum SQLSessionStateQuickCheck: Sendable { + private static let atSign = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let equals = UInt16(UnicodeScalar("=").value) + private static let shortestWord = 3 + private static let longestWord = 32 + + private static let postgresWords: Set = [ + "TEMP", "TEMPORARY", "SET", "SET_CONFIG", "PREPARE", "DECLARE", "LISTEN", "UNLISTEN", "DISCARD", "DEALLOCATE", + "PG_ADVISORY_LOCK", "PG_ADVISORY_LOCK_SHARED", "PG_TRY_ADVISORY_LOCK", "PG_TRY_ADVISORY_LOCK_SHARED", + "SYS_ADVISORY_LOCK", "SYS_ADVISORY_LOCK_SHARED", "SYS_TRY_ADVISORY_LOCK", "SYS_TRY_ADVISORY_LOCK_SHARED", + ] + private static let mySQLWords: Set = ["TEMP", "TEMPORARY", "SET", "PREPARE", "LOCK", "GET_LOCK"] + private static let sqlServerWords: Set = [ + "SET", "EXEC", "EXECUTE", "DECLARE", "SP_SET_SESSION_CONTEXT", "SP_GETAPPLOCK", "SP_SETAPPROLE", + ] + private static let oracleWords: Set = ["SESSION", "ROLE", "PRIVATE"] + private static let sqliteWords: Set = ["TEMP", "TEMPORARY"] + private static let duckDBWords: Set = ["TEMP", "TEMPORARY", "SET"] + + static func mayChangeSession(_ text: String, dialect: SQLSessionDialect) -> Bool { + let units = text as NSString + let length = units.length + let triggers = words(for: dialect) + var lastWord = "" + var lastWordEnd = -1 + var index = 0 + while index < length { + let unit = units.character(at: index) + if isWordUnit(unit) { + let start = index + while index < length, isWordUnit(units.character(at: index)) { + index += 1 + } + let count = index - start + lastWord = count >= shortestWord && count <= longestWord + ? units.substring(with: NSRange(location: start, length: count)).uppercased() + : "" + lastWordEnd = index + if triggers.contains(lastWord) || (dialect == .postgres && lastWord.hasPrefix("PG_TEMP")) { + return true + } + continue + } + if dialect == .mySQL, unit == atSign, + mayAssignMySQLVariable(units, at: index, gluedToInto: lastWord == "INTO" && lastWordEnd == index) { + return true + } + if dialect == .sqlServer, unit == SqlLexer.hash, startsSQLServerName(units, at: index) { + return true + } + index += 1 + } + return false + } + + private static func words(for dialect: SQLSessionDialect) -> Set { + switch dialect { + case .postgres: return postgresWords + case .mySQL: return mySQLWords + case .sqlServer: return sqlServerWords + case .oracle: return oracleWords + case .sqlite: return sqliteWords + case .duckDB: return duckDBWords + } + } + + // ASCII letters and `_`: a digit or `$` ends the run, so `1SET` still holds the `SET` the server reads. + private static func isWordUnit(_ unit: UInt16) -> Bool { + (0x41...0x5A).contains(unit) || (0x61...0x7A).contains(unit) || unit == 0x5F + } + + private static func isNameUnit(_ unit: UInt16) -> Bool { + isWordUnit(unit) || (0x30...0x39).contains(unit) || unit == 0x24 || unit >= 0x80 + } + + // An address glues its `@` to the name before it, as in `ana@example.com`. A variable stands apart, follows + // `INTO`, or is assigned with `:=`. A second `@` makes a system variable, which a statement only reads. + private static func mayAssignMySQLVariable(_ text: NSString, at index: Int, gluedToInto: Bool) -> Bool { + let length = text.length + if index > 0, text.character(at: index - 1) == atSign { return false } + guard index + 1 < length, text.character(at: index + 1) != atSign else { return false } + if gluedToInto { return true } + let previous = index > 0 ? text.character(at: index - 1) : SqlLexer.space + if !isNameUnit(previous), !"._%+-".utf16.contains(previous) { return true } + return assignedWithColonEquals(text, after: index) + } + + private static func assignedWithColonEquals(_ text: NSString, after index: Int) -> Bool { + let length = text.length + var cursor = index + 1 + let opener = text.character(at: cursor) + if opener == SqlLexer.singleQuote || opener == SqlLexer.doubleQuote || opener == SqlLexer.backtick { + cursor += 1 + while cursor < length, text.character(at: cursor) != opener { + cursor += 1 + } + cursor += 1 + } else { + while cursor < length, isNameUnit(text.character(at: cursor)) { + cursor += 1 + } + } + cursor = pastBlanksAndBlockComments(text, from: cursor) + return cursor + 1 < length && text.character(at: cursor) == colon && text.character(at: cursor + 1) == equals + } + + private static func pastBlanksAndBlockComments(_ text: NSString, from start: Int) -> Int { + let length = text.length + var cursor = start + while cursor < length { + let blank = StatementBlank.blankLength(in: text, at: cursor) + if blank > 0 { + cursor += blank + continue + } + guard cursor + 1 < length, text.character(at: cursor) == SqlLexer.slash, + text.character(at: cursor + 1) == SqlLexer.star + else { break } + let close = text.range( + of: "*/", options: [], range: NSRange(location: cursor + 2, length: length - cursor - 2) + ) + guard close.location != NSNotFound else { return length } + cursor = NSMaxRange(close) + } + return cursor + } + + // `#t`, `##t`, `[#t]` and `"#t"`: a `#` that begins a name. One inside a name, `a#b`, is part of it. + private static func startsSQLServerName(_ text: NSString, at index: Int) -> Bool { + let length = text.length + guard index + 1 < length else { return false } + if index > 0 { + let previous = text.character(at: index - 1) + if isNameUnit(previous) || previous == SqlLexer.hash || previous == atSign { return false } + } + let next = text.character(at: index + 1) + return isNameUnit(next) || next == SqlLexer.hash + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift index 3c9ec3fb97..59181a6a75 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift @@ -48,10 +48,24 @@ public protocol SQLStatementBoundaryTracking { /// Returns whether this `;` ends the statement. mutating func observeSemicolon() -> Bool + /// Whether the `;` that just ended the statement can still be taken back by the token after it, as T-SQL takes + /// back the one in `IF @x = 1 SELECT 1; ELSE SELECT 2;` at its `ELSE`. A reader holds the end until the next token + /// that is not whitespace or a comment, then asks ``continuesStatement(with:)``. + var awaitsContinuation: Bool { get } + + /// Whether the first token after a `;` that left ``awaitsContinuation`` set carries the statement on. `word` is + /// that token uppercased, or nil for a token that is not a word. On true the reader keeps the statement open and + /// observes the token as usual; on false it ends the statement at the `;` and resets. + mutating func continuesStatement(with word: String?) -> Bool + /// Whether a block this statement opened is still open, which at the end of a text means everything after it was /// swallowed by a body that never closed. var holdsOpenBlock: Bool { get } + /// Whether the statement is a `COPY ... FROM STDIN` whose rows a psql script holds on the lines after it, which a + /// reader then takes as ``SQLPsqlScript/copyRows(afterSemicolonAt:in:length:)`` rather than as SQL. + var copiesFromStandardInput: Bool { get } + mutating func reset() } @@ -60,17 +74,35 @@ public extension SQLStatementBoundaryTracking { observeOpaqueToken() } + var awaitsContinuation: Bool { + false + } + + mutating func continuesStatement(with word: String?) -> Bool { + false + } + var holdsOpenBlock: Bool { false } + + var copiesFromStandardInput: Bool { + false + } } public enum SQLStatementBoundaries { /// The one place a grammar is matched to its statement boundaries, so no reader can pick a different one. public static func makeTracker(for grammar: SQLLexicalGrammar) -> any SQLStatementBoundaryTracking { - if grammar.contains(.plsqlBlocks) { return PLSQLUnitTracker() } - if grammar.contains(.cqlBatches) { return CQLBatchTracker() } - return SQLRoutineBodyTracker(grammar: grammar) + let tracker: any SQLStatementBoundaryTracking + if grammar.contains(.plsqlBlocks) { + tracker = PLSQLUnitTracker() + } else if grammar.contains(.cqlBatches) { + tracker = CQLBatchTracker() + } else { + tracker = SQLRoutineBodyTracker(grammar: grammar) + } + return grammar.contains(.psqlScripts) ? SQLPsqlStatementTracker(base: tracker) : tracker } /// Whether a statement `grammar` reads can own a `;`: the one that ends a PL/SQL unit or a T-SQL `MERGE`, or one diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+DataStatements.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+DataStatements.swift new file mode 100644 index 0000000000..14d0f04f10 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+DataStatements.swift @@ -0,0 +1,266 @@ +import Foundation + +public extension SQLStatementClassifier { + // Whether a request-form statement only reads or changes rows, or nil for SQL. + typealias RequestDataStatement = (_ statement: String, _ grammar: SQLLexicalGrammar) -> Bool? + + // Whether every statement `sql` runs, under every way the engine could lex it, only reads or changes rows. A + // request form `requestIsData` cannot read, but `request` tiers, is refused. + // `rowChanges` replaces the SQL words that change rows on an engine whose language reads as SQL but differs there: + // SurrealQL's CREATE and RELATE make records. + static func isDataStatement( + _ sql: String, + databaseTypeId: String, + readings: SQLLexicalReadings, + request: RequestClassification? = nil, + requestIsData: RequestDataStatement? = nil, + rowChanges: Set? = nil + ) -> Bool { + guard let lexings = readings.lexings(of: sql) else { return false } + var tiering = SQLStatementTiering(databaseTypeId: databaseTypeId, readings: readings, request: request) + tiering.rowChanges = rowChanges + var seen: Set = [] + return lexings.allSatisfy { lexing in + runnableStatements(of: lexing.text, grammar: lexing.grammar).allSatisfy { statement in + guard seen.insert(SQLLexing(grammar: lexing.grammar, text: statement)).inserted else { return true } + if let answer = requestIsData?(statement, lexing.grammar) { return answer } + if let request, request(statement, lexing.grammar) != nil { return false } + return tiering.isDataStatement(statement, grammar: lexing.grammar) + } + } + } + + // Whether an unqualified table lands in a schema whose tables the server acts on. Oracle and Dameng name the + // session's schema after the user, so SYSTEM there is a login with tables of its own; SYS is the dictionary. + static func isSystemScope(database: String?, schema: String?, databaseTypeId: String) -> Bool { + let schemas = SQLStatementTiering.systemSchemas(for: databaseTypeId) + let namesUser = databaseTypeId == "Oracle" || databaseTypeId == "Dameng" + let readSchema = namesUser ? schema.flatMap { $0.uppercased() == "SYS" ? $0 : nil } : schema + return [database, readSchema].contains { $0.map { schemas.contains($0.uppercased()) } == true } + } +} + +extension SQLStatementTiering { + static let rowChangeKeywords: Set = ["INSERT", "UPDATE", "DELETE", "MERGE", "UPSERT"] + + // `REPLACE [LOW_PRIORITY | DELAYED] INTO t`. ClickHouse's `REPLACE TABLE` and Teradata's `REPLACE VIEW` define. + static let replaceRowWords: Set = ["INTO", "LOW_PRIORITY", "DELAYED"] + + // MySQL makes INTO optional in `REPLACE t VALUES (...)`. + static let mySQLFamily: Set = ["MySQL", "MariaDB", "TiDB", "OceanBase"] + + static let postgreSQLFamily: Set = ["PostgreSQL", "CockroachDB", "PGlite", "KingbaseES", "Redshift"] + + // An UPDATE of `pg_settings` runs SET; KingbaseES names its catalog `sys_` as well. + static func systemRelationPrefixes(for databaseTypeId: String) -> [String] { + guard postgreSQLFamily.contains(databaseTypeId) else { return [] } + return databaseTypeId == "KingbaseES" ? ["PG_", "SYS_"] : ["PG_"] + } + + // SQL Server's msdb holds the SQL Agent jobs it runs. + static func systemSchemas(for databaseTypeId: String) -> Set { + if mySQLFamily.contains(databaseTypeId) { return ["MYSQL", "PERFORMANCE_SCHEMA", "SYS"] } + switch databaseTypeId { + case "KingbaseES": return ["PG_CATALOG", "SYS_CATALOG"] + case "CockroachDB": return ["PG_CATALOG", "SYSTEM", "CRDB_INTERNAL"] + case "PostgreSQL", "PGlite", "Redshift": return ["PG_CATALOG"] + case "SQL Server": return ["MSDB"] + case "Oracle": return ["SYS", "SYSTEM"] + case "Dameng": return ["SYS"] + case "ClickHouse": return ["SYSTEM"] + // A row change there passes the server's MODIFY check: `UPDATE system_auth.roles SET is_superuser = true`, + // ScyllaDB's `system.roles` and its live `system.config`, Cassandra's writable `system_views` cache tables. + case "Cassandra": return cassandraSystemKeyspaces + // ScyllaDB 2026.1 writes its table audit backend to `audit.audit_log`, which the server does not mark internal. + case "ScyllaDB": return cassandraSystemKeyspaces.union(["AUDIT"]) + default: return [] + } + } + + static let cassandraSystemKeyspaces: Set = [ + "SYSTEM", "SYSTEM_SCHEMA", "SYSTEM_AUTH", "SYSTEM_DISTRIBUTED", "SYSTEM_DISTRIBUTED_EVERYWHERE", + "SYSTEM_TRACES", "SYSTEM_VIEWS", "SYSTEM_VIRTUAL_SCHEMA", "SYSTEM_REPLICATED_KEYS", + ] + + // A ClickHouse ALTER action that is not a row change, after a comma at the statement's depth. + static let clickHouseAlterActions: Set = [ + "ADD", "DROP", "MODIFY", "RENAME", "CLEAR", "COMMENT", "MATERIALIZE", "FREEZE", "UNFREEZE", "ATTACH", "DETACH", + "MOVE", "REPLACE", "FETCH", "RESET", "REMOVE", "APPLY", + ] + + func isDataStatement(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + // A SurrealQL stored body tiers as a read, as any unlisted call does, but what it runs is not read. + if callsSurrealStoredCode(StatementCode(statement, grammar: grammar)) { return false } + if statementClassification(statement, grammar: grammar).tier == .safe { return true } + if let batch = CQLBatch.statements(in: statement, grammar: grammar) { + return batch.allSatisfy { isDataStatement($0, grammar: grammar) } + } + guard !SQLProceduralBlock.runs(statement, grammar: grammar) else { return false } + return changesOnlyRows(StatementCode(statement, grammar: grammar), grammar: grammar, depth: 0) + } + + private func changesOnlyRows(_ code: StatementCode, grammar: SQLLexicalGrammar, depth: Int) -> Bool { + guard depth <= Self.deepestNesting else { return false } + let calls = SQLFunctionCallScanner.calls(in: code.text, code: code.code as NSString) + guard !changesSessionOrServer(code, calls: calls, grammar: grammar), + dynamicSQLClassification(code, grammar: grammar, depth: depth) == nil, + Self.conditionalCommentClassification(code.statement, grammar: grammar) == nil + else { return false } + if SQLExplainStatement.keywords.contains(code.keyword) { + return explainedStatementChangesOnlyRows(code, grammar: grammar, depth: depth) + } + if code.keyword == "WITH" { + return commonTableExpressionsChangeOnlyRows(code, grammar: grammar, depth: depth) + } + return keywordChangesOnlyRows(code, grammar: grammar) + } + + // A plain EXPLAIN already read as safe; what is left runs its statement, writes its plan or stores statistics. + private func explainedStatementChangesOnlyRows(_ code: StatementCode, grammar: SQLLexicalGrammar, depth: Int) -> Bool { + guard let explain = SQLExplainStatement.read(code.statement, grammar: grammar), + case .explains(let start, true) = explain.reading, + !code.tokens.contains(where: { $0.location < start && $0.word == "INTO" }) + else { return false } + let explained = StatementCode(code.text.substring(from: start), grammar: grammar) + if explain.keyword == "ANALYZE", explained.keyword == "TABLE" { return false } + return changesOnlyRows(explained, grammar: grammar, depth: depth + 1) + } + + private func commonTableExpressionsChangeOnlyRows( + _ code: StatementCode, + grammar: SQLLexicalGrammar, + depth: Int + ) -> Bool { + guard depth <= Self.deepestNesting, + let list = SQLCommonTableExpressions.read(code.statement, grammar: grammar) + else { return false } + let main = StatementCode(code.text.substring(from: list.mainStatement), grammar: grammar) + let bodies = list.bodies.map { StatementCode(code.text.substring(with: $0), grammar: grammar) } + return ([main] + bodies).allSatisfy { part in + part.keyword == "WITH" + ? commonTableExpressionsChangeOnlyRows(part, grammar: grammar, depth: depth + 1) + : keywordChangesOnlyRows(part, grammar: grammar) + } + } + + private func keywordChangesOnlyRows(_ code: StatementCode, grammar: SQLLexicalGrammar) -> Bool { + let keyword = code.keyword + // `SELECT ... INTO` makes a table on PostgreSQL and SQL Server, and sets a variable on MySQL. + if isQuery(keyword) { return !code.contains("INTO") } + let changesRows: Bool + if keyword == "REPLACE" { + changesRows = Self.mySQLFamily.contains(databaseTypeId) + || Self.word(after: keyword, in: code).map(Self.replaceRowWords.contains) == true + } else if keyword == "ALTER" { + changesRows = databaseTypeId == Self.clickHouse && Self.changesClickHouseRows(code) + } else { + changesRows = (rowChanges ?? Self.rowChangeKeywords).contains(keyword) + && !Self.updatesStatistics(code) && !Self.insertsProcedureResults(code) + } + return changesRows && !namesSystemRelation(code, grammar: grammar) + } + + // A call that sets the session, signals the server or runs work elsewhere, which a SELECT or an INSERT can carry. A + // sequence's next value is a row's key, so it passes. + private func changesSessionOrServer( + _ code: StatementCode, + calls: [SQLFunctionCallScanner.Call], + grammar: SQLLexicalGrammar + ) -> Bool { + let callsOne = calls.contains { call in + guard case .named(let name, _) = call.callee else { return true } + if SQLStateChangingFunctions.sequences.contains(name.uppercased().lowercased()) { return false } + return SQLStateChangingFunctions.contains(name) + || (databaseTypeId == Self.duckDB && SQLStateChangingFunctions.isDuckDBCall(name)) + } + return callsOne + || SQLStateChangingFunctions.namesStateChangingTable(in: code.text, tokens: code.tokens, grammar: grammar) + } + + // Any qualifier naming a system schema, or on the PostgreSQL family any `pg_` relation, since an unqualified one + // resolves to pg_catalog first. A row change that only reads one in a subquery is refused too. + private func namesSystemRelation(_ code: StatementCode, grammar: SQLLexicalGrammar) -> Bool { + let schemas = Self.systemSchemas(for: databaseTypeId) + let prefixes = Self.systemRelationPrefixes(for: databaseTypeId) + guard !schemas.isEmpty || !prefixes.isEmpty else { return false } + return code.tokens.indices.contains { index in + if !prefixes.isEmpty, Self.spellsEscapedName(at: index, in: code) { return true } + guard let name = Self.identifier(at: index, in: code, grammar: grammar) else { return false } + if schemas.contains(name), code.isSymbol(Self.period, at: index + 1) { return true } + let isCall = code.tokens.indices.contains(index + 1) && code.tokens[index + 1].kind == .open + return !isCall && prefixes.contains(where: name.hasPrefix) + } + } + + // A word, or the name inside an identifier quote, in capitals. + private static func identifier(at index: Int, in code: StatementCode, grammar: SQLLexicalGrammar) -> String? { + let token = code.tokens[index] + if let word = token.word { return word } + guard token.kind == .quoted, + let span = SQLNonCodeSpan.span(at: token.location, in: code.text, grammar: grammar), + identifierQuotes.contains(code.text.character(at: span.start)) + else { return nil } + let inner = NSRange(location: span.start + 1, length: max(0, span.contentEnd - span.start - 1)) + return code.text.substring(with: inner).uppercased() + } + + // PostgreSQL's `U&"pg\005fsettings"` spells a name in escapes this reader does not decode: the word U, then `&`, + // then the quoted name, with nothing between them. + private static func spellsEscapedName(at index: Int, in code: StatementCode) -> Bool { + let tokens = code.tokens + return index >= 2 && tokens[index].kind == .quoted && tokens[index - 2].word == "U" + && code.isSymbol(ampersand, at: index - 1) + && tokens[index - 1].location == tokens[index - 2].location + 1 + && tokens[index].location == tokens[index - 1].location + 1 + } + + private static let ampersand = UInt16(UnicodeScalar("&").value) + + private static func word(after keyword: String, in code: StatementCode) -> String? { + guard let verb = code.tokens.firstIndex(where: { $0.word == keyword }), + code.tokens.indices.contains(verb + 1) + else { return nil } + return code.tokens[verb + 1].word + } + + // `ALTER TABLE t [ON CLUSTER c] UPDATE ... WHERE` and `... DELETE WHERE` are the row changes ClickHouse's grid + // writes with. Any other action in the same ALTER defines. + private static func changesClickHouseRows(_ code: StatementCode) -> Bool { + let tokens = code.tokens + guard let verb = tokens.firstIndex(where: { $0.word == "ALTER" }), + tokens.indices.contains(verb + 2), tokens[verb + 1].word == "TABLE" + else { return false } + let depth = tokens[verb].depth + var index = verb + 2 + while tokens.indices.contains(index + 2), code.isSymbol(period, at: index + 1) { + index += 2 + } + index += 1 + if tokens.indices.contains(index + 2), tokens[index].word == "ON", tokens[index + 1].word == "CLUSTER" { + index += 3 + } + guard tokens.indices.contains(index), tokens[index].word == "UPDATE" || tokens[index].word == "DELETE" else { + return false + } + return !tokens.indices.dropFirst(index + 1).contains { later in + tokens[later].kind == .comma && tokens[later].depth == depth && tokens.indices.contains(later + 1) + && tokens[later + 1].word.map(clickHouseAlterActions.contains) == true + } + } + + // T-SQL's `UPDATE STATISTICS t` rebuilds statistics. A table named statistics is followed by SET, and T-SQL's own + // reader cuts that SET off as a statement of its own. + private static func updatesStatistics(_ code: StatementCode) -> Bool { + guard code.keyword == "UPDATE", + let verb = code.tokens.firstIndex(where: { $0.word == "UPDATE" }), + code.tokens.indices.contains(verb + 1), code.tokens[verb + 1].word == "STATISTICS" + else { return false } + return !(code.tokens.indices.contains(verb + 2) && code.tokens[verb + 2].word == "SET") + } + + // T-SQL's `INSERT INTO t EXEC proc` runs the procedure and stores what it returns. + private static func insertsProcedureResults(_ code: StatementCode) -> Bool { + guard code.keyword == "INSERT" else { return false } + return code.tokens.contains { $0.depth == 0 && ($0.word == "EXEC" || $0.word == "EXECUTE") } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+Surfaces.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+Surfaces.swift index dbb2cd9348..3535708a15 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+Surfaces.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier+Surfaces.swift @@ -139,6 +139,18 @@ extension SQLStatementTiering { } } + // SurrealQL's fn:: and api:: calls run a body the database stores, which can define, grant or write, and file:: + // reads and writes a bucket. Measured on 2.7.0 and 3.3.5: `SELECT fn::mk()` ran the DEFINE TABLE in its body. + func callsSurrealStoredCode(_ code: StatementCode) -> Bool { + guard databaseTypeId == Self.surrealDB else { return false } + return code.tokens.indices.contains { index in + code.tokens[index].word.map(Self.surrealStoredCodeNamespaces.contains) == true + && code.isSymbol(Self.colon, at: index + 1) && code.isSymbol(Self.colon, at: index + 2) + } + } + + private static let surrealStoredCodeNamespaces: Set = ["FN", "API", "FILE"] + // KingbaseES has it in Oracle mode only, and its type id cannot say which mode a server runs. static let enginesWithSequencePseudoColumns: Set = [ "Oracle", "SAP HANA", "Snowflake", "Dameng", "MariaDB", "KingbaseES", diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier.swift index bec2495864..1830fa21c3 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementClassifier.swift @@ -121,6 +121,7 @@ struct SQLStatementTiering { let databaseTypeId: String let readings: SQLLexicalReadings let request: SQLStatementClassifier.RequestClassification? + var rowChanges: Set? // Each nested statement copies and reads what is left of the text, so nesting past this is refused. static let deepestNesting = 32 @@ -169,7 +170,9 @@ struct SQLStatementTiering { guard case .explains(let start, true) = explain.reading else { return false } return statementDeletesEveryRow((statement as NSString).substring(from: start), grammar: grammar) } - return SQLUnfilteredWrite.writes(in: statement, grammar: grammar).contains(.delete) + let readsRecordIds = databaseTypeId == Self.surrealDB + return SQLUnfilteredWrite.writes(in: statement, grammar: grammar, readsRecordIds: readsRecordIds) + .contains(.delete) } func sqlClassification(_ statement: String, grammar: SQLLexicalGrammar, depth: Int = 0) -> StatementClassification { @@ -277,6 +280,11 @@ extension SQLStatementTiering { if isQuery(keyword) { return code.contains("INTO") ? .write : .safe } + // `INSERT OVERWRITE` empties the table first on Snowflake, Databricks and Hive; Hive's `... DIRECTORY` writes files. + if keyword == "INSERT", let verb = code.tokens.firstIndex(where: { $0.kind != .open }), + code.tokens.indices.contains(verb + 1), code.tokens[verb + 1].word == "OVERWRITE" { + return StatementClassification(tier: .destructive, reachesFilesystemOrExecutesCode: code.contains("DIRECTORY")) + } if keyword == "VACUUM" { // SQLite's `VACUUM [schema] INTO 'file'` writes a file whatever stands between the two words. return StatementClassification(tier: .write, reachesFilesystemOrExecutesCode: code.contains("INTO")) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementCursorIndex.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementCursorIndex.swift new file mode 100644 index 0000000000..af948724c7 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementCursorIndex.swift @@ -0,0 +1,74 @@ +import Foundation + +// Answers `SQLStatementScanner.locatedStatementAtCursor` for a document that changes a little at a time: the scan +// resumes from the last recorded statement boundary before the caret instead of from the top. +public struct SQLStatementCursorIndex: Sendable { + public static let defaultCheckpointSpacing = 4_096 + + public let grammar: SQLLexicalGrammar + public private(set) var unitsScanned = 0 + + private let checkpointSpacing: Int + // In offset order, and so in `decidedThrough` order too. + private var checkpoints: [SQLStatementScanner.Checkpoint] = [] + + public init(grammar: SQLLexicalGrammar, checkpointSpacing: Int = SQLStatementCursorIndex.defaultCheckpointSpacing) { + self.grammar = grammar + self.checkpointSpacing = max(1, checkpointSpacing) + } + + public var checkpointCount: Int { + checkpoints.count + } + + // Call before the next lookup whenever the text changed at or after `location`. + public mutating func textDidChange(at location: Int) { + while let last = checkpoints.last, last.decidedThrough >= location { + checkpoints.removeLast() + } + } + + public mutating func statementAtCursor(_ cursorPosition: Int, in text: String) -> SQLStatementScanner.LocatedStatement { + let length = (text as NSString).length + let cursor = min(max(0, cursorPosition), length) + let resume = lastCheckpoint(before: cursor) + let start = resume?.offset ?? 0 + let spacing = checkpointSpacing + + var lastRecorded = checkpoints.last?.offset ?? 0 + var recorded: [SQLStatementScanner.Checkpoint] = [] + var result = SQLStatementScanner.LocatedStatement(sql: "", offset: 0, hasContent: false) + SQLStatementScanner.scan( + sql: text, + cursorPosition: cursor, + grammar: grammar, + resumingAt: resume, + onCheckpoint: { checkpoint in + guard checkpoint.offset - lastRecorded >= spacing else { return } + recorded.append(checkpoint) + lastRecorded = checkpoint.offset + }, + onStatement: { statement in + result = statement + return false + } + ) + checkpoints.append(contentsOf: recorded) + unitsScanned += max(cursor, NSMaxRange(result.range)) - start + return result + } + + private func lastCheckpoint(before offset: Int) -> SQLStatementScanner.Checkpoint? { + var low = 0 + var high = checkpoints.count + while low < high { + let middle = (low + high) / 2 + if checkpoints[middle].offset < offset { + low = middle + 1 + } else { + high = middle + } + } + return low > 0 ? checkpoints[low - 1] : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScan.swift new file mode 100644 index 0000000000..4596a59245 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScan.swift @@ -0,0 +1,357 @@ +import Foundation + +/// One pass of ``SQLStatementScanner`` over a text. +/// +/// Where a statement ends is the tracker's decision, except at a `GO` line, which ends the batch whatever the tracker +/// thinks and is a segment of its own that runs nothing, and at a client delimiter, which ends the statement wherever +/// it stands in code. A `DELIMITER` line and a psql `\restrict` line are segments that run nothing, and a +/// `COPY ... FROM STDIN` runs on through the rows after it. +/// +/// A custom delimiter does not stop a `;` from ending a statement where the tracker says it does: the server still +/// splits the text it gets there, so reading it as one statement would let a gate miss the second. +struct SQLStatementScan { + struct Events { + let checkpoint: (SQLStatementScanner.Checkpoint) -> Void + let batchSeparator: (SQLBatchSeparator) -> Void + let scriptText: (NSRange) -> Void + let statement: (SQLStatementScanner.LocatedStatement) -> Bool + } + + private enum Step { + case declined + case consumed + case finished + } + + private let text: NSString + private let length: Int + private let grammar: SQLLexicalGrammar + private let cursor: Int? + private let events: Events + + private var tracker: any SQLStatementBoundaryTracking + private var currentStart: Int + private var index: Int + private var hasStatementContent = false + private var lastEndedStatement: SQLStatementScanner.EndedStatement? + private var delimiter: SQLClientDelimiter? + + // Where a `;` ended the statement while the tracker could still take it back at the next token, and the further + // `;`s after it, which T-SQL also reads as part of an `IF`'s body before its `ELSE`. + private var heldEnd: Int? + private var heldSemicolons: [Int] = [] + + private static let smallD = UInt16(UnicodeScalar("d").value) + + init( + text: NSString, + cursorPosition: Int?, + grammar: SQLLexicalGrammar, + resume: SQLStatementScanner.Checkpoint?, + events: Events + ) { + self.text = text + self.length = text.length + self.grammar = grammar + self.cursor = cursorPosition.map { min(max(0, $0), text.length) } + self.events = events + self.tracker = SQLStatementBoundaries.makeTracker(for: grammar) + self.currentStart = min(max(0, resume?.offset ?? 0), text.length) + self.index = currentStart + self.lastEndedStatement = resume?.endedStatement + self.delimiter = resume?.delimiter + } + + var holdsOpenBlock: Bool { + tracker.holdsOpenBlock + } + + /// Returns false when the scan stopped early, because the cursor's segment was found or a reader asked to stop. + mutating func run() -> Bool { + while index < length { + let unit = text.character(at: index) + var step = readClientLine(unit) + if step == .declined { step = readDelimiter(unit) } + if step == .declined { step = readNonCode(unit) } + if step == .declined { step = readBatchSeparator(unit) } + if step == .declined { step = readWord(unit) } + if step == .declined { step = readSemicolon(unit) } + if step == .declined { step = readSlashLine(unit) } + switch step { + case .finished: + return false + case .consumed: + continue + case .declined: + guard readBlankOrSymbol(unit) else { return false } + } + } + guard settleHeldEnd(word: nil, decidedThrough: length) else { return false } + if currentStart < length { + _ = events.statement(SQLStatementScanner.LocatedStatement( + sql: text.substring(with: NSRange(location: currentStart, length: length - currentStart)), + offset: currentStart, + hasContent: hasStatementContent, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters, + clientDelimiter: delimiter + )) + } + return true + } + + // MARK: - Tokens + + private mutating func readClientLine(_ unit: UInt16) -> Step { + guard !hasStatementContent else { return .declined } + if grammar.contains(.delimiterDirective), unit | 0x20 == Self.smallD, + let directive = SQLDelimiterDirective.line(at: index, in: text, length: length) { + if let set = directive.delimiter { + delimiter = SQLClientDelimiter(set) + } + return finishClientLine(at: NSMaxRange(directive.range)) + } + if grammar.contains(.psqlScripts), unit == SqlLexer.backslash, + let line = SQLPsqlScript.restrictLine(at: index, in: text, length: length) { + events.scriptText(line) + return finishClientLine(at: NSMaxRange(line)) + } + return .declined + } + + private mutating func readDelimiter(_ unit: UInt16) -> Step { + guard let delimiter, delimiter.starts(in: text, at: index, length: length) else { return .declined } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + let end = index + delimiter.units.count + guard finishSegment(at: end, hasContent: hasStatementContent, delimiterLength: delimiter.units.count) else { + return .finished + } + endStatement(at: end, decidedThrough: end - 1) + return .consumed + } + + private mutating func readNonCode(_ unit: UInt16) -> Step { + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) else { return .declined } + if !span.kind.isComment { + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + } + switch span.kind { + case .lineComment, .blockComment: + tracker.observeGap() + case .executableComment: + hasStatementContent = true + case .quoted: + hasStatementContent = true + tracker.observeQuotedToken(opener: text.character(at: span.start)) + case .parameter: + hasStatementContent = true + tracker.observeOpaqueToken() + } + index = max(span.end, index + 1) + return .consumed + } + + private mutating func readBatchSeparator(_ unit: UInt16) -> Step { + guard grammar.contains(.batchSeparatorLines), + let separator = SQLBatchSeparator.line(at: index, in: text, length: length, grammar: grammar) + else { + return .declined + } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + events.batchSeparator(separator) + if hasStatementContent { + guard finishSegment(at: index, hasContent: true) else { return .finished } + } + let separatorEnd = NSMaxRange(separator.range) + guard finishSegment(at: separatorEnd, hasContent: false, standsForEndedStatement: true) else { + return .finished + } + lastEndedStatement = nil + endStatement(at: separatorEnd, decidedThrough: separatorEnd) + return .consumed + } + + private mutating func readWord(_ unit: UInt16) -> Step { + guard SqlBlockStructure.startsWord(text, at: index, length: length, grammar: grammar) else { return .declined } + guard heldEnd != nil || tracker.needsWords else { + hasStatementContent = true + index += 1 + while index < length, SqlBlockStructure.continuesWord(text.character(at: index), grammar: grammar), + delimiter?.starts(in: text, at: index, length: length) != true { + index += 1 + } + return .consumed + } + let read = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) + let word = delimiter?.cut(read, readAt: index, in: text, length: length) ?? read + guard settleHeldEnd(word: word.text, decidedThrough: word.end) else { return .finished } + hasStatementContent = true + if tracker.needsWords { + tracker.observeWord(word.text) + } + index = word.end + return .consumed + } + + private mutating func readSemicolon(_ unit: UInt16) -> Step { + guard unit == SqlLexer.semicolon else { return .declined } + if heldEnd != nil { + _ = tracker.observeSemicolon() + heldSemicolons.append(index) + index += 1 + return .consumed + } + guard tracker.observeSemicolon() else { + hasStatementContent = true + index += 1 + return .consumed + } + if tracker.awaitsContinuation { + heldEnd = index + 1 + index += 1 + return .consumed + } + if tracker.copiesFromStandardInput { + let rows = SQLPsqlScript.copyRows(afterSemicolonAt: index, in: text, length: length) + let end = NSMaxRange(rows) + let segment = NSRange(location: currentStart, length: end - currentStart) + // The rows run to `end` as written: a last row's trailing tab is a column. + let start = currentStart + StatementBlank.contentRange(of: text.substring(with: segment)).location + events.scriptText(NSRange(location: start, length: end - start)) + guard finishSegment(at: end, hasContent: hasStatementContent, copyRows: rows) else { return .finished } + endStatement(at: end, decidedThrough: end + 1) + return .consumed + } + guard finishSegment(at: index + 1, hasContent: hasStatementContent) else { return .finished } + // A delimiter such as `;;` was looked for past this `;` before the `;` alone was read. + endStatement(at: index + 1, decidedThrough: index + max(0, (delimiter?.units.count ?? 1) - 1)) + return .consumed + } + + private mutating func readSlashLine(_ unit: UInt16) -> Step { + guard grammar.contains(.slashLineTerminators), unit == SqlLexer.slash, + SQLStatementScanner.isSlashLine(text, at: index, length: length) + else { + return .declined + } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + if hasStatementContent { + guard finishSegment(at: index, hasContent: true) else { return .finished } + } + guard finishSegment(at: index + 1, hasContent: false, standsForEndedStatement: true) else { return .finished } + endStatement( + at: index + 1, + decidedThrough: SQLStatementScanner.endOfSlashLine(text, at: index, length: length) + ) + return .consumed + } + + /// Returns false when the scan is over. + private mutating func readBlankOrSymbol(_ unit: UInt16) -> Bool { + let blankLength = StatementBlank.blankLength(in: text, at: index) + if blankLength > 0 { + tracker.observeGap() + index += blankLength + return true + } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return false } + hasStatementContent = true + tracker.observeSymbol(unit) + index += 1 + return true + } + + // MARK: - Segments + + // A caret on a SQL*Plus `/` line stands for the statement the slash ends, where `Cmd+Enter` is expected to act. + // A caret on a `GO` line stands for the last statement of its batch, and for nothing when the batch is empty. + // Returns false once the scan has reported what it was asked for. + private mutating func finishSegment( + at end: Int, + hasContent: Bool, + standsForEndedStatement: Bool = false, + delimiterLength: Int = 0, + copyRows: NSRange? = nil, + isClientLine: Bool = false + ) -> Bool { + let range = NSRange(location: currentStart, length: end - currentStart) + if hasContent { + lastEndedStatement = SQLStatementScanner.EndedStatement( + range: range, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters, + delimiterLength: delimiterLength, + copyRows: copyRows, + clientDelimiter: delimiter + ) + } + if let cursor { + guard cursor >= currentStart, cursor <= end else { + currentStart = end + return true + } + if standsForEndedStatement, let ended = lastEndedStatement { + _ = events.statement(ended.located(in: text)) + return false + } + } + let statement = SQLStatementScanner.LocatedStatement( + sql: text.substring(with: range), + offset: currentStart, + hasContent: hasContent, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters, + delimiterLength: delimiterLength, + copyRows: copyRows, + clientDelimiter: delimiter, + isClientLine: isClientLine + ) + if cursor != nil { + _ = events.statement(statement) + return false + } + currentStart = end + return events.statement(statement) + } + + private mutating func endStatement(at end: Int, decidedThrough: Int) { + tracker.reset() + hasStatementContent = false + checkpoint(at: end, decidedThrough: decidedThrough) + index = end + } + + private func checkpoint(at offset: Int, decidedThrough: Int) { + events.checkpoint(SQLStatementScanner.Checkpoint( + offset: offset, + decidedThrough: decidedThrough, + endedStatement: lastEndedStatement, + delimiter: delimiter + )) + } + + // A `DELIMITER` or `\restrict` line ends a segment that runs nothing. + private mutating func finishClientLine(at end: Int) -> Step { + guard finishSegment(at: end, hasContent: false, isClientLine: true) else { return .finished } + endStatement(at: end, decidedThrough: end) + return .consumed + } + + // Settles a held end at the first token after it; returns false once the scan has reported its statement. + private mutating func settleHeldEnd(word: String?, decidedThrough: Int) -> Bool { + guard let end = heldEnd else { return true } + heldEnd = nil + let semicolons = heldSemicolons + heldSemicolons.removeAll() + if tracker.continuesStatement(with: word) { return true } + guard finishSegment(at: end, hasContent: hasStatementContent) else { return false } + tracker.reset() + hasStatementContent = false + checkpoint(at: end, decidedThrough: decidedThrough) + for semicolon in semicolons { + guard finishSegment(at: semicolon + 1, hasContent: false) else { return false } + checkpoint(at: semicolon + 1, decidedThrough: decidedThrough) + } + return true + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift index 062c6802fb..65270ddfd5 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift @@ -7,6 +7,20 @@ public enum SQLStatementScanner { public let hasContent: Bool public let terminator: SQLStatementTerminator public let acceptsBindParameters: Bool + + /// How many units at the end of ``sql`` are a client delimiter such as `//`, which no server receives. Zero when + /// a `;` or nothing ends the statement. + public let delimiterLength: Int + + /// For a `COPY ... FROM STDIN`, everything after its `;` that a psql script holds as rows, in the document's + /// coordinates. None of it is SQL. + public let copyRows: NSRange? + + /// The client delimiter in force once the statement ends. + public let clientDelimiter: SQLClientDelimiter? + + /// A `DELIMITER` or psql `\restrict` line, which the client reads and no server receives. + public let isClientLine: Bool private let isAllContent: Bool public init( @@ -14,13 +28,21 @@ public enum SQLStatementScanner { offset: Int, hasContent: Bool = true, terminator: SQLStatementTerminator = .separator, - acceptsBindParameters: Bool = true + acceptsBindParameters: Bool = true, + delimiterLength: Int = 0, + copyRows: NSRange? = nil, + clientDelimiter: SQLClientDelimiter? = nil, + isClientLine: Bool = false ) { self.sql = sql self.offset = offset self.hasContent = hasContent self.terminator = terminator self.acceptsBindParameters = acceptsBindParameters + self.delimiterLength = delimiterLength + self.copyRows = copyRows + self.clientDelimiter = clientDelimiter + self.isClientLine = isClientLine isAllContent = false } @@ -31,6 +53,10 @@ public enum SQLStatementScanner { hasContent = true terminator = .separator acceptsBindParameters = true + delimiterLength = 0 + copyRows = nil + clientDelimiter = nil + isClientLine = false isAllContent = true } @@ -44,10 +70,16 @@ public enum SQLStatementScanner { /// `offset` is the index just past the previous semicolon, so in a script written one statement per line it /// lands on the newline that ended the previous line. A decoration or a gutter anchor placed from ``range`` /// therefore starts a line early, and uses this instead. + /// + /// A client delimiter is left out: Run Current scans the text again on its own, where no `DELIMITER` line + /// precedes it, so the delimiter would reach the server. public var contentRange: NSRange { guard !isAllContent else { return range } - let content = StatementBlank.contentRange(of: sql) - return NSRange(location: offset + content.location, length: content.length) + let text = sql as NSString + let content = StatementBlank.contentRange(of: text.substring(to: max(0, text.length - delimiterLength))) + // A COPY's rows run to their end as written: a last row's trailing tab is a column. + let end = copyRows.map { max(NSMaxRange(content), NSMaxRange($0) - offset) } ?? NSMaxRange(content) + return NSRange(location: offset + content.location, length: end - content.location) } } @@ -60,7 +92,7 @@ public enum SQLStatementScanner { /// The range is relative to the text the scan was given. A run started from a selection or from a single /// statement scans a fragment, so those callers shift the range onto the tab's whole query with ``offset(by:)`` /// before it travels any further. Everything downstream may then assume tab coordinates. - public struct ExecutableStatement: Sendable { + public struct ExecutableStatement: Sendable, Equatable { public let sql: String public let range: NSRange @@ -68,10 +100,28 @@ public enum SQLStatementScanner { /// ``SQLStatementBoundaryTracking/acceptsBindParameters``. public let acceptsBindParameters: Bool - public init(sql: String, range: NSRange, acceptsBindParameters: Bool = true) { + /// Where the scan went on after this statement: past its `;`, a client delimiter or a `COPY`'s rows. A run + /// that splits the rest of a script again starts there, with ``clientDelimiter`` in force. + public let segmentEnd: Int + public let clientDelimiter: SQLClientDelimiter? + + // A `COPY ... FROM STDIN`'s rows as psql sends them, empty for a block with none; nil for any other statement. + public let copyData: String? + + public init( + sql: String, + range: NSRange, + acceptsBindParameters: Bool = true, + segmentEnd: Int? = nil, + clientDelimiter: SQLClientDelimiter? = nil, + copyData: String? = nil + ) { self.sql = sql self.range = range self.acceptsBindParameters = acceptsBindParameters + self.segmentEnd = segmentEnd ?? NSMaxRange(range) + self.clientDelimiter = clientDelimiter + self.copyData = copyData } public func offset(by delta: Int) -> ExecutableStatement { @@ -79,7 +129,10 @@ public enum SQLStatementScanner { return ExecutableStatement( sql: sql, range: NSRange(location: range.location + delta, length: range.length), - acceptsBindParameters: acceptsBindParameters + acceptsBindParameters: acceptsBindParameters, + segmentEnd: segmentEnd + delta, + clientDelimiter: clientDelimiter, + copyData: copyData ) } } @@ -179,6 +232,16 @@ public enum SQLStatementScanner { return separators } + /// The text of a psql script a formatter keeps as written, in document order: each `COPY ... FROM STDIN` with the + /// rows after it through the `\.` line, and pg_dump's `\restrict` and `\unrestrict` lines. Empty for an engine + /// whose scripts are not psql's. + public static func psqlScriptText(in sql: String, grammar: SQLLexicalGrammar) -> [NSRange] { + guard grammar.contains(.psqlScripts) else { return [] } + var ranges: [NSRange] = [] + scan(sql: sql, cursorPosition: nil, grammar: grammar, onScriptText: { ranges.append($0) }) { _ in true } + return ranges + } + /// Returns statements as the driver receives them, for driver execution. public static func allStatements(in sql: String, grammar: SQLLexicalGrammar) -> [String] { executableStatements(in: sql, grammar: grammar).map(\.sql) @@ -215,10 +278,14 @@ public enum SQLStatementScanner { /// /// A `;` that belongs to the statement stays, but only when there is something before it: a unit reduced to its /// terminator is as empty as any other. + /// + /// A client delimiter never reaches the server, and a `COPY ... FROM STDIN` carries its rows apart from its text, + /// in ``ExecutableStatement/copyData``. public static func executableStatement(from located: LocatedStatement) -> ExecutableStatement? { + guard !located.isClientLine else { return nil } let rawSQL = located.sql - var content = StatementBlank.trimming(rawSQL[...]) - if content.last == ";" { + var content = StatementBlank.trimming(rawSQL[.. String? { + guard let rows = located.copyRows else { return nil } + let text = located.sql as NSString + let local = NSRange(location: rows.location - located.offset, length: rows.length) + return text.substring(with: SQLPsqlScript.copyData(inRows: local, of: text)) + } + + /// The statements after `ran`, one of those `sql` was split into, split again under `grammar`, up to `end`: once + /// a statement changes how the session reads literals, the rest no longer splits as planned. The split starts at + /// ``ExecutableStatement/segmentEnd``, so a `COPY`'s rows are never read as statements, and keeps the client + /// delimiter in force there. + public static func executableStatements( + in sql: String, + after ran: ExecutableStatement, + through end: Int, + grammar: SQLLexicalGrammar + ) -> [ExecutableStatement] { + let text = sql as NSString + let limit = min(end, text.length) + guard ran.segmentEnd < limit else { return [] } + let resume = Checkpoint( + offset: ran.segmentEnd, + decidedThrough: ran.segmentEnd, + endedStatement: nil, + delimiter: ran.clientDelimiter ) + var results: [ExecutableStatement] = [] + scan(sql: text.substring(to: limit), cursorPosition: nil, grammar: grammar, resumingAt: resume) { located in + if located.hasContent, let statement = executableStatement(from: located) { + results.append(statement) + } + return true + } + return results + } + + private static func sentEnd(of located: LocatedStatement) -> String.Index { + let length = located.sql.utf16.count + let end = located.copyRows.map { $0.location - located.offset } ?? length - located.delimiterLength + return String.Index(utf16Offset: min(max(0, end), length), in: located.sql) } /// Whether the text ends inside a block it opened, so the last statement swallowed everything after its opener. @@ -248,7 +360,7 @@ public enum SQLStatementScanner { var results: [String] = [] scan(sql: sql, cursorPosition: nil, grammar: grammar) { statement in guard statement.hasContent else { return true } - let trimmed = StatementBlank.trimming(statement.sql) + let trimmed = StatementBlank.trimming(String(statement.sql[.. LocatedStatement { + LocatedStatement( + sql: text.substring(with: range), + offset: range.location, + hasContent: true, + terminator: terminator, + acceptsBindParameters: acceptsBindParameters, + delimiterLength: delimiterLength, + copyRows: copyRows, + clientDelimiter: clientDelimiter + ) + } + } - /// Walks the document once and hands each segment to `onStatement`, which returns false to stop. - /// - /// Segments tile the document: a statement runs from the end of the previous one to its terminator, so it carries - /// the whitespace before it. With a `cursorPosition`, only the segment holding it is reported, or the last one - /// when the cursor sits past every terminator. Where a statement ends is the tracker's decision; this loop only - /// lexes, so a `;` inside a string, a comment or a quoted body never reaches it. A `GO` line is the exception: it - /// ends the batch, so it ends whatever statement is open whatever the tracker thinks, and is a segment of its own - /// that holds nothing to run. - private static func scan( + // A point where the scan holds no state but the last statement with content and the client delimiter in force, + // so it can start again there. + struct Checkpoint: Sendable, Equatable { + let offset: Int + // The last offset whose text decided this boundary: an edit there or earlier moves it. + let decidedThrough: Int + let endedStatement: EndedStatement? + var delimiter: SQLClientDelimiter? + } + + // MARK: - Scanning + + // Segments tile the document: each runs from the end of the previous one to its terminator, whitespace included. + // With a cursor only the segment holding it is reported, or the last one when it sits past every terminator. + // See ``SQLStatementScan`` for where a segment ends. + static func scan( sql: String, cursorPosition: Int?, grammar: SQLLexicalGrammar, - onBatchSeparator: (SQLBatchSeparator) -> Void = { _ in }, + resumingAt resume: Checkpoint? = nil, + onCheckpoint: @escaping (Checkpoint) -> Void = { _ in }, + onBatchSeparator: @escaping (SQLBatchSeparator) -> Void = { _ in }, + onScriptText: @escaping (NSRange) -> Void = { _ in }, onFinish: (_ holdsOpenBlock: Bool) -> Void = { _ in }, - onStatement: (LocatedStatement) -> Bool + onStatement: @escaping (LocatedStatement) -> Bool ) { - let nsQuery = sql as NSString - let length = nsQuery.length - guard length > 0 else { return } - - let safePosition = cursorPosition.map { min(max(0, $0), length) } - - var tracker = SQLStatementBoundaries.makeTracker(for: grammar) - var currentStart = 0 - var hasStatementContent = false - var i = 0 - - var lastStatementWithContent: LocatedStatement? - - /// Reports the segment ending at `end` and starts the next one there. Returns false when the scan is done. - /// - /// A caret on a SQL*Plus `/` line stands for the statement the slash ends, which is where a reader who has - /// just typed the slash expects `Cmd+Enter` to act. A caret on a `GO` line stands for the last statement of the - /// batch it ends, and for nothing when that batch is empty: the batch before is not the one the line ends. - func finishSegment(at end: Int, hasContent: Bool, standsForEndedStatement: Bool = false) -> Bool { - let statement = LocatedStatement( - sql: nsQuery.substring(with: NSRange(location: currentStart, length: end - currentStart)), - offset: currentStart, - hasContent: hasContent, - terminator: tracker.terminator, - acceptsBindParameters: tracker.acceptsBindParameters + let text = sql as NSString + guard text.length > 0 else { return } + var scan = SQLStatementScan( + text: text, + cursorPosition: cursorPosition, + grammar: grammar, + resume: resume, + events: SQLStatementScan.Events( + checkpoint: onCheckpoint, + batchSeparator: onBatchSeparator, + scriptText: onScriptText, + statement: onStatement ) - if hasContent { - lastStatementWithContent = statement - } - if let cursor = safePosition { - guard cursor >= currentStart, cursor <= end else { - currentStart = end - return true - } - _ = onStatement(standsForEndedStatement ? lastStatementWithContent ?? statement : statement) - return false - } - currentStart = end - return onStatement(statement) - } - - while i < length { - let ch = nsQuery.character(at: i) - - if let span = SQLNonCodeSpan.span(at: i, in: nsQuery, grammar: grammar) { - switch span.kind { - case .lineComment, .blockComment: - tracker.observeGap() - case .executableComment: - hasStatementContent = true - case .quoted: - hasStatementContent = true - tracker.observeQuotedToken(opener: nsQuery.character(at: span.start)) - case .parameter: - hasStatementContent = true - tracker.observeOpaqueToken() - } - i = max(span.end, i + 1) - continue - } - - if grammar.contains(.batchSeparatorLines), - let separator = SQLBatchSeparator.line(at: i, in: nsQuery, length: length, grammar: grammar) { - onBatchSeparator(separator) - if hasStatementContent { - guard finishSegment(at: i, hasContent: true) else { return } - } - let separatorEnd = NSMaxRange(separator.range) - guard finishSegment(at: separatorEnd, hasContent: false, standsForEndedStatement: true) else { return } - lastStatementWithContent = nil - tracker.reset() - hasStatementContent = false - i = separatorEnd - continue - } - - if SqlBlockStructure.startsWord(nsQuery, at: i, length: length, grammar: grammar) { - hasStatementContent = true - if tracker.needsWords { - let word = SqlBlockStructure.readKeyword(nsQuery, at: i, length: length, grammar: grammar) - tracker.observeWord(word.text) - i = word.end - } else { - i += 1 - while i < length, SqlBlockStructure.continuesWord(nsQuery.character(at: i), grammar: grammar) { - i += 1 - } - } - continue - } - - if ch == SqlLexer.semicolon { - guard tracker.observeSemicolon() else { - hasStatementContent = true - i += 1 - continue - } - guard finishSegment(at: i + 1, hasContent: hasStatementContent) else { return } - tracker.reset() - hasStatementContent = false - i += 1 - continue - } - - if grammar.contains(.slashLineTerminators), ch == SqlLexer.slash, isSlashLine(nsQuery, at: i, length: length) { - if hasStatementContent { - guard finishSegment(at: i, hasContent: true) else { return } - } - guard finishSegment(at: i + 1, hasContent: false, standsForEndedStatement: true) else { return } - tracker.reset() - hasStatementContent = false - i += 1 - continue - } - - let blankLength = StatementBlank.blankLength(in: nsQuery, at: i) - if blankLength > 0 { - tracker.observeGap() - i += blankLength - continue - } - hasStatementContent = true - tracker.observeSymbol(ch) - i += 1 - } - - if currentStart < length { - let statement = LocatedStatement( - sql: nsQuery.substring(with: NSRange(location: currentStart, length: length - currentStart)), - offset: currentStart, - hasContent: hasStatementContent, - terminator: tracker.terminator, - acceptsBindParameters: tracker.acceptsBindParameters - ) - _ = onStatement(statement) - } - onFinish(tracker.holdsOpenBlock) + ) + guard scan.run() else { return } + onFinish(scan.holdsOpenBlock) } /// Whether the `/` at `offset` stands alone on its line, which is what makes it SQL*Plus's terminator rather than @@ -439,11 +463,17 @@ public enum SQLStatementScanner { before -= 1 } guard before < 0 || text.character(at: before) == SqlLexer.newline else { return false } + let after = endOfSlashLine(text, at: offset, length: length) + return after == length || text.character(at: after) == SqlLexer.newline + } + + // The last offset `isSlashLine` reads after the slash: its line break, or the end of the text. + static func endOfSlashLine(_ text: NSString, at offset: Int, length: Int) -> Int { var after = offset + 1 while after < length, isLineBlank(text.character(at: after)) { after += 1 } - return after == length || text.character(at: after) == SqlLexer.newline + return after } private static func isLineBlank(_ character: UInt16) -> Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift index 4639b4203e..e69c29480a 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnfilteredWrite.swift @@ -6,14 +6,28 @@ public enum SQLUnfilteredWrite: Sendable, Equatable { case delete public static func writes(in sql: String, grammar: SQLLexicalGrammar) -> [SQLUnfilteredWrite] { + writes(in: sql, grammar: grammar, readsRecordIds: false) + } + + static func writes(in sql: String, grammar: SQLLexicalGrammar, readsRecordIds: Bool) -> [SQLUnfilteredWrite] { SQLStatementScanner.executableStatements(in: sql, grammar: grammar).flatMap { statement in // On T-SQL a statement can begin inside another without a `;`. SQLUnterminatedStatements.runnable(in: statement.sql, grammar: grammar).flatMap { text in - writes(inStatement: text as NSString, grammar: grammar) + writes(inStatement: text as NSString, grammar: grammar, readsRecordIds: readsRecordIds) } } } + // SurrealQL's `DELETE person:tobie` has no WHERE and removes the one record its id names. + private static func writes( + inStatement source: NSString, + grammar: SQLLexicalGrammar, + readsRecordIds: Bool + ) -> [SQLUnfilteredWrite] { + if readsRecordIds, SurrealRecordDelete.namesOneRecord(source, grammar: grammar) { return [] } + return writes(inStatement: source, grammar: grammar) + } + private static let period = UInt16(UnicodeScalar(".").value) private static let whereLength = "WHERE".utf16.count @@ -152,3 +166,145 @@ public enum SQLUnfilteredWrite: Sendable, Equatable { return previous.kind == .other && source.character(at: previous.location) == period } } + +// SurrealDB 2.3 and 3.0 (`parse_delete_stmt`) read `DELETE [FROM] [ONLY]`, the targets, then WITH, WHERE, RETURN, +// TIMEOUT, PARALLEL and EXPLAIN. A `table:key` target with the key right after the colon names one record +// (`parse_record_id_or_range`); a `..` or `>` after the key makes it a range. A table, a parameter, a subquery, an +// edge (`t:a->e`), a field (`t:a.f`), a second target or a key this does not read is every row. +private struct SurrealRecordDelete { + private static let clauses: Set = ["WITH", "WHERE", "RETURN", "TIMEOUT", "PARALLEL", "EXPLAIN"] + private static let openBracket = UInt16(UnicodeScalar("[").value) + private static let openBrace = UInt16(UnicodeScalar("{").value) + private static let closers: [UInt16: UInt16] = [ + SqlLexer.openParen: SqlLexer.closeParen, + openBracket: UInt16(UnicodeScalar("]").value), + openBrace: UInt16(UnicodeScalar("}").value), + ] + private static let closing = Set(closers.values) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let period = UInt16(UnicodeScalar(".").value) + private static let plus = UInt16(UnicodeScalar("+").value) + private static let smallU = UInt16(UnicodeScalar("u").value) + + let text: NSString + let grammar: SQLLexicalGrammar + var index = 0 + + static func namesOneRecord(_ text: NSString, grammar: SQLLexicalGrammar) -> Bool { + var reader = SurrealRecordDelete(text: text, grammar: grammar) + return reader.readsOneRecordDelete() + } + + private mutating func readsOneRecordDelete() -> Bool { + skipBlanks() + guard word() == "DELETE" else { return false } + skipKeyword("FROM") + skipKeyword("ONLY") + skipBlanks() + guard readsName(), peek() == Self.colon else { return false } + index += 1 + guard readsKey() else { return false } + skipBlanks() + guard let next = peek() else { return true } + if next == SqlLexer.semicolon { + index += 1 + skipBlanks() + return peek() == nil + } + return word().map(Self.clauses.contains) == true + } + + // `parse_record_id_key`: an integer with or without a sign, a name, a `u"..."` UUID, or an array or object. A + // generated key such as `rand()` is left unread. + private mutating func readsKey() -> Bool { + guard let first = peek() else { return false } + if first == Self.plus || first == SqlLexer.dash { + index += 1 + let digits = index + while let next = peek(), next >= 0x30, next <= 0x39 { + index += 1 + } + return index > digits + } + if first == Self.smallU, index + 1 < text.length, + [SqlLexer.singleQuote, SqlLexer.doubleQuote].contains(text.character(at: index + 1)) { + return readsQuoted(at: index + 1) + } + if first == Self.openBracket || first == Self.openBrace { + return readsValues() + } + return readsName() + } + + // A string is a value, so `person:"tobie"` is not a key SurrealDB parses. + private mutating func readsName() -> Bool { + if let first = peek(), first == SqlLexer.backtick || first == SqlLexer.openAngleQuote { + return readsQuoted(at: index) + } + return word() != nil + } + + private mutating func readsQuoted(at start: Int) -> Bool { + guard let span = SQLNonCodeSpan.span(at: start, in: text, grammar: grammar), span.kind == .quoted, + span.isTerminated + else { return false } + index = span.end + return true + } + + // Whatever an array or object key holds, it is one key. A `..` inside is still read as a range. + private mutating func readsValues() -> Bool { + var expected: [UInt16] = [] + while let next = peek() { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + if let closer = Self.closers[next] { + expected.append(closer) + } else if Self.closing.contains(next) { + guard expected.popLast() == next else { return false } + if expected.isEmpty { + index += 1 + return true + } + } else if next == Self.period, index + 1 < text.length, text.character(at: index + 1) == Self.period { + return false + } + index += 1 + } + return false + } + + private mutating func skipKeyword(_ keyword: String) { + skipBlanks() + let start = index + if word() != keyword { index = start } + } + + private mutating func skipBlanks() { + while index < text.length { + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + } else if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar), span.kind.isComment { + index = max(span.end, index + 1) + } else { + return + } + } + } + + private mutating func word() -> String? { + let start = index + while let next = peek(), SqlDollarQuote.isIdentifierPart(next) { + index += 1 + } + guard index > start else { return nil } + return text.substring(with: NSRange(location: start, length: index - start)).uppercased() + } + + private func peek() -> UInt16? { + index < text.length ? text.character(at: index) : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift index 38f254c5bd..851741b05f 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift @@ -20,11 +20,13 @@ public enum SQLUnterminatedStatements { /// Every text a gate has to tier for `statement`: the statement itself, then, when `grammar` needs no terminator, /// the text of each statement that begins inside it, less the statements nested in that one, which come on their /// own. A grammar that takes a statement in place of a value or a table adds what ``SQLEmbeddedStatements`` finds, - /// and one with a pipe operator each statement ``SQLPipeOperatorChain`` finds. + /// and one with a pipe operator each statement ``SQLPipeOperatorChain`` finds. A grammar that needs a terminator + /// also adds the pieces a `;` inside the statement's parentheses separates, from ``SQLParenthesizedSemicolons``. public static func runnable(in statement: String, grammar: SQLLexicalGrammar) -> [String] { guard grammar.contains(.unterminatedStatements) else { return [statement] + SQLEmbeddedStatements.texts(in: statement, grammar: grammar) + SQLPipeOperatorChain.statements(in: statement, grammar: grammar) + + SQLParenthesizedSemicolons.pieces(in: statement, grammar: grammar) } let text = statement as NSString let tokens = TSQLTokens.read(text, grammar: grammar) @@ -32,6 +34,27 @@ public enum SQLUnterminatedStatements { return [statement] + ownTexts(of: statementSpans(in: tokens, length: text.length), in: text) } + // A scanner statement on T-SQL keeps a `;` outside parentheses only when a block holds it, so anything after one but + // the block's `END`, an `ELSE` or another `;` is a second statement the block runs. A stored routine runs nothing. + public static func blockHoldsSeveralStatements(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + guard grammar.contains(.unterminatedStatements) else { return false } + let tokens = TSQLTokens.read(statement as NSString, grammar: grammar) + guard !definesRoutine(tokens) else { return false } + return tokens.indices.dropLast().contains { index in + guard tokens[index].kind == .terminator, tokens[index].depth == 0 else { return false } + let next = tokens[index + 1] + guard next.kind != .terminator else { return false } + switch next.word { + case "ELSE": + return false + case "END": + return tokens.indices.contains(index + 2) && tokens[index + 2].word == "CONVERSATION" + default: + return true + } + } + } + /// The reserved keywords that begin a T-SQL statement. `ELSE` and `FETCH` are left out because they also continue /// a `CASE` and an `OFFSET` clause, and the statement after an `ELSE` begins with a keyword of its own. `WITH` is /// left out because a common table expression cannot follow a statement without a `;`, and the word also opens diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift index 0a11e84e6d..6968b11fef 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift @@ -23,7 +23,8 @@ public enum SqlBlockStructure { /// What an `END` means, given the word after it. public enum EndFollower: Equatable, Sendable { - /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and the word is theirs. + /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and T-SQL's + /// `END CONVERSATION` is a statement that closes nothing. Either way the word is theirs. case closesControlFlow /// `END CASE` closes the `CASE` that opened a block, and the word is theirs. case closesCaseStatement @@ -48,6 +49,10 @@ public enum SqlBlockStructure { /// second opener. private static let controlFlowFollowers: Set = ["IF", "LOOP", "WHILE", "FOR", "REPEAT"] + // T-SQL's `BEGIN DIALOG`, `BEGIN CONVERSATION TIMER` and `END CONVERSATION` are statements, not a block's ends. + private static let serviceBrokerFollowers: Set = ["DIALOG", conversationKeyword] + private static let conversationKeyword = "CONVERSATION" + /// The keywords a statement has to open with before a `BEGIN` inside it is read as a routine body. /// /// Only ``SQLRoutineBodyTracker`` consults this, and the reason is safety rather than grammar. An anonymous @@ -56,7 +61,9 @@ public enum SqlBlockStructure { /// single statement whose first word is `BEGIN`, and `QueryClassifier` tiers a statement by its leading keyword, /// so a swallowed `DROP` would be tiered a plain write and skip the destructive confirmation. Folding cannot /// execute anything, so it takes every block. Oracle, whose anonymous blocks are the everyday form, has its own - /// grammar in ``PLSQLUnitTracker`` and a classifier that reads the whole block. + /// grammar in ``PLSQLUnitTracker`` and a classifier that reads the whole block. T-SQL reserves `BEGIN`, so its + /// blocks open outside a routine too, and the gate tiers every statement inside one through + /// ``SQLUnterminatedStatements``. private static let routineDefinitionOpeners: Set = ["CREATE", "ALTER", "REPLACE"] /// What such a statement has to define, for the same reason. `BEGIN` is not reserved on PostgreSQL, MySQL or @@ -111,6 +118,28 @@ public enum SqlBlockStructure { return transactionFollowers.contains(keyword) } + public static func beginStartsStatement(followedBy keyword: String?, grammar: SQLLexicalGrammar) -> Bool { + if beginStartsTransaction(followedBy: keyword) { return true } + return readsTSQLBlocks(grammar) && keyword.map(serviceBrokerFollowers.contains) == true + } + + // T-SQL closes every block with a bare `END`: in `END IF @x = 1 ...` the `IF` begins the next statement. + public static func endingFollowedBy( + _ keyword: String, + closesCaseExpression: Bool, + grammar: SQLLexicalGrammar + ) -> EndFollower { + guard readsTSQLBlocks(grammar) else { + return endingFollowedBy(keyword, closesCaseExpression: closesCaseExpression) + } + return !closesCaseExpression && keyword == conversationKeyword ? .closesControlFlow : .closesBlock + } + + // The grammar whose statements need no `;` between them is T-SQL's, which reserves `BEGIN`, `CASE` and `END`. + private static func readsTSQLBlocks(_ grammar: SQLLexicalGrammar) -> Bool { + grammar.contains(.unterminatedStatements) + } + /// What an `END` followed by `keyword` closes. /// /// - Parameter closesCaseExpression: whether a `CASE` is open that has held no `;`. Only the statement form can @@ -260,7 +289,7 @@ public enum SqlBlockStructure { let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return true } guard text.character(at: cursor) != SqlLexer.semicolon else { return true } - return beginStartsTransaction(followedBy: readKeyword(text, at: cursor, length: length).text) + return beginStartsStatement(followedBy: readKeyword(text, at: cursor, length: length).text, grammar: grammar) } /// Goes by the word after the `END` alone: only folding asks, and it keeps no record of which `CASE` is open. @@ -273,7 +302,7 @@ public enum SqlBlockStructure { let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return .closesBlock(resumeAt: offset) } let follower = readKeyword(text, at: cursor, length: length) - switch endingFollowedBy(follower.text, closesCaseExpression: false) { + switch endingFollowedBy(follower.text, closesCaseExpression: false, grammar: grammar) { case .closesControlFlow: return .none case .closesCaseStatement: diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift index de2f991f56..bcc1e93a50 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift @@ -41,13 +41,19 @@ public enum SqlDollarQuote { } /// Resolves a `$` at `pos` to a dollar-quote opener, a positional parameter like `$1`, or a non-tag dollar. A `$` - /// glued to a preceding identifier is not an opener. Returns `needsMoreData` when the buffer ends mid-tag; a - /// whole-string caller treats that as `notOpener`. + /// glued to a preceding identifier, as `continuesName` reads the unit before it, is not an opener. Returns + /// `needsMoreData` when the buffer ends mid-tag; a whole-string caller treats that as `notOpener`. /// /// A tag starts with a letter or an underscore and continues with letters, digits and underscores, where a letter /// is any non-ASCII character as well, which is how PostgreSQL accepts `$ü$`. - public static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int, style: Style) -> Opener { - if pos > 0, isIdentifierContinuation(buffer.character(at: pos - 1)) { + public static func scanOpener( + at pos: Int, + in buffer: NSString, + bufLen: Int, + style: Style, + continuesName: (unichar) -> Bool = SqlDollarQuote.isIdentifierContinuation + ) -> Opener { + if pos > 0, continuesName(buffer.character(at: pos - 1)) { return .notOpener } guard pos + 1 < bufLen else { return .needsMoreData } diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/RelayPollState.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/RelayPollState.swift index bf0f4180de..915b186cbc 100644 --- a/apple/Packages/TableProCore/Sources/TableProSSHTransport/RelayPollState.swift +++ b/apple/Packages/TableProCore/Sources/TableProSSHTransport/RelayPollState.swift @@ -5,40 +5,10 @@ import Foundation -public enum RelayFDState: Equatable, Sendable { - case idle - case readable - case drainThenStop - case stop -} - -public func relayFDState(_ revents: Int16) -> RelayFDState { - if revents & Int16(POLLERR | POLLNVAL) != 0 { return .stop } - if revents & Int16(POLLHUP) != 0 { return .drainThenStop } - if revents & Int16(POLLIN) != 0 { return .readable } - return .idle -} - -/// Outcome of waiting for a specific direction on the SSH transport. Distinct from -/// `RelayFDState`, which classifies a read-side poll where "no bits set" means idle. -/// Here "no bits set" means the wait expired without the requested direction becoming -/// ready, which must not be mistaken for readiness. -public enum TransportPollOutcome: Equatable, Sendable { - case ready - case timedOut - case hangup -} - -public func transportPollOutcome(revents: Int16, requestedEvents: Int16) -> TransportPollOutcome { - if revents & Int16(POLLERR | POLLNVAL | POLLHUP) != 0 { return .hangup } - if revents & requestedEvents != 0 { return .ready } - return .timedOut -} - -/// Waits for libssh2's requested block directions to become ready on the transport. -/// When libssh2 reports no direction, there is nothing to wait on, so this yields -/// briefly instead of returning immediately, which would let a caller retry in a -/// zero-yield spin. +// Waits for libssh2's requested block directions to become ready on the transport. +// When libssh2 reports no direction, there is nothing to wait on, so this yields +// briefly instead of returning immediately, which would let a caller retry in a +// zero-yield spin. public func pollReady(fd: Int32, directions: RelayDirections, timeoutMs: Int32) -> Bool { var events: Int16 = 0 if directions.contains(.inbound) { events |= Int16(POLLIN) } @@ -54,3 +24,18 @@ public func pollReady(fd: Int32, directions: RelayDirections, timeoutMs: Int32) } private let directionlessYieldMicroseconds: UInt32 = 1_000 + +public enum TransportRoom: Equatable, Sendable { + case room + case full + case gone +} + +// A socket polls writable once its send low-water mark is free (2048 bytes unless set), which holds a small +// packet whole. A reset or closed socket reports a hangup or an error instead. +public func transportRoom(_ descriptor: Int32) -> TransportRoom { + var pollFD = pollfd(fd: descriptor, events: Int16(POLLOUT), revents: 0) + guard poll(&pollFD, 1, 0) == 1 else { return .full } + if pollFD.revents & Int16(POLLERR | POLLHUP | POLLNVAL) != 0 { return .gone } + return pollFD.revents & Int16(POLLOUT) != 0 ? .room : .full +} diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHChannelIO.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHChannelIO.swift new file mode 100644 index 0000000000..c89d65cf16 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHChannelIO.swift @@ -0,0 +1,52 @@ +// +// SSHChannelIO.swift +// TableProSSHTransport +// + +import Foundation + +public struct RelayDirections: OptionSet, Sendable { + public let rawValue: Int32 + + public init(rawValue: Int32) { + self.rawValue = rawValue + } + + public static let inbound = RelayDirections(rawValue: 1 << 0) + public static let outbound = RelayDirections(rawValue: 1 << 1) +} + +public enum ChannelReadResult: Equatable, Sendable { + case bytes(Int) + case wouldBlock + case closed + // The session is gone, not only this channel: the relay ends .transportHangup so the owner marks the tunnel dead now, not at the next keep-alive. + case transportFailed +} + +public enum ChannelWriteResult: Equatable, Sendable { + case bytes(Int) + case wouldBlock + case closed + case transportFailed +} + +// One channel of a session, called only on its `SSHSessionPump`'s queue. +public protocol SSHChannelIO: Sendable { + func read(into buffer: UnsafeMutablePointer, count: Int) -> ChannelReadResult + func write(_ buffer: UnsafePointer, count: Int) -> ChannelWriteResult + func blockDirections() -> RelayDirections + // Data or an end of file another call on the session already took off the transport for this channel. + func hasQueuedInput() -> Bool + // Room left in the server's window for this channel's data. A `write` that answered `.wouldBlock` with + // room has a packet on its way that only another `write` on this channel finishes; without room it waits + // for the server to open the window. + func hasSendWindow() -> Bool +} + +public enum RelayTermination: Equatable, Sendable { + case localClosed + case transportHangup + case channelClosed + case cancelled +} diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHChannelRelay.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHChannelRelay.swift deleted file mode 100644 index ea3e7cc078..0000000000 --- a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHChannelRelay.swift +++ /dev/null @@ -1,234 +0,0 @@ -// -// SSHChannelRelay.swift -// TableProSSHTransport -// - -import Foundation - -public struct RelayDirections: OptionSet, Sendable { - public let rawValue: Int32 - - public init(rawValue: Int32) { - self.rawValue = rawValue - } - - public static let inbound = RelayDirections(rawValue: 1 << 0) - public static let outbound = RelayDirections(rawValue: 1 << 1) -} - -public enum ChannelReadResult: Equatable, Sendable { - case bytes(Int) - case wouldBlock - case closed - // The session is gone, not only this channel: the relay ends .transportHangup so the owner marks the tunnel dead now, not at the next keep-alive. - case transportFailed -} - -public enum ChannelWriteResult: Equatable, Sendable { - case bytes(Int) - case wouldBlock - case closed - case transportFailed -} - -public protocol SSHChannelIO { - func read(into buffer: UnsafeMutablePointer, count: Int) -> ChannelReadResult - func write(_ buffer: UnsafePointer, count: Int) -> ChannelWriteResult - func blockDirections() -> RelayDirections -} - -public enum RelayTermination: Equatable, Sendable { - case localClosed - case transportHangup - case channelClosed - case cancelled -} - -// A hangup drains buffered data before the loop ends, so the last bytes reach the client. -public struct SSHChannelRelay { - public let localFD: Int32 - public let transportFD: Int32 - public let channelIO: any SSHChannelIO - public let bufferSize: Int - public let isActive: () -> Bool - - /// Counts what crosses this relay, for the connection activity readout. Absent for a jump hop, - /// whose relay carries the same payload a second time on its way to the next hop: counting both - /// would report every byte twice for a connection that goes through a bastion. - public var byteCounter: (any RelayByteObserver)? - - public init( - localFD: Int32, - transportFD: Int32, - channelIO: any SSHChannelIO, - bufferSize: Int, - isActive: @escaping () -> Bool, - byteCounter: (any RelayByteObserver)? = nil - ) { - self.localFD = localFD - self.transportFD = transportFD - self.channelIO = channelIO - self.bufferSize = bufferSize - self.isActive = isActive - self.byteCounter = byteCounter - } - - private static let pollTimeoutMs: Int32 = 500 - private static let writeWaitTimeoutMs: Int32 = 1_000 - - /// How many channel reads one pump round may make before looking at the local fd again. - /// - /// libssh2 buffers whole SSH packets as it decrypts them, so a channel that has fallen behind - /// holds more than one buffer's worth and the transport reports nothing new until that backlog - /// is drained. Reading one buffer per poll round pays the 500ms timeout for every buffer, which - /// measured 1,504ms for 98,304 queued bytes against a real sshd where draining took 502ms. - /// The cap is what keeps a server that streams without pause from starving local to channel; - /// a round that ends on the cap rather than on EAGAIN polls with no timeout, so the cap costs - /// one round trip through `poll` and never a wait. - private static let maxChannelReadsPerRound = 16 - - public func run() -> RelayTermination { - let buffer = UnsafeMutablePointer.allocate(capacity: bufferSize) - defer { buffer.deallocate() } - var pollTimeoutMs = Self.pollTimeoutMs - - while isActive() { - var pollFDs = [ - pollfd(fd: localFD, events: Int16(POLLIN), revents: 0), - pollfd(fd: transportFD, events: Int16(POLLIN), revents: 0), - ] - - let pollResult = poll(&pollFDs, 2, pollTimeoutMs) - pollTimeoutMs = Self.pollTimeoutMs - if pollResult < 0 { - if errno == EINTR { continue } - return .cancelled - } - - let localState = relayFDState(pollFDs[0].revents) - let transportState = relayFDState(pollFDs[1].revents) - - if transportState == .stop { return .transportHangup } - if localState == .stop { return .localClosed } - - if transportState != .idle || pollResult == 0 { - switch pumpChannelToLocal(buffer, transportReadable: transportState != .idle) { - case .drained: break - case .cappedWithMoreQueued: pollTimeoutMs = 0 - case .channelClosed: return .channelClosed - case .localClosed: return .localClosed - case .transportHangup: return .transportHangup - } - } - - if localState == .readable || localState == .drainThenStop { - switch pumpLocalToChannel(buffer) { - case .drained, .cappedWithMoreQueued: break - case .channelClosed: return .channelClosed - case .localClosed: return .localClosed - case .transportHangup: return .transportHangup - } - } - - if transportState == .drainThenStop { return .transportHangup } - if localState == .drainThenStop { return .localClosed } - } - - return .cancelled - } - - private enum PumpOutcome { - case drained - case cappedWithMoreQueued - case channelClosed - case localClosed - case transportHangup - } - - private enum TransportWait { - case ready - case hangup - } - - private func pumpChannelToLocal(_ buffer: UnsafeMutablePointer, transportReadable: Bool) -> PumpOutcome { - for _ in 0 ..< Self.maxChannelReadsPerRound { - switch channelIO.read(into: buffer, count: bufferSize) { - case .bytes(let count): - byteCounter?.recordReceived(count) - guard sendToLocal(buffer, count: count) else { return .localClosed } - case .wouldBlock: - if transportReadable, transportAtEOF() { return .transportHangup } - return .drained - case .closed: - return .channelClosed - case .transportFailed: - return .transportHangup - } - } - return .cappedWithMoreQueued - } - - private func sendToLocal(_ buffer: UnsafeMutablePointer, count: Int) -> Bool { - var totalSent = 0 - while totalSent < count { - // Per send, not SO_NOSIGPIPE: setsockopt fails with EINVAL once a client has already closed, and the send then raises SIGPIPE. - let sent = send(localFD, buffer.advanced(by: totalSent), count - totalSent, MSG_NOSIGNAL) - if sent <= 0 { return false } - totalSent += sent - } - return true - } - - private func pumpLocalToChannel(_ buffer: UnsafeMutablePointer) -> PumpOutcome { - let localRead = recv(localFD, buffer, bufferSize, 0) - if localRead <= 0 { return .localClosed } - byteCounter?.recordSent(localRead) - - var totalWritten = 0 - while totalWritten < Int(localRead) { - switch channelIO.write(buffer.advanced(by: totalWritten), count: Int(localRead) - totalWritten) { - case .bytes(let count): - totalWritten += count - case .wouldBlock: - switch waitForTransport(channelIO.blockDirections()) { - case .ready: break - case .hangup: return .transportHangup - } - case .closed: - return .channelClosed - case .transportFailed: - return .transportHangup - } - } - return .drained - } - - private func waitForTransport(_ directions: RelayDirections) -> TransportWait { - var events: Int16 = 0 - if directions.contains(.inbound) { events |= Int16(POLLIN) } - if directions.contains(.outbound) { events |= Int16(POLLOUT) } - guard events != 0 else { return .ready } - - while true { - var pollFD = pollfd(fd: transportFD, events: events, revents: 0) - let rc = poll(&pollFD, 1, Self.writeWaitTimeoutMs) - if rc < 0 { - if errno == EINTR { continue } - return .hangup - } - switch transportPollOutcome(revents: pollFD.revents, requestedEvents: events) { - case .hangup: - return .hangup - case .ready: - return .ready - case .timedOut: - guard isActive() else { return .hangup } - } - } - } - - private func transportAtEOF() -> Bool { - var byte: UInt8 = 0 - return recv(transportFD, &byte, 1, MSG_PEEK | MSG_DONTWAIT) == 0 - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHJumpRelayFence.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHJumpRelayFence.swift index 4a545d1478..c7105bcac0 100644 --- a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHJumpRelayFence.swift +++ b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHJumpRelayFence.swift @@ -6,10 +6,9 @@ import Foundation import os -// Cancelling a Swift task does not cancel the GCD block that runs a hop's `SSHChannelRelay`, so -// teardown wakes its descriptors and waits here for its last libssh2 call before freeing the channel -// or session. The stop action and the relay's final close share one lock, so a late stop cannot -// `shutdown` a descriptor number the kernel already handed to someone else. +// A hop's relay runs on its session's queue, outside any Swift task, so teardown stops it and waits +// here for its last libssh2 call before freeing the channel or session. The stop action and the +// finish share one lock, so a stop that comes after the finish does nothing. public final class SSHJumpRelayFence: @unchecked Sendable { private struct State { var stopAction: (@Sendable () -> Void)? diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHListenerAcceptor.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHListenerAcceptor.swift new file mode 100644 index 0000000000..ee7959c49e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHListenerAcceptor.swift @@ -0,0 +1,103 @@ +// +// SSHListenerAcceptor.swift +// TableProSSHTransport +// + +import Foundation +import os + +public struct SSHAcceptedClient: Sendable { + public let descriptor: Int32 + // Taken at accept, before any queue hop, since the client's own connect timeout is already running. + public let acceptedAt: Date +} + +// Accepts on a tunnel's listening socket only when the kernel reports a pending connection, so an +// idle listener costs no wakeups and parks no thread. GCD runs the cancel handler only after a +// running event handler has returned, so once `waitForCancellation()` returns nothing can accept on +// the descriptor and its owner may close it. The acceptor never closes it. +public final class SSHListenerAcceptor: @unchecked Sendable { + private let source: any DispatchSourceRead + private let isCancelled: OSAllocatedUnfairLock + private let cancelled: DispatchGroup + + public init( + listenFD: Int32, + queue: DispatchQueue, + onAccept: @escaping @Sendable (SSHAcceptedClient) -> Void + ) { + // A drain then ends on EAGAIN instead of blocking the queue in accept. + let flags = fcntl(listenFD, F_GETFL, 0) + if flags >= 0 { + _ = fcntl(listenFD, F_SETFL, flags | O_NONBLOCK) + } + + let isCancelled = OSAllocatedUnfairLock(initialState: false) + let cancelled = DispatchGroup() + let source = DispatchSource.makeReadSource(fileDescriptor: listenFD, queue: queue) + source.setEventHandler { @Sendable in + SSHListenerAcceptor.acceptPending( + listenFD: listenFD, + isCancelled: { isCancelled.withLock { $0 } }, + onAccept: onAccept + ) + } + cancelled.enter() + source.setCancelHandler { @Sendable in + cancelled.leave() + } + + self.source = source + self.isCancelled = isCancelled + self.cancelled = cancelled + } + + // libdispatch crashes on releasing a source that was never activated. + deinit { + cancel() + } + + public func start() { + source.activate() + } + + public func cancel() { + isCancelled.withLock { $0 = true } + source.cancel() + // A source cancelled before start runs its cancel handler only once active. + source.activate() + } + + public func waitForCancellation() async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + cancelled.notify(queue: .global(qos: .utility)) { + continuation.resume() + } + } + } + + private static func acceptPending( + listenFD: Int32, + isCancelled: () -> Bool, + onAccept: (SSHAcceptedClient) -> Void + ) { + while !isCancelled() { + let descriptor = accept(listenFD, nil, nil) + guard descriptor >= 0 else { + switch errno { + case EINTR, ECONNABORTED: + continue + default: + return + } + } + let acceptedAt = Date() + // Darwin hands the listener's O_NONBLOCK to the accepted socket, and the Remote SQLite admission read blocks on it. + let flags = fcntl(descriptor, F_GETFL, 0) + if flags >= 0 { + _ = fcntl(descriptor, F_SETFL, flags & ~O_NONBLOCK) + } + onAccept(SSHAcceptedClient(descriptor: descriptor, acceptedAt: acceptedAt)) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHSessionPump.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHSessionPump.swift new file mode 100644 index 0000000000..22edaea70d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHSessionPump.swift @@ -0,0 +1,784 @@ +// +// SSHSessionPump.swift +// TableProSSHTransport +// + +import Foundation +import os + +// The one reader of an SSH session. libssh2 moves every packet on the transport into the session's own queue +// whenever any channel reads or writes (ssh2_channel_read and ssh2_channel_write loop ssh2_transport_read until +// EAGAIN), so a channel that waits on the transport descriptor alone misses data another call already took off the +// socket. Every libssh2 call on the session runs on `queue`, and after each one the pump serves every channel it +// relays. Readiness comes from dispatch sources on the transport and the local sockets, so an idle session has no +// timer and parks no thread. +public final class SSHSessionPump: @unchecked Sendable { + public let queue: DispatchQueue + + private let transportFD: Int32 + private let maxReadsPerRound: Int + private let callbackQueue = DispatchQueue.global(qos: .utility) + // Entered once for the pump, once per relay and once per transport source; `stop` leaves the first. + private let lifetime = DispatchGroup() + private let outsideSenders = OSAllocatedUnfairLock(initialState: 0) + + // Everything below is touched only on `queue`. + private var relays: [PumpedChannel] = [] + private var transportRead: ReadinessSource? + private var transportWrite: ReadinessSource? + private var isStopped = false + private var isPassScheduled = false + private var passNumber = 0 + private var nextStart = 0 + + // A read on one channel can queue data for a channel served earlier in the same pass. + private static let maxSettleSweeps = 4 + private static let contendedReadsPerRound = 4 + + public init(transportFD: Int32, queue: DispatchQueue, maxReadsPerRound: Int = 16) { + self.transportFD = transportFD + self.queue = queue + self.maxReadsPerRound = maxReadsPerRound + lifetime.enter() + } + + // libdispatch crashes on releasing a source that is suspended or was never activated. + deinit { + let ending = relays + relays.removeAll() + ending.forEach { finish($0, .cancelled) } + transportRead?.cancel() + transportWrite?.cancel() + } + + // Runs a libssh2 call on the session's queue, then relays whatever it left queued. + public func perform(_ body: () throws -> T) rethrows -> T { + dispatchPrecondition(condition: .notOnQueue(queue)) + return try queue.sync { + defer { serve() } + return try body() + } + } + + // The same, for a caller on Swift's cooperative pool, which must not block a thread waiting for the queue. + public func performAsync(_ body: @escaping @Sendable () -> T) async -> T { + await withCheckedContinuation { (continuation: CheckedContinuation) in + queue.async { [self] in + let result = body() + serve() + continuation.resume(returning: result) + } + } + } + + // Takes ownership of `localFD`: the pump closes it once the relay ends, then calls `onEnd` once, off the queue. + public func relay( + localFD: Int32, + channel: any SSHChannelIO, + bufferSize: Int, + byteCounter: (any RelayByteObserver)? = nil, + onEnd: @escaping @Sendable (RelayTermination) -> Void + ) { + queue.async { [self] in + guard !isStopped else { + Darwin.close(localFD) + callbackQueue.async { onEnd(.cancelled) } + return + } + let flags = fcntl(localFD, F_GETFL, 0) + if flags >= 0 { + _ = fcntl(localFD, F_SETFL, flags | O_NONBLOCK) + } + startTransportSources() + + let relay = PumpedChannel( + localFD: localFD, + io: channel, + bufferSize: bufferSize, + byteCounter: byteCounter, + onEnd: onEnd + ) + lifetime.enter() + relay.localRead = ReadinessSource( + DispatchSource.makeReadSource(fileDescriptor: localFD, queue: queue), + group: relay.sources + ) { [weak self, weak relay] in + guard let relay else { return } + relay.localReadable = true + self?.serve(only: relay) + } + relay.localWrite = ReadinessSource( + DispatchSource.makeWriteSource(fileDescriptor: localFD, queue: queue), + group: relay.sources + ) { [weak self, weak relay] in + guard let relay else { return } + self?.serve(only: relay) + } + relays.append(relay) + serve() + } + } + + // Ends every relay as `.cancelled`; a relay added later ends at once. The transport descriptor stays open. + public func stop() { + queue.async { [self] in + guard !isStopped else { return } + isStopped = true + let ending = relays + relays.removeAll() + ending.forEach { finish($0, .cancelled) } + transportRead?.cancel() + transportWrite?.cancel() + lifetime.leave() + } + } + + // After `stop`, once no source can fire, every local descriptor is closed and every `onEnd` has run. + public func whenStopped(_ body: @escaping @Sendable () -> Void) { + lifetime.notify(queue: callbackQueue, execute: body) + } + + public func waitUntilStopped() async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + lifetime.notify(queue: callbackQueue) { + continuation.resume() + } + } + } + + // MARK: - Send turns + + // An outside call that has to send on the session, an open or a free, holds a turn while it waits to. libssh2 + // sends one packet at a time per session and refuses every other send until it is out (transport_send_existing), + // so a relay that always had one pending would leave the call no gap. While a turn is held the relays finish the + // packet they have on its way and start no new one. + public func holdSendTurn() { + outsideSenders.withLock { $0 += 1 } + } + + public func releaseSendTurn() { + outsideSenders.withLock { $0 -= 1 } + queue.async { [weak self] in + self?.serve() + } + } + + // MARK: - Serving + + private struct Pass { + let number: Int + let outsideWaiting: Bool + let readBudget: Int + var hasMore = false + } + + private enum Step { + case proceed + case end(RelayTermination) + } + + private func startTransportSources() { + guard transportRead == nil else { return } + transportRead = ReadinessSource( + DispatchSource.makeReadSource(fileDescriptor: transportFD, queue: queue), + group: lifetime + ) { [weak self] in + self?.transportBecameReadable() + } + transportWrite = ReadinessSource( + DispatchSource.makeWriteSource(fileDescriptor: transportFD, queue: queue), + group: lifetime + ) { [weak self] in + self?.serve() + } + } + + private func transportBecameReadable() { + serve() + guard !isStopped, !relays.isEmpty, Self.transportEnded(transportFD) else { return } + let ending = relays + relays.removeAll() + for relay in ending { + _ = relay.sendToLocal() + finish(relay, .transportHangup) + } + rearmTransport() + } + + // A local socket's event serves its own relay first and the rest in one scheduled pass, so a busy channel + // cannot make every other client's event wait behind its reads. That pass always follows: the relay's own call + // may have taken the others' data or a key exchange reply off the transport. + private func serve(only focus: PumpedChannel) { + guard !isStopped, !focus.isFinished else { return } + var pass = newPass() + if let termination = serve(focus, pass: &pass) { + remove(focus, termination) + } else if isLagging(focus, pass: pass), let termination = serve(focus, pass: &pass) { + remove(focus, termination) + } + if pass.hasMore || relays.contains(where: { $0 !== focus }) { + schedulePass() + } + rearmTransport() + } + + // Each pass starts one relay further on, and serves with a smaller read budget while more than one relay has + // work, so a busy channel cannot hold the queue against a quiet one for long. + private func newPass() -> Pass { + passNumber += 1 + let busy = relays.count(where: \.hasWork) + return Pass( + number: passNumber, + outsideWaiting: outsideSenders.withLock { $0 > 0 }, + readBudget: busy > 1 ? min(maxReadsPerRound, Self.contendedReadsPerRound) : maxReadsPerRound + ) + } + + private func serve() { + guard !isStopped else { return } + var pass = newPass() + let first = relays.isEmpty ? 0 : nextStart % relays.count + let order = Array(relays[first...]) + Array(relays[.. RelayTermination? { + if let ending = relay.endingAfterSend { + return finishSend(relay, ending: ending) + } + if case .end(let termination) = relayChannelToLocal(relay, pass: &pass) { + return end(relay, termination) + } + if case .end(let termination) = relayLocalToChannel(relay, pass: &pass) { + return end(relay, termination) + } + if relay.channelAtEOF, relay.toLocal.isEmpty { + return end(relay, .channelClosed) + } + relay.localRead?.arm(relay.toChannel.isEmpty) + relay.localWrite?.arm(!relay.toLocal.isEmpty) + return nil + } + + private func remove(_ relay: PumpedChannel, _ termination: RelayTermination) { + relays.removeAll { $0 === relay } + finish(relay, termination) + } + + // Freeing a channel with its packet half sent would leave every later send on the session refused, and only + // the call that started the packet can finish it: a write, or a read's window adjust. Such a relay ends once + // that call has gone through. + private func end(_ relay: PumpedChannel, _ termination: RelayTermination) -> RelayTermination? { + guard termination != .transportHangup, relay.pendingSend != nil else { + return termination + } + relay.endingAfterSend = termination + relay.localRead?.arm(false) + relay.localWrite?.arm(false) + return nil + } + + private func finishSend(_ relay: PumpedChannel, ending: RelayTermination) -> RelayTermination? { + switch relay.pendingSend { + case .write: + switch relay.io.write(relay.toChannel.head, count: relay.toChannel.count) { + case .bytes, .closed: + relay.pendingSend = nil + case .wouldBlock: + return nil + case .transportFailed: + return .transportHangup + } + case .read: + // What this read returns has nowhere to go: the relay is ending. + switch relay.io.read(into: relay.toChannel.storage, count: relay.toChannel.capacity) { + case .bytes, .closed: + relay.pendingSend = nil + case .wouldBlock: + guard relay.io.blockDirections().contains(.outbound) else { + relay.pendingSend = nil + return ending + } + return nil + case .transportFailed: + return .transportHangup + } + case nil: + break + } + return ending + } + + private func relayChannelToLocal(_ relay: PumpedChannel, pass: inout Pass) -> Step { + if !relay.toLocal.isEmpty { + switch relay.sendToLocal() { + case .drained: break + case .blocked: return .proceed + case .closed: return .end(.localClosed) + } + } + guard !relay.channelAtEOF else { return .proceed } + + for _ in 0 ..< pass.readBudget { + let owned = relay.pendingSend == .read + let outboundBefore = sessionWaitsToSend + switch relay.io.read(into: relay.toLocal.storage, count: relay.toLocal.capacity) { + case .bytes(let count): + relay.ignoresQueuedHint = false + relay.readWaitsOnSend = false + if owned { relay.pendingSend = nil } + relay.byteCounter?.recordReceived(count) + relay.toLocal.fill(count) + switch relay.sendToLocal() { + case .drained: continue + case .blocked: return .proceed + case .closed: return .end(.localClosed) + } + case .wouldBlock: + let outboundAfter = relay.io.blockDirections().contains(.outbound) + relay.lastReadWasEmpty = true + relay.readWaitsOnSend = outboundAfter + if outboundAfter, owned || !outboundBefore { + relay.pendingSend = .read + } else if owned { + relay.pendingSend = nil + } + return .proceed + case .closed: + relay.readWaitsOnSend = false + if owned { relay.pendingSend = nil } + relay.channelAtEOF = true + return .proceed + case .transportFailed: + return .end(.transportHangup) + } + } + pass.hasMore = true + return .proceed + } + + private func relayLocalToChannel(_ relay: PumpedChannel, pass: inout Pass) -> Step { + var reads = 0 + while true { + while !relay.toChannel.isEmpty { + let owned = relay.pendingSend == .write + if !owned { + guard mayStartPacket(relay, pass: &pass) else { return .proceed } + } + let outboundBefore = sessionWaitsToSend + switch relay.io.write(relay.toChannel.head, count: relay.toChannel.count) { + case .bytes(let count): + // Sending or finishing a packet was this relay's turn. + relay.writeBlocked = false + relay.refusedBehindPacket = false + if owned { relay.pendingSend = nil } + relay.startedPacketInPass = pass.number + relay.toChannel.consume(count) + case .wouldBlock: + // With no window libssh2 built no packet, and the window opens on input. + let hasWindow = relay.io.hasSendWindow() + relay.writeBlocked = true + relay.writeWaitsOnWindow = !hasWindow + let outboundAfter = relay.io.blockDirections().contains(.outbound) + // A write that started its packet owns it until a call returns bytes: libssh2 may clear the + // session's outbound bit under it (a write with no window sets the directions outright). + if owned || (outboundAfter && !outboundBefore) { + relay.pendingSend = .write + relay.refusedBehindPacket = false + relay.startedPacketInPass = pass.number + } else { + relay.refusedBehindPacket = hasWindow && outboundBefore + } + return .proceed + case .closed: + relay.writeBlocked = false + if owned { relay.pendingSend = nil } + return .end(.channelClosed) + case .transportFailed: + return .end(.transportHangup) + } + } + guard relay.localReadable else { return .proceed } + guard reads < maxReadsPerRound else { + pass.hasMore = true + return .proceed + } + reads += 1 + switch relay.receiveFromLocal() { + case .bytes(let count): + relay.byteCounter?.recordSent(count) + case .wouldBlock: + relay.localReadable = false + return .proceed + case .closed: + return .end(.localClosed) + } + } + } + + // Turns: a relay refused behind another's packet goes before any relay that was not, and one that already + // had its turn this pass lets any other that waits go first. While an outside call holds a turn no relay starts + // a packet at all; that call comes back through `perform` or `releaseSendTurn`. A pass is scheduled for the + // relay held back only when no packet is pending, as one that is pending ends through its own owner. + private func mayStartPacket(_ relay: PumpedChannel, pass: inout Pass) -> Bool { + guard !turnAllows(relay, pass: pass) else { return true } + if !pass.outsideWaiting, !packetPending { + pass.hasMore = true + } + return false + } + + private func turnAllows(_ relay: PumpedChannel, pass: Pass) -> Bool { + if pass.outsideWaiting { return false } + var othersWait = false + var othersRefused = false + for other in relays where other !== relay && !other.isFinished { + othersWait = othersWait || other.waitsToSend + othersRefused = othersRefused || other.refusedBehindPacket + } + let hadTurn = relay.startedPacketInPass == pass.number && othersWait + let cedes = !relay.refusedBehindPacket && othersRefused + return !hadTurn && !cedes + } + + // A writer blocked on something that is gone, or a reader that waited on a send, retries in this pass; a + // writer only when its turn allows, or settle would sweep it again and again. + private func isLagging(_ relay: PumpedChannel, pass: Pass) -> Bool { + guard relay.endingAfterSend == nil, !relay.isFinished else { return false } + if relay.writeBlocked, relay.pendingSend == nil { + let obstacleGone = relay.writeWaitsOnWindow ? relay.io.hasSendWindow() : relay.refusedBehindPacket && !packetPending + if obstacleGone, turnAllows(relay, pass: pass) { return true } + } + if relay.readWaitsOnSend, relay.pendingSend == nil, !packetPending, relay.canTakeInput { return true } + return relay.isLaggingOnInput + } + + // libssh2 keeps its one pending packet by the address of the call that started it. A relay that started it + // knows; otherwise the session's outbound bit tells, and stays set until the packet is out. + private var sessionWaitsToSend: Bool { + relays.first?.io.blockDirections().contains(.outbound) ?? false + } + + private var relayOwnsPacket: Bool { + relays.contains { $0.pendingSend != nil } + } + + private var packetPending: Bool { + relayOwnsPacket || sessionWaitsToSend + } + + // Read and write sources fire for as long as the descriptor stays ready. While a packet waits to go out, the + // write source is armed only for a relay that owns it, since nothing else here can finish it, and the read source + // waits too: libssh2 reads nothing that needs a send. A packet an open or a free owns is finished by that call, + // which comes back through `perform`. + private func rearmTransport() { + guard let transportRead, let transportWrite else { return } + if relayOwnsPacket { + transportWrite.arm(true) + transportRead.arm(false) + } else if sessionWaitsToSend { + transportWrite.arm(false) + transportRead.arm(false) + } else { + transportWrite.arm(false) + transportRead.arm(relays.contains(where: \.wantsTransportInput)) + } + } + + private func schedulePass() { + guard !isPassScheduled else { return } + isPassScheduled = true + queue.async { [weak self] in + guard let self else { return } + isPassScheduled = false + serve() + } + } + + private func finish(_ relay: PumpedChannel, _ termination: RelayTermination) { + relay.isFinished = true + relay.cancelSources() + let localFD = relay.localFD + let onEnd = relay.onEnd + let lifetime = self.lifetime + relay.sources.notify(queue: callbackQueue) { + Darwin.close(localFD) + onEnd(termination) + lifetime.leave() + } + } + + private static func transportEnded(_ descriptor: Int32) -> Bool { + var byte: UInt8 = 0 + let peeked = recv(descriptor, &byte, 1, MSG_PEEK | MSG_DONTWAIT) + if peeked == 0 { return true } + return peeked < 0 && errno != EAGAIN && errno != EINTR + } +} + +// One relayed channel, touched only on the pump's queue. +private final class PumpedChannel: @unchecked Sendable { + let localFD: Int32 + let io: any SSHChannelIO + let byteCounter: (any RelayByteObserver)? + let onEnd: @Sendable (RelayTermination) -> Void + let toLocal: RelayBuffer + let toChannel: RelayBuffer + let sources = DispatchGroup() + + enum PendingSend { + case write + case read + } + + var localRead: ReadinessSource? + var localWrite: ReadinessSource? + var localReadable = true + var channelAtEOF = false + var writeBlocked = false + var writeWaitsOnWindow = false + // Its last write was turned away because another call's packet was on its way. + var refusedBehindPacket = false + var readWaitsOnSend = false + // The session's one pending packet, when this relay's own write or window adjust started it. + var pendingSend: PendingSend? + var startedPacketInPass = 0 + var endingAfterSend: RelayTermination? + var isFinished = false + var lastReadWasEmpty = false + // libssh2 counts extended data a plain read never returns, so a hint the read disproved is set aside until data flows. + var ignoresQueuedHint = false + + init( + localFD: Int32, + io: any SSHChannelIO, + bufferSize: Int, + byteCounter: (any RelayByteObserver)?, + onEnd: @escaping @Sendable (RelayTermination) -> Void + ) { + self.localFD = localFD + self.io = io + self.byteCounter = byteCounter + self.onEnd = onEnd + toLocal = RelayBuffer(capacity: bufferSize) + toChannel = RelayBuffer(capacity: bufferSize) + } + + var canTakeInput: Bool { + endingAfterSend == nil && toLocal.isEmpty && !channelAtEOF + } + + var hasWork: Bool { + !toChannel.isEmpty || (canTakeInput && io.hasQueuedInput()) + } + + var waitsToSend: Bool { + guard endingAfterSend == nil else { return false } + if pendingSend != nil || refusedBehindPacket { return true } + return !toChannel.isEmpty && (!writeWaitsOnWindow || io.hasSendWindow()) + } + + var isLaggingOnInput: Bool { + canTakeInput && !ignoresQueuedHint && io.hasQueuedInput() + } + + var wantsTransportInput: Bool { + canTakeInput || !toChannel.isEmpty || endingAfterSend != nil + } + + enum LocalSend { + case drained + case blocked + case closed + } + + enum LocalReceive { + case bytes(Int) + case wouldBlock + case closed + } + + func sendToLocal() -> LocalSend { + while !toLocal.isEmpty { + // Per send, not SO_NOSIGPIPE: setsockopt fails with EINVAL once a client has already closed, and the send then raises SIGPIPE. + let sent = send(localFD, toLocal.head, toLocal.count, MSG_NOSIGNAL) + if sent > 0 { + toLocal.consume(sent) + } else if sent < 0, errno == EINTR { + continue + } else if sent < 0, errno == EAGAIN { + return .blocked + } else { + return .closed + } + } + return .drained + } + + func receiveFromLocal() -> LocalReceive { + while true { + let received = recv(localFD, toChannel.storage, toChannel.capacity, 0) + if received > 0 { + toChannel.fill(received) + return .bytes(received) + } + if received < 0, errno == EINTR { continue } + if received < 0, errno == EAGAIN { return .wouldBlock } + return .closed + } + } + + func cancelSources() { + localRead?.cancel() + localWrite?.cancel() + } +} + +private final class RelayBuffer { + let storage: UnsafeMutablePointer + let capacity: Int + private var start = 0 + private var end = 0 + + init(capacity: Int) { + storage = .allocate(capacity: capacity) + self.capacity = capacity + } + + deinit { + storage.deallocate() + } + + var isEmpty: Bool { start == end } + var head: UnsafeMutablePointer { storage + start } + var count: Int { end - start } + + func fill(_ count: Int) { + start = 0 + end = count + } + + func consume(_ count: Int) { + start = min(end, start + count) + if start == end { + start = 0 + end = 0 + } + } +} + +// A dispatch source the pump suspends while it has no use for the event, since read and write sources fire for as +// long as the descriptor stays ready. Touched only on the pump's queue. +private final class ReadinessSource: @unchecked Sendable { + private enum State { + case inactive + case armed + case disarmed + case cancelled + } + + private let source: any DispatchSourceProtocol + private var state = State.inactive + + init(_ source: any DispatchSourceProtocol, group: DispatchGroup, handler: @escaping @Sendable () -> Void) { + self.source = source + group.enter() + source.setEventHandler(handler: handler) + source.setCancelHandler { @Sendable in + group.leave() + } + } + + func arm(_ wanted: Bool) { + switch (state, wanted) { + case (.inactive, true): + source.activate() + state = .armed + case (.disarmed, true): + source.resume() + state = .armed + case (.armed, false): + source.suspend() + state = .disarmed + default: + break + } + } + + // GCD runs a cancel handler only once the source is active and not suspended. + func cancel() { + let previous = state + guard previous != .cancelled else { return } + state = .cancelled + source.cancel() + switch previous { + case .inactive: source.activate() + case .disarmed: source.resume() + case .armed, .cancelled: break + } + } +} + +// One outside call's turn on a session (`SSHSessionPump.holdSendTurn`): held while the call waits to send, let go +// once it waits only for the server or is done. +public final class SSHSendTurn: @unchecked Sendable { + private let pump: SSHSessionPump + private let isHeld = OSAllocatedUnfairLock(initialState: false) + + public init(pump: SSHSessionPump) { + self.pump = pump + } + + deinit { + end() + } + + public func waitsToSend(_ waits: Bool) { + let changed = isHeld.withLock { held -> Bool in + guard held != waits else { return false } + held = waits + return true + } + guard changed else { return } + if waits { + pump.holdSendTurn() + } else { + pump.releaseSendTurn() + } + } + + public func end() { + waitsToSend(false) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeBrowserCallback.swift b/apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeBrowserCallback.swift new file mode 100644 index 0000000000..dd9ad0a3ad --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeBrowserCallback.swift @@ -0,0 +1,23 @@ +import Foundation +import TableProHTTP + +// The identity provider sends the SAML token to http://localhost:/?token=..., with no state to check, +// so only that exact shape ends the wait and anything else is answered and ignored. +public enum SnowflakeBrowserCallback: Sendable, Equatable { + case token(String) + case ignore + + public static func parse(requestHead: Data) -> SnowflakeBrowserCallback { + guard LoopbackRedirectRequest.isComplete(requestHead), + let target = LoopbackRedirectRequest.getTarget(LoopbackRedirectRequest.text(of: requestHead)), + let components = URLComponents(string: target), + components.host == nil, + components.path == "/", + let token = LoopbackRedirectRequest.singleValue(named: "token", in: components.queryItems ?? []), + !token.isEmpty + else { + return .ignore + } + return .token(token) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSpannerCore/SQL/SpannerRowEditSQL.swift b/apple/Packages/TableProCore/Sources/TableProSpannerCore/SQL/SpannerRowEditSQL.swift index c1abd4de35..e2a7d1059e 100644 --- a/apple/Packages/TableProCore/Sources/TableProSpannerCore/SQL/SpannerRowEditSQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProSpannerCore/SQL/SpannerRowEditSQL.swift @@ -24,25 +24,27 @@ public struct SpannerRowChange: Sendable, Equatable { public let originalRow: [SpannerCell]? // Keyed by name, as the writer collapses a row's cell changes to one value per column. public let expressionColumns: Set + // The columns left to their default, by name like `expressionColumns`. Every other cell is a value. + public let defaultColumns: Set public init( rowIndex: Int, kind: Kind, cellChanges: [CellChange] = [], originalRow: [SpannerCell]? = nil, - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) { self.rowIndex = rowIndex self.kind = kind self.cellChanges = cellChanges self.originalRow = originalRow self.expressionColumns = expressionColumns + self.defaultColumns = defaultColumns } } public enum SpannerRowEditSQL { - public static let defaultSentinel = "__DEFAULT__" - private static let logger = Logger(subsystem: "com.TablePro", category: "SpannerRowEditSQL") public static func statements( @@ -115,7 +117,7 @@ internal struct SpannerRowEditTarget { func insert(_ change: SpannerRowChange, row: [SpannerCell]?) -> SpannerRenderedStatement? { let cells = row.map { zip(columns, $0).map { SpannerRowChange.CellChange(column: $0.0, value: $0.1) } } ?? Self.lastValuePerColumn(change.cellChanges) - let written = cells.filter { !Self.isDefaultSentinel($0.value) } + let written = cells.filter { !change.defaultColumns.contains($0.column) } guard written.isEmpty else { return Self.insertStatement( table: table, cells: written, dialect: dialect, expressionColumns: change.expressionColumns @@ -132,7 +134,7 @@ internal struct SpannerRowEditTarget { var parameters: [SpannerCell] = [] let assignments = Self.lastValuePerColumn(change.cellChanges).map { cell in let column = dialect.quoteIdentifier(cell.column) - guard !Self.isDefaultSentinel(cell.value) else { return "\(column) = DEFAULT" } + guard !change.defaultColumns.contains(cell.column) else { return "\(column) = DEFAULT" } if let sql = Self.expression(cell, in: change.expressionColumns) { return "\(column) = \(sql)" } parameters.append(cell.value) return "\(column) = ?" @@ -191,10 +193,6 @@ internal struct SpannerRowEditTarget { return (terms.joined(separator: " AND "), parameters) } - private static func isDefaultSentinel(_ cell: SpannerCell) -> Bool { - cell == .text(SpannerRowEditSQL.defaultSentinel) - } - private static func lastValuePerColumn(_ cells: [SpannerRowChange.CellChange]) -> [SpannerRowChange.CellChange] { var order: [String] = [] var values: [String: SpannerCell] = [:] diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift index 78795f78de..69df7386a3 100644 --- a/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift @@ -7,6 +7,7 @@ public struct PullResult: Sendable { public let deletedRecordIDs: [CKRecord.ID] public let newToken: CKServerChangeToken? // Records the zone holds that the fetch could not read, so their absence from `changedRecords` proves nothing. + // Until `UnreadRecordLedger` releases one, `newToken` stops before it, so the next pull lists it again. public let unreadRecordIDs: [CKRecord.ID] public init( @@ -34,6 +35,11 @@ public actor CloudKitSyncEngine { private static let maxRetries = 3 + // Under the 400 items per operation CloudKit's limitExceeded documentation gives as its guideline. + private static let refetchBatchSize = 200 + + private var unreadLedger = UnreadRecordLedger() + /// What a build without the iCloud entitlement reports, in CloudKit's own terms, so it is /// classified like the server's answer to the same fault. private static var unavailable: CKError { CKError(.missingEntitlement) } @@ -80,6 +86,15 @@ public actor CloudKitSyncEngine { Self.logger.trace("Created or confirmed sync zone: \(Self.zoneName)") } + // Every device saves the same id, so a save replaces the subscription rather than adding one. + public func saveSubscription() async throws { + guard let database else { throw Self.unavailable } + try await withRetry { + _ = try await database.save(SyncPushSubscription.make()) + } + Self.logger.info("Saved the CloudKit change subscription") + } + // MARK: - Push @discardableResult @@ -236,7 +251,7 @@ public actor CloudKitSyncEngine { public func pull(since token: CKServerChangeToken?) async throws -> PullResult { var changedRecords: [CKRecord] = [] var deletedRecordIDs: [CKRecord.ID] = [] - var unreadRecordIDs: [CKRecord.ID] = [] + var pages: [PullAcknowledgement.Page] = [] var cursor = token while true { @@ -246,17 +261,89 @@ public actor CloudKitSyncEngine { changedRecords.append(contentsOf: page.result.changedRecords) deletedRecordIDs.append(contentsOf: page.result.deletedRecordIDs) - unreadRecordIDs.append(contentsOf: page.result.unreadRecordIDs) + pages.append(PullAcknowledgement.Page( + startToken: cursor, + changed: Set(page.result.changedRecords.map(\.recordID)), + deleted: Set(page.result.deletedRecordIDs), + unread: page.result.unreadRecordIDs + )) cursor = page.result.newToken ?? cursor - guard page.moreComing, page.result.newToken != nil else { - return PullResult( - changedRecords: changedRecords, - deletedRecordIDs: deletedRecordIDs, - newToken: cursor, - unreadRecordIDs: unreadRecordIDs - ) + guard page.moreComing, page.result.newToken != nil else { break } + } + + let unread = Set(PullAcknowledgement.settle(pages, finalToken: cursor).unread).union(unreadLedger.released) + let refetched = await refetchByID(Array(unread)) + changedRecords.append(contentsOf: refetched.records) + unreadLedger.forget(refetched.resolved) + var settled = PullAcknowledgement.settle( + pages, finalToken: cursor, resolved: refetched.resolved.union(unreadLedger.released) + ) + let releasing = unreadLedger.countPull(holding: settled.unread) + if !releasing.isEmpty { + let names = releasing.map(\.recordName).sorted().joined(separator: ", ") + Self.logger.error( + "Released \(releasing.count, privacy: .public) records unread after \(UnreadRecordLedger.heldPulls, privacy: .public) pulls; fetching them by ID: \(names, privacy: .private)" + ) + settled = PullAcknowledgement.settle( + pages, finalToken: cursor, resolved: refetched.resolved.union(unreadLedger.released) + ) + } + if !settled.unread.isEmpty { + Self.logger.error( + "\(settled.unread.count, privacy: .public) changed records could not be read; the change token stays before them" + ) + } + return PullResult( + changedRecords: changedRecords, + deletedRecordIDs: deletedRecordIDs, + newToken: settled.token, + unreadRecordIDs: settled.unread + unreadLedger.released.sorted { $0.recordName < $1.recordName } + ) + } + + private struct Refetch { + var records: [CKRecord] = [] + // Read, or gone from the zone, so it holds the token no longer. + var resolved: Set = [] + } + + // A failed batch leaves its records unread, which holds the token rather than failing the pages already read. + private func refetchByID(_ recordIDs: [CKRecord.ID]) async -> Refetch { + var outcome = Refetch() + var start = 0 + while start < recordIDs.count { + let end = min(start + Self.refetchBatchSize, recordIDs.count) + await refetch(Array(recordIDs[start.. 1 { + let middle = batch.count / 2 + await refetch(Array(batch[.. { + let startToken: Token? + let changed: Set + let deleted: Set + let unread: [CKRecord.ID] + } + + struct Settled { + let token: Token? + let unread: [CKRecord.ID] + } + + // A record a later page changed or deleted was read at a state no older than its failure. One in `resolved` + // holds nothing either: it was read by ID, is gone, or was released by `UnreadRecordLedger`. + static func settle( + _ pages: [Page], + finalToken: Token?, + resolved: Set = [] + ) -> Settled { + var token = finalToken + var unread: [CKRecord.ID] = [] + var readLater = resolved + for page in pages.reversed() { + let stillUnread = page.unread.filter { !readLater.contains($0) } + if !stillUnread.isEmpty { + token = page.startToken + unread.insert(contentsOf: stillUnread, at: 0) + } + readLater.formUnion(page.changed) + readLater.formUnion(page.deleted) + } + var listed: Set = [] + return Settled(token: token, unread: unread.filter { listed.insert($0).inserted }) + } +} + +// A record CloudKit keeps failing to deliver holds the token for `heldPulls` pulls in a row. Then it is released: the +// token moves past it, so sync and the full-fetch deletion check go on, and each later pull fetches it by ID until it +// reads or is gone. Kept for the life of the engine. +struct UnreadRecordLedger { + static let heldPulls = 3 + + private var pullsHeld: [CKRecord.ID: Int] = [:] + private(set) var released: Set = [] + + mutating func forget(_ recordIDs: Set) { + released.subtract(recordIDs) + } + + // Counts one more pull for each record that still holds the token; a record that stopped holding starts over. + // Returns the records this pull releases. + mutating func countPull(holding recordIDs: [CKRecord.ID]) -> Set { + let holding = Set(recordIDs) + pullsHeld = pullsHeld.filter { holding.contains($0.key) } + var releasing: Set = [] + for recordID in holding { + let pulls = (pullsHeld[recordID] ?? 0) + 1 + if pulls >= Self.heldPulls { + releasing.insert(recordID) + pullsHeld[recordID] = nil + } else { + pullsHeld[recordID] = pulls + } + } + released.formUnion(releasing) + return releasing + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncAdmission.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncAdmission.swift index 3d681701fd..27082b71b9 100644 --- a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncAdmission.swift +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncAdmission.swift @@ -7,6 +7,8 @@ public enum SyncTrigger: Equatable, Sendable { case launch /// The app came forward, or the system woke it to refresh. case activation + /// CloudKit's silent notice that another device changed the private database. + case remoteChange /// Something synced was edited on this device. case localChange /// `CKAccountChanged`: the person signed in or out, or the account became ready. @@ -65,7 +67,7 @@ public enum SyncAdmission: Equatable, Sendable { return .full case .localChange: return isDue ? .full : .none - case .activation, .networkRestored: + case .activation, .remoteChange, .networkRestored: return isDue ? .full : .downloadOnly } case .signedOut, .accountNotReady, .accountRestricted: diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift index 546eb9078f..dcf9039e9b 100644 --- a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift @@ -223,6 +223,12 @@ public final class SyncMetadataStorage: @unchecked Sendable { set { userDefaults.set(newValue, forKey: key("lastAccountId")) } } + // Device-local: every device saves the change subscription once for the account it syncs with. + public var subscribedAccountId: String? { + get { userDefaults.string(forKey: key("subscribedAccountId")) } + set { userDefaults.set(newValue, forKey: key("subscribedAccountId")) } + } + @discardableResult public func adoptAccount(_ accountId: String) -> SyncAccountChange { guard let recorded = lastAccountId else { @@ -281,6 +287,7 @@ public final class SyncMetadataStorage: @unchecked Sendable { zoneState = .unknown userDefaults.removeObject(forKey: key("lastSyncDate")) userDefaults.removeObject(forKey: key("lastAccountId")) + userDefaults.removeObject(forKey: key("subscribedAccountId")) for type in SyncRecordType.allCases { clearDirty(type: type) diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushChannel.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushChannel.swift new file mode 100644 index 0000000000..30773dcf8b --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushChannel.swift @@ -0,0 +1,87 @@ +import CloudKit +import Foundation +import os + +// Registration for CloudKit's change notices and the subscription that sends them. Nothing here runs while sync is off. +@MainActor +public final class SyncPushChannel { + private static let logger = Logger(subsystem: "com.TablePro", category: "SyncPush") + + private let registrar: SyncPushRegistrar + private let metadata: SyncMetadataStorage + private var isRequested = false + private var isRegistered = false + private var missedNotice = false + private var refusedAccount: String? + public private(set) var noticesReceived = 0 + + public init(registrar: SyncPushRegistrar, metadata: SyncMetadataStorage) { + self.registrar = registrar + self.metadata = metadata + } + + public func register() { + guard !isRequested else { return } + isRequested = true + registrar.register() + } + + // Only a registration this process asked for is taken back, so a launch with sync off calls nothing. + public func unregister() { + isRegistered = false + missedNotice = false + guard isRequested else { return } + isRequested = false + registrar.unregister() + } + + public func registrationDidFinish(registered: Bool) { + guard isRequested else { return } + isRegistered = registered + } + + public func noticeArrived() { + missedNotice = false + noticesReceived += 1 + } + + // While notices arrive, a download they did not start finds nothing new. One that does, with no notice since + // the last download began, means they stopped arriving. The subscription is saved again in case it is gone. + public func activationFoundUnannouncedChanges(noticesSince count: Int) -> Bool { + guard isLive, noticesReceived == count else { return false } + missedNotice = true + forgetSubscription() + return true + } + + public var isLive: Bool { + guard isRegistered, !missedNotice, let account = metadata.lastAccountId else { return false } + return metadata.subscribedAccountId == account + } + + // A failed save never fails the run. One CloudKit refuses is not sent again for that account until the next launch. + public func saveSubscriptionIfNeeded(_ save: () async throws -> Void) async { + guard let account = metadata.lastAccountId, + metadata.subscribedAccountId != account, + refusedAccount != account else { return } + do { + try await save() + metadata.subscribedAccountId = account + } catch { + Self.logger.error("Could not save the CloudKit change subscription: \(error.localizedDescription, privacy: .private)") + // Only a fault of the build or the app version stays until relaunch; anything else is tried on the next run. + switch SyncFailure(error) { + case .blocked(.unavailable), .blocked(.appUpdateRequired): + refusedAccount = account + default: + break + } + } + } + + // Turning sync on, or finding TablePro's data removed from iCloud, saves the subscription again in case it is gone. + public func forgetSubscription() { + metadata.subscribedAccountId = nil + refusedAccount = nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushRegistrar.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushRegistrar.swift new file mode 100644 index 0000000000..1e4f295552 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushRegistrar.swift @@ -0,0 +1,13 @@ +// Each app supplies its own: AppKit and UIKit answer a registration through the app delegate. +public struct SyncPushRegistrar: Sendable { + public let register: @MainActor @Sendable () -> Void + public let unregister: @MainActor @Sendable () -> Void + + public init( + register: @escaping @MainActor @Sendable () -> Void, + unregister: @escaping @MainActor @Sendable () -> Void + ) { + self.register = register + self.unregister = unregister + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushSubscription.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushSubscription.swift new file mode 100644 index 0000000000..c998385588 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncPushSubscription.swift @@ -0,0 +1,27 @@ +import CloudKit +import Foundation + +public enum SyncPushSubscription { + public static let id = "TableProSyncChanges" + + // A database subscription outlives a deleted or purged zone and announces the zone coming back. + public static func make() -> CKDatabaseSubscription { + let subscription = CKDatabaseSubscription(subscriptionID: id) + let notificationInfo = CKSubscription.NotificationInfo() + // Silent: no alert, badge or sound, so the system asks the person nothing. + notificationInfo.shouldSendContentAvailable = true + subscription.notificationInfo = notificationInfo + return subscription + } + + // The payload is only a signal: it starts the same download an activation does, and nothing in it is read. + public static func announcesChange( + _ userInfo: [AnyHashable: Any], + containerIdentifier: String = CloudKitSyncEngine.defaultContainerID + ) -> Bool { + guard let notification = CKNotification(fromRemoteNotificationDictionary: userInfo) as? CKDatabaseNotification else { + return false + } + return notification.subscriptionID == id && notification.containerIdentifier == containerIdentifier + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift new file mode 100644 index 0000000000..d9b8f9c169 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift @@ -0,0 +1,32 @@ +// +// LibPQSystemTrustRule.swift +// TableProTLSClientIdentity +// + +import Foundation + +// One rule for the driver and for pg_dump and pg_restore, so a session and its backup trust the same roots. +public enum LibPQSystemTrustRule { + // Verify Identity with no CA file, over TCP, with no root file the user named for libpq. + public static func usesSystemTrustAnchors( + verifiesHostname: Bool, + caCertificatePath: String, + host: String, + environment: [String: String], + fileExists: (String) -> Bool + ) -> Bool { + // libpq opens no TLS on a socket, and an empty host is its default socket. + guard verifiesHostname, caCertificatePath.isEmpty, !host.isEmpty, !host.hasPrefix("/") else { return false } + if let named = environment["PGSSLROOTCERT"], !named.isEmpty { return false } + return !fileExists(defaultRootCertificatePath(environment: environment)) + } + + // libpq reads HOME first and the password database only when it is unset or empty. + public static func defaultRootCertificatePath(environment: [String: String]) -> String { + let home = environment["HOME"].flatMap { $0.isEmpty ? nil : $0 } ?? NSHomeDirectory() + return URL(fileURLWithPath: home, isDirectory: true) + .appendingPathComponent(".postgresql", isDirectory: true) + .appendingPathComponent("root.crt") + .path + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift new file mode 100644 index 0000000000..ec8d75984d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift @@ -0,0 +1,284 @@ +#if os(macOS) +import Foundation +import os +import Security + +public enum SystemTrustDomain: Sendable { + case user + case admin + case system + + var securityDomain: SecTrustSettingsDomain { + switch self { + case .user: .user + case .admin: .admin + case .system: .system + } + } +} + +public enum SystemTrustResult: Sendable, Equatable { + case trustRoot + case trustAsRoot + case deny + case unspecified +} + +public struct SystemTrustSetting: Sendable, Equatable { + public var appliesToSSL: Bool + public var namesApplication: Bool + public var hostname: String? + // Nil when the entry carries no result, which Apple reads as TrustRoot. + public var result: SystemTrustResult? + + public init( + appliesToSSL: Bool = true, + namesApplication: Bool = false, + hostname: String? = nil, + result: SystemTrustResult? = nil + ) { + self.appliesToSSL = appliesToSSL + self.namesApplication = namesApplication + self.hostname = hostname + self.result = result + } +} + +public protocol SystemTrustSource { + // Empty when the domain has no trust settings. + func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] + // Nil when the domain holds no settings for the certificate. + func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? + func anchorCertificates() throws -> [SecCertificate] +} + +public enum SystemTrustAnchorsError: Error, Sendable, Equatable { + case settingsUnreadable(OSStatus) + case noTrustedRoots + case exportFailed(OSStatus) + case fileUnwritable +} + +extension SystemTrustAnchorsError: LocalizedError { + public var message: String { + String( + localized: """ + Could not read this Mac's trusted certificates to verify the server. Choose the server's CA \ + certificate under SSL, then connect again. + """ + ) + } + + public var errorDescription: String? { + message + } +} + +// For a TLS library with no trust store of its own, such as the bundled OpenSSL. The roots come from the trust +// settings, never from evaluating each root alone: a SHA-1 self-signed root fails as a leaf and still anchors chains. +public enum SystemTrustAnchors { + public static let shared = SystemTrustAnchorCache(source: SecuritySystemTrustSource(), clock: ContinuousClock()) + + static let logger = Logger(subsystem: "com.TablePro", category: "SystemTrustAnchors") + + // Per-user settings override admin ones, which override the system's. The first domain with a result decides. + public static func trustedRoots(from source: any SystemTrustSource) throws -> [SecCertificate] { + var results: [Data: SystemTrustResult] = [:] + var decided: [(der: Data, certificate: SecCertificate)] = [] + for domain in [SystemTrustDomain.user, .admin, .system] { + let (certificates, fromAnchorList) = try certificates(in: domain, from: source) + for certificate in certificates { + let der = SecCertificateCopyData(certificate) as Data + guard results[der] == nil else { continue } + let result: SystemTrustResult? = fromAnchorList ? .trustRoot : decision(for: certificate, in: domain, from: source) + guard let result else { continue } + results[der] = result + decided.append((der, certificate)) + } + } + return decided.filter { results[$0.der] == .trustRoot || results[$0.der] == .trustAsRoot }.map(\.certificate) + } + + public static func pem(from source: any SystemTrustSource) throws -> Data { + let roots = try trustedRoots(from: source) + guard !roots.isEmpty else { throw SystemTrustAnchorsError.noTrustedRoots } + var exported: CFData? + let status = SecItemExport(roots as CFArray, .formatPEMSequence, [], nil, &exported) + guard status == errSecSuccess, let exported, CFDataGetLength(exported) > 0 else { + logger.error("Exporting the trusted roots failed: \(status, privacy: .public)") + throw SystemTrustAnchorsError.exportFailed(status) + } + return exported as Data + } + + static func effectiveResult(of settings: [SystemTrustSetting]) -> SystemTrustResult? { + guard !settings.isEmpty else { return .trustRoot } + for setting in settings where setting.appliesToSSL && !setting.namesApplication { + let result = setting.result ?? .trustRoot + // A PEM cannot carry a hostname limit, so a scoped trust is left out and a scoped deny covers every host. + if setting.hostname != nil { + if result == .deny { return .deny } + continue + } + if result != .unspecified { return result } + } + return nil + } + + // The system domain is the one that must answer. If it cannot, the anchors macOS ships stand in for it. + private static func certificates( + in domain: SystemTrustDomain, + from source: any SystemTrustSource + ) throws -> (certificates: [SecCertificate], fromAnchorList: Bool) { + do { + let certificates = try source.certificates(in: domain) + if domain == .system, certificates.isEmpty { + logger.error("The system trust settings list no certificates; using the system anchors") + return (try source.anchorCertificates(), true) + } + return (certificates, false) + } catch { + logger.error( + "Reading the \(String(describing: domain), privacy: .public) trust settings failed: \(String(describing: error), privacy: .private)" + ) + guard domain == .system else { throw error } + return (try source.anchorCertificates(), true) + } + } + + // A setting that cannot be read could be a deny, so it counts as one. + private static func decision( + for certificate: SecCertificate, + in domain: SystemTrustDomain, + from source: any SystemTrustSource + ) -> SystemTrustResult? { + do { + guard let settings = try source.settings(for: certificate, in: domain) else { return nil } + return effectiveResult(of: settings) + } catch { + logger.error("A certificate's \(String(describing: domain), privacy: .public) trust settings could not be read") + return .deny + } + } +} + +// The bytes are kept for a minute, so a root just trusted or distrusted in Keychain Access applies on a later connect +// without a relaunch. A failed build is never kept. +public final class SystemTrustAnchorCache: @unchecked Sendable where C.Duration == Duration { + private let source: any SystemTrustSource + private let clock: C + private let lifetime: Duration + private let lock = NSLock() + private var cached: (pem: Data, builtAt: C.Instant)? + + public init(source: any SystemTrustSource, clock: C, lifetime: Duration = .seconds(60)) { + self.source = source + self.clock = clock + self.lifetime = lifetime + } + + public func pem() throws -> Data { + try lock.withLock { + if let cached, cached.builtAt.duration(to: clock.now) < lifetime { + return cached.pem + } + let built = try SystemTrustAnchors.pem(from: source) + cached = (built, clock.now) + return built + } + } + + public func writeFile(in parent: URL = FileManager.default.temporaryDirectory) throws -> SystemTrustAnchorFile { + try SystemTrustAnchorFile.write(pem(), in: parent) + } +} + +// One file per use, in a folder only this user can open, removed when the use ends. +public struct SystemTrustAnchorFile: Sendable { + public let url: URL + let directory: URL + + static func write(_ pem: Data, in parent: URL) throws -> SystemTrustAnchorFile { + let directory = parent.appendingPathComponent("tablepro-trust-\(UUID().uuidString)", isDirectory: true) + let url = directory.appendingPathComponent("roots.pem") + do { + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: false, + attributes: [.posixPermissions: 0o700] + ) + guard FileManager.default.createFile( + atPath: url.path, + contents: pem, + attributes: [.posixPermissions: 0o600] + ) else { + throw SystemTrustAnchorsError.fileUnwritable + } + } catch { + try? FileManager.default.removeItem(at: directory) + SystemTrustAnchors.logger.error("Writing the trusted roots file failed") + throw SystemTrustAnchorsError.fileUnwritable + } + return SystemTrustAnchorFile(url: url, directory: directory) + } + + public func remove() { + try? FileManager.default.removeItem(at: directory) + } +} + +public struct SecuritySystemTrustSource: SystemTrustSource, Sendable { + public init() {} + + public func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] { + var certificates: CFArray? + let status = SecTrustSettingsCopyCertificates(domain.securityDomain, &certificates) + if status == errSecNoTrustSettings { return [] } + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + return (certificates as? [SecCertificate]) ?? [] + } + + public func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? { + var settings: CFArray? + let status = SecTrustSettingsCopyTrustSettings(certificate, domain.securityDomain, &settings) + if status == errSecItemNotFound || status == errSecNoTrustSettings { return nil } + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + let entries = (settings as? [[String: Any]]) ?? [] + return entries.map(Self.setting(from:)) + } + + public func anchorCertificates() throws -> [SecCertificate] { + var anchors: CFArray? + let status = SecTrustCopyAnchorCertificates(&anchors) + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + return (anchors as? [SecCertificate]) ?? [] + } + + public static func setting(from entry: [String: Any]) -> SystemTrustSetting { + let hostname = entry[kSecTrustSettingsPolicyString as String] as? String + return SystemTrustSetting( + appliesToSSL: entry[kSecTrustSettingsPolicy as String].map(isSSLPolicy) ?? true, + namesApplication: entry[kSecTrustSettingsApplication as String] != nil, + hostname: hostname?.isEmpty == false ? hostname : nil, + result: (entry[kSecTrustSettingsResult as String] as? NSNumber).map { result(of: $0.uint32Value) } + ) + } + + private static func isSSLPolicy(_ value: Any) -> Bool { + guard CFGetTypeID(value as CFTypeRef) == SecPolicyGetTypeID() else { return false } + let policy = unsafeDowncast(value as AnyObject, to: SecPolicy.self) + let properties = SecPolicyCopyProperties(policy) as? [String: Any] + return properties?[kSecPolicyOid as String] as? String == kSecPolicyAppleSSL as String + } + + // An unknown value could mean distrust, so it counts as a deny. + private static func result(of rawValue: UInt32) -> SystemTrustResult { + switch SecTrustSettingsResult(rawValue: rawValue) { + case .trustRoot?: .trustRoot + case .trustAsRoot?: .trustAsRoot + case .unspecified?: .unspecified + default: .deny + } + } +} +#endif diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift index e00a7551a5..cb810a30f5 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift @@ -4,6 +4,8 @@ import TableProWireIO enum ParcelFlavor: UInt16 { case request = 1 case response = 4 + case abort = 6 + case cancel = 7 case success = 8 case failure = 9 case record = 10 diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift index 157d694a8c..27b0827601 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift @@ -41,7 +41,8 @@ enum RecordDecoder { case TeradataType.decimal: return try reader.take(decimalByteWidth(precision: column.dataLength >> 8)) default: - throw TeradataWireError.unsupported("record field type \(column.baseCode)") + // NUMBER's length byte is not documented well enough to read, and LOBs need the multipart request. + throw TeradataWireError.unreadableColumn(name: column.name, typeCode: column.typeCode) } } @@ -115,7 +116,7 @@ enum RecordDecoder { let day = Int(packed % 100) return .text(String(format: "%04d-%02d-%02d", year, month, day)) default: - throw TeradataWireError.unsupported("record field type \(column.baseCode)") + throw TeradataWireError.unreadableColumn(name: column.name, typeCode: column.typeCode) } } } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift index 155fe14060..e6b8c0ca40 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift @@ -58,6 +58,11 @@ public enum TeradataColumnType { } public static func wireTypeName(_ code: UInt16) -> String { + knownWireTypeName(code) ?? "VARCHAR" + } + + // Codes from the CLIv2 data type table; JSON's from the embedded SQL SQLDA table, which CLIv2 does not list. + public static func knownWireTypeName(_ code: UInt16) -> String? { switch code & 0xFFFE { case TeradataType.byteint: return "BYTEINT" case TeradataType.smallint: return "SMALLINT" @@ -65,7 +70,7 @@ public enum TeradataColumnType { case TeradataType.bigint: return "BIGINT" case TeradataType.float: return "FLOAT" case TeradataType.decimal: return "DECIMAL" - case 604: return "NUMBER" + case TeradataType.number: return "NUMBER" case TeradataType.char: return "CHAR" case TeradataType.varchar: return "VARCHAR" case TeradataType.longVarchar: return "LONG VARCHAR" @@ -76,10 +81,19 @@ public enum TeradataColumnType { case 764: return "TIMESTAMP" case 768: return "TIME WITH TIME ZONE" case 772: return "TIMESTAMP WITH TIME ZONE" - case 400: return "BLOB" - case 416: return "CLOB" - case 880: return "JSON" - default: return "VARCHAR" + case 776, 780, 784, 788, 792, 796, 800, 804, 808, 812, 816, 820, 824: return "INTERVAL" + case 832: return "PERIOD(DATE)" + case 836: return "PERIOD(TIME)" + case 840: return "PERIOD(TIME WITH TIME ZONE)" + case 844: return "PERIOD(TIMESTAMP)" + case 848: return "PERIOD(TIMESTAMP WITH TIME ZONE)" + case 400, 404, 408, 412: return "BLOB" + case 416, 420, 424: return "CLOB" + case 432, 436, 440: return "UDT" + case 504, 508: return "ARRAY" + case 852, 856, 860: return "XML" + case 880, 884, 888: return "JSON" + default: return nil } } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index ed5608fe9f..3483dee05c 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -1,7 +1,8 @@ import Darwin import Foundation -public final class TeradataConnection { +// One queue drives it. Only `cancel()` comes from elsewhere, and what it reads and sends is under socketLock or wireLock. +public final class TeradataConnection: @unchecked Sendable { private let config: TeradataConnectionConfig private let socketLock = NSLock() private var _socket: (any TeradataTransport)? @@ -20,18 +21,90 @@ public final class TeradataConnection { private var gssSequence: UInt64 = 0 private var confidentialityBypassed = false + // Orders every post-logon send with the counters it consumes, since Stop sends its abort from another thread. + private let wireLock = NSLock() + // Under socketLock, which is never held across I/O. + private var _phase = RequestPhase.idle + private var _stopRequested = false + private var _operationActive = false + // The request a stop is waiting on the server for, so the stop's deadline closes nothing that came after it. + private var _stoppingRequest: UInt32? + // Under socketLock: the semantics the session runs with, and what its requests have left open. + private var _sessionMode: TeradataTransactionMode + private var _sessionModeSource: TeradataSessionModeSource + private var _transaction: TeradataTransactionTracker + // Nil until a switch the connection made itself, and again once a request of the user's may have moved it. + private var _defaultDatabase: String? + + // 3110: "The transaction was aborted by the user." + public static let userAbortCode = 3_110 + private static let abortLockWait: TimeInterval = 0.25 + // A server or network that stops answering would otherwise hold a stop for the whole 20-second receive timeout. + static let defaultStopGrace: TimeInterval = 5 + private let stopGrace: TimeInterval + private let stopDeadlineQueue = DispatchQueue(label: "com.TablePro.teradata.stop-deadline") + + private enum RequestPhase: Equatable { + case idle + case connecting + case awaitingResponse(UInt32) + case fetching(UInt32) + } + + private enum StopAction { + case nothing + case abort(UInt32) + case endOutput(UInt32) + case close + } + + private enum Exchange { + case answered + case notSent + case outputEnded + // The consumer threw on a reply, and the request has ended on the wire. + case refused(any Error) + } + public init(config: TeradataConnectionConfig) { self.config = config + stopGrace = Self.defaultStopGrace + let configured = Self.configuredMode(config.transactionMode) + _sessionMode = configured.mode + _sessionModeSource = configured.source + _transaction = TeradataTransactionTracker(mode: configured.mode) } - init(config: TeradataConnectionConfig, transport: any TeradataTransport) { + init( + config: TeradataConnectionConfig, + transport: any TeradataTransport, + stopGrace: TimeInterval = TeradataConnection.defaultStopGrace + ) { self.config = config self._socket = transport + self.stopGrace = stopGrace confidentialityBypassed = true + let configured = Self.configuredMode(config.transactionMode) + _sessionMode = configured.mode + _sessionModeSource = configured.source + _transaction = TeradataTransactionTracker(mode: configured.mode) + } + + private static func configuredMode( + _ mode: TeradataTransactionMode + ) -> (mode: TeradataTransactionMode, source: TeradataSessionModeSource) { + mode == .default ? (.default, .unresolved) : (mode, .configured) } public var isConnected: Bool { socket != nil } + // `.default` when the session could not say which semantics it runs with; it then gets Teradata's. + public var sessionTransactionMode: TeradataTransactionMode { socketLock.withLock { _sessionMode } } + + public var sessionModeSource: TeradataSessionModeSource { socketLock.withLock { _sessionModeSource } } + + public var transactionState: TeradataTransactionState { socketLock.withLock { _transaction.state } } + // Set when a request failed partway, so the session it ran on was given up rather than ended. public var hasLostConnection: Bool { socketLock.withLock { _hasLostConnection } } @@ -49,6 +122,7 @@ public final class TeradataConnection { socketLock.withLock { _socket = socket _hasLostConnection = false + _phase = .connecting } var established = false defer { @@ -57,6 +131,7 @@ public final class TeradataConnection { self.socket = nil connectDeadline = nil } + socketLock.withLock { _phase = .idle } } serverIP = Self.resolveAddress(config.host) ?? config.host _ = try deadline.remainingMilliseconds() @@ -66,16 +141,12 @@ public final class TeradataConnection { let params = try Td2Token.parseServerParams(serverToken) try runKeyExchange(params) try runConnect() - - if let database = config.database, !database.isEmpty { - _ = try execute(TeradataSchemaQueries.setDatabase(database)) - } + try prepareSession() _ = try deadline.remainingMilliseconds() established = true } - /// Keeps the transport's connect deadline active through the driver's required VERSION probe. - /// Only then may ordinary query I/O use the session's 20-second socket timeout. + // The connect deadline holds through the driver's VERSION probe; only then does query I/O get the 20-second timeout. public func finishConnecting() throws { guard let socket, let connectDeadline else { throw TeradataWireError.connectionFailed("not connecting") @@ -85,34 +156,268 @@ public final class TeradataConnection { self.connectDeadline = nil } + // DEFAULT is the site's default, so the session is asked which one it got. An ANSI session opens a transaction + // with its first request and counts DATABASE as DDL, after which only COMMIT is legal, so the setup is committed. + func prepareSession() throws { + let (mode, source) = try resolveSessionMode() + var ranRequest = config.transactionMode == .default + if let database = config.database, !database.isEmpty { + _ = try execute(TeradataSchemaQueries.setDatabase(database)) + ranRequest = true + } + if mode == .ansi, ranRequest { + _ = try execute(TeradataTransactionSQL.commit(.ansi)) + } + socketLock.withLock { + _sessionMode = mode + _sessionModeSource = source + _transaction = TeradataTransactionTracker(mode: mode) + _defaultDatabase = config.database.flatMap { $0.isEmpty ? nil : $0 } + } + } + + private func resolveSessionMode() throws -> (TeradataTransactionMode, TeradataSessionModeSource) { + guard config.transactionMode == .default else { return (config.transactionMode, .configured) } + if let mode = try probeSessionMode( + TeradataSessionModeProbe.helpSession, read: TeradataSessionModeProbe.mode(fromHelpSession:) + ) { + return (mode, .helpSession) + } + if let mode = try probeSessionMode( + TeradataSessionModeProbe.sessionInfo, read: TeradataSessionModeProbe.mode(fromSessionInfo:) + ) { + return (mode, .sessionInfoView) + } + return (.default, .unresolved) + } + + // A refusal, or a reply this decoder cannot read, leaves the session usable for the next way of asking. + private func probeSessionMode( + _ sql: String, + read: (TeradataResultSet) -> TeradataTransactionMode? + ) throws -> TeradataTransactionMode? { + do { + return read(try execute(sql)) + } catch TeradataWireError.cancelled { + throw TeradataWireError.cancelled + } catch { + guard isConnected else { throw error } + return nil + } + } + + public func beginTransaction() throws { + guard let statement = TeradataTransactionSQL.begin(sessionTransactionMode) else { return } + _ = try execute(statement) + } + + public func commitTransaction() throws { + _ = try execute(TeradataTransactionSQL.commit(sessionTransactionMode)) + } + + public func rollbackTransaction() throws { + do { + _ = try execute(TeradataTransactionSQL.rollback(sessionTransactionMode)) + } catch TeradataWireError.server(let code, _) where TeradataTransactionSQL.rollbackAnswerCodes.contains(code) { + // The failure parcel a rollback answers with is its success. + } + socketLock.withLock { _transaction.rolledBack() } + } + + // After DATABASE inside a transaction only the closing statement is legal (ANSI counts it as DDL; Teradata mode + // is not documented either way), so a switch never joins an open one, and in ANSI it is committed at once. + // The host re-sends the switch before every statement, so one to where the session already is sends nothing. + public func switchDatabase(to database: String) throws { + let (mode, state, current) = socketLock.withLock { (_sessionMode, _transaction.state, _defaultDatabase) } + guard state != .open else { + // Teradata compares database names without regard to case. + guard let current, current.caseInsensitiveCompare(database) == .orderedSame else { + throw TeradataWireError.openTransactionBlocksDatabaseSwitch + } + return + } + _ = try execute(TeradataSchemaQueries.setDatabase(database)) + socketLock.withLock { _defaultDatabase = database } + if mode == .ansi { + _ = try execute(TeradataTransactionSQL.commit(.ansi)) + } + } + public func execute(_ sql: String) throws -> TeradataResultSet { + var columns: [TeradataColumn] = [] + var rows: [[TeradataValue]] = [] + let activityCount = try perform(sql, onColumns: { columns = $0 }, onRows: { batch in + rows.append(contentsOf: batch) + return true + }) + return TeradataResultSet(columns: columns, rows: rows, activityCount: activityCount) + } + + // The reply that passes the cap ends the output on the server, so the rows past it are never fetched. + public func execute(_ sql: String, rowCap: Int?) throws -> TeradataCappedResult { + guard let rowCap, rowCap > 0 else { + return TeradataCappedResult(resultSet: try execute(sql), isTruncated: false) + } + var columns: [TeradataColumn] = [] + var rows: [[TeradataValue]] = [] + var isTruncated = false + let activityCount = try perform(sql, onColumns: { columns = $0 }, onRows: { batch in + guard rows.count + batch.count > rowCap else { + rows.append(contentsOf: batch) + return true + } + rows.append(contentsOf: batch.prefix(rowCap - rows.count)) + isTruncated = true + return false + }) + return TeradataCappedResult( + resultSet: TeradataResultSet(columns: columns, rows: rows, activityCount: activityCount), + isTruncated: isTruncated) + } + + // `onColumns` runs once before any rows, with none for a request that has no result set; false from `onRows` ends the output. + public func stream( + _ sql: String, + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws { + _ = try perform(sql, onColumns: onColumns, onRows: onRows) + } + + private func perform( + _ sql: String, + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws -> Int { guard socket != nil else { throw TeradataWireError.connectionFailed("not connected") } + let kinds = TeradataStatementKind.kinds(in: sql) + let movesDatabase = TeradataStatementKind.mayMoveDefaultDatabase(sql) let requestNumber = nextRequest() let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() + TeradataMessages.responseParcel().encoded() - let parcels: [Parcel] + var reader = TeradataResultParser.Reader() + let outcome: Exchange do { - parcels = try exchange(body, requestNumber: requestNumber) + outcome = try exchange(body, requestNumber: requestNumber) { parcels in + try reader.read(parcels, onColumns: onColumns, onRows: onRows) + } } catch { + endRequest() // The reply still on its way would be read as the next request's. dropTransport() throw error } - return try TeradataResultParser.parse(parcels) + let stopped = endRequest() + switch outcome { + case .notSent: + throw TeradataWireError.cancelled + case .refused(let error): + guard case .server(let code, _)? = error as? TeradataWireError else { + // Refused by its reader rather than the server, so the request ran. + recordRan(kinds, movesDatabase: movesDatabase) + throw error + } + let abortRolledBack = socketLock.withLock { () -> Bool in + let heldTransaction = _transaction.abortRollsBackTransaction + _transaction.failed(kinds, code: code) + return heldTransaction + } + if stopped, code == Self.userAbortCode { + throw abortRolledBack ? TeradataWireError.rolledBackByStop : TeradataWireError.cancelled + } + throw error + case .outputEnded where stopped: + recordRan(kinds, movesDatabase: movesDatabase) + throw TeradataWireError.cancelled + case .answered, .outputEnded: + recordRan(kinds, movesDatabase: movesDatabase) + try reader.finish(onColumns: onColumns) + return reader.activityCount + } + } + + private func recordRan(_ kinds: [TeradataStatementKind], movesDatabase: Bool) { + socketLock.withLock { + _transaction.ran(kinds) + if movesDatabase { _defaultDatabase = nil } + } } - private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> [Parcel] { - try sendEncrypted(kind: .start, body: body, requestNumber: requestNumber) - var parcels: [Parcel] = [] + private func exchange( + _ body: [UInt8], + requestNumber: UInt32, + consume: ([Parcel]) throws -> Bool + ) throws -> Exchange { + let sent = try wireLock.withLock { () -> Bool in + // A stop that landed before the request went out refuses it, and the session stays as it was. + let refused = socketLock.withLock { () -> Bool in + guard !_stopRequested else { return true } + _phase = .awaitingResponse(requestNumber) + return false + } + guard !refused else { return false } + try sendEncryptedLocked(kind: .start, body: body, requestNumber: requestNumber) + return true + } + guard sent else { return .notSent } while true { let batch = try readReply() - parcels.append(contentsOf: batch) - if Self.responseIsComplete(batch) { return parcels } + let complete = Self.responseIsComplete(batch) + // Once the first reply is out an abort no longer takes effect, so a stop ends the output instead. + socketLock.withLock { _phase = .fetching(requestNumber) } + let wantsMore: Bool + do { + wantsMore = try consume(batch) + } catch { + // Ending the output keeps the session; a failed end leaves the rest to be read as the next request's. + if !complete, (try? endOutput(requestNumber: requestNumber)) == nil { throw error } + return .refused(error) + } + if complete { return .answered } + if !wantsMore || socketLock.withLock({ _stopRequested }) { + try endOutput(requestNumber: requestNumber) + return .outputEnded + } try sendContinue(requestNumber: requestNumber) } } + // The Cancel parcel in place of the Response parcel deletes the request's spool, and the server answers with + // End Request. One that does not throws, and the caller gives the transport up. + private func endOutput(requestNumber: UInt32) throws { + try sendEncrypted(kind: .cont, body: TeradataMessages.cancelParcel().encoded(), requestNumber: requestNumber) + guard Self.responseIsComplete(try readReply()) else { + throw TeradataWireError.malformed("the server kept answering a cancelled request") + } + } + + // A caller's operation, so a stop that lands before its request goes out still reaches it. + public func beginOperation() { + socketLock.withLock { + _operationActive = true + _stopRequested = false + } + } + + public func endOperation() { + socketLock.withLock { + _operationActive = false + _stopRequested = false + } + } + + @discardableResult + private func endRequest() -> Bool { + socketLock.withLock { + let stopped = _stopRequested + _phase = .idle + _stopRequested = false + _stoppingRequest = nil + return stopped + } + } + private func dropTransport() { let dropped = socketLock.withLock { let transport = _socket @@ -129,19 +434,80 @@ public final class TeradataConnection { return } if sessionNumber != 0 { - let message = LanMessage( - kind: .logoff, body: TeradataMessages.logoffParcel().encoded(), - sessionNumber: sessionNumber, byteVar: 0, - authentication: TeradataMessages.authBytes(nextMessage()), hostCharSet: charsetCode) - try? socket.send(message.encoded()) + wireLock.withLock { + let message = LanMessage( + kind: .logoff, body: TeradataMessages.logoffParcel().encoded(), + sessionNumber: sessionNumber, byteVar: 0, + authentication: TeradataMessages.authBytes(nextMessage()), hostCharSet: charsetCode) + try? socket.send(message.encoded()) + } } socket.close() self.socket = nil connectDeadline = nil } + // A request the server is still working on is aborted on the session, which rolls back its transaction, answers it + // with failure 3110 and stays logged on. Past the first reply the request's own loop ends the output instead, and a + // request not yet sent is refused. A connect has no request to stop, so it gives the transport up, as does an + // abort that cannot be sent or a server that has not ended the request within the grace. public func cancel() { - socket?.cancel() + let action: StopAction = socketLock.withLock { + switch _phase { + case .idle: + if _operationActive { _stopRequested = true } + return .nothing + case .connecting: + return .close + case .awaitingResponse(let request): + _stopRequested = true + _stoppingRequest = request + return .abort(request) + case .fetching(let request): + _stopRequested = true + _stoppingRequest = request + return .endOutput(request) + } + } + switch action { + case .nothing: + return + case .close: + socket?.cancel() + case .abort(let request): + guard sendAbort(for: request) else { + socket?.cancel() + return + } + closeUnlessEnded(request) + case .endOutput(let request): + closeUnlessEnded(request) + } + } + + private func closeUnlessEnded(_ request: UInt32) { + stopDeadlineQueue.asyncAfter(deadline: .now() + stopGrace) { [weak self] in + guard let self else { return } + // Shut down under the lock, so a request that ended meanwhile cannot hand the transport to a new one. + socketLock.withLock { + guard _stoppingRequest == request else { return } + _socket?.cancel() + } + } + } + + // A long start still going out holds the lock, and the caller may be the main thread. + private func sendAbort(for request: UInt32) -> Bool { + guard wireLock.lock(before: Date(timeIntervalSinceNow: Self.abortLockWait)) else { return false } + defer { wireLock.unlock() } + // Past the first reply the request's own loop ends the output; a request already answered needs nothing. + guard socketLock.withLock({ _phase == .awaitingResponse(request) }) else { return true } + do { + try sendEncryptedLocked(kind: .abort, body: TeradataMessages.abortParcel().encoded(), requestNumber: request) + return true + } catch { + return false + } } static func responseIsComplete(_ parcels: [Parcel]) -> Bool { @@ -252,6 +618,12 @@ public final class TeradataConnection { } private func sendEncrypted(kind: MessageKind, body: [UInt8], requestNumber: UInt32) throws { + try wireLock.withLock { + try sendEncryptedLocked(kind: kind, body: body, requestNumber: requestNumber) + } + } + + private func sendEncryptedLocked(kind: MessageKind, body: [UInt8], requestNumber: UInt32) throws { let message = LanMessage( kind: kind, body: body, sessionNumber: sessionNumber, requestNumber: requestNumber, byteVar: 0, authentication: TeradataMessages.authBytes(nextMessage()), @@ -340,3 +712,9 @@ public final class TeradataConnection { return String(cString: buffer) } } + +public struct TeradataCappedResult: Sendable { + public let resultSet: TeradataResultSet + // The answer held more rows than the cap, and its output was ended there. + public let isTruncated: Bool +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift index 042633063c..d0ec9d359e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift @@ -119,6 +119,14 @@ enum TeradataMessages { Parcel(.logoff) } + static func abortParcel() -> Parcel { + Parcel(.abort) + } + + static func cancelParcel() -> Parcel { + Parcel(.cancel) + } + static func charsetCode(fromConfigResponse body: [UInt8], name: String) -> UInt8? { let needle = Array(name.utf8) guard !needle.isEmpty else { return nil } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift index b8c409d28f..13073c15a2 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift @@ -2,36 +2,62 @@ import Foundation import TableProWireIO enum TeradataResultParser { - static func parse(_ parcels: [Parcel]) throws -> TeradataResultSet { - for parcel in parcels where parcel.flavor == ParcelFlavor.failure.rawValue - || parcel.flavor == ParcelFlavor.error.rawValue - || parcel.flavor == ParcelFlavor.statementError.rawValue { - let (code, message) = errorDetail(parcel) - throw TeradataWireError.server(code: code, message: message) - } + // The answer is its first result set; a later statement's rows join only when they decode the same way. + struct Reader { + private var resultColumns: [ColumnMeta]? + private var statementColumns: [ColumnMeta] = [] + private var statementJoins = false + private var firstActivityCount: Int? - let activityCount = parcels - .first { $0.flavor == ParcelFlavor.success.rawValue } - .map { readActivityCount(fromSuccess: $0.body) } ?? 0 + var activityCount: Int { firstActivityCount ?? 0 } - var columns: [ColumnMeta] = [] - if let prepInfoX = parcels.first(where: { $0.flavor == ParcelFlavor.prepInfoX.rawValue }) { - columns = try parsePrepInfo(prepInfoX.body, extended: true) - } else if let prepInfo = parcels.first(where: { $0.flavor == ParcelFlavor.prepInfo.rawValue }) { - columns = try parsePrepInfo(prepInfo.body, extended: false) - } - - var rows: [[TeradataValue]] = [] - if !columns.isEmpty { - for record in parcels where record.flavor == ParcelFlavor.record.rawValue { - rows.append(try RecordDecoder.decode(recordBody: record.body, columns: columns)) + // False when the consumer wants no more rows. + mutating func read( + _ parcels: [Parcel], + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws -> Bool { + if let failure = parcels.first(where: TeradataResultParser.isError) { + let (code, message) = TeradataResultParser.errorDetail(failure) + throw TeradataWireError.server(code: code, message: message) + } + var rows: [[TeradataValue]] = [] + for parcel in parcels { + switch parcel.knownFlavor { + case .success?: + if firstActivityCount == nil { + firstActivityCount = TeradataResultParser.readActivityCount(fromSuccess: parcel.body) + } + case .prepInfo?, .prepInfoX?: + let columns = try TeradataResultParser.parsePrepInfo( + parcel.body, extended: parcel.flavor == ParcelFlavor.prepInfoX.rawValue) + if let resultColumns { + statementJoins = !resultColumns.isEmpty + && TeradataResultParser.decodeAlike(columns, resultColumns) + } else { + resultColumns = columns + statementJoins = !columns.isEmpty + try onColumns(columns.map { + TeradataColumn(name: $0.name, typeCode: $0.typeCode, dataLength: $0.dataLength) + }) + } + statementColumns = columns + case .record? where statementJoins: + rows.append(try RecordDecoder.decode(recordBody: parcel.body, columns: statementColumns)) + default: + break + } } + guard !rows.isEmpty else { return true } + return try onRows(rows) } - let resultColumns = columns.map { - TeradataColumn(name: $0.name, typeCode: $0.typeCode, dataLength: $0.dataLength) + // A request that described no result set still answers its consumer, with no columns. + mutating func finish(onColumns: ([TeradataColumn]) throws -> Void) throws { + guard resultColumns == nil else { return } + resultColumns = [] + try onColumns([]) } - return TeradataResultSet(columns: resultColumns, rows: rows, activityCount: activityCount) } static func errorDetail(_ parcel: Parcel) -> (code: Int, message: String) { @@ -45,6 +71,18 @@ enum TeradataResultParser { return (code, message.isEmpty ? decodedText(body) : message) } + private static func isError(_ parcel: Parcel) -> Bool { + parcel.flavor == ParcelFlavor.failure.rawValue + || parcel.flavor == ParcelFlavor.error.rawValue + || parcel.flavor == ParcelFlavor.statementError.rawValue + } + + // A record decodes by each field's base type and declared length, so two descriptions alike in those read it alike. + private static func decodeAlike(_ lhs: [ColumnMeta], _ rhs: [ColumnMeta]) -> Bool { + lhs.count == rhs.count + && zip(lhs, rhs).allSatisfy { $0.baseCode == $1.baseCode && $0.dataLength == $1.dataLength } + } + private static func decodedText(_ bytes: some Collection) -> String { String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift index a82563f33b..af2f79984c 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift @@ -8,6 +8,10 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case server(code: Int, message: String) case unsupported(String) case cancelled + // Stop aborted the request, and the server rolled back the transaction it ran in. + case rolledBackByStop + case unreadableColumn(name: String, typeCode: UInt16) + case openTransactionBlocksDatabaseSwitch public var description: String { switch self { @@ -17,9 +21,21 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case .server(let code, let message): return "\(message) (\(code))" case .unsupported(let what): return "unsupported: \(what)" case .cancelled: return "cancelled" + case .rolledBackByStop: return "stopped, and the server rolled back the transaction (\(TeradataConnection.userAbortCode))" + case .unreadableColumn(let name, let typeCode): return Self.unreadableColumnMessage(name: name, typeCode: typeCode) + case .openTransactionBlocksDatabaseSwitch: + return String(localized: "Commit or roll back the open transaction before switching database.") } } + private static func unreadableColumnMessage(name: String, typeCode: UInt16) -> String { + let typeName = TeradataColumnType.knownWireTypeName(typeCode) ?? String(typeCode) + let format = typeCode & 0xFFFE == TeradataType.number + ? String(localized: "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to DECIMAL or VARCHAR in the query.") + : String(localized: "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to VARCHAR in the query.") + return String(format: format, name, typeName) + } + public var errorDescription: String? { description } } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataStatementKind.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataStatementKind.swift new file mode 100644 index 0000000000..9e331afb1f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataStatementKind.swift @@ -0,0 +1,178 @@ +import Foundation + +// What a statement does to the session's transaction, read from its first keyword. Anything not known to be a read or +// a transaction statement counts as a change, so a misread keeps a transaction reported open rather than hidden. +enum TeradataStatementKind: Equatable { + case read + case change + case begin + case end + case commit + case rollback + + static func kinds(in request: String) -> [TeradataStatementKind] { + TeradataRequestTokens.statements(in: request).map { kind(of: $0[...]) } + } + + // `DATABASE x`, `SET SESSION DATABASE x` and its `SS` abbreviation move the session's default database, and a + // macro or procedure can hold one where the request text does not show it. + static func mayMoveDefaultDatabase(_ request: String) -> Bool { + TeradataRequestTokens.statements(in: request).contains { statement in + let words = statement.prefix(3).map(\.word) + switch words.first { + case "DATABASE"?, "EXEC"?, "EXECUTE"?, "CALL"?: + return true + case "SET"?: + return words.count > 2 && words[1] == "SESSION" && words[2] == "DATABASE" + case "SS"?: + return words.count > 1 && words[1] == "DATABASE" + default: + return false + } + } + } + + private static let readKeywords: Set = ["SELECT", "SEL", "HELP", "SHOW", "EXPLAIN"] + private static let writeKeywords: Set = ["INSERT", "INS", "UPDATE", "UPD", "DELETE", "DEL", "MERGE"] + private static let lockTypes: Set = ["ACCESS", "READ", "WRITE", "EXCLUSIVE", "SHARE", "CHECKSUM", "LOAD"] + private static let lockTypeModifiers: Set = ["MODE", "NOWAIT", "OVERRIDE"] + + private static func kind(of tokens: ArraySlice) -> TeradataStatementKind { + let statement = tokens.drop { $0 == .symbol(UInt8(ascii: "(")) } + guard case .word(let keyword)? = statement.first else { return .change } + let rest = statement.dropFirst() + // `SELECT AND CONSUME` removes the row it reads from a queue table. + if keyword == "SELECT" || keyword == "SEL", rest.first?.word == "AND", rest.dropFirst().first?.word == "CONSUME" { + return .change + } + if readKeywords.contains(keyword) { return .read } + switch keyword { + case "WITH": + return rest.contains { token in token.word.map { writeKeywords.contains($0) } ?? false } ? .change : .read + case "LOCKING", "LOCK": + return lockedStatementKind(rest) + case "BT": + return .begin + case "ET": + return .end + case "BEGIN": + return rest.first?.word == "TRANSACTION" ? .begin : .change + case "END": + return rest.first?.word == "TRANSACTION" ? .end : .change + case "COMMIT": + return .commit + case "ROLLBACK", "ABORT": + return .rollback + default: + return .change + } + } + + // `LOCKING ROW FOR ACCESS SELECT ...`: the statement the modifier applies to follows the lock type. + private static func lockedStatementKind(_ tokens: ArraySlice) -> TeradataStatementKind { + var sawLockType = false + for index in tokens.indices { + guard let word = tokens[index].word else { continue } + if sawLockType { + if lockTypeModifiers.contains(word) { continue } + return kind(of: tokens[index...]) + } + sawLockType = lockTypes.contains(word) + } + return .change + } +} + +// Splits a request at each `;` outside quotes and comments, keeping each statement's words uppercased. +enum TeradataRequestTokens { + enum Token: Equatable { + case word(String) + case quoted + case symbol(UInt8) + + var word: String? { + guard case .word(let word) = self else { return nil } + return word + } + } + + static func statements(in request: String) -> [[Token]] { + let bytes = Array(request.utf8) + var statements: [[Token]] = [] + var current: [Token] = [] + var index = 0 + while index < bytes.count { + let byte = bytes[index] + let following = index + 1 < bytes.count ? bytes[index + 1] : 0 + switch byte { + case UInt8(ascii: ";"): + if !current.isEmpty { statements.append(current) } + current = [] + index += 1 + case UInt8(ascii: "'"), UInt8(ascii: "\""): + current.append(.quoted) + index = endOfQuoted(bytes, from: index) + case UInt8(ascii: "-") where following == UInt8(ascii: "-"): + index = endOfLineComment(bytes, from: index) + case UInt8(ascii: "/") where following == UInt8(ascii: "*"): + index = endOfBlockComment(bytes, from: index) + case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\n"), UInt8(ascii: "\r"), 0x0B, 0x0C: + index += 1 + default: + guard isWordByte(byte) else { + current.append(.symbol(byte)) + index += 1 + continue + } + let start = index + while index < bytes.count, isWordByte(bytes[index]) { index += 1 } + current.append(.word(String(bytes: bytes[start.. Bool { + switch byte { + case UInt8(ascii: "a")...UInt8(ascii: "z"), UInt8(ascii: "A")...UInt8(ascii: "Z"), + UInt8(ascii: "0")...UInt8(ascii: "9"), UInt8(ascii: "_"), UInt8(ascii: "$"), UInt8(ascii: "#"): + return true + default: + return byte >= 0x80 + } + } + + // A doubled quote stands for itself; an unclosed one runs to the end. + private static func endOfQuoted(_ bytes: [UInt8], from start: Int) -> Int { + let quote = bytes[start] + var index = start + 1 + while index < bytes.count { + guard bytes[index] == quote else { + index += 1 + continue + } + guard index + 1 < bytes.count, bytes[index + 1] == quote else { return index + 1 } + index += 2 + } + return index + } + + // A carriage return ends the comment too, which leans toward reading more of the request as code. + private static func endOfLineComment(_ bytes: [UInt8], from start: Int) -> Int { + var index = start + 2 + while index < bytes.count, bytes[index] != UInt8(ascii: "\n"), bytes[index] != UInt8(ascii: "\r") { + index += 1 + } + return index + } + + private static func endOfBlockComment(_ bytes: [UInt8], from start: Int) -> Int { + var index = start + 2 + while index + 1 < bytes.count { + if bytes[index] == UInt8(ascii: "*"), bytes[index + 1] == UInt8(ascii: "/") { return index + 2 } + index += 1 + } + return bytes.count + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTransaction.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTransaction.swift new file mode 100644 index 0000000000..ca89d7403c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTransaction.swift @@ -0,0 +1,129 @@ +import Foundation + +public enum TeradataTransactionState: Sendable, Equatable { + case idle + case open + case unknown +} + +public enum TeradataSessionModeSource: Sendable, Equatable { + case configured + case helpSession + case sessionInfoView + // Neither read answered, so the session runs with Teradata semantics as before and its state is unknown. + case unresolved +} + +// BT and ET are invalid in an ANSI session, and COMMIT in a Teradata one. ANSI opens a transaction with the first +// request, so there is nothing to send to begin one. +enum TeradataTransactionSQL { + // ROLLBACK and ABORT answer with a failure parcel even when they worked: 3514, or 3513 with a message. + static let rollbackAnswerCodes: Set = [3_513, 3_514] + + static func begin(_ mode: TeradataTransactionMode) -> String? { + mode == .ansi ? nil : "BEGIN TRANSACTION" + } + + static func commit(_ mode: TeradataTransactionMode) -> String { + mode == .ansi ? "COMMIT WORK" : "END TRANSACTION" + } + + static func rollback(_ mode: TeradataTransactionMode) -> String { + mode == .ansi ? "ROLLBACK WORK" : "ROLLBACK" + } +} + +enum TeradataSessionModeProbe { + static let helpSession = "HELP SESSION" + static let sessionInfo = "SELECT Transaction_Mode FROM DBC.SessionInfoV WHERE SessionNo = SESSION" + + static func mode(fromHelpSession result: TeradataResultSet) -> TeradataTransactionMode? { + switch value(of: "Transaction Semantics", in: result) { + case "ANSI"?: return .ansi + case "TERADATA"?: return .tera + default: return nil + } + } + + static func mode(fromSessionInfo result: TeradataResultSet) -> TeradataTransactionMode? { + switch value(of: "Transaction_Mode", in: result) { + case "A"?: return .ansi + case "T"?: return .tera + default: return nil + } + } + + // Matched by name, ignoring case, spaces and underscores, since the column's place in HELP SESSION is not fixed. + private static func value(of column: String, in result: TeradataResultSet) -> String? { + let wanted = normalized(column) + guard let index = result.columns.firstIndex(where: { normalized($0.name) == wanted }), + let row = result.rows.first, index < row.count, + case .text(let text) = row[index] else { return nil } + return text.trimmingCharacters(in: .whitespaces).uppercased() + } + + private static func normalized(_ name: String) -> String { + name.uppercased().filter { $0 != " " && $0 != "_" } + } +} + +// Tracks the session's transaction from the requests that ran on it and how each ended. +struct TeradataTransactionTracker { + let mode: TeradataTransactionMode + private var explicitDepth = 0 + private var holdsChanges = false + + init(mode: TeradataTransactionMode) { + self.mode = mode + } + + // An ANSI session that only read holds read locks, which a commit releases harmlessly. + var state: TeradataTransactionState { + switch mode { + case .ansi: return holdsChanges ? .open : .idle + case .tera: return explicitDepth > 0 ? .open : .idle + case .default: return .unknown + } + } + + // An abort rolls back the whole transaction; outside BT a Teradata-mode request is a transaction of its own. + var abortRollsBackTransaction: Bool { + mode == .ansi || explicitDepth > 0 + } + + mutating func ran(_ kinds: [TeradataStatementKind]) { + for kind in kinds { + switch kind { + case .read: + continue + case .change: + holdsChanges = true + case .begin: + if mode == .ansi { holdsChanges = true } else { explicitDepth += 1 } + case .end: + if mode == .ansi { holdsChanges = true } else { explicitDepth = max(explicitDepth - 1, 0) } + case .commit: + if mode == .ansi { holdsChanges = false } + case .rollback: + // Only `ROLLBACK ... WHERE` with a false condition answers success, and it rolled nothing back. + continue + } + } + } + + mutating func failed(_ kinds: [TeradataStatementKind], code: Int) { + if code == TeradataConnection.userAbortCode + || (kinds.contains(.rollback) && TeradataTransactionSQL.rollbackAnswerCodes.contains(code)) { + rolledBack() + return + } + // In Teradata semantics a failure rolls the whole transaction back. In ANSI the docs disagree on whether it + // ends the transaction, so what it held is kept. + if mode != .ansi { explicitDepth = 0 } + } + + mutating func rolledBack() { + explicitDepth = 0 + holdsChanges = false + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift index 44c90928d9..7b41f5257e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift @@ -21,6 +21,7 @@ enum TeradataType { static let integer: UInt16 = 496 static let smallint: UInt16 = 500 static let bigint: UInt16 = 600 + static let number: UInt16 = 604 static let byteint: UInt16 = 756 static let float: UInt16 = 480 static let decimal: UInt16 = 484 diff --git a/apple/Packages/TableProCore/Sources/TableProTheme/Import/VSCodeThemeMapping.swift b/apple/Packages/TableProCore/Sources/TableProTheme/Import/VSCodeThemeMapping.swift index 6269a02475..5399f05179 100644 --- a/apple/Packages/TableProCore/Sources/TableProTheme/Import/VSCodeThemeMapping.swift +++ b/apple/Packages/TableProCore/Sources/TableProTheme/Import/VSCodeThemeMapping.swift @@ -22,7 +22,7 @@ private extension VSCodeColorSource { } internal extension ThemeSlot { - // A list that stops short leaves the slot to `fallback`, which derives the stripe and the bools the way VS Code paints them. + // A list that stops short leaves the slot to `fallback`, which derives the stripe and the values the way VS Code paints them. var vscodeSources: [VSCodeColorSource] { switch self { case .windowBackdrop: @@ -104,6 +104,13 @@ internal extension ThemeSlot { return [.sqlToken("constant.language.boolean.true.sql")] case .gridBoolFalse: return [.sqlToken("constant.language.boolean.false.sql")] + case .gridString: + return [.sqlToken("string.quoted.single.sql")] + case .gridNumber: + return [.sqlToken("constant.numeric.sql")] + // The SQL grammar scopes no date, binary, JSON, UUID or enum literal. + case .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: + return [] case .gridRowNumber: return [.color("editorLineNumber.foreground")] case .gridModified: diff --git a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeFileFormat.swift b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeFileFormat.swift index 4bdaab4028..c0938288d5 100644 --- a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeFileFormat.swift +++ b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeFileFormat.swift @@ -1,7 +1,9 @@ import Foundation public enum ThemeSchema { - public static let current = 3 + public static let current = 4 + // Every release before format 4 reads and writes format 3 only. + public static let oldestReadable = 3 } public enum ThemeIdentifier { @@ -45,12 +47,12 @@ public enum ThemeFileFormat { guard let schema = root[MetadataKey.schema.rawValue]?.intValue else { if let legacy = root["version"]?.intValue { - throw ThemeLoadError.schemaTooOld(found: legacy, supported: ThemeSchema.current) + throw ThemeLoadError.schemaTooOld(found: legacy, supported: ThemeSchema.oldestReadable) } throw ThemeLoadError.missingSchema } - guard schema >= ThemeSchema.current else { - throw ThemeLoadError.schemaTooOld(found: schema, supported: ThemeSchema.current) + guard schema >= ThemeSchema.oldestReadable else { + throw ThemeLoadError.schemaTooOld(found: schema, supported: ThemeSchema.oldestReadable) } guard schema <= ThemeSchema.current else { throw ThemeLoadError.schemaTooNew(found: schema, supported: ThemeSchema.current) @@ -80,7 +82,7 @@ public enum ThemeFileFormat { name: name, author: author, appearance: appearance, - overrides: try colors(in: root) + overrides: try colors(in: root, schema: schema) ) } @@ -89,7 +91,7 @@ public enum ThemeFileFormat { let name = try validatedName(theme.name, policy: .strict) var root: [String: Any] = [ - MetadataKey.schema.rawValue: ThemeSchema.current, + MetadataKey.schema.rawValue: schema(of: theme), MetadataKey.id.rawValue: theme.id, MetadataKey.name.rawValue: name, MetadataKey.author.rawValue: try validatedAuthor(theme.author, policy: .strict), @@ -108,6 +110,11 @@ public enum ThemeFileFormat { ) } + // The oldest format holding the theme's keys, so an older build still reads a theme that sets no newer key. + private static func schema(of theme: ThemeDefinition) -> Int { + theme.overrides.keys.reduce(ThemeSchema.oldestReadable) { max($0, $1.introducedInSchema) } + } + // MARK: - Reading // `JSONDecoder` over `JSONSerialization`: its error names the character and where it is. @@ -122,7 +129,7 @@ public enum ThemeFileFormat { } } - private static func colors(in root: [String: ThemeJSONNode]) throws -> [ThemeSlot: ThemeColor] { + private static func colors(in root: [String: ThemeJSONNode], schema: Int) throws -> [ThemeSlot: ThemeColor] { let leaves = root .filter { MetadataKey(rawValue: $0.key) == nil } .flatMap { key, node in node.leaves(prefix: key) } @@ -130,12 +137,18 @@ public enum ThemeFileFormat { var parsed: [ThemeSlot: ThemeColor] = [:] var unknown: [String] = [] + var newer: [ThemeSlot] = [] for leaf in leaves { guard let slot = ThemeSlot(rawValue: leaf.path) else { unknown.append(leaf.path) continue } + // Every build that reads the file's format has to read all of it. + guard slot.introducedInSchema <= schema else { + newer.append(slot) + continue + } guard parsed[slot] == nil else { throw ThemeLoadError.duplicateKey(leaf.path) } guard let raw = leaf.value else { throw ThemeLoadError.invalidColor(leaf.path) } let color = try ThemeColor(validating: raw) @@ -143,6 +156,9 @@ public enum ThemeFileFormat { } guard unknown.isEmpty else { throw ThemeLoadError.unknownKeys(unknown) } + if let needed = newer.map(\.introducedInSchema).max() { + throw ThemeLoadError.keysNeedNewerFormat(newer.map(\.rawValue), format: needed) + } return parsed } diff --git a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeLoadError.swift b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeLoadError.swift index 5acd16d076..65d717afc8 100644 --- a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeLoadError.swift +++ b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeLoadError.swift @@ -10,6 +10,7 @@ public enum ThemeLoadError: LocalizedError, Equatable, Sendable { case invalidAuthor case invalidColor(String) case unknownKeys([String]) + case keysNeedNewerFormat([String], format: Int) case duplicateKey(String) case reservedIdentifier(String) case invalidIdentifier(String) @@ -42,13 +43,13 @@ public enum ThemeLoadError: LocalizedError, Equatable, Sendable { return String(localized: "The file does not declare a theme format version.") case let .schemaTooOld(found, supported): return String( - format: String(localized: "Theme format %1$d is no longer supported. This version of TablePro reads format %2$d."), + format: String(localized: "Theme format %1$d is no longer supported. This version of TablePro reads format %2$d and later."), found, supported ) case let .schemaTooNew(found, supported): return String( - format: String(localized: "Theme format %1$d was made for a newer TablePro. This version reads format %2$d."), + format: String(localized: "Theme format %1$d was made for a newer TablePro. This version reads format %2$d and earlier."), found, supported ) @@ -69,6 +70,9 @@ public enum ThemeLoadError: LocalizedError, Equatable, Sendable { case let .unknownKeys(names): let shown = ListFormatter.localizedString(byJoining: names.prefix(3).map { "\"\($0)\"" }) return String(format: String(localized: "TablePro does not use these theme keys: %@."), shown) + case let .keysNeedNewerFormat(names, format): + let shown = ListFormatter.localizedString(byJoining: names.prefix(3).map { "\"\($0)\"" }) + return String(format: String(localized: "These theme keys need theme format %1$d: %2$@."), format, shown) case let .duplicateKey(path): return String(format: String(localized: "The theme sets \"%@\" more than once."), path) case let .reservedIdentifier(identifier): diff --git a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlot.swift b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlot.swift index 4b4489b75d..41d9ac2b39 100644 --- a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlot.swift +++ b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlot.swift @@ -34,6 +34,13 @@ public enum ThemeSlot: String, CaseIterable, Sendable { case gridNullValue = "dataGrid.nullValue" case gridBoolTrue = "dataGrid.boolTrue" case gridBoolFalse = "dataGrid.boolFalse" + case gridString = "dataGrid.string" + case gridNumber = "dataGrid.number" + case gridDateTime = "dataGrid.dateTime" + case gridBinary = "dataGrid.binary" + case gridJSON = "dataGrid.json" + case gridUUID = "dataGrid.uuid" + case gridEnum = "dataGrid.enum" case gridRowNumber = "dataGrid.rowNumber" case gridModified = "dataGrid.modified" case gridInserted = "dataGrid.inserted" @@ -52,8 +59,9 @@ public enum ThemeSlot: String, CaseIterable, Sendable { return .syntax case .gridBackground, .gridText, .gridAlternateRow, .gridHeaderBackground, .gridHeaderText, .gridLine, .gridSelection, .gridSelectionText, .gridInactiveSelection, .gridFocusBorder, - .gridNullValue, .gridBoolTrue, .gridBoolFalse, .gridRowNumber, - .gridModified, .gridInserted, .gridDeleted, .gridDeletedText: + .gridNullValue, .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum, + .gridRowNumber, .gridModified, .gridInserted, .gridDeleted, .gridDeletedText: return .dataGrid } } @@ -69,8 +77,11 @@ public enum ThemeSlot: String, CaseIterable, Sendable { .syntaxNull, .syntaxOperator, .syntaxFunction, .syntaxType: return .syntax case .gridBackground, .gridText, .gridAlternateRow, .gridHeaderBackground, .gridHeaderText, - .gridLine, .gridNullValue, .gridBoolTrue, .gridBoolFalse, .gridRowNumber: + .gridLine, .gridRowNumber: return .dataGrid + case .gridNullValue, .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: + return .values case .gridSelection, .gridSelectionText, .gridInactiveSelection, .gridFocusBorder: return .selection case .gridModified, .gridInserted, .gridDeleted, .gridDeletedText: @@ -94,6 +105,7 @@ public enum ThemeSlot: String, CaseIterable, Sendable { .syntaxNull, .syntaxOperator, .syntaxFunction, .syntaxType, .gridText, .gridAlternateRow, .gridHeaderText, .gridLine, .gridSelection, .gridSelectionText, .gridInactiveSelection, .gridFocusBorder, .gridNullValue, .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum, .gridRowNumber, .gridModified, .gridInserted, .gridDeleted, .gridDeletedText: return false } @@ -110,6 +122,7 @@ public enum ThemeSlot: String, CaseIterable, Sendable { .syntaxNull, .syntaxOperator, .syntaxFunction, .syntaxType, .gridBackground, .gridText, .gridHeaderBackground, .gridHeaderText, .gridLine, .gridSelectionText, .gridFocusBorder, .gridNullValue, .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum, .gridRowNumber: return false } @@ -126,8 +139,8 @@ public enum ThemeSlot: String, CaseIterable, Sendable { case .editorLineNumber: return String(localized: "Line Number") case .editorInvisibles: return String(localized: "Invisibles") case .syntaxKeyword: return String(localized: "Keyword") - case .syntaxString: return String(localized: "String") - case .syntaxNumber: return String(localized: "Number") + case .syntaxString, .gridString: return String(localized: "String") + case .syntaxNumber, .gridNumber: return String(localized: "Number") case .syntaxComment: return String(localized: "Comment") case .syntaxNull: return String(localized: "NULL") case .syntaxOperator: return String(localized: "Operator") @@ -143,6 +156,11 @@ public enum ThemeSlot: String, CaseIterable, Sendable { case .gridNullValue: return String(localized: "NULL Value") case .gridBoolTrue: return String(localized: "Bool True") case .gridBoolFalse: return String(localized: "Bool False") + case .gridDateTime: return String(localized: "Date & Time") + case .gridBinary: return String(localized: "Binary") + case .gridJSON: return String(localized: "JSON") + case .gridUUID: return String(localized: "UUID") + case .gridEnum: return String(localized: "Enum") case .gridRowNumber: return String(localized: "Row Number") case .gridModified: return String(localized: "Modified") case .gridInserted: return String(localized: "Inserted") @@ -184,6 +202,13 @@ public enum ThemeSlot: String, CaseIterable, Sendable { case .gridNullValue: return String(localized: "Data Grid NULL Value") case .gridBoolTrue: return String(localized: "Data Grid Bool True") case .gridBoolFalse: return String(localized: "Data Grid Bool False") + case .gridString: return String(localized: "Data Grid String") + case .gridNumber: return String(localized: "Data Grid Number") + case .gridDateTime: return String(localized: "Data Grid Date & Time") + case .gridBinary: return String(localized: "Data Grid Binary") + case .gridJSON: return String(localized: "Data Grid JSON") + case .gridUUID: return String(localized: "Data Grid UUID") + case .gridEnum: return String(localized: "Data Grid Enum") case .gridRowNumber: return String(localized: "Data Grid Row Number") case .gridModified: return String(localized: "Data Grid Modified") case .gridInserted: return String(localized: "Data Grid Inserted") @@ -191,4 +216,21 @@ public enum ThemeSlot: String, CaseIterable, Sendable { case .gridDeletedText: return String(localized: "Data Grid Deleted Text") } } + + public var introducedInSchema: Int { + switch self { + case .windowBackdrop, + .editorBackground, .editorText, .editorCursor, .editorSelection, + .editorCurrentLine, .editorCurrentStatement, .editorLineNumber, .editorInvisibles, + .syntaxKeyword, .syntaxString, .syntaxNumber, .syntaxComment, + .syntaxNull, .syntaxOperator, .syntaxFunction, .syntaxType, + .gridBackground, .gridText, .gridAlternateRow, .gridHeaderBackground, .gridHeaderText, + .gridLine, .gridSelection, .gridSelectionText, .gridInactiveSelection, .gridFocusBorder, + .gridNullValue, .gridBoolTrue, .gridBoolFalse, .gridRowNumber, + .gridModified, .gridInserted, .gridDeleted, .gridDeletedText: + return 3 + case .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: + return 4 + } + } } diff --git a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotFallback.swift b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotFallback.swift index 83cbe6112d..ed55776c13 100644 --- a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotFallback.swift +++ b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotFallback.swift @@ -34,8 +34,9 @@ public extension ThemeSlot { return .defaultSyntax case .syntaxOperator: return .slot(.editorText) - // A bool in labelColor next to themed cell text reads as a different table. - case .gridBoolTrue, .gridBoolFalse: + // A value in labelColor next to themed cell text reads as a different table. + case .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: return .slot(.gridText) case .gridAlternateRow: return .derived(base: .gridBackground, rule: .stripe(toward: .gridText)) diff --git a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotSection.swift b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotSection.swift index f64c4e6109..adca33977a 100644 --- a/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotSection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTheme/ThemeSlotSection.swift @@ -1,11 +1,12 @@ import Foundation -// The theme editor's grouping: selection colors and row change colors, each chosen as a set, get sections of their own. +// The theme editor's grouping: value colors, selection colors and row change colors, each chosen as a set, get sections of their own. public enum ThemeSlotSection: String, CaseIterable, Sendable { case window case editor case syntax case dataGrid + case values case selection case changes @@ -15,6 +16,7 @@ public enum ThemeSlotSection: String, CaseIterable, Sendable { case .editor: return String(localized: "Editor") case .syntax: return String(localized: "Syntax") case .dataGrid: return String(localized: "Data Grid") + case .values: return String(localized: "Values") case .selection: return String(localized: "Selection") case .changes: return String(localized: "Changes") } diff --git a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoStatementClient.swift b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoStatementClient.swift index c13edc6ba4..d63785c92b 100644 --- a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoStatementClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoStatementClient.swift @@ -1,9 +1,19 @@ import Foundation import os -public enum TrinoStreamElement: Sendable { - case columns([TrinoColumnDescriptor]) - case rows([[TrinoValue]]) +// How a reader that holds a page tells the coordinator it is still there. Past `query.client.timeout` (5 minutes by +// default) without contact, the coordinator abandons the query. +public enum TrinoHeartbeat: Sendable, Equatable { + // Trino 475 and later answer HEAD on the next page's URI, with no body. + case head + // Asking again for the page already received returns it from the coordinator's cache and counts as contact. + case repeatLastPage + + // `version` is what `flavor.versionStatement` returned, such as 475 or 476-e.1. + public init(flavor: TrinoServerFlavor, version: String?) { + let release = version.flatMap { Int($0.prefix { $0.isNumber }) } ?? 0 + self = flavor == .trino && release >= 475 ? .head : .repeatLastPage + } } public final class TrinoStatementClient: @unchecked Sendable { @@ -41,8 +51,9 @@ public final class TrinoStatementClient: @unchecked Sendable { var rows: [[TrinoValue]] = [] let outcome = try await runStatement( sql, + heartbeat: .repeatLastPage, onColumns: { columns = $0 }, - onPage: { rows.append(contentsOf: $0) } + onPage: { page, _ in rows.append(contentsOf: page) } ) return TrinoResultSet( columns: descriptors(from: columns), @@ -53,26 +64,20 @@ public final class TrinoStatementClient: @unchecked Sendable { ) } - /// The paging loop runs in an unstructured task, so terminating the stream has to cancel it - /// explicitly. Without that the loop keeps fetching pages nobody reads, and its own - /// `abortIfCancelled` never fires because nothing ever cancels the task it runs in. - public func executeStreamed(_ sql: String) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - let client = self - let statementTask = Task { - do { - _ = try await client.runStatement( - sql, - onColumns: { continuation.yield(.columns(client.descriptors(from: $0))) }, - onPage: { continuation.yield(.rows($0)) } - ) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - continuation.onTermination = { @Sendable _ in statementTask.cancel() } - } + // The next page is asked for only once `onPage` returns, so a reader that waits there holds the query where it + // is. While it waits it calls `keepAlive`. A cancelled caller stops at the next request and sends Trino the DELETE. + public func executeStreamed( + _ sql: String, + heartbeat: TrinoHeartbeat, + onColumns: ([TrinoColumnDescriptor]) async throws -> Void, + onPage: (_ page: [[TrinoValue]], _ keepAlive: () async throws -> Void) async throws -> Void + ) async throws { + _ = try await runStatement( + sql, + heartbeat: heartbeat, + onColumns: { try await onColumns(self.descriptors(from: $0)) }, + onPage: onPage + ) } /// Stops every statement running on this client, not only the one that started last: the app @@ -93,8 +98,9 @@ public final class TrinoStatementClient: @unchecked Sendable { private func runStatement( _ sql: String, - onColumns: ([TrinoColumn]) -> Void, - onPage: ([[TrinoValue]]) -> Void + heartbeat: TrinoHeartbeat, + onColumns: ([TrinoColumn]) async throws -> Void, + onPage: ([[TrinoValue]], () async throws -> Void) async throws -> Void ) async throws -> StatementOutcome { guard let statementURL = config.statementURL else { throw TrinoError.invalidConfiguration("Invalid \(config.flavor.productName) server URL") @@ -107,9 +113,16 @@ public final class TrinoStatementClient: @unchecked Sendable { defer { lock.withLock { running[ObjectIdentifier(statement)] = nil } } do { - return try await drive(statement, url: statementURL, sql: sql, onColumns: onColumns, onPage: onPage) + return try await drive( + statement, + url: statementURL, + sql: sql, + heartbeat: heartbeat, + onColumns: onColumns, + onPage: onPage + ) } catch { - releaseIfCancelled(statement) + releaseIfCancelled(statement, after: error) throw error } } @@ -118,8 +131,9 @@ public final class TrinoStatementClient: @unchecked Sendable { _ statement: TrinoRunningStatement, url statementURL: URL, sql: String, - onColumns: ([TrinoColumn]) -> Void, - onPage: ([[TrinoValue]]) -> Void + heartbeat: TrinoHeartbeat, + onColumns: ([TrinoColumn]) async throws -> Void, + onPage: ([[TrinoValue]], () async throws -> Void) async throws -> Void ) async throws -> StatementOutcome { var httpResponse = try await sendWithRetry( makeRequest(method: .post, url: statementURL, headers: initialHeaders(), body: Data(sql.utf8)), @@ -134,10 +148,15 @@ public final class TrinoStatementClient: @unchecked Sendable { var columns = results.columns var columnsEmitted = false if let columns { - onColumns(columns) + try await onColumns(columns) columnsEmitted = true } - emitRows(results, columns: columns, onPage: onPage) + let first = results + try await emitRows(first, columns: columns) { rows in + try await onPage(rows) { + try await self.sendHeartbeat(heartbeat, for: statement, pageURL: nil, nextUri: first.nextUri) + } + } var updateType = results.updateType var updateCount = results.updateCount let queryId = results.id @@ -159,10 +178,15 @@ public final class TrinoStatementClient: @unchecked Sendable { columns = results.columns } if !columnsEmitted, let columns { - onColumns(columns) + try await onColumns(columns) columnsEmitted = true } - emitRows(results, columns: columns, onPage: onPage) + let page = results + try await emitRows(page, columns: columns) { rows in + try await onPage(rows) { + try await self.sendHeartbeat(heartbeat, for: statement, pageURL: nextURL, nextUri: page.nextUri) + } + } if let type = results.updateType { updateType = type } @@ -175,7 +199,11 @@ public final class TrinoStatementClient: @unchecked Sendable { return StatementOutcome(updateType: updateType, updateCount: updateCount, queryId: queryId) } - private func emitRows(_ results: TrinoQueryResults, columns: [TrinoColumn]?, onPage: ([[TrinoValue]]) -> Void) { + private func emitRows( + _ results: TrinoQueryResults, + columns: [TrinoColumn]?, + onPage: ([[TrinoValue]]) async throws -> Void + ) async throws { guard let data = results.data, let columns, !data.isEmpty else { return } var page: [[TrinoValue]] = [] page.reserveCapacity(data.count) @@ -188,7 +216,33 @@ public final class TrinoStatementClient: @unchecked Sendable { } page.append(decoded) } - onPage(page) + try await onPage(page) + } + + // A heartbeat never asks for the next page: that would hand the reader rows it has no room for. + private func sendHeartbeat( + _ heartbeat: TrinoHeartbeat, + for statement: TrinoRunningStatement, + pageURL: URL?, + nextUri: String? + ) async throws { + if heartbeat == .head, !statement.refusesHeadHeartbeat, let nextUri { + do { + _ = try await sendWithRetry( + makeRequest(method: .head, url: try followURL(nextUri), headers: followHeaders()), + for: statement + ) + return + } catch { + // A proxy can refuse HEAD where the coordinator would answer it; the page already received works anywhere. + try abortIfCancelled(statement) + if case TrinoError.httpStatus(let code, _) = error, [404, 405, 501].contains(code) { + statement.refuseHeadHeartbeat() + } + } + } + guard let pageURL else { return } + _ = try await sendWithRetry(makeRequest(method: .get, url: pageURL, headers: followHeaders()), for: statement) } private func descriptors(from columns: [TrinoColumn]) -> [TrinoColumnDescriptor] { @@ -202,8 +256,10 @@ public final class TrinoStatementClient: @unchecked Sendable { throw TrinoError.cancelled } - private func releaseIfCancelled(_ statement: TrinoRunningStatement) { - guard statement.isCancelled || Task.isCancelled, let uri = statement.claimRelease() else { return } + // A reader that stops taking pages throws CancellationError from its callback, before its task is cancelled. + private func releaseIfCancelled(_ statement: TrinoRunningStatement, after error: Error) { + guard statement.isCancelled || Task.isCancelled || error is CancellationError, + let uri = statement.claimRelease() else { return } fireDelete(uri) } @@ -396,6 +452,15 @@ private final class TrinoRunningStatement: @unchecked Sendable { private var cancelled = false private var released = false private var nextUri: String? + private var headRefused = false + + var refusesHeadHeartbeat: Bool { + lock.withLock { headRefused } + } + + func refuseHeadHeartbeat() { + lock.withLock { headRefused = true } + } var isCancelled: Bool { lock.withLock { cancelled } diff --git a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift index 484a7599de..013b77421f 100644 --- a/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProTrinoCore/TrinoTransport.swift @@ -43,6 +43,7 @@ public struct TrinoHTTPRequest: Sendable { public enum Method: String, Sendable { case get = "GET" case post = "POST" + case head = "HEAD" case delete = "DELETE" } diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateAuth.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateAuth.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateAuth.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateAuth.swift index bf2a74a611..f49ff8e3fc 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateAuth.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateAuth.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWeaviateCore public enum WeaviateFieldID { public static let authMethod = "wvAuthMethod" diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateFilterBuilder.swift similarity index 90% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateFilterBuilder.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateFilterBuilder.swift index 43d7f01a2e..0682d56b10 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateFilterBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateFilterBuilder.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWeaviateCore public enum WeaviateFilterError: Error, LocalizedError, Equatable { case unsupportedOperator(String) @@ -55,51 +56,6 @@ public enum WeaviateFilterError: Error, LocalizedError, Equatable { } } -/// Weaviate has no array value field: an `int[]` property filters with `valueInt`, so the kind is -/// always the element's. -public enum WeaviateValueKind: String, Sendable, Equatable { - case text - case uuid - case int - case number - case boolean - case date - - public var graphQLField: String { - switch self { - case .text, .uuid: return "valueText" - case .int: return "valueInt" - case .number: return "valueNumber" - case .boolean: return "valueBoolean" - case .date: return "valueDate" - } - } - - public var isOrdered: Bool { - self == .int || self == .number || self == .date - } - - /// `Like` compiles to a regex over the inverted index, which Weaviate only builds for text. - public var acceptsPatternMatch: Bool { - self == .text - } - - public static func forDataType(_ dataType: String) -> WeaviateValueKind { - var name = dataType.trimmingCharacters(in: .whitespaces).lowercased() - while name.hasSuffix("[]") { - name = String(name.dropLast(2)) - } - switch name { - case "int": return .int - case "number": return .number - case "boolean", "bool": return .boolean - case "date": return .date - case "uuid": return .uuid - default: return .text - } - } -} - public enum WeaviateFilterBuilder { private static let rawFilterColumn = "__RAW__" diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateGraphQL+Query.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateGraphQL+Query.swift new file mode 100644 index 0000000000..1afa23befa --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateGraphQL+Query.swift @@ -0,0 +1,73 @@ +import Foundation +import TableProWeaviateCore + +extension WeaviateGraphQL { + public static func getQuery( + collection: String, + properties: [String], + limit: Int, + offset: Int, + sorts: [WeaviateSortSpec], + filters: [WeaviateFilterSpec], + logicMode: String, + schema: [String: WeaviateProperty], + includeVector: Bool = true + ) throws -> String { + let types = schema.mapValues(\.dataType) + let fields = properties + .filter { $0 != WeaviateSchema.uuidColumn && $0 != WeaviateSchema.vectorColumn } + .compactMap { selection(for: $0, schema: schema) } + .joined(separator: " ") + var args: [String] = ["limit: \(max(limit, 0))", "offset: \(max(offset, 0))"] + if let whereClause = try WeaviateFilterBuilder.graphQLWhere( + filters: filters, logicMode: logicMode, types: types + ) { + args.append("where: \(whereClause)") + } + let sortArgs = sorts.compactMap { sort -> String? in + guard let path = sortPath(for: sort) else { return nil } + let order = sort.ascending ? "asc" : "desc" + return "{ path: [\"\(WeaviateFilterBuilder.escape(path))\"] order: \(order) }" + } + if !sortArgs.isEmpty { + args.append("sort: [\(sortArgs.joined(separator: " "))]") + } + let argumentList = args.joined(separator: ", ") + let additional = includeVector ? "id vector" : "id" + return """ + { Get { \(collection)(\(argumentList)) { \(fields) _additional { \(additional) } } } } + """ + } + + /// A structured property needs its own sub-selection, and an `object` with no declared nested + /// properties has nothing to select, so it is left out rather than failing the query. + private static func selection(for name: String, schema: [String: WeaviateProperty]) -> String? { + guard let property = schema[name] else { return name } + switch WeaviatePropertyShape.of(property) { + case .scalar: + return name + case .geoCoordinates: + return "\(name) { latitude longitude }" + case .phoneNumber: + return "\(name) { input internationalFormatted nationalFormatted countryCode national valid defaultCountry }" + case .object: + let nested = property.nestedProperties + .compactMap { selection(for: $0.name, schema: [$0.name: $0]) } + .joined(separator: " ") + return nested.isEmpty ? nil : "\(name) { \(nested) }" + case .crossReference(let targets): + let fragments = targets.map { "... on \($0) { _additional { id } }" }.joined(separator: " ") + return "\(name) { \(fragments) }" + } + } + + /// `vector` is a grid column rather than a property, so Weaviate has nothing to sort on. Every + /// real property is passed through: a type it cannot sort, such as uuid, is reported by the + /// server, which beats painting a sort chevron over rows in insertion order. + private static func sortPath(for sort: WeaviateSortSpec) -> String? { + if sort.column == WeaviateSchema.uuidColumn { + return WeaviateSchema.uuidGraphQLPath + } + return sort.column == WeaviateSchema.vectorColumn ? nil : sort.column + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateObjectCodec.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateObjectCodec.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateObjectCodec.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateObjectCodec.swift index c64d73eaaa..b468f49cdd 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateObjectCodec.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateObjectCodec.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWeaviateCore public struct WeaviateObject: Sendable, Equatable { public let uuid: String diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateOperations.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateOperations.swift similarity index 98% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateOperations.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateOperations.swift index 2c83f47e0b..4b81447a1a 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateOperations.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateOperations.swift @@ -1,6 +1,6 @@ // // WeaviateOperations.swift -// TableProWeaviateCore +// TableProWeaviateClient // import Foundation diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviatePathEncoding.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviatePathEncoding.swift similarity index 100% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviatePathEncoding.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviatePathEncoding.swift diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateStatementGenerator.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateStatementGenerator.swift new file mode 100644 index 0000000000..a958e54f9d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/WeaviateStatementGenerator.swift @@ -0,0 +1,170 @@ +import Foundation +import TableProWeaviateCore + +public struct WeaviateCellChange: Sendable, Equatable { + public let column: String + public let newText: String? + + public init(column: String, newText: String?) { + self.column = column + self.newText = newText + } +} + +public struct WeaviateTrackedChange: Sendable, Equatable { + public enum Kind: String, Sendable, Equatable { + case insert + case update + case delete + } + + public let kind: Kind + public let uuid: String? + public let values: [String: String?] + public let cellChanges: [WeaviateCellChange] + + public init( + kind: Kind, + uuid: String?, + values: [String: String?], + cellChanges: [WeaviateCellChange] + ) { + self.kind = kind + self.uuid = uuid + self.values = values + self.cellChanges = cellChanges + } +} + +public enum WeaviateSkipReason: String, Sendable, Equatable { + case missingUUID + case noEditableColumns + case payloadNotEncodable +} + +public struct WeaviateSkippedChange: Sendable, Equatable { + public let kind: WeaviateTrackedChange.Kind + public let reason: WeaviateSkipReason + + public init(kind: WeaviateTrackedChange.Kind, reason: WeaviateSkipReason) { + self.kind = kind + self.reason = reason + } +} + +/// A skipped change writes nothing while the grid reports the save succeeded, so the driver has to +/// be able to say what it dropped. Same reason the MongoDB generator logs its own skips. +public struct WeaviateWriteBatch: Sendable, Equatable { + public let requests: [WeaviateWriteRequest] + public let skipped: [WeaviateSkippedChange] + + public init(requests: [WeaviateWriteRequest], skipped: [WeaviateSkippedChange]) { + self.requests = requests + self.skipped = skipped + } +} + +public enum WeaviateStatementGenerator { + public static func generate( + collection: String, + columns: [String], + typeNames: [String], + changes: [WeaviateTrackedChange] + ) -> WeaviateWriteBatch { + let types = Dictionary(zip(columns, typeNames), uniquingKeysWith: { first, _ in first }) + var requests: [WeaviateWriteRequest] = [] + var skipped: [WeaviateSkippedChange] = [] + for change in changes { + let request: WeaviateWriteRequest? + switch change.kind { + case .insert: + request = insert(collection: collection, types: types, change: change) + case .update: + request = update(collection: collection, types: types, change: change) + case .delete: + request = delete(collection: collection, change: change) + } + if let request { + requests.append(request) + } else { + skipped.append(WeaviateSkippedChange(kind: change.kind, reason: reason(for: change))) + } + } + return WeaviateWriteBatch(requests: requests, skipped: skipped) + } + + private static func reason(for change: WeaviateTrackedChange) -> WeaviateSkipReason { + if change.kind != .insert, change.uuid?.isEmpty ?? true { + return .missingUUID + } + if change.kind == .update, editablePatch(from: change).isEmpty { + return .noEditableColumns + } + return .payloadNotEncodable + } + + private static func editablePatch(from change: WeaviateTrackedChange) -> [String: String?] { + var patch: [String: String?] = [:] + for cell in change.cellChanges where !WeaviateSchema.immutableColumns.contains(cell.column) { + patch[cell.column] = cell.newText + } + return patch + } + + private static func insert( + collection: String, + types: [String: String], + change: WeaviateTrackedChange + ) -> WeaviateWriteRequest? { + var payload: [String: Any] = [ + "class": collection, + "properties": properties(from: change.values, types: types) + ] + if let uuid = change.uuid, !uuid.isEmpty { + payload["id"] = uuid + } + guard let body = try? WeaviateJSON.text(payload) else { return nil } + return WeaviateWriteRequest(method: "POST", path: "/v1/objects", body: body) + } + + private static func update( + collection: String, + types: [String: String], + change: WeaviateTrackedChange + ) -> WeaviateWriteRequest? { + guard let uuid = change.uuid, !uuid.isEmpty else { return nil } + let patch = editablePatch(from: change) + guard !patch.isEmpty else { return nil } + let payload: [String: Any] = [ + "class": collection, + "properties": properties(from: patch, types: types) + ] + guard let body = try? WeaviateJSON.text(payload) else { return nil } + return WeaviateWriteRequest(method: "PATCH", path: objectPath(collection: collection, uuid: uuid), body: body) + } + + // The server reads no `class` parameter on `/v1/objects/{id}` and deletes the id in every collection that holds it + // (Weaviate 1.40 `usecases/objects/delete.go`), so a row is named by its collection's route. + private static func delete(collection: String, change: WeaviateTrackedChange) -> WeaviateWriteRequest? { + guard let uuid = change.uuid, !uuid.isEmpty else { return nil } + return WeaviateWriteRequest(method: "DELETE", path: objectPath(collection: collection, uuid: uuid), body: nil) + } + + private static func objectPath(collection: String, uuid: String) -> String { + "/v1/objects/\(WeaviatePathEncoding.segment(collection))/\(WeaviatePathEncoding.segment(uuid))" + } + + private static func properties(from values: [String: String?], types: [String: String]) -> [String: Any] { + var result: [String: Any] = [:] + for (column, text) in values { + if WeaviateSchema.immutableColumns.contains(column) { continue } + if column.hasPrefix("\(WeaviateObjectCodec.additionalKey).") { continue } + if let text { + result[column] = WeaviateJSON.parsedValue(text, typeName: types[column] ?? "text") + } else { + result[column] = NSNull() + } + } + return result + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateClient.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateClient.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateClient.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateClient.swift index c4b998249a..745fae5bbb 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateClient.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateClient.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWeaviateCore public final class WeaviateClient: @unchecked Sendable { public let settings: WeaviateConnectionSettings diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateObjectPages.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateObjectPages.swift similarity index 98% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateObjectPages.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateObjectPages.swift index 833bc9c40b..43ed28915a 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateObjectPages.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateObjectPages.swift @@ -1,4 +1,5 @@ import Foundation +import TableProWeaviateCore public struct WeaviateObjectPages: AsyncSequence, Sendable { public typealias Element = [WeaviateObject] diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateTransport.swift similarity index 99% rename from apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift rename to apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateTransport.swift index c6078c2ebc..a91d99c8b5 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/Wire/WeaviateTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateClient/Wire/WeaviateTransport.swift @@ -1,5 +1,6 @@ import Foundation import Security +import TableProWeaviateCore public struct WeaviateHTTPRequest: Sendable, Equatable { public let method: String diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift index 7fa048ddc5..4dcb2c66f6 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateGraphQL.swift @@ -1,75 +1,6 @@ import Foundation public enum WeaviateGraphQL { - public static func getQuery( - collection: String, - properties: [String], - limit: Int, - offset: Int, - sorts: [WeaviateSortSpec], - filters: [WeaviateFilterSpec], - logicMode: String, - schema: [String: WeaviateProperty], - includeVector: Bool = true - ) throws -> String { - let types = schema.mapValues(\.dataType) - let fields = properties - .filter { $0 != WeaviateSchema.uuidColumn && $0 != WeaviateSchema.vectorColumn } - .compactMap { selection(for: $0, schema: schema) } - .joined(separator: " ") - var args: [String] = ["limit: \(max(limit, 0))", "offset: \(max(offset, 0))"] - if let whereClause = try WeaviateFilterBuilder.graphQLWhere( - filters: filters, logicMode: logicMode, types: types - ) { - args.append("where: \(whereClause)") - } - let sortArgs = sorts.compactMap { sort -> String? in - guard let path = sortPath(for: sort) else { return nil } - let order = sort.ascending ? "asc" : "desc" - return "{ path: [\"\(WeaviateFilterBuilder.escape(path))\"] order: \(order) }" - } - if !sortArgs.isEmpty { - args.append("sort: [\(sortArgs.joined(separator: " "))]") - } - let argumentList = args.joined(separator: ", ") - let additional = includeVector ? "id vector" : "id" - return """ - { Get { \(collection)(\(argumentList)) { \(fields) _additional { \(additional) } } } } - """ - } - - /// A structured property needs its own sub-selection, and an `object` with no declared nested - /// properties has nothing to select, so it is left out rather than failing the query. - private static func selection(for name: String, schema: [String: WeaviateProperty]) -> String? { - guard let property = schema[name] else { return name } - switch WeaviatePropertyShape.of(property) { - case .scalar: - return name - case .geoCoordinates: - return "\(name) { latitude longitude }" - case .phoneNumber: - return "\(name) { input internationalFormatted nationalFormatted countryCode national valid defaultCountry }" - case .object: - let nested = property.nestedProperties - .compactMap { selection(for: $0.name, schema: [$0.name: $0]) } - .joined(separator: " ") - return nested.isEmpty ? nil : "\(name) { \(nested) }" - case .crossReference(let targets): - let fragments = targets.map { "... on \($0) { _additional { id } }" }.joined(separator: " ") - return "\(name) { \(fragments) }" - } - } - - /// `vector` is a grid column rather than a property, so Weaviate has nothing to sort on. Every - /// real property is passed through: a type it cannot sort, such as uuid, is reported by the - /// server, which beats painting a sort chevron over rows in insertion order. - private static func sortPath(for sort: WeaviateSortSpec) -> String? { - if sort.column == WeaviateSchema.uuidColumn { - return WeaviateSchema.uuidGraphQLPath - } - return sort.column == WeaviateSchema.vectorColumn ? nil : sort.column - } - public static func looksLikeGraphQL(_ text: String) -> Bool { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.hasPrefix("{") { return true } diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift index 773b8cf34d..f66e1849fb 100644 --- a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateQuery.swift @@ -186,175 +186,3 @@ public enum WeaviateWriteCodec { statement.hasPrefix(writeTag) } } - -public struct WeaviateCellChange: Sendable, Equatable { - public let column: String - public let newText: String? - - public init(column: String, newText: String?) { - self.column = column - self.newText = newText - } -} - -public struct WeaviateTrackedChange: Sendable, Equatable { - public enum Kind: String, Sendable, Equatable { - case insert - case update - case delete - } - - public let kind: Kind - public let uuid: String? - public let values: [String: String?] - public let cellChanges: [WeaviateCellChange] - - public init( - kind: Kind, - uuid: String?, - values: [String: String?], - cellChanges: [WeaviateCellChange] - ) { - self.kind = kind - self.uuid = uuid - self.values = values - self.cellChanges = cellChanges - } -} - -public enum WeaviateSkipReason: String, Sendable, Equatable { - case missingUUID - case noEditableColumns - case payloadNotEncodable -} - -public struct WeaviateSkippedChange: Sendable, Equatable { - public let kind: WeaviateTrackedChange.Kind - public let reason: WeaviateSkipReason - - public init(kind: WeaviateTrackedChange.Kind, reason: WeaviateSkipReason) { - self.kind = kind - self.reason = reason - } -} - -/// A skipped change writes nothing while the grid reports the save succeeded, so the driver has to -/// be able to say what it dropped. Same reason the MongoDB generator logs its own skips. -public struct WeaviateWriteBatch: Sendable, Equatable { - public let requests: [WeaviateWriteRequest] - public let skipped: [WeaviateSkippedChange] - - public init(requests: [WeaviateWriteRequest], skipped: [WeaviateSkippedChange]) { - self.requests = requests - self.skipped = skipped - } -} - -public enum WeaviateStatementGenerator { - public static func generate( - collection: String, - columns: [String], - typeNames: [String], - changes: [WeaviateTrackedChange] - ) -> WeaviateWriteBatch { - let types = Dictionary(zip(columns, typeNames), uniquingKeysWith: { first, _ in first }) - var requests: [WeaviateWriteRequest] = [] - var skipped: [WeaviateSkippedChange] = [] - for change in changes { - let request: WeaviateWriteRequest? - switch change.kind { - case .insert: - request = insert(collection: collection, types: types, change: change) - case .update: - request = update(collection: collection, types: types, change: change) - case .delete: - request = delete(collection: collection, change: change) - } - if let request { - requests.append(request) - } else { - skipped.append(WeaviateSkippedChange(kind: change.kind, reason: reason(for: change))) - } - } - return WeaviateWriteBatch(requests: requests, skipped: skipped) - } - - private static func reason(for change: WeaviateTrackedChange) -> WeaviateSkipReason { - if change.kind != .insert, change.uuid?.isEmpty ?? true { - return .missingUUID - } - if change.kind == .update, editablePatch(from: change).isEmpty { - return .noEditableColumns - } - return .payloadNotEncodable - } - - private static func editablePatch(from change: WeaviateTrackedChange) -> [String: String?] { - var patch: [String: String?] = [:] - for cell in change.cellChanges where !WeaviateSchema.immutableColumns.contains(cell.column) { - patch[cell.column] = cell.newText - } - return patch - } - - private static func insert( - collection: String, - types: [String: String], - change: WeaviateTrackedChange - ) -> WeaviateWriteRequest? { - var payload: [String: Any] = [ - "class": collection, - "properties": properties(from: change.values, types: types) - ] - if let uuid = change.uuid, !uuid.isEmpty { - payload["id"] = uuid - } - guard let body = try? WeaviateJSON.text(payload) else { return nil } - return WeaviateWriteRequest(method: "POST", path: "/v1/objects", body: body) - } - - private static func update( - collection: String, - types: [String: String], - change: WeaviateTrackedChange - ) -> WeaviateWriteRequest? { - guard let uuid = change.uuid, !uuid.isEmpty else { return nil } - let patch = editablePatch(from: change) - guard !patch.isEmpty else { return nil } - let payload: [String: Any] = [ - "class": collection, - "properties": properties(from: patch, types: types) - ] - guard let body = try? WeaviateJSON.text(payload) else { return nil } - return WeaviateWriteRequest( - method: "PATCH", - path: "/v1/objects/\(WeaviatePathEncoding.segment(uuid))", - query: ["class": collection], - body: body - ) - } - - private static func delete(collection: String, change: WeaviateTrackedChange) -> WeaviateWriteRequest? { - guard let uuid = change.uuid, !uuid.isEmpty else { return nil } - return WeaviateWriteRequest( - method: "DELETE", - path: "/v1/objects/\(WeaviatePathEncoding.segment(uuid))", - query: ["class": collection], - body: nil - ) - } - - private static func properties(from values: [String: String?], types: [String: String]) -> [String: Any] { - var result: [String: Any] = [:] - for (column, text) in values { - if WeaviateSchema.immutableColumns.contains(column) { continue } - if column.hasPrefix("\(WeaviateObjectCodec.additionalKey).") { continue } - if let text { - result[column] = WeaviateJSON.parsedValue(text, typeName: types[column] ?? "text") - } else { - result[column] = NSNull() - } - } - return result - } -} diff --git a/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateValueKind.swift b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateValueKind.swift new file mode 100644 index 0000000000..e3d913b6e7 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWeaviateCore/WeaviateValueKind.swift @@ -0,0 +1,46 @@ +import Foundation + +/// Weaviate has no array value field: an `int[]` property filters with `valueInt`, so the kind is +/// always the element's. +public enum WeaviateValueKind: String, Sendable, Equatable { + case text + case uuid + case int + case number + case boolean + case date + + public var graphQLField: String { + switch self { + case .text, .uuid: return "valueText" + case .int: return "valueInt" + case .number: return "valueNumber" + case .boolean: return "valueBoolean" + case .date: return "valueDate" + } + } + + public var isOrdered: Bool { + self == .int || self == .number || self == .date + } + + /// `Like` compiles to a regex over the inverted index, which Weaviate only builds for text. + public var acceptsPatternMatch: Bool { + self == .text + } + + public static func forDataType(_ dataType: String) -> WeaviateValueKind { + var name = dataType.trimmingCharacters(in: .whitespaces).lowercased() + while name.hasSuffix("[]") { + name = String(name.dropLast(2)) + } + switch name { + case "int": return .int + case "number": return .number + case "boolean", "bool": return .boolean + case "date": return .date + case "uuid": return .uuid + default: return .text + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProWireIO/SpoolFile.swift b/apple/Packages/TableProCore/Sources/TableProWireIO/SpoolFile.swift new file mode 100644 index 0000000000..a3240ce7d0 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProWireIO/SpoolFile.swift @@ -0,0 +1,178 @@ +#if canImport(Darwin) +import Darwin +import Foundation + +public enum SpoolFileError: Error, Equatable, Sendable, CustomStringConvertible, LocalizedError { + case cannotCreate(errno: Int32) + case writeFailed(errno: Int32) + case readFailed(errno: Int32) + + public var description: String { + switch self { + case .cannotCreate(let code): return "could not create a temporary file: \(String(cString: strerror(code)))" + case .writeFailed(let code): return "could not write a temporary file: \(String(cString: strerror(code)))" + case .readFailed(let code): return "could not read a temporary file: \(String(cString: strerror(code)))" + } + } + + // The system's own localized text for the errno. + public var errorDescription: String? { + switch self { + case .cannotCreate(let code), .writeFailed(let code), .readFailed(let code): + return NSError(domain: NSPOSIXErrorDomain, code: Int(code)).localizedDescription + } + } +} + +// Records read back in append order, in a file mkostemp makes 0600 and that is unlinked at once, so nothing is left +// after a crash or quit. The descriptor closes only in deinit, so no read meets a reused number. +public final class SpoolFile: @unchecked Sendable { + private static let headerSize = 8 + private static let readChunk = 64 * 1_024 + + private let descriptor: Int32 + // One thread appends while another reads; the lock covers what the two share and the truncate, not reads or writes. + private let lock = NSLock() + private var written = 0 + private var appending = false + private var readBuffer: [UInt8] = [] + private var readStart = 0 + private var fileReadOffset = 0 + + public init(directory: URL = FileManager.default.temporaryDirectory, prefix: String = "TablePro-spool") throws { + var template = Array(directory.appending(path: "\(prefix)-XXXXXX").path(percentEncoded: false).utf8CString) + let descriptor = template.withUnsafeMutableBufferPointer { buffer in + buffer.baseAddress.map { mkostemp($0, O_CLOEXEC) } ?? -1 + } + guard descriptor >= 0 else { throw SpoolFileError.cannotCreate(errno: errno) } + guard template.withUnsafeBufferPointer({ unlink($0.baseAddress) }) == 0 else { + let code = errno + close(descriptor) + throw SpoolFileError.cannotCreate(errno: code) + } + self.descriptor = descriptor + } + + deinit { + close(descriptor) + } + + // The bytes on disk, length prefixes included. Zero again once the reader has taken every record. + public var byteCount: Int { + lock.withLock { written } + } + + // A failed append leaves every earlier record readable and adds nothing. + public func append(_ record: some ContiguousBytes) throws { + let offset = lock.withLock { + appending = true + return written + } + do { + let end = try record.withUnsafeBytes { body in + var header = UInt64(body.count).bigEndian + if body.count <= Self.readChunk { + var bytes = withUnsafeBytes(of: &header) { [UInt8]($0) } + bytes.append(contentsOf: body) + try bytes.withUnsafeBytes { try write($0, at: offset) } + } else { + try withUnsafeBytes(of: &header) { try write($0, at: offset) } + try write(body, at: offset + Self.headerSize) + } + return offset + Self.headerSize + body.count + } + lock.withLock { + written = end + appending = false + } + } catch { + lock.withLock { appending = false } + throw error + } + } + + // The next record, or nil once every record appended so far has been read. + public func next() throws -> [UInt8]? { + let end = lock.withLock { written } + let position = fileReadOffset - (readBuffer.count - readStart) + guard position < end else { return nil } + let header = try take(Self.headerSize, end: end) + let length = header.reduce(0) { $0 << 8 | Int($1) } + guard (0...(end - position - Self.headerSize)).contains(length) else { + throw SpoolFileError.readFailed(errno: EIO) + } + let record = try take(length, end: end) + if fileReadOffset - (readBuffer.count - readStart) == end { + emptyFileIfIdle(at: end) + } + return record + } + + // Only while no append runs and none has landed since `end`, so the writer's next offset is the new start. + private func emptyFileIfIdle(at end: Int) { + lock.withLock { + guard !appending, written == end, ftruncate(descriptor, 0) == 0 else { return } + written = 0 + fileReadOffset = 0 + readStart = 0 + readBuffer.removeAll() + } + } + + private func take(_ count: Int, end: Int) throws -> [UInt8] { + let buffered = readBuffer.count - readStart + if buffered >= count { + defer { readStart += count } + return Array(readBuffer[readStart..<(readStart + count)]) + } + if count - buffered > Self.readChunk { + var record = Array(readBuffer[readStart...]) + readBuffer.removeAll(keepingCapacity: true) + readStart = 0 + record.append(contentsOf: repeatElement(0, count: count - buffered)) + try read(into: &record, from: buffered, at: fileReadOffset) + fileReadOffset += count - buffered + return record + } + readBuffer.removeSubrange(0.. Int in + guard let base = raw.baseAddress else { return -1 } + return pread(descriptor, base + start + done, count - done, off_t(offset + done)) + } + if read < 0, errno == EINTR { continue } + guard read > 0 else { throw SpoolFileError.readFailed(errno: read == 0 ? EIO : errno) } + done += read + } + } + + private func write(_ bytes: UnsafeRawBufferPointer, at offset: Int) throws { + guard let base = bytes.baseAddress else { return } + var done = 0 + while done < bytes.count { + let wrote = pwrite(descriptor, base + done, bytes.count - done, off_t(offset + done)) + if wrote < 0, errno == EINTR { continue } + guard wrote > 0 else { throw SpoolFileError.writeFailed(errno: wrote == 0 ? EIO : errno) } + done += wrote + } + } +} +#endif diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift index b2d62173dd..e03d45832a 100644 --- a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift @@ -3,13 +3,47 @@ import Foundation import TableProSQLGrammar import Testing -private struct StubCatalog: SQLCompletionCatalog { +struct StubCatalog: SQLCompletionCatalog { let tables: [String] let columns: [String: [String]] var values: [String] = [] var databases: [String] = [] var schemas: [String] = [] var schemaTables: [String: [String]] = [:] + var keys: [String: [SQLCompletionForeignKey]] = [:] + // Like the apps' catalogs: two tables in scope qualify every column. + var qualifiesInScope = false + // Like the iOS catalog, which quotes a mixed-case name itself because nothing spells it later. + var writesNames = false + let keyReads = KeyReads() + + final class KeyReads: @unchecked Sendable { + private let lock = NSLock() + private var tables: [String] = [] + + func record(_ table: String) { + lock.withLock { tables.append(table) } + } + + var all: [String] { + lock.withLock { tables } + } + } + + func foreignKeys(of tableName: String, schema: String?) -> [SQLCompletionForeignKey] { + keyReads.record(tableName) + return keys[tableName.lowercased()] ?? [] + } + + func writtenText(of identifier: SQLCompletionIdentifier) -> String? { + guard writesNames else { return nil } + return identifier.parts.map { part in + switch part { + case .catalog(let name): return name == name.lowercased() ? name : "\"\(name)\"" + case .statement(let text, let isQuoted): return isQuoted ? "\"\(text)\"" : text + } + }.joined(separator: ".") + } func resolveAlias(_ aliasOrName: String, in references: [TableReference]) -> String? { let name = aliasOrName.lowercased() @@ -41,9 +75,23 @@ private struct StubCatalog: SQLCompletionCatalog { func allowedValues(forColumn column: String, in references: [TableReference]) -> [String] { values } func allColumnsInScope(for references: [TableReference]) -> [SQLCompletionItem] { - references.flatMap { reference in + references.flatMap { reference -> [SQLCompletionItem] in guard let derived = reference.derivedColumns else { - return columnCompletionItems(for: reference.tableName, schema: reference.schema) + let items = columnCompletionItems(for: reference.tableName, schema: reference.schema) + guard qualifiesInScope, references.count > 1 else { return items } + return items.map { column in + let parts: [SQLCompletionIdentifier.Part] = [ + .statement(reference.identifier, quoted: reference.isIdentifierQuoted), .catalog(column.label), + ] + var item = SQLCompletionItem( + label: "\(reference.identifier).\(column.label)", + kind: .column, + insertText: writtenText(of: SQLCompletionIdentifier(parts)), + identifier: SQLCompletionIdentifier(parts) + ) + item.isQualifiedReference = true + return item + } } return derived.map { SQLCompletionItem.column($0, dataType: nil, tableName: reference.identifier) } } diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLContextAnalyzerJoinScopeTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLContextAnalyzerJoinScopeTests.swift new file mode 100644 index 0000000000..44e9356f49 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLContextAnalyzerJoinScopeTests.swift @@ -0,0 +1,117 @@ +import Foundation +@testable import TableProAutocomplete +import TableProSQLGrammar +import Testing + +struct SQLContextAnalyzerJoinScopeTests { + private let analyzer = SQLContextAnalyzer() + + // `|` marks the cursor. + private func scope(_ marked: String, grammar: SQLLexicalGrammar = .ansi) -> SQLJoinScope? { + let marker = (marked as NSString).range(of: "|") + let text = (marked as NSString).replacingCharacters(in: marker, with: "") + return analyzer.analyze(query: text, cursorPosition: marker.location, grammar: grammar).joinScope + } + + private func names(_ marked: String, grammar: SQLLexicalGrammar = .ansi) -> [String]? { + guard let scope = scope(marked, grammar: grammar) else { return nil } + return [scope.joined.identifier] + scope.earlier.map(\.identifier) + } + + @Test("Right after a join's ON, the joined table and the earlier ones, nearest first", arguments: [ + ("SELECT * FROM orders o JOIN customers c ON |", ["c", "o"]), + ("SELECT * FROM orders o JOIN customers c ON cu|", ["c", "o"]), + ("SELECT * FROM orders JOIN customers ON |", ["customers", "orders"]), + ("SELECT * FROM orders o LEFT OUTER JOIN customers AS c ON |", ["c", "o"]), + ("SELECT * FROM customers c JOIN orders o ON o.customer_id = c.id JOIN payments p ON |", ["p", "o", "c"]), + ("SELECT * FROM a JOIN b USING (id) JOIN c ON |", ["c", "b", "a"]), + ("SELECT * FROM a JOIN b ON a.x IS DISTINCT FROM b.x JOIN c ON |", ["c", "b", "a"]), + ("SELECT * FROM a STRAIGHT_JOIN b ON |", ["b", "a"]), + ("UPDATE a JOIN b ON | SET a.x = 1", ["b", "a"]), + ("DELETE FROM t USING a JOIN b ON |", ["b", "a"]), + ("UPDATE t SET x = 1 FROM a JOIN b ON |", ["b", "a"]), + ]) + func joinScope(_ marked: String, _ expected: [String]) { + #expect(names(marked) == expected) + } + + @Test("A table joined after the cursor is not in scope") + func laterJoinIsOutOfScope() { + #expect(names("SELECT * FROM orders o JOIN customers c ON | JOIN payments p ON p.order_id = o.id") == ["c", "o"]) + } + + @Test("Only the tables since the last comma may be named, as the server reads the ON") + func commaEndsTheScope() { + #expect(names("SELECT * FROM orders o, customers c JOIN payments p ON |") == ["p", "c"]) + } + + @Test("A subquery reads its own FROM list") + func subquery() { + #expect(names("SELECT * FROM x WHERE id IN (SELECT 1 FROM orders o JOIN customers c ON |)") == ["c", "o"]) + #expect(names("WITH r AS (SELECT * FROM orders o JOIN customers c ON |) SELECT * FROM r") == ["c", "o"]) + } + + @Test("Inside a join group the ON names the group's own tables") + func joinGroup() { + #expect(names("SELECT * FROM a JOIN (b JOIN c ON |) ON a.id = b.id") == ["c", "b"]) + #expect(names("SELECT * FROM (a JOIN b ON a.id = b.id) JOIN c ON |") == ["c", "b", "a"]) + } + + @Test("A table hint between the table and its ON is skipped", arguments: [ + ("SELECT * FROM a WITH (NOLOCK) JOIN b WITH (NOLOCK) ON |", "SQL Server"), + ("SELECT * FROM a JOIN b USE INDEX (ix_b) ON |", "MySQL"), + ]) + func tableHint(_ marked: String, _ engine: String) { + #expect(names(marked, grammar: .engine(engine)) == ["b", "a"]) + } + + @Test("A schema and a quoted name are kept") + func schemaAndQuotes() throws { + let found = try #require(scope("SELECT * FROM sales.orders \"O\" JOIN \"Customers\" ON |", grammar: .engine("PostgreSQL"))) + #expect(found.joined.tableName == "Customers") + #expect(found.joined.alias == nil) + #expect(found.joined.isIdentifierQuoted) + #expect(found.earlier.first?.tableName == "orders") + #expect(found.earlier.first?.schema == "sales") + #expect(found.earlier.first?.alias == "O") + #expect(found.earlier.first?.isIdentifierQuoted == true) + } + + @Test("No scope for an ON that belongs to no join, or once the condition has started", arguments: [ + "CREATE INDEX idx ON |", + "CREATE UNIQUE INDEX idx ON |", + "GRANT SELECT ON |", + "CREATE TRIGGER trg AFTER INSERT ON |", + "CREATE TRIGGER trg AFTER UPDATE ON |", + "MERGE INTO orders o USING customers c ON |", + "SELECT * FROM orders o ON |", + "SELECT * FROM orders o, customers c ON |", + "SELECT * FROM orders o JOIN ON |", + "SELECT * FROM orders o JOIN customers c ON o.|", + "SELECT * FROM orders o JOIN customers c ON \"|", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = |", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id |", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id AND |", + "SELECT * FROM orders o JOIN customers c ON|", + "SELECT * FROM orders o CROSS APPLY f(o.id) x ON |", + ]) + func noScope(_ marked: String) { + #expect(scope(marked) == nil) + } + + @Test("A derived table or a CTE has no keys, on either side", arguments: [ + "SELECT * FROM orders o JOIN (SELECT * FROM customers) c ON |", + "SELECT * FROM (SELECT * FROM orders) o JOIN customers c ON |", + "WITH recent AS (SELECT * FROM orders) SELECT * FROM recent r JOIN customers c ON |", + "WITH customers AS (SELECT 1 AS id) SELECT * FROM orders o JOIN customers c ON |", + "SELECT * FROM orders o JOIN generate_series(1, 3) g ON |", + ]) + func derivedSides(_ marked: String) { + #expect(scope(marked) == nil) + } + + @Test("A derived table before a stored one leaves the stored ones in scope") + func derivedEarlierTable() { + #expect(names("SELECT * FROM customers c JOIN (SELECT * FROM x) d ON d.id = c.id JOIN orders o ON |") == ["o", "c"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift index 12c08eee63..203d93370d 100644 --- a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift @@ -155,6 +155,46 @@ struct SQLIdentifierSpellingTests { #expect(applied([item], .lower, quoting: .enclosed)[0].insertText == "o.\"MixedCase\"") } + @Test("A join condition spells each column and keeps each qualifier as written") + func joinConditionSides() { + let item = SQLCompletionItem( + label: "O.CUSTOMER_ID = c.ID AND O.Region = c.REGION", + kind: .joinCondition, + columnPairs: [ + SQLCompletionColumnPair( + left: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("CUSTOMER_ID")]), + right: SQLCompletionIdentifier([.statement("c", quoted: false), .catalog("ID")]) + ), + SQLCompletionColumnPair( + left: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("Region")]), + right: SQLCompletionIdentifier([.statement("c", quoted: false), .catalog("REGION")]) + ), + ] + ) + let spelled = applied([item], .lower, prefix: "O")[0] + #expect(spelled.label == "O.customer_id = c.id AND O.Region = c.region") + #expect(spelled.insertText == "\"O\".customer_id = c.id AND \"O\".\"Region\" = c.region") + #expect(spelled.filterText == item.filterText) + + let opened = applied([item], .lower, quoting: .opened)[0] + #expect(opened.insertText == item.insertText) + } + + @Test("On a lower-folding engine a mixed-case key column is quoted and a quoted alias keeps its quotes") + func joinConditionOnLowerFolding() { + let item = SQLCompletionItem( + label: "O.CustomerId = c.id", + kind: .joinCondition, + columnPairs: [SQLCompletionColumnPair( + left: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("CustomerId")]), + right: SQLCompletionIdentifier([.statement("c", quoted: false), .catalog("id")]) + )] + ) + let spelled = spelling(.asStored, rules: lowerFolding).applied(to: [item], typedPrefix: "", quoting: .none)[0] + #expect(spelled.insertText == "\"O\".\"CustomerId\" = c.id") + #expect(spelled.label == "O.CustomerId = c.id") + } + @Test("Keywords and items without a catalog name are untouched") func otherItemsUntouched() { let keyword = SQLCompletionItem.keyword("SELECT") diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLJoinConditionCompletionTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLJoinConditionCompletionTests.swift new file mode 100644 index 0000000000..9a969331e1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLJoinConditionCompletionTests.swift @@ -0,0 +1,228 @@ +import Foundation +@testable import TableProAutocomplete +import TableProSQLGrammar +import Testing + +struct SQLJoinConditionCompletionTests { + private static func key( + _ name: String, _ columns: [String], _ table: String, _ referenced: [String], schema: String? = nil + ) -> SQLCompletionForeignKey { + SQLCompletionForeignKey( + name: name, columns: columns, referencedTable: table, referencedSchema: schema, referencedColumns: referenced + ) + } + + // A fresh catalog per use, so one test's key reads never show in another's. + private static var shop: StubCatalog { + StubCatalog( + tables: ["orders", "customers", "order_items", "payments", "flights", "airports", "employees"], + columns: [ + "orders": ["id", "customer_id", "tenant_id", "total"], + "customers": ["id", "name"], + "order_items": ["order_id", "tenant_id", "qty"], + "payments": ["id", "order_id", "customer_id"], + "flights": ["id", "origin", "destination"], + "airports": ["code", "name"], + "employees": ["id", "manager_id"], + ], + keys: [ + "orders": [key("orders_customer_fk", ["customer_id"], "customers", ["id"])], + "order_items": [key("items_order_fk", ["order_id", "tenant_id"], "orders", ["id", "tenant_id"])], + "payments": [ + key("payments_order_fk", ["order_id"], "orders", ["id"]), + key("payments_customer_fk", ["customer_id"], "customers", ["id"]), + ], + "flights": [ + key("flights_origin_fk", ["origin"], "airports", ["code"]), + key("flights_destination_fk", ["destination"], "airports", ["code"]), + ], + "employees": [key("employees_manager_fk", ["manager_id"], "employees", ["id"])], + ] + ) + } + + // `|` marks the cursor. + private func items(_ marked: String, catalog: StubCatalog? = nil) async -> [SQLCompletionItem] { + let provider = SQLCompletionProvider(catalog: catalog ?? Self.shop, grammar: .engine("SQLite"), vocabulary: .generic) + let marker = (marked as NSString).range(of: "|") + let text = (marked as NSString).replacingCharacters(in: marker, with: "") + return await provider.completionSession(text: text, cursorPosition: marker.location).items + } + + private func conditions(_ marked: String, catalog: StubCatalog? = nil) async -> [String] { + await items(marked, catalog: catalog).filter { $0.kind == .joinCondition }.map(\.label) + } + + @Test("Right after a join's ON the condition from the foreign key leads, the earlier table first") + func conditionLeads() async { + let first = await items("SELECT * FROM orders o JOIN customers c ON |").first + #expect(first?.label == "o.customer_id = c.id") + #expect(first?.insertText == "o.customer_id = c.id") + #expect(first?.kind == .joinCondition) + #expect(first?.detail == "orders_customer_fk") + } + + @Test("A key from the joined table to an earlier one is written with the earlier table first too") + func reverseDirection() async { + #expect(await items("SELECT * FROM customers c JOIN orders o ON |").first?.label == "c.id = o.customer_id") + } + + @Test("A table with no alias is named as written") + func noAlias() async { + #expect(await items("SELECT * FROM orders JOIN customers ON |").first?.label == "orders.customer_id = customers.id") + } + + @Test("A composite key joins its pairs with AND in key order") + func compositeKey() async { + #expect(await items("SELECT * FROM orders o JOIN order_items i ON |").first?.label + == "o.id = i.order_id AND o.tenant_id = i.tenant_id") + } + + @Test("Two keys between one pair give one condition each") + func twoKeys() async { + #expect(await conditions("SELECT * FROM flights f JOIN airports a ON |") + == ["f.origin = a.code", "f.destination = a.code"]) + } + + @Test("A self-join offers the key both ways") + func selfJoin() async { + #expect(await conditions("SELECT * FROM employees e JOIN employees m ON |") + == ["e.manager_id = m.id", "e.id = m.manager_id"]) + } + + @Test("Earlier tables are paired nearest first") + func nearestFirst() async { + let marked = "SELECT * FROM customers c JOIN orders o ON o.customer_id = c.id JOIN payments p ON |" + #expect(await conditions(marked) == ["o.id = p.order_id", "c.id = p.customer_id"]) + } + + @Test("A table joined after the cursor is neither paired nor read") + func laterTableIsNotRead() async { + let catalog = Self.shop + let marked = "SELECT * FROM orders o JOIN customers c ON | JOIN payments p ON p.order_id = o.id" + #expect(await conditions(marked, catalog: catalog) == ["o.customer_id = c.id"]) + #expect(catalog.keyReads.all.sorted() == ["customers", "orders"]) + } + + @Test("Typing the earlier alias keeps the condition first") + func qualifierPrefix() async { + #expect(await items("SELECT * FROM orders o JOIN customers c ON o|").first?.label == "o.customer_id = c.id") + } + + @Test("No condition and no key read where the ON belongs to no join or the condition has started", arguments: [ + "CREATE INDEX idx ON |", + "SELECT * FROM orders o JOIN customers c ON o.|", + "SELECT * FROM orders o JOIN customers c ON \"|", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id AND |", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id |", + "SELECT * FROM orders o JOIN (SELECT * FROM customers) c ON |", + "WITH customers AS (SELECT 1 AS id) SELECT * FROM orders o JOIN customers c ON |", + "SELECT * FROM orders o WHERE |", + ]) + func noCondition(_ marked: String) async { + let catalog = Self.shop + #expect(await conditions(marked, catalog: catalog).isEmpty) + #expect(catalog.keyReads.all.isEmpty) + } + + @Test("A catalog that writes its own names quotes each side, and the label stays plain") + func catalogWrittenCondition() async throws { + var catalog = StubCatalog( + tables: ["orders", "customers"], + columns: ["orders": ["Id", "CustomerId"], "customers": ["Id"]], + keys: ["orders": [Self.key("fk", ["CustomerId"], "customers", ["Id"])]] + ) + catalog.writesNames = true + let first = try #require(await items("SELECT * FROM orders \"O\" JOIN customers c ON |", catalog: catalog).first) + #expect(first.label == "O.CustomerId = c.Id") + #expect(first.insertText == "\"O\".\"CustomerId\" = c.\"Id\"") + } + + @Test("A key column spelled in another case takes the table's spelling") + func keyColumnTakesTableSpelling() async { + let catalog = StubCatalog( + tables: ["orders", "customers"], + columns: ["orders": ["id", "customer_id"], "customers": ["id"]], + keys: ["orders": [Self.key("fk", ["CUSTOMER_ID"], "Customers", ["ID"])]] + ) + #expect(await conditions("SELECT * FROM orders o JOIN customers c ON |", catalog: catalog) == ["o.customer_id = c.id"]) + } + + // MARK: - The qualified columns of the same ON + + @Test("A column the catalog already qualifies is offered once, with the catalog's spelling") + func qualifiedColumnOnce() async throws { + var catalog = StubCatalog(tables: ["orders", "customers"], columns: ["orders": ["Id", "total"], "customers": ["Id"]]) + catalog.qualifiesInScope = true + catalog.writesNames = true + let items = await items("SELECT * FROM orders o JOIN customers c ON |", catalog: catalog) + + #expect(items.filter { $0.label == "o.Id" }.count == 1) + let column = try #require(items.first { $0.label == "o.Id" }) + #expect(column.insertText == "o.\"Id\"") + #expect(column.isQualifiedReference) + #expect(column.sortPriority == 80) + } + + @Test("A qualified column the catalog writes keeps a quoted alias and the column's quotes") + func writtenQualifiedColumn() async { + var catalog = StubCatalog(tables: ["orders"], columns: ["orders": ["Id"]]) + catalog.writesNames = true + let items = await items("SELECT * FROM orders \"O\" ON |", catalog: catalog) + #expect(items.first { $0.label == "O.Id" }?.insertText == "\"O\".\"Id\"") + } + + // MARK: - Pairing + + private func side(_ table: String, alias: String, schema: String? = nil, keys: [SQLCompletionForeignKey] = []) -> SQLJoinConditions.Side { + SQLJoinConditions.Side( + reference: TableReference(tableName: table, alias: alias, schema: schema), + keys: keys, + columns: [] + ) + } + + @Test("Keys keep the driver's order, an earlier table's own keys before the joined table's") + func pairingOrder() { + let joined = side("airports", alias: "a", keys: [Self.key("hub_fk", ["hub_flight"], "flights", ["id"])]) + let earlier = side("flights", alias: "f", keys: [ + Self.key("destination_fk", ["destination"], "airports", ["code"]), + Self.key("origin_fk", ["origin"], "airports", ["code"]), + ]) + let names = SQLJoinConditions.conditions(joined: joined, earlier: [earlier]).map(\.constraintName) + #expect(names == ["destination_fk", "origin_fk", "hub_fk"]) + } + + @Test("A key into another schema pairs only with a table of that schema") + func pairingSchema() { + let toSales = Self.key("fk", ["customer_id"], "customers", ["id"], schema: "sales") + let orders = side("orders", alias: "o", keys: [toSales]) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c", schema: "sales"), earlier: [orders]).count == 1) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c", schema: "public"), earlier: [orders]).isEmpty) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c"), earlier: [orders]).count == 1) + + let salesOrders = side("orders", alias: "o", schema: "sales", keys: [toSales]) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c"), earlier: [salesOrders]).isEmpty) + let ownNamespace = Self.key("fk", ["user_id"], "users", ["id"]) + let otherOrders = side("orders", alias: "o", schema: "otherdb", keys: [ownNamespace]) + #expect(SQLJoinConditions.conditions(joined: side("users", alias: "u"), earlier: [otherOrders]).isEmpty) + #expect(SQLJoinConditions.conditions(joined: side("users", alias: "u", schema: "otherdb"), earlier: [otherOrders]).count == 1) + let localOrders = side("orders", alias: "o", keys: [ownNamespace]) + #expect(SQLJoinConditions.conditions(joined: side("users", alias: "u", schema: "otherdb"), earlier: [localOrders]).isEmpty) + } + + @Test("Driver rows group into one key per name and target, pairs in key order") + func grouping() { + let rows = [ + SQLCompletionForeignKey.Row(name: "items_fk", column: "order_id", referencedTable: "orders", referencedSchema: nil, referencedColumn: "id"), + SQLCompletionForeignKey.Row(name: "origin_fk", column: "origin", referencedTable: "airports", referencedSchema: nil, referencedColumn: "code"), + SQLCompletionForeignKey.Row(name: "items_fk", column: "tenant_id", referencedTable: "orders", referencedSchema: nil, referencedColumn: "tenant_id"), + SQLCompletionForeignKey.Row(name: "items_fk", column: "tenant_id", referencedTable: "orders", referencedSchema: nil, referencedColumn: "tenant_id"), + SQLCompletionForeignKey.Row(name: "destination_fk", column: "destination", referencedTable: "airports", referencedSchema: nil, referencedColumn: "code"), + ] + let keys = SQLCompletionForeignKey.grouping(rows) + #expect(keys.map(\.name) == ["items_fk", "origin_fk", "destination_fk"]) + #expect(keys.first?.columns == ["order_id", "tenant_id"]) + #expect(keys.first?.referencedColumns == ["id", "tenant_id"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift index 7442a5c272..57d0d8e57f 100644 --- a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift @@ -98,6 +98,27 @@ struct BigtableBytesTests { #expect(throws: BigtableError.self) { try BigtableBytes.parse("b64:***") } } + @Test("Text that starts with a byte order mark shows with it, and reads back to the same bytes") + func byteOrderMarkKept() throws { + let marked = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + let text = try #require(BigtableBytes.printableText(marked)) + #expect(Data(text.utf8) == marked) + #expect(try BigtableBytes.parse(BigtableBytes.display(marked)) == marked) + #expect(BigtableBytes.display(marked) != BigtableBytes.display(Data("abc".utf8))) + #expect(BigtableBytes.printableText(Data([0x61, 0xFF])) == nil) + } + + @Test("A key and a qualifier that start with a byte order mark read back from a request's own text") + func byteOrderMarkRequestRoundTrip() throws { + let marked = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + let column = BigtableColumn(family: "cf", qualifier: marked) + let request = BigtableRequest(verb: .delete, table: "events", key: marked, column: column.name) + let parsed = try BigtableRequest.parse(request.text) + #expect(parsed.key == marked) + let columnName = try #require(parsed.column) + #expect(try BigtableColumn(name: columnName).qualifier == marked) + } + @Test("Prefix end increments the last byte that can grow") func prefixEnd() { #expect(BigtableBytes.prefixEnd(Data("ab".utf8)) == Data("ac".utf8)) diff --git a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift index e9bf7ce67b..2a89954929 100644 --- a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift @@ -149,4 +149,31 @@ struct BigtableRowWritesTests { func delete() { #expect(BigtableRowWrites.delete(table: "events", key: Data("a b".utf8)).text == #"DELETE events/"a b""#) } + + @Test("A row's locator names its key over the _key text") + func locatorNamesTheKey() throws { + let real = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + let key = try #require(try BigtableRowWrites.originalKey(locator: real.base64EncodedString(), keyCell: .text("abc"))) + #expect(key == real) + let written = try BigtableRequest.parse(BigtableRowWrites.delete(table: "events", key: key).text) + #expect(written.key == real) + } + + @Test("Without a locator the key comes from the _key text") + func keyTextWithoutLocator() throws { + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: .text("b64:/wA=")) == Data([0xFF, 0x00])) + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: .text("k")) == Data("k".utf8)) + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: nil) == nil) + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: .absent) == nil) + } + + @Test("A locator that is not base64 is refused rather than read as some other key") + func damagedLocatorRefused() { + #expect(throws: BigtableError.self) { + try BigtableRowWrites.originalKey(locator: "***", keyCell: .text("k")) + } + #expect(throws: BigtableError.self) { + try BigtableRowWrites.originalKey(locator: "", keyCell: .text("k")) + } + } } diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraCellLifetimeTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraCellLifetimeTests.swift new file mode 100644 index 0000000000..430c0cb539 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraCellLifetimeTests.swift @@ -0,0 +1,81 @@ +// +// CassandraCellLifetimeTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraCellLifetimeTests { + @Test("A cell read with its TTL() is expiring, without one permanent, and as null unknown") + func readLifetimes() { + #expect(CassandraCellLifetime(valueIsNull: false, timeToLive: "86400") == .expiring(seconds: 86_400)) + #expect(CassandraCellLifetime(valueIsNull: false, timeToLive: nil) == .permanent) + #expect(CassandraCellLifetime(valueIsNull: true, timeToLive: nil) == nil) + #expect(CassandraCellLifetime(valueIsNull: true, timeToLive: "30") == nil) + #expect(CassandraCellLifetime(valueIsNull: false, timeToLive: "soon") == nil) + } + + @Test("What is left is the TTL less the whole seconds since the read, and at least one", arguments: [ + (100, Duration.seconds(10), 90), + (100, Duration.milliseconds(10_900), 90), + (100, Duration.zero, 100), + (5, Duration.seconds(30), 1), + (5, Duration.seconds(5), 1), + (100, Duration.seconds(-3), 100), + ]) + func remaining(seconds: Int, elapsed: Duration, expected: Int) { + #expect(CassandraCellLifetime.remaining(seconds: seconds, after: elapsed) == expected) + } + + @Test("An edit keeps what an expiring cell had left, and a permanent one only when the table has a default TTL") + func rowTimesToLive() throws { + let readAt = ContinuousClock.now + let later = readAt.advanced(by: .seconds(30)) + let lifetimes: [CassandraCellLifetime?] = [.expiring(seconds: 100), .permanent, nil] + let columns = ["note", "address", "owner"] + + func row(defaultTimeToLive: Int?) throws -> CassandraRowLifetimes { + let layout = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: defaultTimeToLive)) + return try #require(CassandraRowLifetimes( + columns: columns, lifetimes: lifetimes, readAt: readAt, layout: layout + )) + } + + let noDefault = try row(defaultTimeToLive: 0) + #expect(noDefault.timeToLive(of: "note", at: later) == 70) + #expect(noDefault.timeToLive(of: "address", at: later) == nil) + #expect(noDefault.timeToLive(of: "owner", at: later) == nil) + #expect(noDefault.timeToLive(of: "Count", at: later) == nil) + #expect(try row(defaultTimeToLive: 3_600).timeToLive(of: "address", at: later) == 0) + #expect(try row(defaultTimeToLive: nil).timeToLive(of: "address", at: later) == 0) + } + + @Test("A cell whose TTL ran out since the read has expired, and no other cell has") + func expiredCells() throws { + let readAt = ContinuousClock.now + let layout = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: 0)) + let row = try #require(CassandraRowLifetimes( + columns: ["note", "address"], lifetimes: [.expiring(seconds: 100), .permanent], readAt: readAt, layout: layout + )) + #expect(!row.hasExpired("note", at: readAt.advanced(by: .seconds(99)))) + #expect(row.hasExpired("note", at: readAt.advanced(by: .seconds(100)))) + #expect(!row.hasExpired("address", at: readAt.advanced(by: .seconds(1_000)))) + #expect(!row.hasExpired("owner", at: readAt.advanced(by: .seconds(1_000)))) + } + + @Test("Nothing is kept for a row every cell of which a plain write keeps the same") + func rowsWithNothingToKeep() throws { + let noDefault = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: 0)) + #expect(CassandraRowLifetimes( + columns: ["note", "owner"], lifetimes: [.permanent, nil], readAt: .now, layout: noDefault + ) == nil) + #expect(CassandraRowLifetimes( + columns: ["note"], lifetimes: [.permanent, .permanent], readAt: .now, layout: noDefault + ) == nil) + let withDefault = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: 60)) + #expect(CassandraRowLifetimes( + columns: ["note", "owner"], lifetimes: [.permanent, nil], readAt: .now, layout: withDefault + ) != nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift new file mode 100644 index 0000000000..8b0a3fa3f1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift @@ -0,0 +1,97 @@ +// +// CassandraConditionalWriteTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraConditionalWriteTests { + private static let update = CassandraConditionalWrite.updateStatement( + target: "\"shop\".\"orders\"", assignments: ["\"note\" = ?"], keyCondition: "\"id\" = ?" + ) + private static let insert = CassandraConditionalWrite.insertStatement( + target: "\"shop\".\"orders\"", names: ["\"id\"", "\"note\""], values: ["?", "?"] + ) + + // Without the condition CQL writes the cell anyway and recreates the row with every other column null. + @Test("A grid edit only writes a row that still exists") + func updateIsConditionedOnTheRow() { + #expect(Self.update == "UPDATE \"shop\".\"orders\" SET \"note\" = ? WHERE \"id\" = ? IF EXISTS") + #expect(Self.insert == "INSERT INTO \"shop\".\"orders\" (\"id\", \"note\") VALUES (?, ?) IF NOT EXISTS") + } + + @Test("An edit of a row deleted since it was read is refused, not reported as saved") + func updateOfAMissingRowIsRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.update, firstColumn: "[applied]", firstValue: "false" + ) == .rowMissing) + } + + @Test("A new row whose key is taken is refused") + func insertOfATakenKeyIsRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.insert, firstColumn: "[applied]", firstValue: "false" + ) == .keyTaken) + } + + @Test("A write the condition let through is no refusal") + func appliedWriteIsNotRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.update, firstColumn: "[applied]", firstValue: "true" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: Self.insert, firstColumn: "[applied]", firstValue: "true" + ) == nil) + } + + // A condition on a column answers with the row's current values, which the console shows as the server sent them. + @Test("An unapplied write with a condition of its own is left to the result") + func otherConditionsAreLeftToTheResult() { + #expect(CassandraConditionalWrite.refusal( + statement: "UPDATE t SET a = 1 WHERE k = 1 IF a = 0", firstColumn: "[applied]", firstValue: "false" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: "DELETE FROM t WHERE k = 1 IF EXISTS", firstColumn: "[applied]", firstValue: "false" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: "UPDATE t SET a = 1 WHERE k = 1", firstColumn: "a", firstValue: "false" + ) == nil) + } + + @Test("An edit that keeps a TTL names it before SET and is refused the same way") + func updateWithTimeToLive() { + let update = CassandraConditionalWrite.updateStatement( + target: "\"shop\".\"orders\"", assignments: ["\"note\" = ?"], keyCondition: "\"id\" = ?", timeToLive: 300 + ) + #expect(update == "UPDATE \"shop\".\"orders\" USING TTL 300 SET \"note\" = ? WHERE \"id\" = ? IF EXISTS") + #expect(CassandraConditionalWrite.refusal(statement: update, firstColumn: "[applied]", firstValue: "false") + == .rowMissing) + } + + @Test("An unapplied batch of the row's UPDATEs, static ones unconditioned, is refused as a missing row") + func batchOfRowUpdatesIsRefused() { + let batch = CassandraConditionalWrite.batch([ + "UPDATE \"t\" USING TTL 60 SET \"a\" = ? WHERE \"k\" = ? AND \"c\" = ? IF EXISTS", + "UPDATE \"t\" USING TTL 0 SET \"b\" = ? WHERE \"k\" = ? AND \"c\" = ? IF EXISTS", + "UPDATE \"t\" USING TTL 30 SET \"s\" = ? WHERE \"k\" = ?", + ]) + #expect(batch.hasPrefix("BEGIN BATCH UPDATE")) + #expect(batch.hasSuffix("; APPLY BATCH")) + #expect(CassandraConditionalWrite.refusal(statement: batch, firstColumn: "[applied]", firstValue: "false") + == .rowMissing) + #expect(CassandraConditionalWrite.refusal(statement: batch, firstColumn: "[applied]", firstValue: "true") + == nil) + } + + @Test("A batch with a condition of its own, an INSERT or no condition is left to the result", arguments: [ + "BEGIN BATCH UPDATE t SET a = 1 WHERE k = 1 IF a = 0; UPDATE t SET b = 1 WHERE k = 1 IF EXISTS; APPLY BATCH", + "BEGIN BATCH INSERT INTO t (k) VALUES (1) IF NOT EXISTS; UPDATE t SET b = 1 WHERE k = 1 IF EXISTS APPLY BATCH", + "BEGIN BATCH INSERT INTO t (k, a) VALUES (1, 2) USING TTL 5; UPDATE t SET b = 1 WHERE k = 1; APPLY BATCH", + "BEGIN BATCH UPDATE t SET a = 'IF EXISTS' WHERE k = 1; APPLY BATCH", + ]) + func otherBatchesAreLeftToTheResult(batch: String) { + #expect(CassandraConditionalWrite.refusal(statement: batch, firstColumn: "[applied]", firstValue: "false") + == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeReadTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeReadTests.swift new file mode 100644 index 0000000000..5ef19efe07 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeReadTests.swift @@ -0,0 +1,63 @@ +// +// CassandraLifetimeReadTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraLifetimeReadTests { + private static let target = "\"shop\".\"visits\"" + + @Test("A browse of named columns reads TTL() after them for each one that takes it") + func namedColumns() throws { + let layout = try #require(CassandraTableLayoutTests.visits()) + let plan = try #require(CassandraLifetimeRead.plan( + cql: "SELECT \"id\", \"day\", \"note\", \"tags\" FROM \"shop\".\"visits\" WHERE \"note\" = ? ALLOW FILTERING", + target: Self.target, + layout: layout + )) + #expect(plan.cql + == "SELECT \"id\", \"day\", \"note\", \"tags\", TTL(\"note\") FROM \"shop\".\"visits\" WHERE \"note\" = ? ALLOW FILTERING") + #expect(plan.shownColumns == ["id", "day", "note", "tags"]) + #expect(plan.lifetimeColumns == ["note"]) + #expect(plan.lifetimeValueIndices == [2]) + #expect(plan.keyIndices == ["id": 0, "day": 1]) + } + + @Test("SELECT * is spelled out in the order the server returns it, so the TTL() selectors can follow") + func wildcard() throws { + let layout = try #require(CassandraTableLayoutTests.visits()) + let plan = try #require(CassandraLifetimeRead.plan(cql: "SELECT * FROM \"visits\"", target: "\"visits\"", layout: layout)) + #expect(plan.shownColumns == ["id", "day", "owner", "Count", "address", "note", "tags"]) + #expect(plan.cql == "SELECT \"id\", \"day\", \"owner\", \"Count\", \"address\", \"note\", \"tags\", " + + "TTL(\"owner\"), TTL(\"Count\"), TTL(\"address\"), TTL(\"note\") FROM \"visits\"") + #expect(plan.lifetimeValueIndices == [2, 3, 4, 5]) + } + + @Test("A name holding a quote and a comma is read back whole") + func quotedNames() throws { + let layout = try #require(CassandraTableLayoutTests.visits( + extra: [.init(name: "a\"b, c", type: "text", kind: .regular, position: -1)] + )) + let plan = try #require(CassandraLifetimeRead.plan( + cql: "SELECT \"id\", \"day\", \"a\"\"b, c\" FROM \"shop\".\"visits\"", target: Self.target, layout: layout + )) + #expect(plan.shownColumns == ["id", "day", "a\"b, c"]) + #expect(plan.cql == "SELECT \"id\", \"day\", \"a\"\"b, c\", TTL(\"a\"\"b, c\") FROM \"shop\".\"visits\"") + } + + @Test("Nothing is added to another table, another shape of statement, a read without the key or without a TTL", arguments: [ + "SELECT * FROM \"shop\".\"other\"", + "SELECT * FROM \"shop\".\"visits\" LIMIT 5", + "SELECT id, day, note FROM \"shop\".\"visits\"", + "SELECT \"note\" FROM \"shop\".\"visits\"", + "SELECT \"id\", \"day\", \"tags\" FROM \"shop\".\"visits\"", + "SELECT \"id\", \"day\", \"note FROM \"shop\".\"visits\"", + "select * from \"shop\".\"visits\"", + ]) + func untouchedStatements(cql: String) throws { + let layout = try #require(CassandraTableLayoutTests.visits()) + #expect(CassandraLifetimeRead.plan(cql: cql, target: Self.target, layout: layout) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeWriteTests.swift new file mode 100644 index 0000000000..282485c749 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeWriteTests.swift @@ -0,0 +1,150 @@ +// +// CassandraLifetimeWriteTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraLifetimeWriteTests { + private typealias Write = CassandraLifetimeWrite + private static let target = "\"shop\".\"visits\"" + private static let row = Write.KeyCondition(cql: "\"id\" = ? AND \"day\" = ?", values: ["k", "d"]) + private static let partition = Write.KeyCondition(cql: "\"id\" = ?", values: ["k"]) + + private static func set(_ column: String, _ value: String, ttl: Int?, isStatic: Bool = false) -> Write.Assignment { + Write.Assignment(cql: "\"\(column)\" = ?", parameters: [value], timeToLive: ttl, isStatic: isStatic) + } + + @Test("An edit with no TTL to keep is the statement it always was") + func noTimeToLive() { + let write = Write.update( + target: Self.target, assignments: [Self.set("note", "n", ttl: nil)], row: Self.row, partition: Self.partition + ) + #expect(write.cql == CassandraConditionalWrite.updateStatement( + target: Self.target, assignments: ["\"note\" = ?"], keyCondition: Self.row.cql + )) + #expect(write.parameters == ["n", "k", "d"]) + } + + @Test("Cells that keep one TTL are one UPDATE with it") + func oneTimeToLive() { + let write = Write.update( + target: Self.target, + assignments: [Self.set("note", "n", ttl: 120), Self.set("address", "a", ttl: 120)], + row: Self.row, + partition: Self.partition + ) + #expect(write.cql == "UPDATE \"shop\".\"visits\" USING TTL 120 SET \"note\" = ?, \"address\" = ? " + + "WHERE \"id\" = ? AND \"day\" = ? IF EXISTS") + #expect(write.parameters == ["n", "a", "k", "d"]) + } + + @Test("Cells that keep different TTLs are one batch on the row, a static-only UPDATE naming the partition") + func severalTimesToLive() { + let write = Write.update( + target: Self.target, + assignments: [ + Self.set("note", "n", ttl: 120), + Self.set("address", "a", ttl: 0), + Self.set("Count", "c", ttl: nil), + Self.set("owner", "o", ttl: 50, isStatic: true), + Self.set("extra", "e", ttl: 120), + ], + row: Self.row, + partition: Self.partition + ) + #expect(write.cql == "BEGIN BATCH " + + "UPDATE \"shop\".\"visits\" USING TTL 120 SET \"note\" = ?, \"extra\" = ? WHERE \"id\" = ? AND \"day\" = ? IF EXISTS; " + + "UPDATE \"shop\".\"visits\" USING TTL 0 SET \"address\" = ? WHERE \"id\" = ? AND \"day\" = ? IF EXISTS; " + + "UPDATE \"shop\".\"visits\" SET \"Count\" = ? WHERE \"id\" = ? AND \"day\" = ? IF EXISTS; " + + "UPDATE \"shop\".\"visits\" USING TTL 50 SET \"owner\" = ? WHERE \"id\" = ?; " + + "APPLY BATCH") + #expect(write.parameters == ["n", "e", "k", "d", "a", "k", "d", "c", "k", "d", "o", "k"]) + #expect(CassandraConditionalWrite.refusal(statement: write.cql, firstColumn: "[applied]", firstValue: "false") + == .rowMissing) + } + + private static func cell( + _ column: String, + ttl: Int? = nil, + isKey: Bool = false, + isNull: Bool = false, + isStatic: Bool = false + ) -> Write.RestoredCell { + Write.RestoredCell( + column: column, value: column, isKey: isKey, isNull: isNull, isStatic: isStatic, timeToLive: ttl + ) + } + + private static let keys = Write.RowKeys(row: row, partition: partition) + + @Test("A restored row with no TTL to keep is the INSERT it always was") + func restoreWithoutTimesToLive() { + let cells = [Self.cell("id", isKey: true), Self.cell("day", isKey: true), Self.cell("note"), Self.cell("Count", isNull: true)] + let expected = "INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"note\", \"Count\") VALUES (?, ?, ?, ?)" + for keys in [nil, Self.keys] { + let write = Write.restore(target: Self.target, cells: cells, keys: keys) + #expect(write.cql == expected) + #expect(write.parameters == ["id", "day", "note", "Count"]) + } + let unkeyed = Write.restore( + target: Self.target, cells: [Self.cell("id", isKey: true), Self.cell("note", ttl: 40)], keys: nil + ) + #expect(unkeyed.cql == "INSERT INTO \"shop\".\"visits\" (\"id\", \"note\") VALUES (?, ?)") + } + + @Test("A restored row whose cells share a TTL is one INSERT with it") + func restoreOneTimeToLive() { + let write = Write.restore( + target: Self.target, + cells: [Self.cell("id", isKey: true), Self.cell("day", isKey: true), Self.cell("note", ttl: 90), + Self.cell("Count", isNull: true), Self.cell("owner", ttl: 90, isStatic: true)], + keys: Self.keys + ) + #expect(write.cql == "INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"note\", \"Count\", \"owner\") " + + "VALUES (?, ?, ?, ?, ?) USING TTL 90") + #expect(write.parameters == ["id", "day", "note", "Count", "owner"]) + } + + // The INSERT also writes the row marker, so it takes the cells that live longest. + @Test("A restored row with several TTLs is a batch whose INSERT carries the longest-living cells") + func restoreSeveralTimesToLive() { + let write = Write.restore( + target: Self.target, + cells: [ + Self.cell("id", isKey: true), Self.cell("day", isKey: true), + Self.cell("owner", ttl: 50, isStatic: true), Self.cell("note", ttl: 120), + Self.cell("address", ttl: 0), Self.cell("Count", isNull: true), Self.cell("tags"), + ], + keys: Self.keys + ) + #expect(write.cql == "BEGIN BATCH " + + "INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"address\", \"Count\") VALUES (?, ?, ?, ?) USING TTL 0; " + + "UPDATE \"shop\".\"visits\" USING TTL 50 SET \"owner\" = ? WHERE \"id\" = ?; " + + "UPDATE \"shop\".\"visits\" USING TTL 120 SET \"note\" = ? WHERE \"id\" = ? AND \"day\" = ?; " + + "UPDATE \"shop\".\"visits\" SET \"tags\" = ? WHERE \"id\" = ? AND \"day\" = ?; " + + "APPLY BATCH") + #expect(write.parameters == [ + "id", "day", "address", "Count", "owner", "k", "note", "k", "d", "tags", "k", "d", + ]) + #expect(CassandraConditionalWrite.refusal(statement: write.cql, firstColumn: "[applied]", firstValue: "false") + == nil) + } + + @Test("Without a cell that never expires, the INSERT takes the table's default, else the longest TTL") + func restoreInsertChoice() { + let base = [Self.cell("id", isKey: true), Self.cell("day", isKey: true)] + let toDefault = Write.restore( + target: Self.target, cells: base + [Self.cell("note", ttl: 30), Self.cell("tags")], keys: Self.keys + ) + #expect(toDefault.cql.hasPrefix("BEGIN BATCH INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"tags\") VALUES (?, ?, ?); ")) + let longest = Write.restore( + target: Self.target, cells: base + [Self.cell("note", ttl: 30), Self.cell("address", ttl: 900)], keys: Self.keys + ) + #expect(longest.cql.hasPrefix( + "BEGIN BATCH INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"address\") VALUES (?, ?, ?) USING TTL 900; " + )) + #expect(longest.cql.contains("UPDATE \"shop\".\"visits\" USING TTL 30 SET \"note\" = ? WHERE \"id\" = ? AND \"day\" = ?; ")) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraRecentEntriesTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraRecentEntriesTests.swift new file mode 100644 index 0000000000..ba4c1d4cb8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraRecentEntriesTests.swift @@ -0,0 +1,49 @@ +// +// CassandraRecentEntriesTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraRecentEntriesTests { + @Test("Past the capacity the entries written longest ago are dropped first") + func dropsOldestPastCapacity() { + var entries = CassandraRecentEntries(capacity: 4) + for key in 1...5 { + entries.set("v\(key)", for: key) + } + #expect(entries.count <= 4) + #expect(entries[1] == nil) + #expect(entries[2] == nil) + #expect(entries[3] == "v3") + #expect(entries[5] == "v5") + + for key in 6...100 { + entries.set("v\(key)", for: key) + #expect(entries.count <= 4) + } + #expect(entries[100] == "v100") + #expect(entries[99] == "v99") + } + + @Test("Writing a key again replaces its value and keeps it among the newest") + func rewriteRefreshes() { + var entries = CassandraRecentEntries(capacity: 4) + entries.set(1, for: "a") + entries.set(2, for: "b") + entries.set(3, for: "c") + entries.set(10, for: "a") + entries.set(4, for: "d") + #expect(entries["a"] == 10) + #expect(entries["b"] == nil) + #expect(entries["c"] == 3) + #expect(entries["d"] == 4) + #expect(entries.count == 3) + entries.removeValue(for: "a") + #expect(entries["a"] == nil) + entries.removeAll() + #expect(entries["c"] == nil) + #expect(entries["d"] == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraTableLayoutTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraTableLayoutTests.swift new file mode 100644 index 0000000000..57d6a0e8b3 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraTableLayoutTests.swift @@ -0,0 +1,66 @@ +// +// CassandraTableLayoutTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraTableLayoutTests { + static func visits(defaultTimeToLive: Int? = 0, extra: [CassandraTableLayout.Column] = []) -> CassandraTableLayout? { + CassandraTableLayout(columns: [ + .init(name: "note", type: "text", kind: .regular, position: -1), + .init(name: "day", type: "date", kind: .clustering, position: 0), + .init(name: "tags", type: "set", kind: .regular, position: -1), + .init(name: "owner", type: "text", kind: .staticColumn, position: -1), + .init(name: "id", type: "uuid", kind: .partitionKey, position: 0), + .init(name: "Count", type: "int", kind: .regular, position: -1), + .init(name: "address", type: "frozen
", kind: .regular, position: -1), + ] + extra, defaultTimeToLive: defaultTimeToLive) + } + + @Test("SELECT * order: the key in order, then static and regular columns by name") + func selectOrder() throws { + let layout = try #require(Self.visits()) + #expect(layout.selectOrder == ["id", "day", "owner", "Count", "address", "note", "tags"]) + #expect(layout.partitionKey == ["id"]) + #expect(layout.primaryKey == ["id", "day"]) + #expect(layout.staticColumns == ["owner"]) + } + + @Test("TTL() is read for regular and static cells that hold one value", arguments: [ + ("note", true), ("owner", true), ("Count", true), ("address", true), + ("tags", false), ("id", false), ("day", false), ("missing", false), + ]) + func acceptedColumns(column: String, accepted: Bool) throws { + let layout = try #require(Self.visits()) + #expect(layout.acceptsTimeToLive(column) == accepted) + } + + @Test("Counters, non-frozen collections and user types, and custom types take no TTL()", arguments: [ + ("counter", false), ("list", false), ("map", false), ("address", false), + ("'org.apache.cassandra.db.marshal.DynamicCompositeType'", false), + ("frozen>", true), ("tuple", true), ("vector", true), ("duration", true), + (" TIMESTAMP ", true), + ]) + func typesThatHoldOneCell(type: String, accepted: Bool) throws { + let layout = try #require(Self.visits(extra: [.init(name: "probe", type: type, kind: .regular, position: -1)])) + #expect(layout.acceptsTimeToLive("probe") == accepted) + } + + @Test("No layout for a table without columns or with a COMPACT STORAGE value column") + func unusableTables() { + #expect(CassandraTableLayout(columns: [], defaultTimeToLive: 0) == nil) + #expect(Self.visits(extra: [.init(name: "value", type: "empty", kind: .regular, position: -1)]) == nil) + } + + @Test("A system_schema row needs a name, a type and a kind CQL has") + func columnRows() { + #expect(CassandraTableLayout.column(name: "note", type: "text", kind: "regular", position: "-1") + == .init(name: "note", type: "text", kind: .regular, position: -1)) + #expect(CassandraTableLayout.column(name: "id", type: "int", kind: "partition_key", position: "0")?.kind + == .partitionKey) + #expect(CassandraTableLayout.column(name: "note", type: "text", kind: "other", position: "0") == nil) + #expect(CassandraTableLayout.column(name: nil, type: "text", kind: "regular", position: "0") == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraWireTextTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraWireTextTests.swift new file mode 100644 index 0000000000..49956b11f8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraWireTextTests.swift @@ -0,0 +1,32 @@ +// +// CassandraWireTextTests.swift +// TableProCassandraCoreTests +// + +import Foundation +import TableProCassandraCore +import Testing + +struct CassandraWireTextTests { + // Foundation drops this mark, and the key without it names another row. + @Test("A leading byte order mark stays part of the text") + func byteOrderMarkIsKept() { + let bytes: [UInt8] = [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63] + let fromData = CassandraWireText.text(Data(bytes)) + let fromBuffer = bytes.withUnsafeBytes { CassandraWireText.text($0) } + #expect(fromData == "\u{FEFF}abc") + #expect(fromBuffer == "\u{FEFF}abc") + #expect(fromData.map { Array($0.utf8) } == bytes) + } + + @Test("Text that is not UTF-8 is not read as text", arguments: [[0xFF], [0x61, 0xC3], [0xED, 0xA0, 0x80]] as [[UInt8]]) + func invalidText(bytes: [UInt8]) { + #expect(CassandraWireText.text(Data(bytes)) == nil) + } + + @Test("Empty and plain text read as written") + func plainText() { + #expect(CassandraWireText.text(Data()) == "") + #expect(CassandraWireText.text(Data("héllo".utf8)) == "héllo") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProConnectionLibraryTests/ConnectionDestinationTests.swift b/apple/Packages/TableProCore/Tests/TableProConnectionLibraryTests/ConnectionDestinationTests.swift index 024f2d9d23..57e3d833a6 100644 --- a/apple/Packages/TableProCore/Tests/TableProConnectionLibraryTests/ConnectionDestinationTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProConnectionLibraryTests/ConnectionDestinationTests.swift @@ -23,7 +23,7 @@ struct ConnectionDestinationTests { #expect(saved.digest != crafted.digest) } - @Test("Engine, TLS, SSH server, jump host, SSH profile and any plugin field each change the digest") + @Test("Engine, TLS, SSH server, jump host and any plugin field each change the digest") func everyRoutingInputChangesTheDigest() { let jumpless = ConnectionDestination.Tunnel.ssh( server: .init(host: "bastion.prod", port: 22), jumps: [] @@ -46,7 +46,6 @@ struct ConnectionDestinationTests { jumps: [.init(host: "hop.attacker.example", port: 22)] ) ), - Self.production(transport: .verifyIdentity, tunnel: .profile(UUID())), Self.production(transport: .verifyIdentity, tunnel: jumpless, fields: ["esSkipTLSVerify": "true"]), Self.production(transport: .verifyIdentity, tunnel: jumpless, fields: ["someFutureField": "x"]), ] diff --git a/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerSignInTests.swift b/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerSignInTests.swift index fe26cf08e1..b8cadb50d5 100644 --- a/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerSignInTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerSignInTests.swift @@ -24,10 +24,15 @@ private struct DriverRequest: Sendable { private final class SigningDriverFactory: DriverFactory, @unchecked Sendable { var mint: @Sendable (String?) async throws -> String? = { stored in stored } + var alias: @Sendable (DatabaseConnection) async -> DatabaseConnection = { $0 } var opensCatalogs = false private(set) var mintRequests: [MintRequest] = [] private(set) var driverRequests: [DriverRequest] = [] + func resolvingEndpointAlias(for connection: DatabaseConnection) async throws -> DatabaseConnection { + await alias(connection) + } + func resolvePassword( for connection: DatabaseConnection, stored: String?, @@ -85,11 +90,19 @@ private func passwordKey(for connection: DatabaseConnection) -> String { "com.TablePro.password.\(connection.id.uuidString)" } -private func rdsConnection(tunneled: Bool) -> DatabaseConnection { +private let replicaEndpoint = "replica-a.abc123.us-east-1.rds.amazonaws.com" + +private func pointingAtReplica(_ connection: DatabaseConnection) -> DatabaseConnection { + var resolved = connection + resolved.host = replicaEndpoint + return resolved +} + +private func rdsConnection(tunneled: Bool, host: String = "mydb.abc123.us-east-1.rds.amazonaws.com") -> DatabaseConnection { DatabaseConnection( name: "RDS", type: mockType, - host: "mydb.abc123.us-east-1.rds.amazonaws.com", + host: host, port: 5_432, username: "iam_user", additionalFields: ["awsAuth": "accessKey"], @@ -255,4 +268,59 @@ struct ConnectionManagerSignInTests { #expect(driver.host == connection.host) #expect(driver.preTunnelHost == nil) } + + @Test("The endpoint a CNAME stands for is what the connect mints for, dials and mints for again") + func aliasReachesMintDriverAndRefresh() async throws { + let factory = SigningDriverFactory() + factory.alias = { pointingAtReplica($0) } + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = rdsConnection(tunneled: false, host: "reader.db.example.com") + + _ = try await manager.connect(connection) + let driver = try #require(factory.driverRequests.first) + let refresh = try #require(driver.refreshPassword) + _ = try await refresh() + + #expect(factory.mintRequests.map(\.host) == [replicaEndpoint, replicaEndpoint]) + #expect(driver.host == replicaEndpoint) + #expect(driver.preTunnelHost == nil) + } + + @Test("Whatever endpoint the factory returns, a tunnel forwards to it and the driver names it behind the tunnel") + func aliasMovesTheTunnelDestination() async throws { + let factory = SigningDriverFactory() + factory.alias = { pointingAtReplica($0) } + let ssh = MockSSHProvider() + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore(), sshProvider: ssh) + let connection = rdsConnection(tunneled: true, host: "reader.db.example.com") + + _ = try await manager.connect(connection) + + #expect(factory.mintRequests.first?.host == replicaEndpoint) + #expect(ssh.receivedDestinations == [.tcp(host: replicaEndpoint, port: 5_432)]) + let driver = try #require(factory.driverRequests.first) + #expect(driver.host == "127.0.0.1") + #expect(driver.preTunnelHost == replicaEndpoint) + } + + @Test("An attempt invalidated during the lookup throws CancellationError and mints nothing") + func attemptInvalidatedDuringTheLookupStops() async throws { + let gate = AsyncGate() + let factory = SigningDriverFactory() + factory.alias = { connection in + await gate.wait() + return pointingAtReplica(connection) + } + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = rdsConnection(tunneled: false, host: "reader.db.example.com") + + let attempt = Task { try await manager.connect(connection) } + await gate.arrived(1) + manager.invalidateAttempt(for: connection.id) + await gate.open() + + await #expect(throws: CancellationError.self) { try await attempt.value } + #expect(factory.mintRequests.isEmpty) + #expect(manager.session(for: connection.id) == nil) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdRangeScanTests.swift b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdRangeScanTests.swift new file mode 100644 index 0000000000..17241c41c8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdRangeScanTests.swift @@ -0,0 +1,276 @@ +import Foundation +import Testing + +@testable import TableProEtcdCore + +// Answers range requests the way etcd does for a request with no sort order: ascending keys, the limit applied as it +// reads, and `more` set when the limit cut the range short. +private final class RecordingKeyspace { + struct Request: Equatable { + let range: EtcdKeyRange + let limit: Int64 + let revision: Int64? + } + + static let revision: Int64 = 7 + static let everyKey = EtcdKeyRange(key: Data([0]), rangeEnd: Data([0])) + + let keys: [Data] + private(set) var requests: [Request] = [] + + init(count: Int) { + keys = (0.. Data { + Data(String(format: "/k/%06d", index).utf8) + } + + var keysRead: Int64 { + requests.reduce(0) { $0 + $1.limit } + } + + func fetch(_ range: EtcdKeyRange, _ limit: Int64, _ revision: Int64?) -> EtcdRangeBatch { + requests.append(Request(range: range, limit: limit, revision: revision)) + let inRange = keys.filter { key in + guard !key.lexicographicallyPrecedes(range.key) else { return false } + guard let end = range.rangeEnd else { return key == range.key } + return end == Data([0]) || key.lexicographicallyPrecedes(end) + } + let taken = Array(inRange.prefix(Int(limit))) + return EtcdRangeBatch(items: taken, more: inRange.count > taken.count, revision: Self.revision) + } +} + +private struct Compacted: Error {} +private struct Refused: Error {} + +struct EtcdRangeScanTests { + @Test("A page deep in the range is found by reading keys up to its offset, never past it") + func pageStartReadsKeysUpToTheOffset() async throws { + let keyspace = RecordingKeyspace(count: 10_000) + + let start = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: 2_500, batchSize: 1_000 + ) { keyspace.fetch($0, $1, $2) } + + let limits = keyspace.requests.map(\.limit) + #expect(limits == [1_000, 1_000, 500]) + #expect(keyspace.keysRead == 2_500) + #expect(start?.range == EtcdKeyRange( + key: RecordingKeyspace.key(2_499) + Data([0]), rangeEnd: Data([0]) + )) + #expect(start?.revision == RecordingKeyspace.revision) + } + + @Test("Every skip request after the first reads at the first one's revision") + func pageStartPinsTheRevision() async throws { + let keyspace = RecordingKeyspace(count: 5_000) + + _ = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: 3_000, batchSize: 1_000 + ) { keyspace.fetch($0, $1, $2) } + + let revisions = keyspace.requests.map(\.revision) + #expect(revisions == [nil, RecordingKeyspace.revision, RecordingKeyspace.revision]) + } + + @Test("The first page needs no skip request") + func firstPageSendsNothing() async throws { + let keyspace = RecordingKeyspace(count: 10) + + let start = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: 0, batchSize: 1_000 + ) { keyspace.fetch($0, $1, $2) } + + #expect(keyspace.requests.isEmpty) + #expect(start == EtcdPageStart(range: RecordingKeyspace.everyKey, revision: nil)) + } + + @Test("A page at or past the last key has no start", arguments: [10, 11, 50]) + func pagePastTheEndHasNoStart(offset: Int) async throws { + let keyspace = RecordingKeyspace(count: 10) + + let start = try await EtcdRangeScan.pageStart( + of: RecordingKeyspace.everyKey, skipping: offset, batchSize: 4 + ) { keyspace.fetch($0, $1, $2) } + + #expect(start == nil) + } + + @Test("A scan asks for one batch at a time and stops when the reader has enough") + func scanStopsWhenTheBodyDoes() async throws { + let keyspace = RecordingKeyspace(count: 10_000) + var batches: [[Data]] = [] + + try await EtcdRangeScan.scan( + RecordingKeyspace.everyKey, + batchSize: 1_000, + maxItems: .max, + key: { (key: Data) in key }, + fetch: { keyspace.fetch($0, $1, $2) }, + body: { (batch: [Data]) in + batches.append(batch) + return batches.count < 2 + } + ) + + let limits = keyspace.requests.map(\.limit) + let delivered = batches.flatMap { $0 } + let expected = (0..<2_000).map(RecordingKeyspace.key) + #expect(limits == [1_000, 1_000]) + #expect(delivered == expected) + } + + @Test("A scan reads the whole range in order, each key once") + func scanReadsEveryKeyOnce() async throws { + let keyspace = RecordingKeyspace(count: 2_345) + var seen: [Data] = [] + + try await EtcdRangeScan.scan( + RecordingKeyspace.everyKey, + batchSize: 1_000, + maxItems: .max, + key: { (key: Data) in key }, + fetch: { keyspace.fetch($0, $1, $2) }, + body: { (batch: [Data]) in + seen += batch + return true + } + ) + + #expect(seen == keyspace.keys) + #expect(keyspace.requests.count == 3) + } + + @Test("A scan stops at its row ceiling") + func scanStopsAtTheCeiling() async throws { + let keyspace = RecordingKeyspace(count: 5_000) + var seen = 0 + + try await EtcdRangeScan.scan( + RecordingKeyspace.everyKey, + batchSize: 1_000, + maxItems: 2_500, + key: { (key: Data) in key }, + fetch: { keyspace.fetch($0, $1, $2) }, + body: { (batch: [Data]) in + seen += batch.count + return true + } + ) + + let limits = keyspace.requests.map(\.limit) + #expect(seen == 2_500) + #expect(limits == [1_000, 1_000, 500]) + } + + @Test("A descending read goes on below the last key it returned") + func descendingContinuationEndsAtTheLastKey() { + let prefix = EtcdKeyRange(key: Data("/k/".utf8), prefix: true) + + #expect(EtcdRangeScan.range(before: Data("/k/5".utf8), in: prefix) + == EtcdKeyRange(key: Data("/k/".utf8), rangeEnd: Data("/k/5".utf8))) + } + + @Test("An ascending page skips its offset in matches and stops reading once full") + func ascendingCollectorStopsWhenFull() { + var collector = EtcdPageCollector(offset: 3, limit: 2, ascending: true) + var accepted: [Bool] = [] + for item in 0..<10 { + let goesOn = collector.add(item) + accepted.append(goesOn) + if !goesOn { break } + } + + #expect(collector.page == [3, 4]) + #expect(accepted == [true, true, true, true, false]) + } + + @Test("A descending page is the tail of the ascending matches, newest first") + func descendingCollectorTakesTheTail() { + var collector = EtcdPageCollector(offset: 2, limit: 3, ascending: false) + for item in 0..<10 { + _ = collector.add(item) + } + + #expect(collector.page == [7, 6, 5]) + } + + @Test("A descending page holds at most twice its offset and limit, however long the range") + func descendingCollectorStaysBounded() { + var collector = EtcdPageCollector(offset: 5, limit: 5, ascending: false) + var largestHeld = 0 + for item in 0..<100_000 { + _ = collector.add(item) + largestHeld = max(largestHeld, collector.heldCount) + } + + #expect(largestHeld <= 20) + #expect(collector.page == [99_994, 99_993, 99_992, 99_991, 99_990]) + } + + @Test("Only etcd's compacted-revision answer counts as compaction") + func compactionIsRecognized() { + #expect(EtcdCompaction.isCompacted(grpcCode: 11, message: "etcdserver: mvcc: required revision has been compacted")) + #expect(!EtcdCompaction.isCompacted(grpcCode: 11, message: "etcdserver: mvcc: required revision is a future revision")) + #expect(!EtcdCompaction.isCompacted(grpcCode: 3, message: "required revision has been compacted")) + } + + @Test("A compacted read starts over once, and a second compaction is the caller's") + func compactedReadStartsOverOnce() async { + var attempts = 0 + var failure: Error? + do { + _ = try await EtcdCompaction.restartingOnce(when: { $0 is Compacted }) { () async throws -> Int in + attempts += 1 + throw Compacted() + } + } catch { + failure = error + } + + #expect(failure is Compacted) + #expect(attempts == 2) + } + + @Test("A read that recovers on its second attempt returns that attempt's answer") + func compactedReadReturnsTheRetry() async throws { + var attempts = 0 + let answer = try await EtcdCompaction.restartingOnce(when: { $0 is Compacted }) { () async throws -> Int in + attempts += 1 + if attempts == 1 { throw Compacted() } + return 42 + } + + #expect(answer == 42) + #expect(attempts == 2) + } + + @Test("Any other failure is not retried") + func otherFailuresAreNotRetried() async { + var attempts = 0 + var failure: Error? + do { + _ = try await EtcdCompaction.restartingOnce(when: { $0 is Compacted }) { () async throws -> Int in + attempts += 1 + throw Refused() + } + } catch { + failure = error + } + + #expect(failure is Refused) + #expect(attempts == 1) + } + + @Test("A descending page past the matches is empty") + func descendingCollectorPastTheEnd() { + var collector = EtcdPageCollector(offset: 8, limit: 5, ascending: false) + for item in 0..<5 { + _ = collector.add(item) + } + + #expect(collector.page.isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseConnectionSettingsTests.swift b/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseConnectionSettingsTests.swift index 01025bddd3..37e1321d1b 100644 --- a/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseConnectionSettingsTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseConnectionSettingsTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProFirebaseClient @testable import TableProFirebaseCore import Testing diff --git a/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseStatementTests.swift index e10df0084a..450410a913 100644 --- a/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseStatementTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProFirebaseCoreTests/FirebaseStatementTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProFirebaseClient @testable import TableProFirebaseCore import TableProGoogleCloud import Testing diff --git a/apple/Packages/TableProCore/Tests/TableProHTTPTests/LoopbackRedirectListenerTests.swift b/apple/Packages/TableProCore/Tests/TableProHTTPTests/LoopbackRedirectListenerTests.swift new file mode 100644 index 0000000000..b618f27224 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProHTTPTests/LoopbackRedirectListenerTests.swift @@ -0,0 +1,305 @@ +import Darwin +import Foundation +@testable import TableProHTTP +import Testing + +private enum LoopbackProbe { + static func status(of target: String, port: UInt16) async -> Int? { + guard let url = URL(string: "http://127.0.0.1:\(port)\(target)") else { return nil } + let session = URLSession(configuration: .ephemeral) + defer { session.finishTasksAndInvalidate() } + let request = URLRequest(url: url, timeoutInterval: 5) + guard let (_, response) = try? await session.data(for: request), + let httpResponse = response as? HTTPURLResponse + else { + return nil + } + return httpResponse.statusCode + } + + static func connectIPv4(port: UInt16) -> Int32? { + let descriptor = socket(AF_INET, SOCK_STREAM, 0) + guard descriptor >= 0 else { return nil } + var address = sockaddr_in() + address.sin_len = UInt8(MemoryLayout.size) + address.sin_family = sa_family_t(AF_INET) + address.sin_port = port.bigEndian + _ = inet_pton(AF_INET, "127.0.0.1", &address.sin_addr) + let result = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { generic in + connect(descriptor, generic, socklen_t(MemoryLayout.size)) + } + } + guard result == 0 else { + close(descriptor) + return nil + } + return descriptor + } + + // The descriptor bound to [::1]:port, or the errno the bind failed with. + static func bindIPv6(port: UInt16, listening: Bool = false) -> Result { + let descriptor = socket(AF_INET6, SOCK_STREAM, 0) + guard descriptor >= 0 else { return .failure(POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO)) } + var address = sockaddr_in6() + address.sin6_len = UInt8(MemoryLayout.size) + address.sin6_family = sa_family_t(AF_INET6) + address.sin6_port = port.bigEndian + _ = inet_pton(AF_INET6, "::1", &address.sin6_addr) + let result = withUnsafePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { generic in + bind(descriptor, generic, socklen_t(MemoryLayout.size)) + } + } + guard result == 0 else { + let failure = errno + close(descriptor) + return .failure(POSIXError(POSIXErrorCode(rawValue: failure) ?? .EIO)) + } + if listening { + _ = listen(descriptor, 4) + } + return .success(descriptor) + } + + static func boundPort(of descriptor: Int32) -> UInt16 { + var address = sockaddr_in6() + var length = socklen_t(MemoryLayout.size) + _ = withUnsafeMutablePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { generic in + getsockname(descriptor, generic, &length) + } + } + return UInt16(bigEndian: address.sin6_port) + } + + // True once the peer closed or reset the connection; false when nothing happened before the timeout. + static func waitForClose(_ descriptor: Int32, seconds: Int) async -> Bool { + await withCheckedContinuation { continuation in + DispatchQueue.global().async { + var timeout = timeval(tv_sec: seconds, tv_usec: 0) + setsockopt(descriptor, SOL_SOCKET, SO_RCVTIMEO, &timeout, socklen_t(MemoryLayout.size)) + var byte: UInt8 = 0 + let received = recv(descriptor, &byte, 1, 0) + let failure = errno + continuation.resume(returning: received == 0 || (received < 0 && failure != EAGAIN && failure != EWOULDBLOCK)) + } + } + } + + static func listener(port: UInt16 = 0, headTimeout: TimeInterval = 5) -> LoopbackRedirectListener { + LoopbackRedirectListener(port: port, headTimeout: headTimeout, answer: answer) + } + + // Settles on a GET to /done and ignores every other request. + static let answer: LoopbackRedirectListener.Answer = { head in + let text = LoopbackRedirectRequest.text(of: head) + guard let target = LoopbackRedirectRequest.getTarget(text), target.hasPrefix("/done") else { + return .ignore(LoopbackRedirectPage.badRequest) + } + return .settle(LoopbackRedirectPage.response(status: "200 OK", title: "Done", message: ""), .success(target)) + } + + static func freeIPv4Port() -> UInt16? { + let descriptor = socket(AF_INET, SOCK_STREAM, 0) + guard descriptor >= 0 else { return nil } + defer { close(descriptor) } + var address = sockaddr_in() + address.sin_len = UInt8(MemoryLayout.size) + address.sin_family = sa_family_t(AF_INET) + _ = inet_pton(AF_INET, "127.0.0.1", &address.sin_addr) + var length = socklen_t(MemoryLayout.size) + let bound = withUnsafeMutablePointer(to: &address) { pointer in + pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { generic in + bind(descriptor, generic, length) == 0 && getsockname(descriptor, generic, &length) == 0 + } + } + return bound ? UInt16(bigEndian: address.sin_port) : nil + } +} + +private final class PortAttempts: @unchecked Sendable { + private let lock = NSLock() + private var tried: [UInt16] = [] + + var ports: [UInt16] { + lock.withLock { tried } + } + + func record(_ port: UInt16) -> Int { + lock.withLock { + tried.append(port) + return tried.count + } + } +} + +private struct SampleTimeout: Error, Equatable {} + +struct LoopbackRedirectListenerTests { + @Test("Requests the answer ignores get a reply and leave the wait open for the redirect") + func strayRequestsAreAnsweredAndIgnored() async throws { + let listener = LoopbackProbe.listener() + let port = try await listener.start() + defer { listener.stop() } + let outcome = Task { try await listener.outcome(timeout: 30, timeoutError: SampleTimeout()) } + + let stray = await LoopbackProbe.status(of: "/auth?code=forged", port: port) + let favicon = await LoopbackProbe.status(of: "/favicon.ico", port: port) + let redirect = await LoopbackProbe.status(of: "/done?code=real", port: port) + + #expect([stray, favicon, redirect] == [400, 400, 200]) + #expect(try await outcome.value == "/done?code=real") + #expect(await LoopbackProbe.status(of: "/done?again", port: port) == nil) + } + + @Test("A redirect that lands before anyone waits is kept for the wait") + func settledBeforeTheWait() async throws { + let listener = LoopbackProbe.listener() + let port = try await listener.start() + defer { listener.stop() } + + #expect(await LoopbackProbe.status(of: "/done", port: port) == 200) + #expect(try await listener.outcome(timeout: 30, timeoutError: SampleTimeout()) == "/done") + } + + @Test("A connection that sends no request is closed at the head deadline") + func idleConnectionIsClosedAtTheDeadline() async throws { + let listener = LoopbackProbe.listener(headTimeout: 0.2) + let port = try await listener.start() + defer { listener.stop() } + + let idle = try #require(LoopbackProbe.connectIPv4(port: port)) + defer { close(idle) } + + #expect(await LoopbackProbe.waitForClose(idle, seconds: 5)) + } + + @Test("Idle sockets filling every slot cannot keep the redirect out") + func fullTableMakesRoomForTheRedirect() async throws { + let listener = LoopbackProbe.listener(headTimeout: 60) + let port = try await listener.start() + defer { listener.stop() } + let outcome = Task { try await listener.outcome(timeout: 30, timeoutError: SampleTimeout()) } + + let idle = (0..<20).compactMap { _ in LoopbackProbe.connectIPv4(port: port) } + defer { idle.forEach { close($0) } } + #expect(idle.count == 20) + + #expect(await LoopbackProbe.status(of: "/done", port: port) == 200) + #expect(try await outcome.value == "/done") + } + + @Test("While it listens, no other socket can take the IPv6 loopback port, and stopping frees it") + func ipv6LoopbackPortIsHeld() async throws { + let listener = LoopbackProbe.listener() + let port = try await listener.start() + + switch LoopbackProbe.bindIPv6(port: port) { + case .success(let descriptor): + close(descriptor) + Issue.record("Another socket took [::1]:\(port) while the listener held it") + case .failure(let error): + #expect(error.code == .EADDRINUSE) + } + + listener.stop() + switch LoopbackProbe.bindIPv6(port: port) { + case .success(let descriptor): + close(descriptor) + case .failure(let error): + Issue.record("[::1]:\(port) stayed taken after stop: \(error.code)") + } + } + + @Test("A port whose IPv6 loopback side another process holds fails the start") + func takenIPv6LoopbackPortFailsTheStart() async throws { + let held = try LoopbackProbe.bindIPv6(port: 0, listening: true).get() + defer { close(held) } + let listener = LoopbackProbe.listener(port: LoopbackProbe.boundPort(of: held)) + + await #expect(throws: LoopbackRedirectListenerError.portInUse) { + try await listener.start() + } + } + + @Test("A kernel-picked port whose IPv6 loopback side is taken is traded for another") + func kernelPickedPortTriesAnotherWhenIPv6IsTaken() async throws { + let attempts = PortAttempts() + let listener = LoopbackRedirectListener( + port: 0, + headTimeout: 5, + holdIPv6: { port in + attempts.record(port) < 3 ? .taken(EADDRINUSE) : LoopbackPortHold.ipv6(port: port) + }, + answer: LoopbackProbe.answer + ) + let port = try await listener.start() + defer { listener.stop() } + + #expect(attempts.ports.count == 3) + #expect(attempts.ports.last == port) + #expect(await LoopbackProbe.status(of: "/done", port: port) == 200) + } + + @Test("A kernel-picked port stops trying after a bounded number of taken ports") + func kernelPickedPortGivesUp() async throws { + let attempts = PortAttempts() + let listener = LoopbackRedirectListener( + port: 0, + headTimeout: 5, + holdIPv6: { port in + _ = attempts.record(port) + return .taken(EADDRINUSE) + }, + answer: LoopbackProbe.answer + ) + + await #expect(throws: LoopbackRedirectListenerError.portInUse) { + try await listener.start() + } + #expect(attempts.ports.count == LoopbackRedirectLimits.maximumKernelPortAttempts) + } + + @Test("A fixed port whose IPv6 loopback side is taken fails without trying another") + func fixedPortDoesNotTryAnother() async throws { + let fixed = try #require(LoopbackProbe.freeIPv4Port()) + let attempts = PortAttempts() + let listener = LoopbackRedirectListener( + port: fixed, + headTimeout: 5, + holdIPv6: { port in + _ = attempts.record(port) + return .taken(EADDRINUSE) + }, + answer: LoopbackProbe.answer + ) + + await #expect(throws: LoopbackRedirectListenerError.portInUse) { + try await listener.start() + } + #expect(attempts.ports == [fixed]) + } + + @Test("Stopping ends the wait") + func stopEndsTheWait() async throws { + let listener = LoopbackProbe.listener() + try await listener.start() + listener.stop() + + await #expect(throws: CancellationError.self) { + try await listener.outcome(timeout: 30, timeoutError: SampleTimeout()) + } + } + + @Test("A wait with no redirect ends with the caller's timeout error") + func waitTimesOut() async throws { + let listener = LoopbackProbe.listener() + try await listener.start() + defer { listener.stop() } + + await #expect(throws: SampleTimeout()) { + try await listener.outcome(timeout: 0.2, timeoutError: SampleTimeout()) + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift new file mode 100644 index 0000000000..d5ea9e7298 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift @@ -0,0 +1,224 @@ +import Foundation +@testable import TableProInfluxDBCore +import Testing + +private func bytes(_ text: String) -> Data { + Data(text.utf8) +} + +private func pieces(_ data: Data, size: Int) -> [Data] { + stride(from: 0, to: data.count, by: size).map { data.subdata(in: $0 ..< min($0 + size, data.count)) } +} + +private func records(_ chunks: [Data]) -> [[String]] { + var reader = InfluxCSV.Reader() + var records: [[String]] = [] + for chunk in chunks { + reader.read(chunk) + records += reader.drain() + } + return records + reader.finish() +} + +private func streamed( + _ body: InfluxGridStream.Body, + _ chunks: [Data], + grid: InfluxGridQuery = cpuExport, + schema: [InfluxColumn] = cpuSchema +) throws -> [InfluxResultTable] { + var stream = InfluxGridStream(body: body, grid: grid, schema: schema) + var batches: [InfluxResultTable] = [] + for chunk in chunks { + batches += try stream.consume(chunk) + } + let rest = try stream.finish() + return batches + rest +} + +// The batches under the header the first one carries, which every other one repeats. +private func joined(_ batches: [InfluxResultTable]) throws -> InfluxResultTable { + let first = try #require(batches.first) + #expect(batches.allSatisfy { $0.columns == first.columns && $0.typeNames == first.typeNames }) + return InfluxResultTable(columns: first.columns, typeNames: first.typeNames, rows: batches.flatMap { $0.rows }) +} + +private let cpuExport = InfluxGridQuery(measurement: "cpu", shape: .all) + +private let cpuSchema = [ + InfluxColumn(name: "time", typeName: InfluxTypeName.timestamp, role: .time), + InfluxColumn(name: "host", typeName: InfluxTypeName.tag, role: .tag), + InfluxColumn(name: "idle", typeName: InfluxTypeName.integer, role: .field), + InfluxColumn(name: "note", typeName: InfluxTypeName.string, role: .field), + InfluxColumn(name: "usage", typeName: InfluxTypeName.float, role: .field), +] + +private let v3Body = bytes( + "host,time,usage,note,idle\r\n" + + "a,2024-10-27T03:33:20,12.5,\"hot, \"\"quoted\"\"\",10\r\n" + + "b,2024-10-27T03:34:20.500Z,,,\r\n" + + ",2024-10-27T03:35:20,7,\"two\r\nlines é\",40\r\n" +) + +private let v2Rows = [ + #"["2024-10-27T03:35:20Z","c",10,"a;b",7]"#, + #"["2024-10-27T03:34:20Z","b",40,"hot, \"quoted\"",50.1]"#, + #"["2024-10-27T03:33:20Z","a",null,null,12.5]"#, +] + +private func v2Chunk(_ rows: ArraySlice, partial: Bool) -> String { + let flag = partial ? #","partial":true"# : "" + return #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","idle","note","usage"],"values":["# + + rows.joined(separator: ",") + "]" + flag + "}]" + flag + "}]}" +} + +private let tooWide = InfluxError.malformedResponse( + String(localized: "the result has far more columns than its rows fill") +) + +struct InfluxGridStreamTests { + @Test func aCREndingOneChunkAndTheLFStartingTheNextAreOneLineEnd() { + #expect(records([bytes("a,b\r"), bytes("\nc,d\r\n")]) == [["a", "b"], ["c", "d"]]) + } + + @Test func anEscapedQuoteSplitBetweenItsTwoQuotesIsOneQuote() { + #expect(records([bytes("\"x\""), bytes("\"y\",z\n")]) == [["x\"y", "z"]]) + } + + @Test func aQuotedLineEndSplitAcrossChunksStaysInTheField() { + #expect(records([bytes("\"a\r"), bytes("\nb\",c\n"), bytes("\"d\n"), bytes("e\"\n")]) == [["a\r\nb", "c"], ["d\ne"]]) + } + + @Test("Every chunk size reads the records one whole body gives", arguments: [1, 2, 3, 7, 64]) + func chunkSizeDoesNotChangeTheRecords(size: Int) { + var body = bytes("host,note\r\na,\"x, \"\"y\"\"\"\r\n\r\nb,\"two\nlines\"\nc,日本\r\"\"\"\",é\r\n") + body.append(contentsOf: [0x61, 0xFF, 0x62, 0x2C, 0x7A]) + #expect(InfluxCSV.records(body) == [ + ["host", "note"], + ["a", "x, \"y\""], + [""], + ["b", "two\nlines"], + ["c", "日本"], + ["\"", "é"], + ["a\u{FFFD}b", "z"], + ]) + #expect(records(pieces(body, size: size)) == InfluxCSV.records(body)) + } + + @Test("A v3 stream reads the rows and types the collected read gives", arguments: [1, 3, 64, 100_000]) + func aV3StreamReadsLikeTheCollectedRead(size: Int) throws { + let collected = try InfluxResponseDecoder.decodeV3CSV(v3Body).typed(by: cpuSchema) + let table = try joined(streamed(.csv, pieces(v3Body, size: size))) + #expect(table == collected) + #expect(table.typeNames == [ + InfluxTypeName.tag, InfluxTypeName.timestamp, InfluxTypeName.float, InfluxTypeName.string, InfluxTypeName.integer, + ]) + #expect(table.column(named: "time") == ["2024-10-27T03:33:20Z", "2024-10-27T03:34:20.500Z", "2024-10-27T03:35:20Z"]) + } + + @Test func aV3StreamSendsBatchesOfFiveThousandRows() throws { + let body = bytes("host,usage\n" + String(repeating: "a,1\n", count: 12_001)) + let batches = try streamed(.csv, [body]) + #expect(batches.map { $0.rows.count } == [5_000, 5_000, 2_001]) + #expect(try joined(batches).typeNames == [InfluxTypeName.tag, InfluxTypeName.float]) + } + + @Test func aWideV3HeaderShrinksTheBatchToAboutAMillionCells() throws { + let columns = (0 ..< 4_096).map { "c\($0)" } + let record = Array(repeating: "1", count: columns.count).joined(separator: ",") + let body = bytes(columns.joined(separator: ",") + "\n" + String(repeating: record + "\n", count: 257)) + let batches = try streamed(.csv, [body], schema: []) + #expect(batches.map { $0.rows.count } == [256, 1]) + } + + @Test("A v3 header of 40,000 columns over 1,000 empty records is refused, as the collected read refuses it") + func shortV3RecordsUnderAWideHeaderAreRefused() { + let header = (0 ..< 40_000).map { "c\($0)" }.joined(separator: ",") + let body = bytes(header + "\n" + String(repeating: "\n", count: 1_000)) + #expect(throws: tooWide) { try streamed(.csv, [body], schema: []) } + } + + @Test func aV3HeaderRecordWithNoRowsKeepsItsColumns() throws { + let body = bytes("usage,time,extra\n") + let batches = try streamed(.csv, [body]) + #expect(batches == [InfluxResultTable( + columns: ["usage", "time", "extra"], + typeNames: [InfluxTypeName.float, InfluxTypeName.timestamp, InfluxTypeName.string], + rows: [] + )]) + let collected = try InfluxResponseDecoder.decodeV3CSV(body).typed(by: cpuSchema) + #expect(batches == [collected]) + } + + @Test func aV3ColumnTheSchemaLacksIsTypedFromTheFirstBatch() throws { + let table = try joined(streamed(.csv, [bytes("time,extra\n2024-10-27T03:33:20Z,1\n2024-10-27T03:34:20Z,2\n")])) + #expect(table.typeNames == [InfluxTypeName.timestamp, InfluxTypeName.integer]) + } + + @Test("An empty body gets the header from the schema", arguments: [InfluxGridStream.Body.csv, .influxQLChunks]) + func anEmptyBodyGetsTheHeaderFromTheSchema(body: InfluxGridStream.Body) throws { + let empty = body == .csv ? Data() : bytes(#"{"results":[{"statement_id":0}]}"# + "\n") + #expect(try streamed(body, [empty]) == [InfluxResultTable( + columns: ["time", "host", "idle", "note", "usage"], + typeNames: [ + InfluxTypeName.timestamp, InfluxTypeName.tag, InfluxTypeName.integer, InfluxTypeName.string, InfluxTypeName.float, + ], + rows: [] + )]) + let page = InfluxGridQuery(measurement: "cpu", projection: ["usage", "time", "gone"], shape: .page(limit: 50, offset: 0)) + #expect(try streamed(body, [empty], grid: page) == [InfluxResultTable( + columns: ["usage", "time", "gone"], + typeNames: [InfluxTypeName.float, InfluxTypeName.timestamp, InfluxTypeName.string], + rows: [] + )]) + } + + @Test("v2 chunks read the rows and types one collected body gives", arguments: [1, 5, 64, 100_000]) + func v2ChunksReadLikeTheCollectedBody(size: Int) throws { + let collected = try InfluxResponseDecoder.decodeV1JSON(bytes(v2Chunk(v2Rows[...], partial: false))) + .typed(by: cpuSchema) + let body = bytes(v2Chunk(v2Rows[..<2], partial: true) + "\n" + v2Chunk(v2Rows[2...], partial: false) + "\n") + let batches = try streamed(.influxQLChunks, pieces(body, size: size)) + #expect(batches.map { $0.rows.count } == [2, 1]) + #expect(try joined(batches) == collected) + #expect(collected.typeNames == [ + InfluxTypeName.timestamp, InfluxTypeName.tag, InfluxTypeName.integer, InfluxTypeName.string, InfluxTypeName.float, + ]) + } + + @Test func aLastV2ChunkWithoutItsNewlineIsStillRead() throws { + let batches = try streamed(.influxQLChunks, [bytes(v2Chunk(v2Rows[...], partial: false))]) + #expect(batches.map { $0.rows.count } == [3]) + } + + @Test( + "An error object after some rows throws", + arguments: [ + #"{"results":[{"statement_id":0,"error":"max-select-point limit exceeded"}]}"#, + #"{"error":"max-select-point limit exceeded"}"#, + ] + ) + func anErrorObjectAfterSomeRowsThrows(errorLine: String) throws { + var stream = InfluxGridStream(body: .influxQLChunks, grid: cpuExport, schema: cpuSchema) + let first = try stream.consume(bytes(v2Chunk(v2Rows[..<2], partial: true) + "\n")) + #expect(first.map { $0.rows.count } == [2]) + #expect(throws: InfluxError.query("max-select-point limit exceeded")) { + try stream.consume(bytes(errorLine + "\n")) + } + } + + @Test func aLaterV2ChunkWithItsColumnsReorderedMapsThemByName() throws { + let later = #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["usage","time"],"values":[[1.5,"2024-10-27T03:30:00Z"]]}]}]}"# + let body = bytes(v2Chunk(v2Rows[..<1], partial: true) + "\n" + later + "\n") + let table = try joined(streamed(.influxQLChunks, [body])) + #expect(table.rows.last == ["2024-10-27T03:30:00Z", nil, nil, nil, "1.5"]) + } + + @Test func aLaterV2ChunkWithAColumnTheFirstLacksThrows() { + let later = #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","extra"],"values":[["2024-10-27T03:30:00Z",1]]}]}]}"# + let body = bytes(v2Chunk(v2Rows[..<1], partial: true) + "\n" + later + "\n") + let refusal = InfluxError.malformedResponse( + String(format: String(localized: "column %@ first appears in a later chunk"), "extra") + ) + #expect(throws: refusal) { try streamed(.influxQLChunks, [body]) } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift index 6a032a6746..8aae352353 100644 --- a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift @@ -224,6 +224,38 @@ struct InfluxRequestTests { #expect(try InfluxCatalogDecoder.v3Databases(body) == ["_internal", "metrics"]) } + @Test func aStreamedV2InfluxQLQueryAsksForChunksOfTenThousand() throws { + let statement = InfluxStatement(language: .influxQL, text: #"SELECT * FROM "cpu""#) + let streamed = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v2, credentials: token, streamed: true + ) + #expect(queryItems(ofForm: streamed.body) == [ + URLQueryItem(name: "q", value: #"SELECT * FROM "cpu""#), + URLQueryItem(name: "db", value: "metrics"), + URLQueryItem(name: "chunked", value: "true"), + URLQueryItem(name: "chunk_size", value: "10000"), + ]) + let collected = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v2, credentials: token + ) + #expect(queryItems(ofForm: collected.body) == [ + URLQueryItem(name: "q", value: #"SELECT * FROM "cpu""#), + URLQueryItem(name: "db", value: "metrics"), + ]) + } + + @Test(arguments: [InfluxQueryLanguage.sql, .influxQL]) + func aStreamedV3QueryIsTheCollectedRequest(language: InfluxQueryLanguage) throws { + let statement = InfluxStatement(language: language, text: #"SELECT * FROM "cpu""#) + let streamed = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v3, credentials: token, streamed: true + ) + let collected = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v3, credentials: token + ) + #expect(streamed == collected) + } + @Test func anInfluxQLFormBodyCarriesTheRetentionPolicy() throws { let request = try InfluxRequests.query( InfluxStatement(language: .influxQL, text: "SHOW MEASUREMENTS"), diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift index bbfe76b995..9c67c9d975 100644 --- a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift @@ -85,4 +85,81 @@ private func decodeOne(baseOffset: Int64, baseTimestamp: Int64, offsetDelta: Int budget: &budget ) #expect(decoded.records.map { $0.offset } == [4, 11, 13]) + #expect(decoded.nextOffset == 14) +} + +@Test func offsetAfterABatchComesFromItsHeader() { + #expect(KafkaOffsetMath.offset(afterBatchAt: 100, lastOffsetDelta: 4) == 105) + #expect(KafkaOffsetMath.offset(afterBatchAt: 100, lastOffsetDelta: 0) == 101) + #expect(KafkaOffsetMath.offset(afterBatchAt: 100, lastOffsetDelta: -1) == 101) + #expect(KafkaOffsetMath.offset(afterBatchAt: .max, lastOffsetDelta: 0) == nil) + #expect(KafkaOffsetMath.offset(afterBatchAt: .max - 1, lastOffsetDelta: 2) == nil) +} + +private func decodeBlob(_ blob: Data) throws -> (records: [KafkaRecord], truncated: Bool, nextOffset: Int64?) { + var budget = DecompressionBudget.standard + return try KafkaRecordBatchDecoder.decode(blob: blob, partition: 0, skipAborted: true, budget: &budget) +} + +private func marker(at offset: Int64, producerId: Int64 = 7) -> Data { + KafkaBatchFixture.batch( + baseOffset: offset, + flags: KafkaBatchFixture.transactional | KafkaBatchFixture.control, + producerId: producerId, + payload: KafkaBatchFixture.record(value: nil) + ) +} + +@Test func aReplyOfOnlyACommitMarkerStillMovesTheReadOn() throws { + let decoded = try decodeBlob(marker(at: 41)) + #expect(decoded.records.isEmpty) + #expect(decoded.nextOffset == 42) +} + +@Test func aTrailingMarkerMovesTheReadPastIt() throws { + let blob = KafkaBatchFixture.batch( + baseOffset: 40, flags: KafkaBatchFixture.transactional, producerId: 7, + payload: KafkaBatchFixture.record(value: Data("row".utf8)) + ) + marker(at: 41) + let decoded = try decodeBlob(blob) + #expect(decoded.records.map { $0.offset } == [40]) + #expect(decoded.nextOffset == 42) +} + +@Test func anAbortedRunMovesTheReadPastIt() throws { + let blob = KafkaBatchFixture.batch( + baseOffset: 10, flags: KafkaBatchFixture.transactional, producerId: 7, recordCount: 3, + payload: KafkaBatchFixture.record(offsetDelta: 0, value: Data("a".utf8)) + + KafkaBatchFixture.record(offsetDelta: 1, value: Data("b".utf8)) + + KafkaBatchFixture.record(offsetDelta: 2, value: Data("c".utf8)) + ) + marker(at: 13) + var budget = DecompressionBudget.standard + let decoded = try KafkaRecordBatchDecoder.decode( + blob: blob, + partition: 0, + abortedTransactions: [KafkaAbortedTransaction(producerId: 7, firstOffset: 10)], + skipAborted: true, + budget: &budget + ) + #expect(decoded.records.isEmpty) + #expect(decoded.nextOffset == 14) +} + +// Compaction keeps an emptied batch's header, last offset included (Kafka protocol docs, Record Batch). +@Test func aBatchCompactionEmptiedMovesTheReadPastItsWholeRange() throws { + let decoded = try decodeBlob(KafkaBatchFixture.batch(baseOffset: 200, recordCount: 5, payload: Data())) + #expect(decoded.records.isEmpty) + #expect(decoded.nextOffset == 205) +} + +@Test func onlyWholeBatchesMoveTheReadOn() throws { + let whole = KafkaBatchFixture.batch(baseOffset: 7, payload: KafkaBatchFixture.record(value: Data("x".utf8))) + let cut = try decodeBlob(whole + whole.prefix(20)) + #expect(cut.records.map { $0.offset } == [7]) + #expect(cut.truncated) + #expect(cut.nextOffset == 8) + + let nothingWhole = try decodeBlob(whole.prefix(whole.count - 1)) + #expect(nothingWhole.records.isEmpty) + #expect(nothingWhole.nextOffset == nil) } diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaTailWalkTests.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaTailWalkTests.swift new file mode 100644 index 0000000000..d58225b0fa --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaTailWalkTests.swift @@ -0,0 +1,117 @@ +import Foundation +import TableProKafkaCore +import Testing + +private struct WalkOutcome { + let records: [Int64] + let reads: [[Int64]] + let start: Int64 +} + +// A partition log given by the offsets that hold a record, read the way the browse engine reads it. +private func walk( + log: [Int64], + end: Int64, + floor: Int64, + wanted: Int +) -> WalkOutcome { + let firstStart = KafkaOffsetMath.stepBack(from: end, by: Int64(wanted), floor: floor) + var tail = KafkaTailWalk(wanted: wanted, end: end, firstStart: firstStart, floor: floor) + var reads: [[Int64]] = [] + while let range = tail.nextRange { + reads.append([range.start, range.end]) + tail.absorb(Array(log.filter { $0 >= range.start && $0 < range.end }.suffix(tail.needed))) + } + return WalkOutcome(records: tail.records, reads: reads, start: tail.start) +} + +// What KafkaRecordOrdering.page takes from a tail window on one partition. +private func page(_ records: [Int64], skip: Int, limit: Int) -> [Int64] { + Array(records.dropLast(skip).suffix(limit)) +} + +struct KafkaTailWalkTests { + // One commit marker after every record: half the offsets hold no record. + private static let markedLog = stride(from: Int64(0), to: 2_000, by: 2).map { $0 } + + @Test("A window with fewer records than offsets steps back until it holds the page") + func stepsBackPastMarkers() { + let outcome = walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 100) + #expect(outcome.records == Array(stride(from: Int64(1_800), through: 1_998, by: 2))) + #expect(outcome.reads == [[1_900, 2_000], [1_775, 1_900]]) + #expect(outcome.start == 1_775) + } + + @Test("Page two follows page one with no record skipped or repeated") + func pagesAreContiguous() { + let first = page(walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 100).records, skip: 0, limit: 100) + let second = page(walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 200).records, skip: 100, limit: 100) + let third = page(walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 300).records, skip: 200, limit: 100) + #expect(first == Array(stride(from: Int64(1_800), through: 1_998, by: 2))) + #expect(second == Array(stride(from: Int64(1_600), through: 1_798, by: 2))) + #expect(third == Array(stride(from: Int64(1_400), through: 1_598, by: 2))) + } + + @Test("A deep page of a sparse log is not empty") + func deepPageOfSparseLog() { + let sparse = stride(from: Int64(0), to: 100_000, by: 10).map { $0 } + let tenth = page(walk(log: sparse, end: 100_000, floor: 0, wanted: 1_000).records, skip: 900, limit: 100) + #expect(tenth.count == 100) + let expectedLast: Int64 = 99_990 - 900 * 10 + #expect(tenth.last == expectedLast) + } + + @Test("The walk stops at the earliest offset with what the log holds") + func stopsAtTheFloor() { + let outcome = walk(log: [503, 507, 990], end: 1_000, floor: 500, wanted: 100) + #expect(outcome.records == [503, 507, 990]) + #expect(outcome.start == 500) + #expect(outcome.reads.last?.first == 500) + } + + @Test("A window that already holds the page reads once") + func denseWindowReadsOnce() { + let dense = (0 ..< 1_000).map { Int64($0) } + let outcome = walk(log: dense, end: 1_000, floor: 0, wanted: 50) + #expect(outcome.reads == [[950, 1_000]]) + #expect(outcome.records == Array(950 ..< 1_000).map { Int64($0) }) + } + + @Test("A walk that finds nothing grows each step by the most it may") + func emptyStretchGrowsFast() { + let outcome = walk(log: [0], end: 10_000, floor: 0, wanted: 100) + #expect(outcome.reads == [[9_900, 10_000], [9_500, 9_900], [7_900, 9_500], [1_500, 7_900], [0, 1_500]]) + #expect(outcome.records == [0]) + } + + @Test("An empty partition and an empty page read nothing past the end") + func emptyCases() { + let empty = walk(log: [], end: 40, floor: 40, wanted: 10) + #expect(empty.reads == [[40, 40]]) + #expect(empty.records.isEmpty) + + var none = KafkaTailWalk(wanted: 0, end: 10, firstStart: 0, floor: 0) + let pending = none.nextRange != nil + #expect(!pending) + none.absorb([1]) + #expect(none.records.isEmpty) + } + + @Test("The next width reaches as far as the last range's density says, and grows by the cap after an empty range") + func nextWidth() { + #expect(KafkaOffsetMath.nextWidth(lastWidth: 100, lastRecords: 50, needed: 50, maximumGrowth: 4) == 125) + #expect(KafkaOffsetMath.nextWidth(lastWidth: 1_000, lastRecords: 1_000, needed: 1, maximumGrowth: 4) == 2) + #expect(KafkaOffsetMath.nextWidth(lastWidth: 1_000, lastRecords: 100, needed: 900, maximumGrowth: 4) == 4_000) + #expect(KafkaOffsetMath.nextWidth(lastWidth: 100, lastRecords: 0, needed: 10, maximumGrowth: 4) == 400) + #expect(KafkaOffsetMath.nextWidth(lastWidth: .max, lastRecords: 0, needed: 1, maximumGrowth: 4) == .max) + } + + @Test("An empty stretch below a dense range is crossed by growing steps, not a few offsets at a time") + func denseThenEmpty() { + let log = [Int64(0)] + (99_001 ..< 100_000).map { Int64($0) } + let outcome = walk(log: log, end: 100_000, floor: 0, wanted: 1_000) + #expect(outcome.reads.map { $0[1] - $0[0] } == [1_000, 2, 8, 32, 128, 512, 2_048, 8_192, 32_768, 55_310]) + #expect(outcome.records.count == 1_000) + #expect(outcome.records.first == 0) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLAlterColumnStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLAlterColumnStatementTests.swift new file mode 100644 index 0000000000..94b157d804 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLAlterColumnStatementTests.swift @@ -0,0 +1,73 @@ +import Foundation +import TableProMSSQLCore +import TableProSQLGrammar +import Testing + +// SQL Server's ALTER TABLE docs: a column altered without COLLATE gets the database's default collation. +struct MSSQLAlterColumnStatementTests { + @Test("Widening a character column keeps the collation it was read with") + func wideningKeepsTheCollation() throws { + let collation = MSSQLSchemaQueries.alterColumnCollation( + type: "varchar(40)", collation: "Latin1_General_CS_AS", previous: "Latin1_General_CS_AS" + ) + let sql = try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "code", type: "varchar(40)", collation: collation, isNullable: true + ) + #expect(sql == "ALTER TABLE [dbo].[items] ALTER COLUMN [code] varchar(40) COLLATE Latin1_General_CS_AS NULL") + } + + @Test("The new definition's collation wins, and the old one stays when the new one is not known") + func collationPrecedence() { + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: "Latin1_General_BIN2", previous: "Latin1_General_CS_AS") + == "Latin1_General_BIN2") + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: nil, previous: "Latin1_General_CS_AS") + == "Latin1_General_CS_AS") + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: "", previous: "Latin1_General_CS_AS") + == "Latin1_General_CS_AS") + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: nil, previous: nil) == nil) + } + + // ALTER COLUMN refuses COLLATE on any other type, text and ntext included. + @Test("A type other than char, varchar, nchar or nvarchar takes no collation", arguments: [ + "int", "text", "ntext", "xml", "dbo.Code", "varbinary(20)", + ]) + func nonCharacterTypeTakesNoCollation(type: String) { + #expect(MSSQLSchemaQueries.alterColumnCollation(type: type, collation: "Latin1_General_CS_AS", previous: "Latin1_General_CS_AS") == nil) + } + + @Test("A column without a collation is altered as before") + func noCollation() throws { + let sql = try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "qty", type: "bigint", collation: nil, isNullable: false + ) + #expect(sql == "ALTER TABLE [dbo].[items] ALTER COLUMN [qty] bigint NOT NULL") + } + + @Test("A collation that is not a collation name fails the statement and names the column") + func collationThatIsNotANameIsRefused() { + let text = "Latin1_General_CS_AS NULL; DROP TABLE t; --" + #expect(throws: MSSQLDefinitionRefusal.collation("code", text)) { + try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "code", type: "varchar(40)", collation: text, isNullable: true + ) + } + } + + @Test("A type that would start a statement fails the statement and names the column") + func typeThatStartsAStatementIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.columnType("code", "varchar(40) DROP TABLE t")) { + try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "code", type: "varchar(40) DROP TABLE t", collation: nil, isNullable: true + ) + } + } + + @Test("A bracket in the column name stays inside the name") + func bracketInColumnName() throws { + let sql = try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "a] int; DROP TABLE t; --", type: "int", collation: nil, isNullable: true + ) + #expect(sql == "ALTER TABLE [dbo].[items] ALTER COLUMN [a]] int; DROP TABLE t; --] int NULL") + #expect(SQLStatementScanner.allStatements(in: sql, grammar: SQLLexicalProfile.sqlServer).count == 1) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift index e95081a1ea..6c8a02c047 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift @@ -4,60 +4,122 @@ import TableProSQLGrammar import Testing struct MSSQLCreateTableStatementTests { - private static func column(_ name: String, type: String = "int", default value: String? = nil) -> MSSQLColumnRow { + private static let serverDefault = "SQL_Latin1_General_CP1_CI_AS" + + private static func column( + _ name: String, + type: String = "int", + length: Int? = nil, + nullable: Bool = true, + default value: String? = nil, + collation: String? = nil + ) -> MSSQLColumnRow { MSSQLColumnRow( name: name, dataType: type, - characterMaxLength: nil, + characterMaxLength: length, numericPrecision: nil, numericScale: nil, - isNullable: true, + isNullable: nullable, defaultValue: value, isIdentity: false, - isPrimaryKey: false + isPrimaryKey: false, + collation: collation + ) + } + + private static func computed( + _ name: String, + definition: String?, + persisted: Bool = false, + nullable: Bool = true + ) -> MSSQLColumnRow { + MSSQLColumnRow( + name: name, + dataType: "numeric", + characterMaxLength: nil, + numericPrecision: 21, + numericScale: 2, + isNullable: nullable, + defaultValue: nil, + isIdentity: false, + isPrimaryKey: false, + isComputed: true, + computedDefinition: definition, + isPersisted: persisted + ) + } + + private static func identity(_ name: String, seed: String?, increment: String?) -> MSSQLColumnRow { + MSSQLColumnRow( + name: name, + dataType: "int", + characterMaxLength: nil, + numericPrecision: 10, + numericScale: 0, + isNullable: false, + defaultValue: nil, + isIdentity: true, + isPrimaryKey: true, + identitySeed: seed, + identityIncrement: increment ) } private static func statement( table: String = "t", columns: [MSSQLColumnRow], - primaryKeyColumns: [String] = [], - foreignKeys: [MSSQLForeignKeyRow] = [] + keyConstraints: [MSSQLKeyConstraintRow] = [], + checkConstraints: [MSSQLCheckConstraintRow] = [], + foreignKeys: [MSSQLForeignKeyRow] = [], + databaseCollation: String? = serverDefault ) throws -> String { try MSSQLSchemaQueries.createTableStatement( schema: "dbo", table: table, columns: columns, - primaryKeyColumns: primaryKeyColumns, - foreignKeys: foreignKeys + keyConstraints: keyConstraints, + checkConstraints: checkConstraints, + foreignKeys: foreignKeys, + databaseCollation: databaseCollation ) } + private static func statementCount(_ ddl: String) -> Int { + SQLStatementScanner.allStatements(in: ddl, grammar: SQLLexicalProfile.sqlServer).count + } + @Test("A bracket in a column name stays inside the name") func bracketInColumnNamesStaysInsideTheName() throws { let ddl = try Self.statement(columns: [ Self.column("a] INT); DROP TABLE [victims]; /*"), Self.column("*/ CREATE TABLE [z] ([y"), ]) - #expect(SQLStatementScanner.allStatements(in: ddl, grammar: SQLLexicalProfile.sqlServer).count == 1) + #expect(Self.statementCount(ddl) == 1) #expect(ddl.contains(" [a]] INT); DROP TABLE [victims]]; /*] INT NULL,\n")) #expect(ddl.contains(" [*/ CREATE TABLE [z]] ([y] INT NULL\n);")) } - @Test("A bracket in the table, key and foreign key names is doubled") + @Test("A bracket in the table, key, constraint and referenced names is doubled") func bracketInKeyNamesIsDoubled() throws { let ddl = try Self.statement( table: "t]x", columns: [Self.column("id]")], - primaryKeyColumns: ["id]"], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "pk]", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "id]"), + MSSQLKeyConstraintRow(name: "uq]", isPrimaryKey: false, indexKind: "NONCLUSTERED", columnName: "id]"), + ], + checkConstraints: [MSSQLCheckConstraintRow(name: "ck]", definition: "([id]]]>(0))")], foreignKeys: [MSSQLForeignKeyRow( - constraintName: "fk]", columnName: "id]", referencedTable: "r]", referencedColumn: "rc]" + constraintName: "fk]", columnName: "id]", referencedTable: "r]", referencedColumn: "rc]", referencedSchema: "s]" )] ) #expect(ddl.hasPrefix("CREATE TABLE [dbo].[t]]x] (\n")) - #expect(ddl.contains(" CONSTRAINT [PK_t]]x] PRIMARY KEY ([id]]])")) - #expect(ddl.contains(" CONSTRAINT [fk]]] FOREIGN KEY ([id]]]) REFERENCES [r]]] ([rc]]])")) - #expect(SQLStatementScanner.allStatements(in: ddl, grammar: SQLLexicalProfile.sqlServer).count == 1) + #expect(ddl.contains(" CONSTRAINT [pk]]] PRIMARY KEY CLUSTERED ([id]]])")) + #expect(ddl.contains(" CONSTRAINT [uq]]] UNIQUE NONCLUSTERED ([id]]])")) + #expect(ddl.contains(" CONSTRAINT [fk]]] FOREIGN KEY ([id]]]) REFERENCES [s]]].[r]]] ([rc]]])")) + #expect(ddl.contains(" CONSTRAINT [ck]]] CHECK ([id]]]>(0))")) + #expect(Self.statementCount(ddl) == 1) } @Test("A default as the catalog stores it is written as it is") @@ -83,4 +145,286 @@ struct MSSQLCreateTableStatementTests { try Self.statement(columns: [Self.column("x", type: "int drop table t")]) } } + + @Test("Every clause lands in one statement: columns, then the primary key, UNIQUE, FOREIGN KEY and CHECK") + func clauseOrder() throws { + let ddl = try Self.statement( + table: "order_lines", + columns: [Self.column("order_id", nullable: false), Self.column("line_no", nullable: false)], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "UQ_line", isPrimaryKey: false, indexKind: "NONCLUSTERED", columnName: "line_no"), + MSSQLKeyConstraintRow(name: "PK_order_lines", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "order_id"), + MSSQLKeyConstraintRow(name: "PK_order_lines", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "line_no"), + ], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_line", definition: "([line_no]>(0))")], + foreignKeys: [MSSQLForeignKeyRow( + constraintName: "FK_order", columnName: "order_id", referencedTable: "orders", referencedColumn: "id", referencedSchema: "dbo" + )] + ) + #expect(ddl == """ + CREATE TABLE [dbo].[order_lines] ( + [order_id] INT NOT NULL, + [line_no] INT NOT NULL, + CONSTRAINT [PK_order_lines] PRIMARY KEY CLUSTERED ([order_id], [line_no]), + CONSTRAINT [UQ_line] UNIQUE NONCLUSTERED ([line_no]), + CONSTRAINT [FK_order] FOREIGN KEY ([order_id]) REFERENCES [dbo].[orders] ([id]), + CONSTRAINT [CK_line] CHECK ([line_no]>(0)) + ); + """) + #expect(Self.statementCount(ddl) == 1) + } + + // Each catalog row is one key column, and writing one constraint per row declared FK_x twice: error 2714 on restore. + @Test("A composite foreign key is one constraint with its columns in key order") + func compositeForeignKey() throws { + let ddl = try Self.statement( + table: "shipments", + columns: [Self.column("order_id"), Self.column("line_no")], + foreignKeys: [ + MSSQLForeignKeyRow( + constraintName: "FK_x", columnName: "order_id", referencedTable: "order_lines", + referencedColumn: "order_id", referencedSchema: "dbo" + ), + MSSQLForeignKeyRow( + constraintName: "FK_x", columnName: "line_no", referencedTable: "order_lines", + referencedColumn: "line_no", referencedSchema: "dbo" + ), + ] + ) + #expect(ddl == """ + CREATE TABLE [dbo].[shipments] ( + [order_id] INT NULL, + [line_no] INT NULL, + CONSTRAINT [FK_x] FOREIGN KEY ([order_id], [line_no]) REFERENCES [dbo].[order_lines] ([order_id], [line_no]) + ); + """) + } + + @Test("A key into another schema names that schema and keeps its ON DELETE action") + func crossSchemaForeignKeyWithCascade() throws { + let ddl = try Self.statement( + columns: [Self.column("order_id")], + foreignKeys: [MSSQLForeignKeyRow( + constraintName: "FK_orders", columnName: "order_id", referencedTable: "orders", referencedColumn: "id", + referencedSchema: "sales", onDelete: "CASCADE", onUpdate: "NO ACTION" + )] + ) + #expect(ddl.contains(" CONSTRAINT [FK_orders] FOREIGN KEY ([order_id]) REFERENCES [sales].[orders] ([id]) ON DELETE CASCADE\n")) + #expect(!ddl.contains("ON UPDATE")) + } + + @Test("Both actions read from the catalog are written after the reference") + func catalogActionsAreWritten() throws { + let row = try #require(MSSQLSchemaQueries.parseForeignKeyRow( + ["FK_a", "a_id", "a", "id", "sales", "SET_NULL", "CASCADE", "t"] + )) + let ddl = try Self.statement(columns: [Self.column("a_id")], foreignKeys: [row]) + #expect(ddl.contains("REFERENCES [sales].[a] ([id]) ON DELETE SET NULL ON UPDATE CASCADE\n")) + } + + @Test("A key with no referenced schema is qualified with the table's own") + func foreignKeyWithoutSchemaUsesTheTablesSchema() throws { + let ddl = try Self.statement( + columns: [Self.column("a_id")], + foreignKeys: [MSSQLForeignKeyRow(constraintName: "FK_a", columnName: "a_id", referencedTable: "a", referencedColumn: "id")] + ) + #expect(ddl.contains("REFERENCES [dbo].[a] ([id])")) + } + + @Test("An action SQL Server does not have fails the statement and names the key") + func unknownActionIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.foreignKeyAction("FK_a", "CASCADE; DROP TABLE t")) { + try Self.statement( + columns: [Self.column("a_id")], + foreignKeys: [MSSQLForeignKeyRow( + constraintName: "FK_a", columnName: "a_id", referencedTable: "a", referencedColumn: "id", + onDelete: "CASCADE; DROP TABLE t" + )] + ) + } + } + + @Test("The primary key keeps its catalog name, its kind and each column's direction") + func primaryKeyKeepsItsName() throws { + let ddl = try Self.statement( + columns: [Self.column("a", nullable: false), Self.column("b", nullable: false)], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "PK__t__3213E83F", isPrimaryKey: true, indexKind: "NONCLUSTERED", columnName: "a"), + MSSQLKeyConstraintRow( + name: "PK__t__3213E83F", isPrimaryKey: true, indexKind: "NONCLUSTERED", columnName: "b", isDescending: true + ), + ] + ) + #expect(ddl.contains(" CONSTRAINT [PK__t__3213E83F] PRIMARY KEY NONCLUSTERED ([a], [b] DESC)\n")) + #expect(!ddl.contains("PK_t]")) + } + + @Test("A UNIQUE constraint is declared with its kind and its columns in key order") + func uniqueConstraint() throws { + let ddl = try Self.statement( + columns: [Self.column("code", type: "varchar", length: 20), Self.column("region")], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "UQ_code", isPrimaryKey: false, indexKind: "CLUSTERED", columnName: "code"), + MSSQLKeyConstraintRow( + name: "UQ_code", isPrimaryKey: false, indexKind: "CLUSTERED", columnName: "region", isDescending: true + ), + ] + ) + #expect(ddl.contains(" CONSTRAINT [UQ_code] UNIQUE CLUSTERED ([code], [region] DESC)\n")) + } + + @Test("A kind other than the two rowstore kinds is left to the server's default") + func unknownKeyKindIsOmitted() throws { + let ddl = try Self.statement( + columns: [Self.column("code")], + keyConstraints: [MSSQLKeyConstraintRow( + name: "UQ_code", isPrimaryKey: false, indexKind: "CLUSTERED; DROP TABLE t; --", columnName: "code" + )] + ) + #expect(ddl.contains(" CONSTRAINT [UQ_code] UNIQUE ([code])\n")) + #expect(Self.statementCount(ddl) == 1) + } + + @Test("A CHECK constraint is written with its name and the expression the catalog stores") + func checkConstraint() throws { + let ddl = try Self.statement( + columns: [Self.column("qty")], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_qty", definition: "([qty]>(0))")] + ) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [qty] INT NULL,\n CONSTRAINT [CK_qty] CHECK ([qty]>(0))\n);") + } + + @Test("A CHECK whose text reaches past one expression fails the statement") + func escapingCheckIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.checkConstraint("t")) { + try Self.statement( + columns: [Self.column("qty")], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_qty", definition: "([qty]>(0))) DROP TABLE t; --")] + ) + } + } + + @Test("A disabled or untrusted CHECK is switched off before the rows, and an untrusted one back on after them") + func uncheckedCheckConstraints() throws { + let checks = [ + MSSQLCheckConstraintRow(name: "CK_off", definition: "([qty]>(0))", isDisabled: true, isNotTrusted: true), + MSSQLCheckConstraintRow(name: "CK_loose", definition: "([qty]<(9))", isNotTrusted: true), + MSSQLCheckConstraintRow(name: "CK_on", definition: "([qty]<>(5))"), + ] + let ddl = try Self.statement(columns: [Self.column("qty")], checkConstraints: checks) + #expect(ddl.hasSuffix(""" + ); + ALTER TABLE [dbo].[t] NOCHECK CONSTRAINT [CK_off]; + ALTER TABLE [dbo].[t] NOCHECK CONSTRAINT [CK_loose]; + """)) + #expect(Self.statementCount(ddl) == 3) + #expect(MSSQLSchemaQueries.checkConstraintReenableStatements(schema: "dbo", table: "t", checkConstraints: checks) + == ["ALTER TABLE [dbo].[t] WITH NOCHECK CHECK CONSTRAINT [CK_loose];"]) + } + + @Test("A CHECK with no readable definition is left out") + func unreadableCheckIsLeftOut() throws { + let ddl = try Self.statement( + columns: [Self.column("qty")], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_qty", definition: nil)] + ) + #expect(!ddl.contains("CHECK")) + } + + @Test("A computed column is written as its expression, not as a plain column of its result type") + func computedColumn() throws { + let ddl = try Self.statement(columns: [ + Self.column("qty"), + Self.computed("total", definition: "([qty]*[price])", nullable: false), + ]) + #expect(ddl.contains(" [total] AS ([qty]*[price])\n")) + #expect(!ddl.contains("NUMERIC")) + } + + @Test("A persisted computed column keeps PERSISTED, and NOT NULL only with it") + func persistedComputedColumn() throws { + let notNull = try Self.statement(columns: [Self.computed("total", definition: "([qty]*[price])", persisted: true, nullable: false)]) + #expect(notNull.contains(" [total] AS ([qty]*[price]) PERSISTED NOT NULL\n")) + + let nullable = try Self.statement(columns: [Self.computed("total", definition: "([qty]*[price])", persisted: true)]) + #expect(nullable.contains(" [total] AS ([qty]*[price]) PERSISTED\n")) + } + + @Test("A computed expression that is not one expression, or cannot be read, fails the statement", arguments: [ + "([qty]*[price])) DROP TABLE t; --", nil, + ]) + func unwritableComputedColumnIsRefused(definition: String?) { + #expect(throws: MSSQLDefinitionRefusal.computedColumn("total")) { + try Self.statement(columns: [Self.computed("total", definition: definition)]) + } + } + + @Test("An identity keeps its seed and increment") + func identitySeedAndIncrement() throws { + let ddl = try Self.statement(columns: [Self.identity("id", seed: "1000", increment: "5")]) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [id] INT IDENTITY(1000,5) NOT NULL\n);") + + let descending = try Self.statement(columns: [Self.identity("id", seed: "-1", increment: "-1")]) + #expect(descending.contains("[id] INT IDENTITY(-1,-1) NOT NULL")) + } + + @Test("An identity without a whole-number seed and increment fails the statement") + func unreadableIdentityIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.identity("id", ",")) { + try Self.statement(columns: [Self.identity("id", seed: nil, increment: nil)]) + } + #expect(throws: MSSQLDefinitionRefusal.identity("id", "1) NOT NULL, [x] INT --,1")) { + try Self.statement(columns: [Self.identity("id", seed: "1) NOT NULL, [x] INT --", increment: "1")]) + } + } + + @Test("A collation other than the database's is written after the type") + func differingCollationIsWritten() throws { + let ddl = try Self.statement(columns: [ + Self.column("code", type: "varchar", length: 20, collation: "Latin1_General_CS_AS"), + ]) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [code] VARCHAR(20) COLLATE Latin1_General_CS_AS NULL\n);") + } + + @Test("The database's own collation is not written") + func databaseCollationIsNotWritten() throws { + let ddl = try Self.statement(columns: [ + Self.column("code", type: "varchar", length: 20, collation: Self.serverDefault), + ]) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [code] VARCHAR(20) NULL\n);") + } + + @Test("Without the database's collation every column states its own") + func unknownDatabaseCollationWritesEveryCollation() throws { + let ddl = try Self.statement( + columns: [Self.column("code", type: "varchar", length: 20, collation: Self.serverDefault)], + databaseCollation: nil + ) + #expect(ddl.contains("[code] VARCHAR(20) COLLATE SQL_Latin1_General_CP1_CI_AS NULL")) + } + + @Test("A collation that is not a collation name fails the statement") + func collationThatIsNotANameIsRefused() { + let text = "Latin1_General_CS_AS NOT NULL, [x] INT" + #expect(throws: MSSQLDefinitionRefusal.collation("code", text)) { + try Self.statement(columns: [Self.column("code", type: "varchar", length: 20, collation: text)]) + } + } + + @Test("A column read from the catalog writes the collation, computed expression and identity it was read with") + func catalogRowsRoundTrip() throws { + let rows: [[String?]] = [ + ["id", "bigint", nil, "19", "0", "NO", nil, "1", "1", "0", "0", nil, nil, nil, "100", "10"], + ["code", "nvarchar", "40", nil, nil, "YES", nil, "0", "0", "0", "0", "Latin1_General_CS_AS", nil, nil, nil, nil], + ["twice", "bigint", nil, "19", "0", "NO", nil, "0", "0", "1", "0", nil, "([id]*(2))", "1", nil, nil], + ] + let ddl = try Self.statement(columns: rows.compactMap(MSSQLSchemaQueries.parseColumnRow)) + #expect(ddl == """ + CREATE TABLE [dbo].[t] ( + [id] BIGINT IDENTITY(100,10) NOT NULL, + [code] NVARCHAR(40) COLLATE Latin1_General_CS_AS NULL, + [twice] AS ([id]*(2)) PERSISTED NOT NULL + ); + """) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift index f2e40d5d1c..fe64211a32 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift @@ -105,6 +105,21 @@ struct MSSQLDefinitionTextTests { #expect(!MSSQLDefinitionText.isCollationName("")) } + @Test("A character type is char, varchar, nchar or nvarchar, or an ISO synonym of one", arguments: [ + "varchar(40)", "NVARCHAR(MAX)", "char", "nchar(10)", "nvarchar (50)", "[nvarchar](50)", "character varying(255)", + "char varying(10)", "national character varying(max)", "national char(10)", "character", + ]) + func characterTypeIsRecognised(type: String) { + #expect(MSSQLDefinitionText.isCharacterType(type)) + } + + @Test("Any other type is not a character type", arguments: [ + "text", "ntext", "national text", "int", "varbinary(20)", "dbo.Code", "[dbo].[varchar]", "xml", "sysname", "", + ]) + func otherTypeIsNotACharacterType(type: String) { + #expect(!MSSQLDefinitionText.isCharacterType(type)) + } + @Test("Only the four referential actions are written") func referentialActions() { #expect(MSSQLDefinitionText.referentialAction("CASCADE") == "CASCADE") diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift index 5ee9d1e2dd..f005fd9221 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift @@ -145,9 +145,122 @@ struct MSSQLSchemaQueriesTests { @Test func testAllColumnsQueryCarriesTheTableNameAfterTheParsedColumns() { let sql = MSSQLSchemaQueries.allColumns(schema: "dbo") - #expect(sql.contains("AS IS_SERVER_GENERATED,\n c.TABLE_NAME")) + #expect(sql.contains("AS IDENTITY_INCREMENT,\n c.TABLE_NAME")) #expect(sql.contains("c.TABLE_SCHEMA = N'dbo'")) - #expect(MSSQLSchemaQueries.allColumnsTableNameIndex == 11) + #expect(MSSQLSchemaQueries.allColumnsTableNameIndex == 16) + } + + // Without COLLATION_NAME the Structure editor never knew a column's collation, and ALTER COLUMN reset it. + @Test func testColumnReadsSelectWhatACreateTableRestates() { + let reads = [ + MSSQLSchemaQueries.columns(schema: "dbo", table: "orders"), + MSSQLSchemaQueries.allColumns(schema: "dbo"), + MSSQLSchemaQueries.temporaryColumns(table: "#orders"), + ] + for sql in reads { + #expect(sql.contains("AS IS_SERVER_GENERATED,\n")) + #expect(sql.contains("c.COLLATION_NAME,")) + #expect(sql.contains("cc.definition AS COMPUTED_DEFINITION,")) + #expect(sql.contains("cc.is_persisted AS IS_PERSISTED,")) + #expect(sql.contains("CONVERT(varchar(40), idc.seed_value) AS IDENTITY_SEED,")) + #expect(sql.contains("CONVERT(varchar(40), idc.increment_value) AS IDENTITY_INCREMENT")) + #expect(sql.contains("computed_columns cc ON cc.object_id = sc.object_id AND cc.column_id = sc.column_id")) + #expect(sql.contains("identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id")) + } + let temporary = MSSQLSchemaQueries.temporaryColumns(table: "#orders") + #expect(temporary.contains("LEFT JOIN tempdb.sys.computed_columns cc")) + #expect(temporary.contains("LEFT JOIN tempdb.sys.identity_columns idc")) + } + + @Test func testParseColumnRowReadsCollationComputedAndIdentity() throws { + let identity = try #require(MSSQLSchemaQueries.parseColumnRow( + ["id", "int", nil, "10", "0", "NO", nil, "1", "1", "0", "0", nil, nil, "0", "1000", "5"] + )) + #expect(identity.identitySeed == "1000") + #expect(identity.identityIncrement == "5") + #expect(identity.collation == nil) + + let text = try #require(MSSQLSchemaQueries.parseColumnRow( + ["code", "varchar", "20", nil, nil, "YES", nil, "0", "0", "0", "0", "Latin1_General_CS_AS", nil, nil, nil, nil] + )) + #expect(text.collation == "Latin1_General_CS_AS") + + let computed = try #require(MSSQLSchemaQueries.parseColumnRow( + ["total", "numeric", nil, "21", "2", "YES", nil, "0", "0", "1", "0", nil, "([qty]*[price])", "1", nil, nil] + )) + #expect(computed.computedDefinition == "([qty]*[price])") + #expect(computed.isPersisted) + } + + @Test func testParseColumnRowOfTheShorterShapeHasNoDefinitionParts() throws { + let row = try #require(MSSQLSchemaQueries.parseColumnRow(["code", "varchar", "20", nil, nil, "YES", nil, "0", "0"])) + #expect(row.collation == nil) + #expect(row.computedDefinition == nil) + #expect(!row.isPersisted) + #expect(row.identitySeed == nil) + } + + @Test func testKeyConstraintsQueryReadsPrimaryAndUniqueKeysInKeyOrder() { + let sql = MSSQLSchemaQueries.keyConstraints(schema: "sales", table: "o'rders") + #expect(sql.contains("FROM sys.key_constraints kc")) + #expect(sql.contains("ON i.object_id = kc.parent_object_id AND i.index_id = kc.unique_index_id")) + #expect(sql.contains("WHERE kc.parent_object_id = OBJECT_ID(N'[sales].[o''rders]')")) + #expect(sql.contains("ORDER BY CASE kc.type WHEN 'PK' THEN 0 ELSE 1 END, kc.name, ic.key_ordinal")) + #expect(!sql.contains("tempdb")) + } + + @Test func testConstraintReadsOfATemporaryTableUseTempdb() { + let keys = MSSQLSchemaQueries.keyConstraints(schema: "dbo", table: "#t") + #expect(keys.contains("FROM tempdb.sys.key_constraints kc")) + #expect(keys.contains("JOIN tempdb.sys.index_columns ic")) + #expect(keys.contains("WHERE kc.parent_object_id = OBJECT_ID(N'tempdb..' + QUOTENAME(N'#t'))")) + + let checks = MSSQLSchemaQueries.checkConstraints(schema: "dbo", table: "#t") + #expect(checks.contains("FROM tempdb.sys.check_constraints cc")) + #expect(checks.contains("WHERE cc.parent_object_id = OBJECT_ID(N'tempdb..' + QUOTENAME(N'#t'))")) + } + + @Test func testCheckConstraintsQueryReadsTheStoredDefinition() { + let sql = MSSQLSchemaQueries.checkConstraints(schema: "dbo", table: "orders") + #expect(sql.contains("SELECT cc.name, cc.definition, cc.is_disabled, cc.is_not_trusted\n")) + #expect(sql.contains("FROM sys.check_constraints cc")) + #expect(sql.contains("WHERE cc.parent_object_id = OBJECT_ID(N'[dbo].[orders]')")) + } + + @Test func testDatabaseCollationIsTempdbsForATemporaryTable() { + #expect(MSSQLSchemaQueries.databaseCollation(forTable: "orders") + == "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(DB_NAME(), 'Collation'))") + #expect(MSSQLSchemaQueries.databaseCollation(forTable: "#orders") + == "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(N'tempdb', 'Collation'))") + } + + // The CREATE TABLE declares both constraints, so their indexes written again would fail on restore. + @Test func testIndexDefinitionsLeaveOutConstraintIndexes() { + let sql = MSSQLSchemaQueries.indexDefinitions(schema: "dbo", table: "orders") + #expect(sql.contains("AND i.is_primary_key = 0")) + #expect(sql.contains("AND i.is_unique_constraint = 0")) + #expect(sql.contains("WHERE i.object_id = OBJECT_ID(N'[dbo].[orders]')")) + #expect(sql.contains("ORDER BY i.index_id, ic.is_included_column, ic.key_ordinal")) + } + + @Test func testParseKeyConstraintRow() throws { + let primary = try #require(MSSQLSchemaQueries.parseKeyConstraintRow(["PK_orders", "PK", "CLUSTERED", "id", "0"])) + #expect(primary == MSSQLKeyConstraintRow(name: "PK_orders", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "id")) + + let unique = try #require(MSSQLSchemaQueries.parseKeyConstraintRow(["UQ_code", "UQ", "NONCLUSTERED", "code", "1"])) + #expect(!unique.isPrimaryKey) + #expect(unique.isDescending) + + #expect(MSSQLSchemaQueries.parseKeyConstraintRow(["PK_orders", "PK", "CLUSTERED", nil, "0"]) == nil) + } + + @Test func testParseCheckConstraintRow() { + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", "([qty]>(0))"]) + == MSSQLCheckConstraintRow(name: "CK_qty", definition: "([qty]>(0))")) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", nil])?.definition == nil) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", "([qty]>(0))", "1", "0"])?.isDisabled == true) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", "([qty]>(0))", "0", "1"])?.isNotTrusted == true) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow([nil, "(1)"]) == nil) } @Test func testParseColumnRowReadsServerGeneratedColumns() { diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedItemEditTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedItemEditTests.swift new file mode 100644 index 0000000000..7a0207a0be --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedItemEditTests.swift @@ -0,0 +1,59 @@ +import Foundation +import TableProMemcachedCore +import Testing + +struct MemcachedItemEditTests { + private func write(value: Bool = false, flags: Bool = false, ttl: Bool = false, unique: UInt64?) -> MemcachedItemEdit? { + MemcachedItemEdit.write(editsValue: value, editsFlags: flags, editsTTL: ttl, unique: unique) + } + + @Test("A Flags edit carries the loaded Value and a Value edit the loaded Flags, so each is a cas on the read's unique") + func keepingALoadedFieldIsACompareAndSwap() { + #expect(write(flags: true, unique: 42) == .compareAndSwap(unique: 42)) + #expect(write(flags: true, ttl: true, unique: 42) == .compareAndSwap(unique: 42)) + #expect(write(value: true, unique: 42) == .compareAndSwap(unique: 42)) + #expect(write(value: true, ttl: true, unique: 42) == .compareAndSwap(unique: 42)) + } + + @Test("Without a unique from the read, an edit that keeps a loaded field asks for a refresh") + func keepingALoadedFieldWithoutAUniqueNeedsARefresh() { + #expect(write(flags: true, unique: nil) == .needsRefresh) + #expect(write(value: true, ttl: true, unique: nil) == .needsRefresh) + } + + @Test("An edit that writes both Value and Flags is a set, a TTL alone a touch, and nothing stored is no write") + func writesThatKeepNothingLoaded() { + #expect(write(value: true, flags: true, unique: nil) == .set) + #expect(write(value: true, flags: true, ttl: true, unique: 42) == .set) + #expect(write(ttl: true, unique: nil) == .touch) + #expect(write(ttl: true, unique: 42) == .touch) + #expect(write(unique: 42) == nil) + } + + @Test("A cas answered EXISTS changed since the read and NOT_FOUND is gone; STORED and other commands' replies pass") + func compareAndSwapReplies() { + let cas = MemcachedCommand.compareAndSwap(key: "k", flags: 0, exptime: 0, unique: 7, value: Data("v".utf8)) + #expect(MemcachedCASRefusal(replyTo: cas, status: "EXISTS") == .changed) + #expect(MemcachedCASRefusal(replyTo: cas, status: "NOT_FOUND") == .notFound) + #expect(MemcachedCASRefusal(replyTo: cas, status: "STORED") == nil) + #expect(MemcachedCASRefusal(replyTo: .touch(key: "k", exptime: 0), status: "NOT_FOUND") == nil) + #expect(MemcachedCASRefusal(replyTo: .delete(key: "k"), status: "NOT_FOUND") == nil) + #expect(MemcachedCASRefusal( + replyTo: .store(.add, key: "k", flags: 0, exptime: 0, value: Data()), status: "EXISTS" + ) == nil) + } + + @Test("A cas statement parses back into its command and sends the unique after the byte count") + func compareAndSwapStatement() throws { + let command = try MemcachedCommandParser.parse( + MemcachedStatements.cas(key: "k", flags: 7, exptime: 60, unique: 9_000_000_000, value: "hello") + ) + #expect(command == .compareAndSwap(key: "k", flags: 7, exptime: 60, unique: 9_000_000_000, value: Data("hello".utf8))) + #expect(String(bytes: command.wireBytes, encoding: .utf8) == "cas k 7 60 5 9000000000\r\nhello\r\n") + + let bytes = Data([0x00, 0xFF, 0x22, 0x0A]) + #expect(try MemcachedCommandParser.parse( + MemcachedStatements.cas(key: "user'1", flags: 0, exptime: 0, unique: 1, bytes: bytes) + ) == .compareAndSwap(key: "user'1", flags: 0, exptime: 0, unique: 1, value: bytes)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift index 62c2c55cf8..6ee0c64ead 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift @@ -12,6 +12,14 @@ struct MemcachedKeyListingTests { #expect(entry?.secondsLeft(at: 1_793_499_867) == 300) } + @Test("A listed key a text command line cannot carry is left out", arguments: [ + "key=x%0D%0Aflush_all exp=-1 la=1 cas=1 fetch=no cls=1 size=1", + "key=two%20words exp=-1 la=1 cas=1 fetch=no cls=1 size=1", + ]) + func unsendableKeyIsLeftOut(line: String) { + #expect(MemcachedKeyListing.metadumpEntry(line) == nil) + } + @Test("exp=-1 means the item never expires") func metadumpNeverExpires() { let entry = MemcachedKeyListing.metadumpEntry("key=%22sp%22 exp=-1 la=1 cas=4 fetch=no cls=1 size=64 flags=0") diff --git a/apple/Packages/TableProCore/Tests/TableProModelsTests/CellTests.swift b/apple/Packages/TableProCore/Tests/TableProModelsTests/CellTests.swift new file mode 100644 index 0000000000..6b6714d5f0 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProModelsTests/CellTests.swift @@ -0,0 +1,133 @@ +import Foundation +import Testing + +@testable import TableProModels + +struct CellTests { + private static let long = String(repeating: "a", count: 10_000) + + @Test("A budget keeps long values whole until it is spent, then the next keeps only a preview") + func budgetKeepsWholeUntilSpent() { + let budget = TextBudget(bytes: 15_000) + let options = StreamOptions(textBudget: budget) + + guard case .text(let kept) = Cell.from(text: Self.long, options: options) else { + Issue.record("the first long value should be kept whole") + return + } + #expect(kept == Self.long) + #expect(!budget.hasRefused) + + guard case .truncatedText(let prefix, let totalBytes, let ref) = Cell.from(text: Self.long, options: options) else { + Issue.record("the second long value should be cut") + return + } + #expect(prefix.utf8.count == Cell.previewByteLimit) + #expect(totalBytes == 10_000) + #expect(ref == nil) + #expect(budget.hasRefused) + } + + @Test("A value within the cut spends nothing") + func shortValuesSpendNothing() { + let budget = TextBudget(bytes: 10_000) + let options = StreamOptions(textBudget: budget) + for _ in 0 ..< 10 { + _ = Cell.from(text: String(repeating: "b", count: Cell.previewByteLimit), options: options) + } + + guard case .text = Cell.from(text: Self.long, options: options) else { + Issue.record("the long value should still fit the budget") + return + } + #expect(!budget.hasRefused) + } + + @Test("Without a budget every long value is cut") + func noBudgetCuts() { + guard case .truncatedText(_, let totalBytes, .none) = Cell.from(text: Self.long, options: .default) else { + Issue.record("the long value should be cut") + return + } + #expect(totalBytes == 10_000) + } + + @Test("A value with a ref stays cut and loadable, and spends none of the budget") + func refValuesSpendNothing() { + let budget = TextBudget(bytes: 10_000) + let options = StreamOptions(textBudget: budget) + let ref = CellRef(table: "notes", column: "body", primaryKey: [PrimaryKeyComponent(column: "id", value: "1")]) + + guard case .truncatedText(_, _, let kept?) = Cell.from(text: Self.long, options: options, ref: ref) else { + Issue.record("a value with a ref should stay cut") + return + } + #expect(kept == ref) + guard case .text = Cell.from(text: Self.long, options: options) else { + Issue.record("the budget should be untouched") + return + } + } + + @Test( + "The preview holds at most the cut in bytes and ends on a whole character", + arguments: ["界", "👍🏽", "e\u{301}", "x"] + ) + func previewIsBoundedInBytes(_ unit: String) { + let value = String(repeating: unit, count: 5_000) + + let preview = Cell.preview(of: value, maxBytes: 4_096) + + #expect(preview.utf8.count <= 4_096) + #expect(preview.utf8.count > 4_096 - unit.utf8.count) + #expect(preview == String(repeating: unit, count: preview.count)) + } + + @Test("A cut value's preview is bounded in bytes, not characters") + func cutPreviewIsBoundedInBytes() { + let value = String(repeating: "界", count: 5_000) + + guard case .truncatedText(let prefix, let totalBytes, _) = Cell.from(text: value, options: .default) else { + Issue.record("the value should be cut") + return + } + #expect(prefix.utf8.count <= Cell.previewByteLimit) + #expect(totalBytes == 15_000) + } + + @Test( + "Oracle's BINARY_DOUBLE, BINARY_FLOAT and BINARY_INTEGER hold numbers, not bytes", + arguments: ["binary_double", "BINARY_FLOAT", "binary_integer"] + ) + func oracleBinaryNumbersStayText(_ typeName: String) { + guard case .text(let value) = Cell.from(legacyValue: "1.5", columnTypeName: typeName, options: .default) else { + Issue.record("\(typeName) should read as text") + return + } + #expect(value == "1.5") + } + + @Test( + "A byte type, with or without a length, reads as binary", + arguments: ["BLOB", "bytea", "VARBINARY(16)", "binary(16)", "LONGBLOB", "image", "BLOB(3)"] + ) + func byteTypesAreBinary(_ typeName: String) { + guard case .binary(let byteCount, _) = Cell.from(legacyValue: "0A0B", columnTypeName: typeName, options: .default) else { + Issue.record("\(typeName) should read as binary") + return + } + #expect(byteCount == 4) + } + + @Test("Display values cut a long value the cell holds whole") + func displayValuesCutKeptText() { + let row = Row(cells: [.text(Self.long), .null, .truncatedText(prefix: "a", totalBytes: 10_000, ref: nil)]) + + let shown = row.displayValues() + + #expect(shown[0]?.utf8.count == Cell.previewByteLimit) + #expect(shown[1] == nil) + #expect(shown[2] == row.cells[2].displayString) + #expect(row.legacyValues[0] == Self.long) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift b/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift index 8775894a00..475c977295 100644 --- a/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLObjectQueryTests.swift @@ -303,6 +303,48 @@ struct MySQLObjectQueryTests { #expect(sql.hasSuffix("ORDER BY EVENT_NAME")) } + private func selectedColumns(_ sql: String) throws -> [String] { + let select = try #require(sql.components(separatedBy: "FROM").first) + return select + .replacingOccurrences(of: "SELECT", with: "") + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + } + + // Export reads these four by position on every flavor, including ones the sidebar leaves out. + @Test("Export's event list keeps the four columns it shipped with") + func eventListKeepsItsShippedProjection() throws { + let sql = MySQLObjectQueries.eventList(schema: "app", exactSchema: false) + #expect(try selectedColumns(sql) == ["EVENT_NAME", "EVENT_TYPE", "STATUS", "EVENT_SCHEMA"]) + #expect(!sql.contains("@@")) + } + + @Test("The sidebar's event read starts with export's columns and appends the schedule columns in order") + func eventDetailsAppendsScheduleColumns() throws { + let sql = MySQLObjectQueries.eventDetails(schema: "app", exactSchema: true) + #expect(try selectedColumns(sql) == [ + "EVENT_NAME", "EVENT_TYPE", "STATUS", "EVENT_SCHEMA", + "EXECUTE_AT", "INTERVAL_VALUE", "INTERVAL_FIELD", "STARTS", "ENDS", + "ON_COMPLETION", "LAST_EXECUTED", "DEFINER", "EVENT_COMMENT", + ]) + #expect(sql.contains("EVENT_SCHEMA = 'app' AND CAST(EVENT_SCHEMA AS BINARY) = CAST('app' AS BINARY)")) + #expect(!sql.contains("EVENT_NAME =")) + #expect(!sql.contains("@@")) + #expect(sql.hasSuffix("ORDER BY EVENT_NAME")) + } + + @Test("A re-read of one event names it as an escaped literal") + func eventDetailsForOneEvent() { + let sql = MySQLObjectQueries.eventDetails(schema: "app", name: "it's", exactSchema: false) + #expect(sql.contains("AND EVENT_NAME = 'it''s'")) + #expect(sql.contains("EVENT_SCHEMA = 'app'")) + } + + @Test("The event scheduler is read in a statement of its own") + func eventSchedulerIsItsOwnStatement() { + #expect(MySQLObjectQueries.eventScheduler == "SELECT @@global.event_scheduler") + } + @Test("A routine's recorded mode is read by schema, name and type, with the name selected for an exact match") func routineCreationSettings() { let sql = MySQLObjectQueries.routineCreationSettings( diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSEndpointHostTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSEndpointHostTests.swift new file mode 100644 index 0000000000..e706bb5b06 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSEndpointHostTests.swift @@ -0,0 +1,218 @@ +import Foundation +@testable import TableProPluginKit +import Testing + +private struct FakeCNAMEResolver: CNAMEResolving { + let records: [String: String] + let failing: Set + let queried: QueriedNames + + init(_ records: [String: String], failing: Set = [], queried: QueriedNames = QueriedNames()) { + self.records = records + self.failing = failing + self.queried = queried + } + + func cnameTarget(of name: String, within timeout: Duration?) async -> CNAMELookup { + queried.append(name) + if failing.contains(name) { + return .failed + } + return records[name].map(CNAMELookup.found) ?? .notFound + } +} + +private final class QueriedNames: @unchecked Sendable { + private let lock = NSLock() + private var storage: [String] = [] + + var names: [String] { + lock.withLock { storage } + } + + func append(_ name: String) { + lock.withLock { storage.append(name) } + } +} + +struct AWSEndpointHostTests { + @Test("Every AWS-issued RDS form is recognised, with its region") + func rdsForms() { + let regions: [String: String?] = [ + "mydb.abc123.us-east-1.rds.amazonaws.com": "us-east-1", + "mydb.cluster-abc123.eu-west-1.rds.amazonaws.com": "eu-west-1", + "mydb.cluster-ro-abc123.ap-south-1.rds.amazonaws.com": "ap-south-1", + "mydb.cluster-custom-abc123.us-west-2.rds.amazonaws.com": "us-west-2", + "proxy.proxy-abc123.us-east-1.rds.amazonaws.com": "us-east-1", + "reader.endpoint.proxy-abc123.us-east-1.rds.amazonaws.com": "us-east-1", + "mydb.abc123.rds.cn-northwest-1.amazonaws.com.cn": "cn-northwest-1", + "mydb.abc123.cn-north-1.rds.amazonaws.com.cn": "cn-north-1", + "mydb.abc123.us-gov-west-1.rds.amazonaws.com": "us-gov-west-1", + "mydb.abc123.rds.us-gov-west-1.amazonaws.com": "us-gov-west-1", + "mydb.abc123.eusc-de-east-1.rds.amazonaws.eu": "eusc-de-east-1", + "mydb.abc123.rds.us-iso-east-1.c2s.ic.gov": "us-iso-east-1", + "mydb.global-abc123.global.rds.amazonaws.com": nil + ] + for (host, region) in regions { + #expect(AWSEndpointHost.isRDSEndpoint(host), "\(host)") + #expect(AWSEndpointHost.rdsRegion(forHost: host) == region, "\(host)") + } + } + + @Test("A name only counts as AWS's when it ends in a partition's suffix") + func lookalikesAreNotRDS() { + for host in [ + "mydb.abc123.us-east-1.rds.amazonaws.com.evil.example", + "mydb.abc123.us-east-1.rds.amazonaws.attacker.com", + "abc123.us-east-1.rds.amazonaws.com", + "mydb.abc123.cn-north-1.rds.amazonaws.com", + "mydb.abc123.us-east-1.rds.amazonaws.com.cn", + "mydb..us-east-1.rds.amazonaws.com", + "ec2-54-1-2-3.compute-1.amazonaws.com", + "a.b.rds.s3.amazonaws.com", + "a.b.us-east-1.rds.s3.amazonaws.com", + "mydb.abc123.east.rds.amazonaws.com", + "my-cluster.abc123.us-east-1.redshift.amazonaws.com", + "reader.db.example.com", + "localhost" + ] { + #expect(!AWSEndpointHost.isRDSEndpoint(host), "\(host)") + #expect(AWSEndpointHost.rdsRegion(forHost: host) == nil, "\(host)") + } + } + + @Test("Case and one trailing dot do not change the answer") + func normalization() { + #expect(AWSEndpointHost.normalized(" MyDB.ABC123.US-EAST-1.RDS.AmazonAWS.com. ") == "mydb.abc123.us-east-1.rds.amazonaws.com") + #expect(AWSEndpointHost.rdsRegion(forHost: "MyDB.ABC123.US-EAST-1.RDS.AmazonAWS.com.") == "us-east-1") + } + + @Test("Only literal loopback addresses and names are loopback") + func loopback() { + for host in [ + "localhost", "LocalHost", "127.0.0.1", "127.9.8.7", "127.1", "0x7f.1", "2130706433", + "::1", "0:0:0:0:0:0:0:1", "::ffff:127.0.0.1", "0.0.0.0" + ] { + #expect(AWSEndpointHost.isLoopback(host), "\(host)") + } + for host in [ + "localhost.localdomain", "localhost.", "127.0.0.1.", "127.0.0.1 ", " localhost", "127.0.0.1.attacker.example", + "127.evil.example.com", "0177.0.0.1", "0177.12.34.56", "[::1]", "::1%lo0", "::1%x,db.example.com", + "127.0.0.1,db.example.com", + "127.0.0.256", "128.0.0.1", + "::2", "::ffff:10.0.0.1", "db.example.com", "" + ] { + #expect(!AWSEndpointHost.isLoopback(host), "\(host)") + } + } + + @Test("Literals, loopback, single labels and .local names are never looked up") + func aliasCandidates() { + for host in ["reader.db.example.com", "READER.db.example.com."] { + #expect(AWSEndpointHost.isAliasCandidate(host), "\(host)") + } + for host in [ + "", "db", "localhost", "127.0.0.1", "10.0.0.5", "10.1", "0177.12.34.56", "::1", "fe80::1", "printer.local", + "/tmp/mysql.sock", "a.example.com,b.example.com" + ] { + #expect(!AWSEndpointHost.isAliasCandidate(host), "\(host)") + } + } + + @Test("A custom name leads to the first RDS endpoint in its chain") + func walksToFirstRDSName() async { + let resolver = FakeCNAMEResolver([ + "reader.db.example.com": "mycluster.cluster-ro-abc123.us-east-1.rds.amazonaws.com", + "mycluster.cluster-ro-abc123.us-east-1.rds.amazonaws.com": "instance-2.abc123.us-east-1.rds.amazonaws.com", + "instance-2.abc123.us-east-1.rds.amazonaws.com": "ec2-54-1-2-3.compute-1.amazonaws.com" + ]) + let found = await AWSEndpointHost.rdsEndpoint(behind: "Reader.DB.example.com.", resolver: resolver, within: nil) + #expect(found == .found("mycluster.cluster-ro-abc123.us-east-1.rds.amazonaws.com")) + } + + @Test("Hops through names of the user's own before reaching RDS") + func walksThroughIntermediateNames() async { + let resolver = FakeCNAMEResolver([ + "db.example.com": "db.internal.example.net", + "db.internal.example.net": "replica-b.abc123.us-east-1.rds.amazonaws.com" + ]) + let found = await AWSEndpointHost.rdsEndpoint(behind: "db.example.com", resolver: resolver, within: nil) + #expect(found == .found("replica-b.abc123.us-east-1.rds.amazonaws.com")) + } + + @Test("No lookup for a host that already is an RDS endpoint, or cannot have a CNAME") + func skipsHostsThatNeedNoLookup() async { + let queried = QueriedNames() + let resolver = FakeCNAMEResolver([:], queried: queried) + for host in ["mydb.abc123.us-east-1.rds.amazonaws.com", "127.0.0.1", "10.0.0.5", "localhost", "db"] { + #expect(await AWSEndpointHost.rdsEndpoint(behind: host, resolver: resolver, within: nil) == .notFound) + } + #expect(queried.names.isEmpty) + } + + @Test("A chain with no RDS endpoint, a loop or too many hops finds nothing") + func chainsThatFindNothing() async { + let noRDS = FakeCNAMEResolver(["db.example.com": "ec2-54-1-2-3.compute-1.amazonaws.com"]) + #expect(await AWSEndpointHost.rdsEndpoint(behind: "db.example.com", resolver: noRDS, within: nil) == .notFound) + + let loopQueried = QueriedNames() + let loop = FakeCNAMEResolver(["a.example.com": "b.example.com", "b.example.com": "a.example.com"], queried: loopQueried) + #expect(await AWSEndpointHost.rdsEndpoint(behind: "a.example.com", resolver: loop, within: nil) == .notFound) + #expect(loopQueried.names == ["a.example.com", "b.example.com"]) + + var long: [String: String] = [:] + for hop in 0 ..< 20 { + long["h\(hop).example.com"] = "h\(hop + 1).example.com" + } + long["h20.example.com"] = "mydb.abc123.us-east-1.rds.amazonaws.com" + let longQueried = QueriedNames() + let chain = FakeCNAMEResolver(long, queried: longQueried) + #expect(await AWSEndpointHost.rdsEndpoint(behind: "h0.example.com", resolver: chain, within: nil) == .notFound) + #expect(longQueried.names.count == 8) + } + + @Test("A spent time budget stops the walk before it asks") + func spentBudgetAsksNothing() async { + let queried = QueriedNames() + let resolver = FakeCNAMEResolver(["db.example.com": "mydb.abc123.us-east-1.rds.amazonaws.com"], queried: queried) + #expect(await AWSEndpointHost.rdsEndpoint(behind: "db.example.com", resolver: resolver, within: .zero) == .failed) + #expect(queried.names.isEmpty) + } + + @Test("A lookup that fails partway is a failure, not a name with no RDS endpoint") + func failedHopIsAFailure() async { + let resolver = FakeCNAMEResolver(["db.example.com": "edge.example.net"], failing: ["edge.example.net"]) + #expect(await AWSEndpointHost.rdsEndpoint(behind: "db.example.com", resolver: resolver, within: nil) == .failed) + } +} + +struct DNSWireNameTests { + private func decode(_ bytes: [UInt8]) -> String? { + bytes.withUnsafeBytes { DNSWireName.decode($0) } + } + + private func wire(_ labels: [String]) -> [UInt8] { + labels.flatMap { [UInt8($0.utf8.count)] + Array($0.utf8) } + [0] + } + + @Test("Decodes an uncompressed name into lowercase labels without the root dot") + func decodesName() { + #expect(decode(wire(["Replica-A", "abc123", "us-east-1", "rds", "amazonaws", "com"])) + == "replica-a.abc123.us-east-1.rds.amazonaws.com") + } + + @Test("Refuses compression pointers, truncation, the bare root, trailing bytes and dots inside a label") + func refusesMalformedNames() { + #expect(decode([0xC0, 0x0C]) == nil) + #expect(decode([3, 0x61, 0x62]) == nil) + #expect(decode([0]) == nil) + #expect(decode(wire(["a", "com"]) + [1]) == nil) + #expect(decode([3, 0x61, 0x2E, 0x62, 0]) == nil) + #expect(decode([1, 0x00, 0]) == nil) + for byte in [UInt8(ascii: " "), UInt8(ascii: "'"), UInt8(ascii: "="), UInt8(ascii: "/"), UInt8(ascii: ":"), 0xC3] { + #expect(decode([3, 0x61, byte, 0x62, 3, 0x63, 0x6F, 0x6D, 0]) == nil) + } + #expect(decode(wire(["_dmarc", "example", "com"])) == "_dmarc.example.com") + #expect(decode([64] + Array(repeating: 0x61, count: 64) + [0]) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSIAMTokenTargetTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSIAMTokenTargetTests.swift index c16eef5fa7..0e01f4c803 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSIAMTokenTargetTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/AWSIAMTokenTargetTests.swift @@ -86,10 +86,13 @@ struct AWSIAMTokenTargetTests { #expect(mixedCase == lowercase) } - @Test("The AWS Region field wins over the region in the host") - func rdsExplicitRegionWins() throws { + @Test("The region in the endpoint's name wins over the AWS Region field, which covers a name with none") + func rdsEndpointRegionWins() throws { let signed = try token(rds(), fields: ["awsRegion": "eu-west-1"]) - #expect(signed.contains("%2Feu-west-1%2Frds-db%2Faws4_request")) + #expect(signed.contains("%2Fus-east-1%2Frds-db%2Faws4_request")) + + let global = try token(rds(host: "app.global-abc123.global.rds.amazonaws.com"), fields: ["awsRegion": "eu-west-1"]) + #expect(global.contains("%2Feu-west-1%2Frds-db%2Faws4_request")) } @Test("A host with no region and no AWS Region field is rejected") diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDefaultStreamRowLimitTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDefaultStreamRowLimitTests.swift new file mode 100644 index 0000000000..2547487c4c --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDefaultStreamRowLimitTests.swift @@ -0,0 +1,70 @@ +import Foundation +import TableProPluginKit +import Testing + +// The default stream dropped `isTruncated`, so a result the driver cut reached the export as a whole one. +struct PluginDefaultStreamRowLimitTests { + private func header(of driver: some PluginDatabaseDriver) async throws -> PluginStreamHeader? { + for try await element in driver.streamRows(query: "SELECT id FROM t") { + if case .header(let header) = element { return header } + } + return nil + } + + @Test("A result the driver cut declares the row count it stopped at") + func cutResultDeclaresItsLimit() async throws { + let header = try await header(of: CutResultDriver(rowCount: 3, isCut: true)) + #expect(header?.rowLimit == 3) + } + + @Test("A whole result declares no limit") + func wholeResultDeclaresNoLimit() async throws { + let header = try await header(of: CutResultDriver(rowCount: 3, isCut: false)) + #expect(header != nil) + #expect(header?.rowLimit == nil) + } +} + +private final class CutResultDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let rowCount: Int + private let isCut: Bool + + init(rowCount: Int, isCut: Bool) { + self.rowCount = rowCount + self.isCut = isCut + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult( + columns: ["id"], + columnTypeNames: ["INTEGER"], + rows: (0.. [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDriverDefaultCancellationTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDriverDefaultCancellationTests.swift new file mode 100644 index 0000000000..bb0ef0397e --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginDriverDefaultCancellationTests.swift @@ -0,0 +1,193 @@ +import Foundation +import TableProPluginKit +import Testing + +// The default row stream ran its query in an unstructured task nothing cancelled, so an export +// stopped on a driver without its own stream kept the query and the connection busy to the end. +struct PluginDriverDefaultCancellationTests { + @Test("The default row stream carries the header and every row the query returned") + func defaultStreamCarriesTheResult() async throws { + let driver = HeldQueryDriver(holdsQuery: false) + var columns: [String] = [] + var rows: [PluginRow] = [] + for try await element in driver.streamRows(query: "SELECT v FROM t") { + switch element { + case .header(let header): columns = header.columns + case .rows(let batch): rows += batch + } + } + #expect(columns == ["v"]) + #expect(rows == [["1"], ["2"]]) + #expect(driver.executeCount == 1) + } + + @Test("Cancelling the consumer of the default row stream cancels the query it runs", .timeLimit(.minutes(1))) + func consumerCancelReachesTheQuery() async { + let driver = HeldQueryDriver(holdsQuery: true) + let consumer = Task { + do { + for try await _ in driver.streamRows(query: "SELECT v FROM t") {} + } catch {} + } + await driver.waitUntilQueryStarts() + consumer.cancel() + await driver.waitUntilQueryIsCancelled() + await consumer.value + #expect(driver.executeCount == 1) + } + + @Test("Dropping the default row stream cancels the query it runs", .timeLimit(.minutes(1))) + func droppedStreamCancelsTheQuery() async { + let driver = HeldQueryDriver(holdsQuery: true) + var stream: AsyncThrowingStream? = driver.streamRows(query: "SELECT v FROM t") + await driver.waitUntilQueryStarts() + #expect(stream != nil) + stream = nil + await driver.waitUntilQueryIsCancelled() + } + + @Test("The per-table foreign key read stops at the next table once its caller is cancelled") + func foreignKeyLoopStopsOnCancel() async { + let driver = CancellingCatalogDriver(tableCount: 50) + let read = Task { try await driver.fetchAllForeignKeys(schema: nil) } + await #expect(throws: CancellationError.self) { + _ = try await read.value + } + #expect(driver.perTableReads == 1) + } + + @Test("The per-table metadata read, which skips unreadable tables, still stops once its caller is cancelled") + func tableMetadataLoopStopsOnCancel() async { + let driver = CancellingCatalogDriver(tableCount: 50) + let read = Task { try await driver.fetchAllTableMetadata(schema: nil) } + await #expect(throws: CancellationError.self) { + _ = try await read.value + } + #expect(driver.perTableReads == 1) + } +} + +private final class HeldQueryDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let holdsQuery: Bool + private let lock = NSLock() + private var executions = 0 + private let started: AsyncStream + private let startedSignal: AsyncStream.Continuation + private let cancelled: AsyncStream + private let cancelledSignal: AsyncStream.Continuation + // Never finished, so only a cancel of the task reading it ends the read. + private let wire: AsyncStream + private let wireWriter: AsyncStream.Continuation + + init(holdsQuery: Bool) { + self.holdsQuery = holdsQuery + (started, startedSignal) = AsyncStream.makeStream() + (cancelled, cancelledSignal) = AsyncStream.makeStream() + (wire, wireWriter) = AsyncStream.makeStream() + } + + var executeCount: Int { lock.withLock { executions } } + + func waitUntilQueryStarts() async { + for await _ in started { return } + } + + func waitUntilQueryIsCancelled() async { + for await _ in cancelled { return } + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { executions += 1 } + guard holdsQuery else { + return PluginQueryResult( + columns: ["v"], columnTypeNames: ["INT"], rows: [["1"], ["2"]], rowsAffected: 0, executionTime: 0 + ) + } + startedSignal.yield() + for await _ in wire {} + if Task.isCancelled { cancelledSignal.yield() } + throw CancellationError() + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +// Each per-table read cancels the task that made it and still answers, the way a driver that never +// checks for cancellation finishes its round trip. +private final class CancellingCatalogDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let tableCount: Int + private let lock = NSLock() + private var reads = 0 + + init(tableCount: Int) { + self.tableCount = tableCount + } + + var perTableReads: Int { lock.withLock { reads } } + + private func cancellingRead() { + lock.withLock { reads += 1 } + withUnsafeCurrentTask { $0?.cancel() } + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + func execute(query: String) async throws -> PluginQueryResult { .empty } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + (0.. [PluginForeignKeyInfo] { + cancellingRead() + return [ + PluginForeignKeyInfo( + name: "fk_\(table)", column: "parent_id", referencedTable: "parent", referencedColumn: "id", + referencedDatabase: nil + ) + ] + } + + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + cancellingRead() + return PluginTableMetadata(tableName: table) + } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift index 4f2d558573..1f17a3ac4b 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift @@ -107,6 +107,17 @@ struct IsNumericLiteralTests { #expect(!StubDriver.isNumericLiteral("12abc")) #expect(!StubDriver.isNumericLiteral("1.2.3")) } + + // Written bare, a zip or a product code is compared as a number and matches `123` and `0123` too. + @Test("Numeric-shaped text with padding zeros is bound as a string", arguments: ["00123", "-007", "0042", "00.5"]) + func paddedNumbersAreQuoted(value: String) { + #expect(StubDriver().escapedParameterValue(value).unicodeScalars.first == "'") + } + + @Test("A number that opens with a single zero is bound bare", arguments: ["0", "0.5", "-0.5", "0e3", "+0", "100"]) + func singleLeadingZeroStaysBare(value: String) { + #expect(StubDriver().escapedParameterValue(value) == value) + } } // MARK: - escapedParameterValue diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginReadOnlyModeTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginReadOnlyModeTests.swift new file mode 100644 index 0000000000..c5a0183869 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginReadOnlyModeTests.swift @@ -0,0 +1,131 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginReadOnlyModeTests { + @Test("A new context asks for no Read-Only mode and no byte limit") + func contextDefaults() { + let context = PluginStatementContext() + + #expect(!context.enforcesReadOnly) + #expect(context.resultByteLimit == nil) + #expect(context == PluginStatementContext(enforcesReadOnly: false, resultByteLimit: nil)) + } + + @Test("The initializer sets both fields, and equality reads them") + func contextInitializerSetsTheFields() { + let context = PluginStatementContext(enforcesReadOnly: true, resultByteLimit: 4_096) + + #expect(context.enforcesReadOnly) + #expect(context.resultByteLimit == 4_096) + #expect(context != PluginStatementContext()) + #expect(PluginStatementContext(resultByteLimit: 1) != PluginStatementContext(resultByteLimit: 2)) + } + + @Test("The Read-Only capability is bit 24 and shares no bit with another capability") + func capabilityBit() { + let others: [PluginCapabilities] = [ + .materializedViews, .foreignTables, .storedProcedures, .userFunctions, .alterTableDDL, .foreignKeyToggle, + .truncateTable, .multiSchema, .parameterizedQueries, .cancelQuery, .batchExecute, .transactions, + .userManagement, .schemaCompare, .dataCompare, .resultSetBatches, .cellExpressions, .uuidCellText, + .temporaryTables, .rowWrites, .repeatingPrimaryKey, .readOnlyStatements, .statementPolicy, + .columnDefaultMarks, + ] + + #expect(PluginCapabilities.enforcesReadOnlyMode.rawValue == 1 << 24) + for other in others { + #expect(other.isDisjoint(with: .enforcesReadOnlyMode), "\(other.rawValue)") + } + } + + @Test("The default bounded read with a context runs the context-free one for a plain read") + func defaultForwardsAPlainRead() async throws { + let driver = BoundedReadDriver() + let context = PluginStatementContext(resultByteLimit: 1_024) + + let result = try await driver.executeBoundedQuery(query: "SELECT 1", rowCap: 5, context: context) + + #expect(result?.rows.count == 1) + #expect(driver.calls == [5]) + } + + @Test( + "The default bounded read with a context hands back what the context-free one cannot honor", + arguments: Restriction.allCases + ) + func defaultRefusesARestrictedRead(restriction: Restriction) async throws { + let driver = BoundedReadDriver() + + let result = try await driver.executeBoundedQuery(query: "SELECT 1", rowCap: 5, context: restriction.context) + + #expect(result == nil) + #expect(driver.calls.isEmpty) + } + + enum Restriction: CaseIterable, Sendable { + case enforcesReadOnly + case readOnly + case readsCatalog + case refusesDestructive + case refusesServerCode + + var context: PluginStatementContext { + var context = PluginStatementContext() + switch self { + case .enforcesReadOnly: context.enforcesReadOnly = true + case .readOnly: context.readOnly = true + case .readsCatalog: context.readsCatalog = true + case .refusesDestructive: context.allowsDestructive = false + case .refusesServerCode: context.allowsServerCode = false + } + return context + } + } +} + +private final class BoundedReadDriver: PluginDatabaseDriver, @unchecked Sendable { + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + private let lock = NSLock() + private var rowCaps: [Int] = [] + + var calls: [Int] { + lock.withLock { rowCaps } + } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + lock.withLock { rowCaps.append(rowCap) } + return PluginQueryResult( + columns: ["one"], + columnTypeNames: ["INTEGER"], + rows: [[.text("1")]], + rowsAffected: 0, + executionTime: 0 + ) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamDeliveryTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamDeliveryTests.swift new file mode 100644 index 0000000000..75f8c50f50 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamDeliveryTests.swift @@ -0,0 +1,198 @@ +// +// PluginRowStreamDeliveryTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct PluginRowStreamDeliveryTests { + private static func rows(_ value: Int) -> PluginStreamElement { + .rows([[.text(String(value))]]) + } + + private static func values(of element: PluginStreamElement) -> [String] { + guard case .rows(let rows) = element else { return [] } + return rows.compactMap { $0.first?.asText } + } + + @Test("A producer on its own thread waits for room and loses no batch") + func blockingDeliveryKeepsEveryBatch() async throws { + let total = PluginRowStream.boundedBufferBatches * 4 + let stream = PluginRowStream.makeBounded { continuation, _ in + Thread.detachNewThread { + for value in 0...makeStream() + var stream: AsyncThrowingStream? = PluginRowStream.makeBounded { continuation, _ in + Thread.detachNewThread { + var delivered = 0 + while PluginRowStream.deliverBlocking(Self.rows(delivered), to: continuation) { delivered += 1 } + ended.continuation.yield(delivered) + ended.continuation.finish() + } + } + var iterator = stream?.makeAsyncIterator() + _ = try await iterator?.next() + iterator = nil + stream = nil + var delivered = Int.max + for await count in ended.stream { delivered = count } + #expect(delivered <= PluginRowStream.boundedBufferBatches + 1) + } + + @Test("A blocked producer gives up when asked to stop, and the batch is not sent") + func blockingDeliveryHonoursStop() { + let (stream, continuation) = AsyncThrowingStream.makeStream( + bufferingPolicy: .bufferingOldest(1) + ) + #expect(PluginRowStream.deliverBlocking(Self.rows(0), to: continuation, stops: { true })) + #expect(!PluginRowStream.deliverBlocking(Self.rows(1), to: continuation, stops: { true })) + _ = stream + } + + @Test( + "A blocked producer keeps the server alive while its reader holds a full buffer, and reads nothing ahead", + .timeLimit(.minutes(1)) + ) + func blockingDeliveryKeepsAliveWhileFull() async throws { + let total = PluginRowStream.boundedBufferBatches * 2 + let beats = AsyncStream.makeStream() + let delivered = Counter() + let stream = PluginRowStream.makeBounded { continuation, _ in + Thread.detachNewThread { + for value in 0...makeStream() + let delivered = Counter() + let stream = PluginRowStream.makeBounded { continuation, _ in + Task { + for value in 0...makeStream( + bufferingPolicy: .bufferingOldest(1) + ) + #expect(PluginRowStream.deliverBlocking(Self.rows(0), to: continuation)) + #expect(throws: KeepAliveFailure.self) { + try PluginRowStream.deliverBlocking(Self.rows(1), to: continuation, keepingAliveEvery: 0) { + throw KeepAliveFailure() + } + } + await #expect(throws: KeepAliveFailure.self) { + try await PluginRowStream.deliver(Self.rows(2), to: continuation, keepingAliveEvery: 0) { + throw KeepAliveFailure() + } + } + continuation.finish() + var received: [String] = [] + for try await element in stream { + received += Self.values(of: element) + } + #expect(received == ["0"]) + } + + @Test("A producer running as a task suspends for room and loses no batch") + func asyncDeliveryKeepsEveryBatch() async throws { + let total = PluginRowStream.boundedBufferBatches * 3 + let stream = PluginRowStream.makeBounded { continuation, _ in + Task { + for value in 0..? = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + continuation.yield(.rows([[.text("a")]])) + continuation.finish() + } + var elements = 0 + if let stream { + for try await _ in stream { elements += 1 } + } + stream = nil + + #expect(elements == 2) + #expect(probe.actions == 0) + #expect(probe.abort?.isAborted == false) + } + + @Test("A stream its producer finished with an error runs no abort action") + func failedStreamRunsNoAbortAction() async { + let probe = AbortProbe() + var stream: AsyncThrowingStream? = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + continuation.finish(throwing: ProducerFailure()) + } + await #expect(throws: ProducerFailure.self) { + if let stream { + for try await _ in stream {} + } + } + stream = nil + + #expect(probe.actions == 0) + } + + @Test("A consumer that lets the stream go before its end runs the abort actions") + func droppedStreamRunsTheAbortActions() async throws { + let probe = AbortProbe() + var stream: AsyncThrowingStream? = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + } + var iterator = stream?.makeAsyncIterator() + _ = try await iterator?.next() + iterator = nil + stream = nil + + #expect(probe.actions == 1) + #expect(probe.abort?.isAborted == true) + } + + @Test("A consumer task cancelled while it waits runs the abort actions") + func cancelledConsumerRunsTheAbortActions() async throws { + let probe = AbortProbe() + let stream = PluginRowStream.make { continuation, abort in + probe.watch(abort) + continuation.yield(.header(Self.header)) + } + let reader = Task { + for try await _ in stream {} + } + reader.cancel() + _ = try? await reader.value + + #expect(probe.actions == 1) + _ = stream + } +} + +private struct ProducerFailure: Error {} + +private final class AbortProbe: @unchecked Sendable { + private let lock = NSLock() + private var watched: PluginStreamAbort? + private var count = 0 + + var abort: PluginStreamAbort? { + lock.withLock { watched } + } + + var actions: Int { + lock.withLock { count } + } + + func watch(_ abort: PluginStreamAbort) { + lock.withLock { watched = abort } + abort.onAbort { [self] in + lock.withLock { count += 1 } + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowWriteTests.swift new file mode 100644 index 0000000000..0954525b55 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowWriteTests.swift @@ -0,0 +1,29 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginRowWriteTests { + @Test("A write built without a count leaves it to the host") + func countDefaultsToNil() { + let lookup = PluginRowLookup(query: "SELECT 1", parameters: [], rowCount: 1, pinsRows: false) + + #expect(PluginRowWrite(statement: "DELETE", rowIndices: [0]).expectedRowCount == nil) + #expect(PluginRowWrite(statement: "DELETE", rowIndices: [0], lookup: lookup).expectedRowCount == nil) + } + + @Test("A write keeps the count and the lookup its driver gave it") + func keepsTheDriversCount() { + let lookup = PluginRowLookup(query: "SELECT 1", parameters: [], rowCount: 1, pinsRows: false) + let write = PluginRowWrite( + statement: "UPDATE t SET a = ? WHERE id = ?", + parameters: [.text("1"), .text("2")], + rowIndices: [3], + lookup: lookup, + expectedRowCount: 1 + ) + + #expect(write.expectedRowCount == 1) + #expect(write.lookup == lookup) + #expect(write.rowIndices == [3]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginServerObjectDefaultsTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginServerObjectDefaultsTests.swift new file mode 100644 index 0000000000..9be7b2e5b9 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginServerObjectDefaultsTests.swift @@ -0,0 +1,97 @@ +import Foundation +import TableProPluginKit +import Testing + +private final class StubServerObjectDriver: PluginDatabaseDriver, @unchecked Sendable { + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +struct PluginServerObjectDefaultsTests { + @Test("A driver that implements nothing declares no kinds and lists no objects") + func defaultsDeclareNothing() async throws { + let driver = StubServerObjectDriver() + #expect(driver.serverObjectKinds.isEmpty) + #expect(try await driver.fetchServerObjects(ofKind: "event", schema: "app").isEmpty) + #expect(try await driver.fetchServerObjects(ofKind: "extension", schema: nil).isEmpty) + } + + @Test("Reading an object again returns it when the listing carried its definition") + func rereadKeepsListedDefinition() async throws { + let listed = PluginServerObjectInfo(kindId: "event", name: "nightly", definition: "CREATE EVENT nightly") + let read = try await StubServerObjectDriver().fetchServerObject(listed) + #expect(read.name == "nightly") + #expect(read.definition == "CREATE EVENT nightly") + } + + @Test("Reading an object again without a definition is unsupported, not missing") + func rereadWithoutDefinitionIsUnsupported() async { + let listed = PluginServerObjectInfo(kindId: "event", name: "nightly", definition: "") + var refusal: Error? + do { + _ = try await StubServerObjectDriver().fetchServerObject(listed) + } catch { + refusal = error + } + guard let sourceError = refusal as? PluginObjectSourceError, case .unsupported(let name) = sourceError else { + Issue.record("expected unsupported, got \(String(describing: refusal))") + return + } + #expect(name == "nightly") + } + + @Test("A kind is listed per container unless it says otherwise") + func kindScopeDefaultsToContainer() { + let kind = PluginServerObjectKind(id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar") + #expect(kind.scope == .container) + } + + @Test("Kinds and objects round-trip through Codable") + func codableRoundTrip() throws { + let kind = PluginServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", + iconName: "puzzlepiece.extension", scope: .database + ) + let decodedKind = try JSONDecoder().decode(PluginServerObjectKind.self, from: JSONEncoder().encode(kind)) + #expect(decodedKind == kind) + + let object = PluginServerObjectInfo( + kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384", detail: "1.3", + isEnabled: true, definition: "CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;", + attributes: [PluginObjectAttribute(label: "Version", value: "1.3")] + ) + let decoded = try JSONDecoder().decode(PluginServerObjectInfo.self, from: JSONEncoder().encode(object)) + #expect(decoded.kindId == object.kindId) + #expect(decoded.name == object.name) + #expect(decoded.schema == object.schema) + #expect(decoded.identity == object.identity) + #expect(decoded.detail == object.detail) + #expect(decoded.isEnabled == object.isEnabled) + #expect(decoded.definition == object.definition) + #expect(decoded.attributes == object.attributes) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift new file mode 100644 index 0000000000..2d29b5378b --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift @@ -0,0 +1,189 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginTypedParameterBindingTests { + private func bound( + _ values: [String], + kinds: [PluginColumnKind?]?, + query: String = "SELECT * FROM t WHERE a = ?" + ) async throws -> String { + let driver = RecordingDriver() + var context = PluginStatementContext() + context.parameterKinds = kinds + _ = try await driver.executeParameterized(query: query, parameters: values.map { .text($0) }, context: context) + return try #require(driver.executed.first) + } + + @Test("A String parameter is quoted whatever it looks like", arguments: ["1E5", "42", "00123", "-7"]) + func stringIsQuoted(value: String) async throws { + #expect(try await bound([value], kinds: [.text]) == "SELECT * FROM t WHERE a = '\(value)'") + } + + @Test("An Integer or Decimal parameter that reads as that number is bare") + func numbersAreBare() async throws { + #expect(try await bound(["42"], kinds: [.integer]) == "SELECT * FROM t WHERE a = 42") + #expect(try await bound(["1E5"], kinds: [.decimal]) == "SELECT * FROM t WHERE a = 1E5") + } + + @Test("Text that is not the declared number is sent as a string for the server to refuse") + func notANumberIsQuoted() async throws { + #expect(try await bound(["1.5"], kinds: [.integer]) == "SELECT * FROM t WHERE a = '1.5'") + #expect(try await bound(["1; DROP TABLE t"], kinds: [.decimal]) == "SELECT * FROM t WHERE a = '1; DROP TABLE t'") + } + + @Test("A Boolean parameter is TRUE or FALSE", arguments: [("true", "TRUE"), ("0", "FALSE"), ("Yes", "TRUE")]) + func booleans(value: String, literal: String) async throws { + #expect(try await bound([value], kinds: [.boolean]) == "SELECT * FROM t WHERE a = \(literal)") + } + + @Test("A Boolean parameter that is no boolean is a string") + func notABooleanIsQuoted() async throws { + #expect(try await bound(["maybe"], kinds: [.boolean]) == "SELECT * FROM t WHERE a = 'maybe'") + } + + @Test("A Date parameter is a quoted string") + func dateIsQuoted() async throws { + #expect(try await bound(["2026-01-01"], kinds: [.other]) == "SELECT * FROM t WHERE a = '2026-01-01'") + } + + @Test("A value with no type keeps the reading by shape") + func untypedKeepsTheShape() async throws { + #expect(try await bound(["42"], kinds: nil) == "SELECT * FROM t WHERE a = 42") + #expect(try await bound(["42", "x"], kinds: [.text, nil], query: "SELECT ?, ?") == "SELECT '42', 'x'") + #expect(try await bound(["42"], kinds: [nil]) == "SELECT * FROM t WHERE a = 42") + } + + @Test("Each parameter takes its own type by position") + func typesFollowPosition() async throws { + let sql = try await bound(["7", "7"], kinds: [.integer, .text], query: "SELECT ?, ?") + #expect(sql == "SELECT 7, '7'") + } + + @Test("A driver that forwards its context overload keeps the types through withHostContext") + func forwardingDriverKeepsTheTypes() async throws { + let driver = ForwardingDriver() + var context = PluginStatementContext() + context.parameterKinds = [.text] + _ = try await driver.executeUserQuery( + query: "SELECT ?", rowCap: nil, parameters: [.text("5")], context: context + ) + #expect(driver.inner.executed == ["SELECT '5'"]) + } +} + +struct PluginRowChangeDefaultMarkTests { + @Test("Only a marked column is the default") + func onlyMarkedColumnsAreDefaults() { + var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + #expect(!change.isColumnDefault(0)) + change.defaultColumns = [1] + #expect(change.isColumnDefault(1)) + #expect(!change.isColumnDefault(0)) + } + + @Test("An engine with no defaults refuses the first change that leaves a cell to one, naming its row") + func defaultsUnsupportedNamesTheRow() { + let plain = PluginRowChange(rowIndex: 3, type: .insert, cellChanges: [], originalRow: nil) + var defaulted = PluginRowChange(rowIndex: 7, type: .update, cellChanges: [], originalRow: nil) + defaulted.defaultColumns = [0] + #expect(PluginRowWriteRefusal.columnDefaultsUnsupported(in: [plain]) == nil) + #expect(PluginRowWriteRefusal.columnDefaultsUnsupported(in: [plain, defaulted])?.rowIndex == 7) + } +} + +struct PluginBoundedStreamDeclaredLimitTests { + private func stream(rows: Int, rowLimit: Int?) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + var header = PluginStreamHeader(columns: ["id"], columnTypeNames: ["INT"]) + header.rowLimit = rowLimit + continuation.yield(.header(header)) + continuation.yield(.rows((0.. PluginQueryResult { + lock.withLock { statements.append(query) } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +// The shape of a driver that overrides the context overload to set its own state, then runs the context-free one. +private final class ForwardingDriver: PluginDatabaseDriver, @unchecked Sendable { + let inner = RecordingDriver() + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + func execute(query: String) async throws -> PluginQueryResult { + try await inner.execute(query: query) + } + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + try await PluginParameterBinder.withHostContext(context) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } + } + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/RDSAuthTokenTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/RDSAuthTokenTests.swift index e5fd04c5ef..b6024ac1a6 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/RDSAuthTokenTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/RDSAuthTokenTests.swift @@ -79,32 +79,32 @@ struct RDSAuthTokenGeneratorTests { } } -struct RDSEndpointTests { +struct RDSRegionTests { @Test("Derives region from cluster hostname") func clusterHostname() { - #expect(RDSEndpoint.region(forHost: "mydb.cluster-abc123.us-east-1.rds.amazonaws.com") == "us-east-1") + #expect(AWSEndpointHost.rdsRegion(forHost: "mydb.cluster-abc123.us-east-1.rds.amazonaws.com") == "us-east-1") } @Test("Derives region from instance hostname") func instanceHostname() { - #expect(RDSEndpoint.region(forHost: "mydb.abc123.us-west-2.rds.amazonaws.com") == "us-west-2") + #expect(AWSEndpointHost.rdsRegion(forHost: "mydb.abc123.us-west-2.rds.amazonaws.com") == "us-west-2") } @Test("Derives region from China partition hostname") func chinaHostname() { - #expect(RDSEndpoint.region(forHost: "mydb.abc.cn-north-1.rds.amazonaws.com.cn") == "cn-north-1") + #expect(AWSEndpointHost.rdsRegion(forHost: "mydb.abc.cn-north-1.rds.amazonaws.com.cn") == "cn-north-1") } @Test("Returns nil for non-RDS hosts") func nonRDSHost() { - #expect(RDSEndpoint.region(forHost: "localhost") == nil) - #expect(RDSEndpoint.region(forHost: "db.example.com") == nil) + #expect(AWSEndpointHost.rdsRegion(forHost: "localhost") == nil) + #expect(AWSEndpointHost.rdsRegion(forHost: "db.example.com") == nil) } @Test("Derives a lowercase region from a hostname typed with capitals") func mixedCaseHostname() { - #expect(RDSEndpoint.region(forHost: "MyDB.abc123.US-EAST-1.RDS.AmazonAWS.com") == "us-east-1") - #expect(RDSEndpoint.region(forHost: "MyDB.Cluster-ABC.EU-West-1.Rds.Amazonaws.Com") == "eu-west-1") + #expect(AWSEndpointHost.rdsRegion(forHost: "MyDB.abc123.US-EAST-1.RDS.AmazonAWS.com") == "us-east-1") + #expect(AWSEndpointHost.rdsRegion(forHost: "MyDB.Cluster-ABC.EU-West-1.Rds.Amazonaws.Com") == "eu-west-1") } } @@ -206,7 +206,11 @@ struct RDSSigningEndpointResolverTests { @Test("An override may not redirect the token away from the host being dialled") func overrideCannotRedirectToAnotherHost() { - #expect(throws: AWSAuthError.rdsEndpointUnresolved(host: "evil.example.com")) { + let refusal = AWSAuthError.rdsEndpointNotDialed( + endpoint: "prod.abc123.us-east-1.rds.amazonaws.com", + host: "evil.example.com" + ) + #expect(throws: refusal) { _ = try resolve( host: "evil.example.com", port: 5_432, @@ -215,6 +219,41 @@ struct RDSSigningEndpointResolverTests { } } + @Test("A DNS name that starts like 127.0.0.1 is not a local forward, so the override is refused") + func loopbackLookalikeNameCannotTakeTheOverride() { + for host in ["127.0.0.1.attacker.example", "127.evil.example.com", "127.0.0.1.nip.io", "127.0.0.1.", "0177.12.34.56", "[::1]", "127.0.0.1,db.example.com"] { + let refusal = AWSAuthError.rdsEndpointNotDialed( + endpoint: "prod.abc123.us-east-1.rds.amazonaws.com", + host: AWSEndpointHost.normalized(host) + ) + #expect(throws: refusal) { + _ = try resolve(host: host, port: 3_306, override: "prod.abc123.us-east-1.rds.amazonaws.com:3306") + } + } + } + + @Test("Every 127/8 literal and the loopback names count as a local forward") + func loopbackLiteralsTakeTheOverride() throws { + for host in ["127.0.0.1", "127.1.2.3", "127.255.255.255", "127.1", "localhost", "LOCALHOST", "::1", "0.0.0.0"] { + let endpoint = try resolve(host: host, port: 54_321, override: "mydb.abc123.us-east-1.rds.amazonaws.com") + #expect(endpoint.host == "mydb.abc123.us-east-1.rds.amazonaws.com") + } + } + + @Test("A trailing dot on the host or the override is not signed") + func trailingDotIsDropped() throws { + let bare = RDSSigningEndpoint(host: "mydb.abc123.us-east-1.rds.amazonaws.com", port: 5_432) + #expect(try resolve(host: "mydb.abc123.us-east-1.rds.amazonaws.com.") == bare) + #expect(try resolve(host: "127.0.0.1", port: 62_000, override: "mydb.abc123.us-east-1.rds.amazonaws.com.:5432") == bare) + #expect(try resolve( + host: "127.0.0.1", + port: 62_000, + preTunnelHost: "mydb.abc123.us-east-1.rds.amazonaws.com.", + preTunnelPort: 5_432 + ) == bare) + #expect(try resolve(override: "mydb.abc123.us-east-1.rds.amazonaws.com.") == bare) + } + @Test("An override that restates the dialled host is accepted") func overrideMatchingDialledHostIsAccepted() throws { let endpoint = try resolve( @@ -244,7 +283,7 @@ struct RDSSigningEndpointResolverTests { preTunnelHost: "mydb.abc123.ap-south-1.rds.amazonaws.com", preTunnelPort: 5_432 ) - #expect(RDSEndpoint.region(forHost: endpoint.host) == "ap-south-1") + #expect(AWSEndpointHost.rdsRegion(forHost: endpoint.host) == "ap-south-1") } @Test("A host typed with capitals resolves to the lowercase endpoint") diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift index 3689fd6156..e13ba5a84a 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift @@ -31,6 +31,32 @@ struct SQLStatementSplittingTests { ).count == 1) } + // Measured on DuckDB 1.5.4 and 2.0.0, the engine whose plugin reads E'...' through the kit. + @Test("A prefix glued to a name is part of it, and the non-ASCII spaces DuckDB blanks end the name") + func prefixAfterAName() { + let duckDB: SQLLexicalFeatures = [.taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes] + for glue in ["$", "\u{E9}", "\u{2019}", "\u{FF0C}"] { + #expect(SQLStatementSplitting.statements( + in: "SELECT x\(glue)e'\\'; DROP TABLE t; SELECT 'x'", + lexicalFeatures: duckDB + ).count == 3, "\(glue)") + } + for space in ["\u{00A0}", "\u{200B}", "\u{2060}", "\u{3000}", "\u{FEFF}"] { + #expect(SQLStatementSplitting.statements( + in: "SELECT varchar\(space)E'\\''; DROP TABLE t; SELECT 'x'", + lexicalFeatures: duckDB + ).count == 3, "\(space.unicodeScalars.first.map { String($0.value, radix: 16) } ?? "")") + } + #expect(SQLStatementSplitting.statements( + in: "SELECT 1 AS a,\u{200B}$$'$$; DROP TABLE t; SELECT 'x'", + lexicalFeatures: duckDB + ).count == 3) + let oracle: SQLLexicalFeatures = [.alternativeQuoting, .dollarAndHashInIdentifiers] + #expect(SQLStatementSplitting.statements(in: "SELECT x#q'['; SELECT 2 --]'", lexicalFeatures: oracle).count == 2) + #expect(SQLStatementSplitting.statements(in: "SELECT x#q'['; SELECT 2 --]'", lexicalFeatures: .alternativeQuoting) + .count == 1) + } + @Test("SQLite's $name(...) parameter keeps a semicolon, and a $ inside a name starts none") func featuresReadParenthesizedParameters() { #expect(SQLStatementSplitting.statements(in: "SELECT $a(;)", lexicalFeatures: .parenthesizedParameterNames) diff --git a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Process/DescriptorReadTests.swift b/apple/Packages/TableProCore/Tests/TableProProcessTests/DescriptorReadTests.swift similarity index 98% rename from apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Process/DescriptorReadTests.swift rename to apple/Packages/TableProCore/Tests/TableProProcessTests/DescriptorReadTests.swift index 0f81756178..0da9b84c6c 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Process/DescriptorReadTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProProcessTests/DescriptorReadTests.swift @@ -1,9 +1,10 @@ +#if canImport(Darwin) import Darwin import Foundation import TableProTestSupport import Testing -@testable import TableProUtilities +@testable import TableProProcess struct DescriptorReadTests { @Test("A read returns what has arrived without waiting for the rest", .timeLimit(.minutes(1))) @@ -134,3 +135,4 @@ struct DescriptorReadTests { #expect(fcntl(descriptor, F_GETFL) & O_NONBLOCK == 0) } } +#endif diff --git a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Process/DescriptorWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProProcessTests/DescriptorWriteTests.swift similarity index 97% rename from apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Process/DescriptorWriteTests.swift rename to apple/Packages/TableProCore/Tests/TableProProcessTests/DescriptorWriteTests.swift index 076ac2d9d1..5b22ac5fef 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Process/DescriptorWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProProcessTests/DescriptorWriteTests.swift @@ -1,9 +1,10 @@ +#if canImport(Darwin) import Darwin import Foundation import TableProTestSupport import Testing -@testable import TableProUtilities +@testable import TableProProcess struct DescriptorWriteTests { private struct Measured: Sendable { @@ -61,3 +62,4 @@ struct DescriptorWriteTests { #expect(error?.code == .EPIPE) } } +#endif diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift index dc0b58944f..2cd5c16aa1 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantClientTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProQdrantClient @testable import TableProQdrantCore import Testing diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift index 33f6cb4555..1dfe686572 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantConsoleTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProQdrantClient @testable import TableProQdrantCore import Testing diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift index d2a4a83fcb..00b6f127cd 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantFilterBuilderTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProQdrantClient @testable import TableProQdrantCore import Testing diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index cd0bbbdaab..a76c201c2c 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProQdrantClient @testable import TableProQdrantCore import Testing @@ -98,14 +99,22 @@ struct QdrantRowWriteTests { #expect(try written(#"["a"]"#) == prefix + #"{"payload":{"mixed":["a"]},"points":[7]}"#) } - @Test("A duplicated row's default marker asks for a new id instead of failing as one") + @Test("A duplicated row's defaulted id asks for a new id instead of failing as one") func duplicatedRowGetsNewID() throws { let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ - QdrantCell(column: "id", text: "__DEFAULT__"), QdrantCell(column: "city", text: "Hanoi") + QdrantCell(column: "id", text: nil), QdrantCell(column: "city", text: "Hanoi") ]) #expect(try writes([change]).first?.contains(#""id":"00000000-0000-0000-0000-000000000001""#) == true) } + @Test("The text __DEFAULT__ in a payload field is stored as that text") + func defaultLookingTextIsAValue() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: "7"), QdrantCell(column: "city", text: "__DEFAULT__") + ]) + #expect(try writes([change]).first?.contains(#""city":"__DEFAULT__""#) == true) + } + @Test("A server that would overwrite a taken id gets no grid insert") func insertNeedsConditionalUpsert() { let change = QdrantRowChange(kind: .insert, rowIndex: 6, id: nil, cells: [QdrantCell(column: "city", text: "Hanoi")]) diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift index f52f77f40b..9b02aa1480 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantSchemaTests.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProQdrantClient @testable import TableProQdrantCore import Testing diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift index 7eed1272c5..be37055ade 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantTestSupport.swift @@ -1,4 +1,5 @@ import Foundation +@testable import TableProQdrantClient @testable import TableProQdrantCore final class RecordingTransport: QdrantTransport, @unchecked Sendable { diff --git a/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift b/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift index 4a85284b43..fa0eeb768c 100644 --- a/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift @@ -163,6 +163,16 @@ struct QueryScriptTests { #expect(QueryScript(units: script.units, unitKind: .statement).replanned(after: 0, session: noEscapes) == nil) } + @Test("A replan after a COPY starts past its rows") + func replanAfterACopy() throws { + let text = "COPY t FROM stdin;\nDROP TABLE users; COMMIT;\n\\.\nSELECT 1;" + let script = QueryScript.plan(text, databaseType: .postgresql, sendsBatchesWhole: false) + #expect(script.units.map(\.sql) == ["COPY t FROM stdin", "SELECT 1"]) + let standard = SQLSessionLexicalFacts(determined: .backslashEscapesInSingleQuotes, enabled: []) + let rest = try #require(script.replanned(after: 0, session: standard)) + #expect(rest.units.map(\.sql) == ["SELECT 1"]) + } + @Test("Redis text runs one command per line or per semicolon, a quoted value kept whole") func redisSplitsIntoCommands() { #expect(plan("RPUSH jobs a\nRPUSH jobs b", .redis).units == [ diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift index 515d4828cd..5dee906b01 100644 --- a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift @@ -103,6 +103,18 @@ struct RedisCommandSplitterTests { } } + @Test("Each command's caret reach gives every caret the command command(at:) gives it", arguments: [ + "GET a; GET b\n\nGET c \n", "GET a\n", "GET a; ", "SET k \"a\nb\"\nGET k", " GET a ;", "SET 😀 1\r\nGET 😀", "", + ]) + func caretReachMatchesCommandAt(script: String) { + let reached = RedisCommandSplitter.commandsWithCaretReach(in: script) + #expect(reached.map { $0.command } == RedisCommandSplitter.commands(in: script)) + for caret in 0...((script as NSString).length + 1) { + let viaReach = reached.first { caret <= $0.reach }?.command + #expect(viaReach == RedisCommandSplitter.command(at: caret, in: script), "caret \(caret)") + } + } + @Test("A caret after the last command, or in text with none, has no command") func caretPastEveryCommand() { #expect(command(at: 6, in: "GET a\n") == nil) diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift index 264c3f6009..227c42183c 100644 --- a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift @@ -12,6 +12,19 @@ struct RedisKeySummaryCommandTests { @Test("a string key is read with GET so the whole value arrives") func stringPreviewReadsWholeValue() { #expect(RedisKeySummary.previewCommand(for: .string, key: "session:1") == ["GET", "session:1"]) + #expect(RedisKeySummary.previewCommand(for: .string, key: "session:1", stringByteLimit: nil) == ["GET", "session:1"]) + } + + @Test("a byte limit reads a string with GETRANGE over that many bytes") + func stringPreviewWithLimitReadsRange() { + #expect(RedisKeySummary.previewCommand(for: .string, key: "page:1", stringByteLimit: 65_536) + == ["GETRANGE", "page:1", "0", "65535"]) + } + + @Test("a byte limit leaves collection previews as they were") + func byteLimitIgnoredForCollections() { + #expect(RedisKeySummary.previewCommand(for: .list, key: "k", stringByteLimit: 10) + == RedisKeySummary.previewCommand(for: .list, key: "k")) } @Test("length command matches the Redis command for each kind") diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisStringPreviewTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisStringPreviewTests.swift new file mode 100644 index 0000000000..0e25632dd6 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisStringPreviewTests.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing + +@testable import TableProRedisCore + +struct RedisStringPreviewTests { + @Test("A value within the limit shows whole and unmarked") + func shortValueIsWhole() { + let shown = RedisStringPreview.shown(Data("hello".utf8), length: 5) + #expect(shown == .text("hello")) + } + + @Test("A value exactly at the limit is not shortened") + func valueAtTheLimitIsWhole() { + let bytes = Data(repeating: 0x61, count: 8) + #expect(RedisStringPreview.shown(bytes, length: 8, byteLimit: 8) == .text("aaaaaaaa")) + } + + @Test("A value past the limit shows its start followed by the marker") + func longValueIsMarked() { + let bytes = Data(repeating: 0x61, count: 8) + #expect(RedisStringPreview.shown(bytes, length: 500_000, byteLimit: 8) == .text("aaaaaaaa...")) + } + + @Test("A range that ends inside a multi-byte character drops that character only") + func cutCharacterIsDropped() { + // "aé€" is 61 C3 A9 E2 82 AC; a range of 5 bytes ends inside the euro sign. + let full = Array("aé€".utf8) + let shown = RedisStringPreview.shown(Data(full.prefix(5)), length: full.count, byteLimit: 5) + #expect(shown == .text("aé...")) + } + + @Test("A range that ends inside a four-byte character drops all of it") + func cutFourByteCharacterIsDropped() { + let full = Array("ab😀".utf8) + let shown = RedisStringPreview.shown(Data(full.prefix(4)), length: full.count, byteLimit: 4) + #expect(shown == .text("ab...")) + } + + @Test("A binary value stays bytes, whole range, without a marker") + func binaryValueStaysBytes() { + let bytes = Data([0x1F, 0x8B, 0x08, 0x00, 0xFF, 0xFE, 0x00, 0x03]) + #expect(RedisStringPreview.shown(bytes, length: 4_096, byteLimit: 8) == .bytes(bytes)) + #expect(RedisStringPreview.shown(bytes, length: 8, byteLimit: 8) == .bytes(bytes)) + } + + @Test("A value that starts with a byte order mark keeps it") + func byteOrderMarkSurvives() { + let bytes = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + #expect(RedisStringPreview.shown(bytes, length: 6) == .text("\u{FEFF}abc")) + } + + @Test("Without a length, a read that filled the range counts as shortened") + func unknownLengthFilledRangeIsShortened() { + #expect(RedisStringPreview.isShortened(length: nil, shownByteCount: 8, byteLimit: 8)) + #expect(!RedisStringPreview.isShortened(length: nil, shownByteCount: 7, byteLimit: 8)) + #expect(RedisStringPreview.shown(Data(repeating: 0x61, count: 8), length: nil, byteLimit: 8) == .text("aaaaaaaa...")) + } + + @Test("The length alone decides when the server gave one") + func lengthDecides() { + #expect(RedisStringPreview.isShortened(length: 9, shownByteCount: 0, byteLimit: 8)) + #expect(!RedisStringPreview.isShortened(length: 8, shownByteCount: 8, byteLimit: 8)) + } + + @Test("The marked text of a cut value is at least as long as the limit") + func markedTextReachesTheLimit() throws { + let full = Array(String(repeating: "€", count: 30).utf8) + let limit = 64 + guard case .text(let text) = RedisStringPreview.shown(Data(full.prefix(limit)), length: full.count, byteLimit: limit) else { + Issue.record("expected text") + return + } + #expect(text.hasSuffix(RedisStringPreview.marker)) + #expect(text.utf8.count >= limit) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisWireTextTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisWireTextTests.swift new file mode 100644 index 0000000000..07990daf06 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisWireTextTests.swift @@ -0,0 +1,38 @@ +import Foundation +import Testing + +@testable import TableProRedisCore + +private let byteOrderMark: [UInt8] = [0xEF, 0xBB, 0xBF] + +struct RedisWireTextTests { + @Test("A leading byte order mark stays part of the text") + func keepsLeadingByteOrderMark() throws { + let bytes = Data(byteOrderMark + Array("user:1".utf8)) + let text = try #require(RedisWireText.decode(bytes)) + #expect(text.unicodeScalars.first == "\u{FEFF}") + #expect(Data(text.utf8) == bytes) + } + + @Test("Plain and non-ASCII text decode as they are", arguments: ["", "user:1", "café ☕", "a\u{0}b"]) + func decodesText(_ value: String) { + #expect(RedisWireText.decode(Data(value.utf8)) == value) + } + + @Test("Bytes that are not UTF-8 are not text", arguments: [ + Data([0xFF, 0xFE]), + Data([0x61, 0xE2, 0x82]), + Data([0xC0, 0x80]), + ]) + func rejectsInvalidBytes(_ bytes: Data) { + #expect(RedisWireText.decode(bytes) == nil) + } + + @Test("A quoted argument keeps a leading byte order mark") + func quoteKeepsByteOrderMark() throws { + let bytes = Data(byteOrderMark + Array("user:1".utf8)) + let quoted = RedisArgumentCodec.quote(bytes) + let parsed = try #require(RedisArgumentCodec.split(quoted)) + #expect(parsed == [bytes]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementParserTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementParserTests.swift new file mode 100644 index 0000000000..d9cee9bd79 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/MongoStatementParserTests.swift @@ -0,0 +1,82 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct MongoStatementParserTests { + private func calls(_ text: String) -> [MongoStatement.Call] { + MongoStatementParser.parse(text) { _ in false }?.calls ?? [] + } + + @Test("A collection call records the collection it reaches and the database the text names", arguments: [ + ("db.orders.insertOne({a: 1})", nil, "orders"), + (#"db["user-events"].find({})"#, nil, "user-events"), + (#"db.getCollection("system.js").insertOne({})"#, nil, "system.js"), + (#"db.getSiblingDB("shop").orders.updateOne({}, {$set: {a: 1}})"#, "shop", "orders"), + (#"db.getMongo().getDB("shop").getCollection("orders").deleteOne({_id: 1})"#, "shop", "orders"), + (#"db.getSiblingDB("admin").getSiblingDB("config").settings.updateOne({}, {$set: {a: 1}})"#, "config", "settings") + ] as [(String, String?, String?)]) + func collectionTargets(text: String, database: String?, collection: String?) throws { + let call = try #require(calls(text).last) + #expect(call.receiver == .collection) + #expect(call.database == database) + #expect(call.collection == collection) + #expect(call.commandName == nil) + } + + @Test("A cursor or an explain keeps the database and collection it came from") + func chainedReceivers() { + let found = calls(#"db.getSiblingDB("shop").orders.find({}).limit(1)"#) + #expect(found.map { $0.receiver } == [MongoStatement.Receiver.collection, .cursor]) + #expect(found.map { $0.database } == ["shop", "shop"]) + #expect(found.map { $0.collection } == ["orders", "orders"]) + #expect(calls("db.orders.explain().find({})").map { $0.receiver } == [MongoStatement.Receiver.collection, .explainer]) + } + + @Test("A database call reaches no collection", arguments: ["db.stats()", #"db.createCollection("orders")"#, "db.killOp(1)"]) + func databaseCallsReachNoCollection(text: String) throws { + let call = try #require(calls(text).first) + #expect(call.receiver == .database) + #expect(call.collection == nil) + #expect(call.database == nil) + } + + @Test("A command's collection is its document's first value, and its database the call's or the document's $db", arguments: [ + (#"db.runCommand({insert: "orders", documents: []})"#, nil, "orders"), + (#"db.adminCommand({insert: "system.version", documents: []})"#, "admin", "system.version"), + (#"db.getSiblingDB("shop").runCommand({update: "orders", updates: []})"#, "shop", "orders"), + (#"db.runCommand({update: "settings", updates: [], $db: "config"})"#, "config", "settings"), + (#"db.adminCommand({insert: "orders", "$db": "shop"})"#, "shop", "orders"), + (#"db.runCommand({insert: "orders", $db: "shop", $db: "admin"})"#, "admin", "orders"), + (#"db.runCommand({insert: "orders", documents: [], insert: "system.js"})"#, nil, "system.js"), + (#"db.runCommand({insert: "orders", insert: 1})"#, nil, nil), + ("db.runCommand({insert: 1})", nil, nil), + (#"db.runCommand({insert: {name: "orders"}})"#, nil, nil) + ] as [(String, String?, String?)]) + func commandTargets(text: String, database: String?, collection: String?) throws { + let call = try #require(calls(text).first) + #expect(call.receiver == .database) + #expect(call.database == database) + #expect(call.collection == collection) + } + + @Test("A command is named by its document's first key, and a document-less call names none") + func commandNames() { + #expect(calls(#"db.runCommand({findAndModify: "orders", query: {}})"#).first?.commandName == "findAndModify") + #expect(calls(#"db.runCommand({a: {toEJSON: MinKey}, insert: "orders"})"#).first?.commandName == "a") + #expect(calls(#"db.runCommand("insert")"#).first?.commandName == nil) + #expect(calls(#"db.runCommand("insert")"#).first?.collection == nil) + } + + @Test("use and show are shell calls that read, and use names the database it moves to") + func shellCommands() throws { + let use = try #require(calls("use admin").first) + #expect(use.name == "use") + #expect(use.receiver == .shell) + #expect(use.isRead) + #expect(use.database == "admin") + let show = try #require(calls("show collections").first) + #expect(show.name == "show") + #expect(show.receiver == .shell) + #expect(show.database == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBareNameCallsTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBareNameCallsTests.swift new file mode 100644 index 0000000000..9cd42c94a0 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLBareNameCallsTests.swift @@ -0,0 +1,221 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// The cases are the ones measured on Oracle 23ai and DM8: a function PURGE and WIPE, a package PK with F, a table +// ORDERS(ID, PURGE). Each answer is whether the engine ran a function. +struct SQLBareNameCallsTests { + private static let orders = SQLTableName(owner: nil, name: "ORDERS") + private static let dual = SQLTableName(owner: nil, name: "DUAL") + private static let ownedOrders = SQLTableName(owner: "PROBE", name: "ORDERS") + private static let catalog = SQLCallableCatalog(functions: ["PURGE", "WIPE"], roots: ["PK", "PROBE"]) + private static let columns: [SQLTableName: Set] = [ + orders: ["ID", "PURGE"], dual: ["DUMMY"], ownedOrders: ["ID", "PURGE"], + ] + + private static func reading(_ sql: String, _ engine: String = "Oracle") -> SQLBareNameReading { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + return SQLBareNameCalls.read(sql, databaseTypeId: engine, readings: readings) + } + + private static func calls(_ sql: String, _ engine: String = "Oracle", catalog: SQLCallableCatalog = catalog) -> Bool { + reading(sql, engine).callsFunction(catalog, columns: columns) + } + + @Test("A name no table in scope has as a column calls the function", arguments: [ + "SELECT purge FROM dual", + "SELECT wipe FROM orders", + "SELECT * FROM orders, (SELECT purge FROM dual)", + "SELECT * FROM orders o JOIN (SELECT purge p FROM dual) d ON 1 = 1", + "SELECT 1 AS wipe FROM dual WHERE wipe = 7", + "SELECT 1 wipe FROM dual GROUP BY wipe", + "SELECT COUNT(*) FROM orders WHERE wipe = 7", + "SELECT \"WIPE\" FROM dual", + "SELECT * FROM orders WHERE id = 1 ORDER BY wipe", + "SELECT id, purge FROM orders UNION ALL SELECT 1, purge FROM dual", + "SELECT * FROM orders WHERE purge BETWEEN wipe AND 1000", + "SELECT CASE WHEN wipe = 7 THEN 1 END FROM dual", + "SELECT t.* FROM (SELECT id x FROM orders) t WHERE wipe = 7", + "SELECT wipe FROM orders PIVOT (SUM(purge) FOR id IN (1 AS a))", + "SELECT id FROM orders o WHERE EXISTS (SELECT 1 FROM orders i WHERE i.id = o.id AND wipe = 7)", + "SELECT id FROM orders\nSELECT purge FROM dual", + "SELECT 1 FROM dual\nWITH c AS (SELECT purge FROM dual) SELECT * FROM c", + "EXPLAIN PLAN FOR SELECT 1 FROM dual\nSELECT purge FROM dual", + "SELECT wipe", + ]) + func bareCalls(sql: String) { + #expect(Self.calls(sql)) + #expect(Self.calls(sql, "Dameng")) + } + + @Test("A column, an alias ORDER BY sees and a name the query defines call nothing", arguments: [ + "SELECT purge FROM orders", + "SELECT o.purge FROM orders o", + "SELECT id FROM orders WHERE EXISTS (SELECT 1 FROM dual WHERE purge = 100)", + "SELECT * FROM orders o, LATERAL (SELECT purge FROM dual)", + "SELECT (SELECT purge FROM dual) FROM orders", + "SELECT 1 AS wipe FROM dual ORDER BY wipe", + "SELECT purge p FROM orders ORDER BY p", + "WITH c AS (SELECT 1 wipe FROM dual) SELECT wipe FROM c", + "WITH c (wipe) AS (SELECT 1 FROM dual) SELECT wipe FROM c", + "SELECT purge FROM (SELECT * FROM orders)", + "SELECT * FROM (SELECT purge FROM orders) WHERE purge > 0", + "SELECT probe.orders.purge FROM probe.orders", + "SELECT \"wipe\" FROM dual", + "SELECT id wipe FROM orders", + "SELECT id FROM orders WHERE id = :wipe", + "SELECT seq.NEXTVAL FROM dual", + "SELECT LISTAGG(id, ',') WITHIN GROUP (ORDER BY id) FROM orders", + "SELECT * FROM wipe", + "SELECT CAST(id AS wipe) FROM orders", + "SELECT id FROM orders o WHERE o.id IN (SELECT id FROM orders o2 WHERE o2.purge = o.purge)", + "SELECT 1 wipe FROM dual UNION SELECT 2 FROM dual ORDER BY wipe", + "EXPLAIN PLAN FOR SELECT purge FROM dual", + ]) + func noCalls(sql: String) { + #expect(!Self.calls(sql)) + } + + @Test("A dotted name no table in scope starts calls without the catalog", arguments: [ + "SELECT pk.f FROM dual", + "SELECT probe.wipe FROM dual", + "SELECT probe.pk.f FROM dual", + "SELECT * FROM orders WHERE id IN (SELECT pk.f FROM dual)", + "SELECT wipe@remote FROM dual", + "WITH FUNCTION f RETURN NUMBER IS BEGIN RETURN 1; END; SELECT f FROM dual", + ]) + func dottedCalls(sql: String) { + let reading = Self.reading(sql) + #expect(reading.callsOutright) + #expect(reading.callsFunction(SQLCallableCatalog(functions: [], roots: []), columns: [:])) + } + + @Test("Dameng calls package PK through an alias whose table has no such column") + func aliasFallsToPackage() { + #expect(Self.calls("SELECT pk.f FROM orders pk", "Dameng")) + #expect(!Self.calls("SELECT pk.id FROM orders pk", "Dameng")) + #expect(!Self.calls("SELECT pk.f FROM orders pk", "Dameng", catalog: SQLCallableCatalog(functions: [], roots: []))) + } + + // Measured on Oracle 23ai: functions RUNNING, FINAL and WIPE did not run for these. + @Test("MATCH_RECOGNIZE's own words, a measure's alias, a pattern variable and RUNNING or FINAL before a call are not calls") + func rowPatternClauseNames() { + let catalog = SQLCallableCatalog( + functions: [ + "WIPE", "RUNNING", "FINAL", "MEASURES", "ONE", "ROW", "ROWS", "PER", "MATCH", "AFTER", "SKIP", "PAST", + "LAST", "NEXT", "FIRST", "SUBSET", "DEFINE", "X", "Y", "U", "EMPTY", "MATCHES", "SHOW", + ], + roots: [] + ) + for sql in [ + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES x.id AS wipe ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1)", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES FINAL LAST(x.purge) AS l, RUNNING COUNT(*) AS c " + + "ALL ROWS PER MATCH SHOW EMPTY MATCHES AFTER MATCH SKIP PAST LAST ROW PATTERN (x y*) SUBSET u = (x, y) " + + "DEFINE x AS RUNNING COUNT(*) < 9, y AS u.purge > PREV(y.purge))", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES x.id wipe AFTER MATCH SKIP TO NEXT ROW PATTERN (x) " + + "DEFINE x AS CAST(x.purge AS NUMBER) > 0)", + ] { + #expect(!Self.calls(sql, catalog: catalog), "\(sql)") + } + #expect(Self.calls( + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS final > 1)", catalog: catalog + )) + } + + @Test("A VALUES row, a MATCH_RECOGNIZE condition and a PARTITION FOR value are read") + func rowValuesAndConditions() { + #expect(Self.calls("SELECT * FROM (VALUES (wipe)) t (a)")) + #expect(Self.calls("SELECT * FROM orders, (VALUES (purge)) v (a)")) + #expect(!Self.calls("SELECT a FROM (VALUES (1)) t (a)")) + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (PATTERN (x) DEFINE x AS wipe = 1)")) + #expect(!Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (PATTERN (x) DEFINE x AS purge = 100)")) + #expect(!Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (PATTERN (up+) DEFINE up AS up.purge > 1)")) + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (PATTERN (up+) DEFINE up AS other.purge > 1)")) + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (MEASURES wipe AS w PATTERN (x) DEFINE x AS 1 = 1)")) + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (MEASURES x.id w, wipe m PATTERN (x) DEFINE x AS 1 = 1)")) + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (PARTITION BY wipe PATTERN (x) DEFINE x AS 1 = 1)")) + // Measured on Oracle 23ai: a section word as an alias or a column opens no section, and WIPE runs. + #expect(Self.calls( + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES x.id AS subset, wipe AS w PATTERN (x) DEFINE x AS 1 = 1)" + )) + #expect(Self.calls( + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES x.id AS define, wipe AS w PATTERN (x) DEFINE x AS 1 = 1)" + )) + #expect(Self.calls( + "SELECT * FROM (SELECT id, 1 AS subset FROM orders) MATCH_RECOGNIZE (ORDER BY id MEASURES x.subset + wipe AS m " + + "PATTERN (x) DEFINE x AS 1 = 1)" + )) + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (MEASURES subset + wipe AS m PATTERN (x) DEFINE x AS 1 = 1)")) + // Out of Oracle's order, so the clause is read whole, a name after AS included. + #expect(Self.calls("SELECT * FROM orders MATCH_RECOGNIZE (DEFINE x AS wipe = 1 PATTERN (x))")) + #expect(Self.calls("SELECT * FROM orders, dual PIVOT (MAX(purge) FOR dummy IN ('X' AS x))")) + #expect(!Self.calls("SELECT * FROM orders, dual PIVOT (MAX(dummy) FOR dummy IN ('X' AS x)) WHERE purge = 1")) + #expect(Self.calls("SELECT * FROM orders PARTITION FOR (wipe)")) + } + + @Test("Dameng matches a function in any case; Oracle only as stored") + func damengFoldsFunctionCase() { + let lower = SQLCallableCatalog(functions: ["wipe"], roots: []) + #expect(Self.calls("SELECT wipe FROM dual", "Dameng", catalog: lower)) + #expect(!Self.calls("SELECT wipe FROM dual", "Oracle", catalog: lower)) + } + + @Test("A name holding a backslash is never written into the columns query") + func backslashNamesAreNotWritten() { + #expect(SQLCallableCatalog.columnsQuery(for: [SQLTableName(owner: nil, name: "A\\B")]) == nil) + } + + @Test("Dameng's TOP count and LIMIT do not hide a name") + func damengTopAndLimit() { + #expect(Self.calls("SELECT TOP 5 wipe FROM dual", "Dameng")) + #expect(Self.calls("SELECT TOP (5) PERCENT wipe FROM dual", "Dameng")) + #expect(!Self.calls("SELECT orders.purge FROM orders LIMIT 10", "Dameng")) + } + + @Test("Only a name the catalog holds asks for columns") + func columnsOnlyOnAHit() { + let reading = Self.reading("SELECT purge FROM orders") + #expect(!reading.namesNothing) + #expect(reading.tablesToResolve(Self.catalog) == [Self.orders]) + #expect(reading.tablesToResolve(SQLCallableCatalog(functions: ["OTHER"], roots: [])).isEmpty) + #expect(Self.reading("SELECT 1 FROM dual").namesNothing) + #expect(Self.reading("UPDATE orders SET purge = wipe").namesNothing) + } + + @Test("A table whose columns nobody read counts a matching name as a call") + func unknownColumnsCall() { + #expect(Self.reading("SELECT purge FROM orders").callsFunction(Self.catalog, columns: [:])) + } + + @Test("Catalog rows read into functions and roots") + func catalogRows() { + let catalog = SQLCallableCatalog(rows: [["F", "PURGE"], ["R", "PK"], ["F", nil], ["X", "Y"], ["F"]]) + #expect(catalog == SQLCallableCatalog(functions: ["PURGE"], roots: ["PK"])) + } + + @Test("The first rank that resolves a table wins, and a remote one has no columns") + func columnRows() { + let tables = [Self.orders, Self.dual, SQLTableName(owner: nil, name: "LINKED")] + let rows: [[String?]] = [ + ["0", "3", nil, "OTHER"], + ["0", "1", nil, "ID"], + ["0", "1", nil, "PURGE"], + ["1", "3", nil, "DUMMY"], + ["2", "2", "REMOTE", nil], + ] + let columns = SQLCallableCatalog.columns(fromRows: rows, tables: tables) + #expect(columns == [Self.orders: ["ID", "PURGE"], Self.dual: ["DUMMY"]]) + } + + @Test("The columns query writes each name as a literal and asks public synonyms only for an unowned name") + func columnsQuery() throws { + let query = try #require(SQLCallableCatalog.columnsQuery(for: [ + SQLTableName(owner: nil, name: "O'NEIL"), SQLTableName(owner: "APP", name: "T"), + ])) + #expect(query.contains("'O''NEIL'")) + #expect(query.contains("o.OWNER = 'APP'")) + #expect(query.components(separatedBy: "s.OWNER = 'PUBLIC'").count == 2) + #expect(SQLCallableCatalog.columnsQuery(for: [SQLTableName(owner: nil, name: "A\u{0}B")]) == nil) + #expect(SQLCallableCatalog.columnsQuery(for: []) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCallableCatalogCacheTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCallableCatalogCacheTests.swift new file mode 100644 index 0000000000..5c2f53df50 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCallableCatalogCacheTests.swift @@ -0,0 +1,103 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLCallableCatalogCacheTests { + private struct Unreachable: Error {} + + // Answers the two catalog queries the way Oracle 23ai answered them for the probe schema. + private final class CatalogServer: @unchecked Sendable { + private let lock = NSLock() + private var queries: [String] = [] + private var failing = false + + var catalogReads: Int { lock.withLock { queries.filter { $0 == SQLCallableCatalog.query }.count } } + var columnReads: Int { lock.withLock { queries.filter { $0 != SQLCallableCatalog.query }.count } } + + func fail() { lock.withLock { failing = true } } + + func read(_ query: String) throws -> [[String?]] { + let failing = lock.withLock { + queries.append(query) + return self.failing + } + if failing { throw Unreachable() } + if query == SQLCallableCatalog.query { + return [["F", "PURGE"], ["F", "WIPE"], ["R", "PK"]] + } + return [["0", "1", nil, "ID"], ["0", "1", nil, "PURGE"]] + } + } + + private static func reading(_ sql: String) -> SQLBareNameReading { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Oracle", declared: nil, session: nil) + return SQLBareNameCalls.read(sql, databaseTypeId: "Oracle", readings: readings) + } + + @Test("A column of the table runs, a function name does not, and the catalog is read once") + func columnsAndFunctions() async throws { + let server = CatalogServer() + let cache = SQLCallableCatalogCache() + let connection = UUID() + + let column = try await cache.callsFunction( + Self.reading("SELECT purge FROM orders"), connectionId: connection, schema: nil, read: server.read + ) + let call = try await cache.callsFunction( + Self.reading("SELECT wipe FROM orders"), connectionId: connection, schema: nil, read: server.read + ) + + #expect(!column) + #expect(call) + #expect(server.catalogReads == 1) + #expect(server.columnReads == 1) + } + + @Test("A query naming no function asks for no columns") + func noHitNoColumns() async throws { + let server = CatalogServer() + let calls = try await SQLCallableCatalogCache().callsFunction( + Self.reading("SELECT id FROM orders"), connectionId: UUID(), schema: nil, read: server.read + ) + + #expect(!calls) + #expect(server.columnReads == 0) + } + + @Test("The catalog is kept per schema for its lifetime, and read again after it") + func lifetimeAndSchema() async throws { + let server = CatalogServer() + let kept = SQLCallableCatalogCache(lifetime: .seconds(3_600)) + let expired = SQLCallableCatalogCache(lifetime: .zero) + let connection = UUID() + let reading = Self.reading("SELECT id FROM orders") + + _ = try await kept.callsFunction(reading, connectionId: connection, schema: nil, read: server.read) + _ = try await kept.callsFunction(reading, connectionId: connection, schema: nil, read: server.read) + #expect(server.catalogReads == 1) + _ = try await kept.callsFunction(reading, connectionId: connection, schema: "HR", read: server.read) + #expect(server.catalogReads == 2) + + _ = try await expired.callsFunction(reading, connectionId: connection, schema: nil, read: server.read) + _ = try await expired.callsFunction(reading, connectionId: connection, schema: nil, read: server.read) + #expect(server.catalogReads == 4) + } + + @Test("A dotted call is refused without reading the catalog, and a failed read throws") + func outrightAndFailure() async throws { + let server = CatalogServer() + let cache = SQLCallableCatalogCache() + + #expect(try await cache.callsFunction( + Self.reading("SELECT pk.f FROM dual"), connectionId: UUID(), schema: nil, read: server.read + )) + #expect(server.catalogReads == 0) + + server.fail() + await #expect(throws: Unreachable.self) { + try await cache.callsFunction( + Self.reading("SELECT id FROM orders"), connectionId: UUID(), schema: nil, read: server.read + ) + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCatalogEffectClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCatalogEffectClassifierTests.swift index 1e65ebbaf7..1e235f5f0f 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCatalogEffectClassifierTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLCatalogEffectClassifierTests.swift @@ -21,6 +21,17 @@ struct SQLCatalogEffectClassifierTests { #expect(kinds("CREATE TABLE copy AS SELECT * FROM t") == .tables) } + @Test("An event changes the triggers and the server objects, an extension reaches the server objects") + func serverObjects() { + #expect(kinds("CREATE EVENT nightly ON SCHEDULE EVERY 1 DAY DO DELETE FROM log", "MySQL") == [.triggers, .serverObjects]) + #expect(kinds("ALTER EVENT nightly DISABLE", "MySQL") == [.triggers, .serverObjects]) + #expect(kinds("DROP EVENT IF EXISTS nightly", "MySQL") == [.triggers, .serverObjects]) + #expect(kinds("CREATE EXTENSION IF NOT EXISTS pgcrypto").contains(.serverObjects)) + #expect(kinds("DROP EXTENSION pgcrypto").contains(.serverObjects)) + #expect(SQLCatalogObjectKinds.objects.contains(.serverObjects)) + #expect(!kinds("CREATE TABLE t (id int)").contains(.serverObjects)) + } + @Test("Row work and reads leave the catalog alone, except SELECT INTO") func rowsAndReads() { #expect(kinds("INSERT INTO t VALUES (1)").isEmpty) @@ -45,6 +56,17 @@ struct SQLCatalogEffectClassifierTests { #expect(SQLCatalogEffectClassifier.effect(of: "rollback", grammar: grammar("MySQL")).endsTransaction) #expect(SQLCatalogEffectClassifier.effect(of: "ROLLBACK TO SAVEPOINT a", grammar: grammar("PostgreSQL")) == .none) #expect(SQLCatalogEffectClassifier.effect(of: "BEGIN", grammar: grammar("PostgreSQL")) == .none) + #expect(SQLCatalogEffectClassifier.effect(of: "ET", grammar: grammar("Teradata")).endsTransaction) + } + + @Test("BT and ET are transaction control in Teradata's grammar alone") + func transactionShortFormsAreTeradatas() { + #expect(SQLCatalogEffectClassifier.effect(of: "et", grammar: grammar("Teradata")).endsTransaction) + #expect(SQLCatalogEffectClassifier.effect(of: "BT", grammar: grammar("Teradata")) == .none) + for engine in ["PostgreSQL", "MySQL", "SQL Server", "Oracle", "Nonesuch"] { + #expect(!SQLCatalogEffectClassifier.effect(of: "ET", grammar: grammar(engine)).endsTransaction, "\(engine)") + } + #expect(SQLLexicalProfile.everyKnownReading.allSatisfy { !$0.contains(.transactionShortForms) }) } @Test("Code the text cannot show refreshes every kind") diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDataStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDataStatementTests.swift new file mode 100644 index 0000000000..5ea34c4c7d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDataStatementTests.swift @@ -0,0 +1,216 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLDataStatementTests { + private static func isData(_ sql: String, _ engine: String = "PostgreSQL") -> Bool { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + return SQLStatementClassifier.isDataStatement(sql, databaseTypeId: engine, readings: readings) + } + + @Test("A read or a row change is a data statement", arguments: [ + ("SELECT * FROM orders WHERE id = 1", "PostgreSQL"), + ("INSERT INTO orders (id, note) VALUES (1, 'a')", "PostgreSQL"), + ("UPDATE orders SET paid = true WHERE id = 1", "PostgreSQL"), + ("DELETE FROM orders WHERE id = 1", "PostgreSQL"), + ("MERGE INTO orders o USING staged s ON o.id = s.id WHEN MATCHED THEN UPDATE SET note = s.note", "PostgreSQL"), + ("WITH gone AS (DELETE FROM orders WHERE id = 1 RETURNING *) INSERT INTO archive SELECT * FROM gone", "PostgreSQL"), + ("WITH recent AS (SELECT id FROM orders) UPDATE orders SET seen = true WHERE id IN (SELECT id FROM recent)", + "PostgreSQL"), + ("EXPLAIN ANALYZE UPDATE orders SET paid = true WHERE id = 1", "PostgreSQL"), + ("EXPLAIN SELECT * FROM orders", "PostgreSQL"), + ("INSERT INTO orders (id) VALUES (nextval('orders_id_seq'))", "PostgreSQL"), + ("SELECT nextval('orders_id_seq')", "PostgreSQL"), + ("SHOW search_path", "PostgreSQL"), + ("REPLACE INTO orders (id) VALUES (1)", "MySQL"), + ("INSERT INTO orders (id) VALUES (1) ON DUPLICATE KEY UPDATE note = 'b'", "MySQL"), + ("UPSERT INTO orders (id, note) VALUES (1, 'a')", "CockroachDB"), + ("INSERT OR REPLACE INTO orders (id) VALUES (1)", "SQLite"), + ("UPDATE statistics SET hits = hits + 1 WHERE id = 1", "PostgreSQL"), + ("BEGIN BATCH INSERT INTO ks.t (id) VALUES (1); UPDATE ks.t SET v = 'b' WHERE id = 2; APPLY BATCH", "Cassandra"), + ("REPLACE orders VALUES (1)", "MySQL"), + ("SELECT * FROM pg_settings WHERE name = 'search_path'", "PostgreSQL"), + ("ALTER TABLE shop.orders UPDATE note = 'a' WHERE id = 1", "ClickHouse"), + ("ALTER TABLE orders ON CLUSTER main DELETE WHERE id = 1", "ClickHouse"), + ]) + func rowWorkIsData(sql: String, engine: String) { + #expect(Self.isData(sql, engine)) + } + + @Test("A definition, a grant, a session or transaction statement and a call are not", arguments: [ + ("GRANT SELECT ON orders TO app", "PostgreSQL"), + ("REVOKE ALL ON orders FROM app", "PostgreSQL"), + ("ALTER ROLE app_user WITH SUPERUSER", "PostgreSQL"), + ("SET session_replication_role = replica", "PostgreSQL"), + ("RESET ALL", "PostgreSQL"), + ("CREATE TABLE notes (id int)", "PostgreSQL"), + ("ALTER TABLE orders ADD COLUMN note text", "PostgreSQL"), + ("COMMENT ON TABLE orders IS 'x'", "PostgreSQL"), + ("BEGIN", "PostgreSQL"), + ("COMMIT", "PostgreSQL"), + ("CALL purge_orders()", "PostgreSQL"), + ("LOCK TABLE orders", "PostgreSQL"), + ("VACUUM orders", "PostgreSQL"), + ("ANALYZE orders", "PostgreSQL"), + ("REFRESH MATERIALIZED VIEW totals", "PostgreSQL"), + ("LISTEN orders_changed", "PostgreSQL"), + ("PREPARE purge AS DELETE FROM orders WHERE id = $1", "PostgreSQL"), + ("SELECT set_config('role', 'admin', false)", "PostgreSQL"), + ("SELECT pg_terminate_backend(42)", "PostgreSQL"), + ("SELECT pg_advisory_lock(1)", "PostgreSQL"), + ("SELECT * INTO orders_copy FROM orders", "PostgreSQL"), + ("EXPLAIN ANALYZE CREATE TABLE copy AS SELECT * FROM orders", "PostgreSQL"), + ("WITH x AS (SELECT 1) SELECT * INTO copy FROM x", "PostgreSQL"), + ("SELECT 1; GRANT ALL ON orders TO app", "PostgreSQL"), + ("USE shop", "MySQL"), + ("SET @limit = 10", "MySQL"), + ("SELECT COUNT(*) INTO @total FROM orders", "MySQL"), + ("LOCK TABLES orders WRITE", "MySQL"), + ("UPDATE STATISTICS dbo.orders", "SQL Server"), + ("INSERT INTO dbo.orders EXEC dbo.load_orders", "SQL Server"), + ("EXEC sp_configure 'show advanced options', 1", "SQL Server"), + ("PRAGMA foreign_keys = OFF", "SQLite"), + ("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'", "Oracle"), + ("BEGIN purge_orders; END;", "Oracle"), + ("OPTIMIZE TABLE orders FINAL", "ClickHouse"), + ("SYSTEM FLUSH LOGS", "ClickHouse"), + ("DEFINE TABLE person SCHEMAFULL", "SurrealDB"), + ("USE ks", "Cassandra"), + ("GRANT SELECT ON ks.t TO app", "Cassandra"), + ("REPLACE VIEW totals AS SELECT 1", "Teradata"), + ("ALTER TABLE orders UPDATE note = 'a' WHERE id = 1, DROP COLUMN note", "ClickHouse"), + ("ALTER TABLE orders ADD COLUMN note String", "ClickHouse"), + ("ALTER TABLE orders UPDATE note = 'a' WHERE id = 1", "MySQL"), + ]) + func schemaPrivilegeAndSessionAreNot(sql: String, engine: String) { + #expect(!Self.isData(sql, engine)) + } + + @Test("A row change on a table the server acts on is not", arguments: [ + ("UPDATE pg_settings SET setting = 'replica' WHERE name = 'session_replication_role'", "PostgreSQL"), + ("UPDATE pg_catalog.pg_authid SET rolsuper = true WHERE rolname = 'app'", "PostgreSQL"), + ("INSERT INTO \"pg_catalog\".\"pg_authid\" (rolname) VALUES ('x')", "PostgreSQL"), + ("UPDATE sys_settings SET setting = 'x' WHERE name = 'search_path'", "KingbaseES"), + ("UPSERT INTO system.users (username) VALUES ('x')", "CockroachDB"), + ("UPDATE mysql.user SET Super_priv = 'Y' WHERE User = 'app'", "MySQL"), + ("UPDATE `performance_schema`.`setup_consumers` SET ENABLED = 'NO'", "MariaDB"), + ("UPDATE orders SET note = set_config('role', 'admin', false) WHERE id = 1", "PostgreSQL"), + ("INSERT INTO log SELECT pg_terminate_backend(pid) FROM pg_stat_activity", "PostgreSQL"), + ("UPDATE U&\"pg\\005fsettings\" SET setting = 'x' WHERE name = 'search_path'", "PostgreSQL"), + ("INSERT INTO msdb.dbo.sysjobsteps (job_id, command) VALUES (1, 'x')", "SQL Server"), + ("UPDATE system.help SET text = 'x'", "Oracle"), + ]) + func systemRelationsAreNot(sql: String, engine: String) { + #expect(!Self.isData(sql, engine)) + } + + @Test("A scope in the engine's own schemas is a system scope, in any case, and only on that engine") + func systemScopesFollowTheEngine() { + #expect(SQLStatementClassifier.isSystemScope(database: "mysql", schema: nil, databaseTypeId: "MariaDB")) + #expect(SQLStatementClassifier.isSystemScope(database: "MSDB", schema: "dbo", databaseTypeId: "SQL Server")) + #expect(SQLStatementClassifier.isSystemScope(database: "app", schema: "pg_catalog", databaseTypeId: "PostgreSQL")) + #expect(!SQLStatementClassifier.isSystemScope(database: "mysql", schema: nil, databaseTypeId: "PostgreSQL")) + #expect(!SQLStatementClassifier.isSystemScope(database: "shop", schema: nil, databaseTypeId: "MySQL")) + #expect(!SQLStatementClassifier.isSystemScope(database: "ORCL", schema: "SYSTEM", databaseTypeId: "Oracle")) + #expect(SQLStatementClassifier.isSystemScope(database: "ORCL", schema: "sys", databaseTypeId: "Oracle")) + } + + @Test("A call that happens to start like a catalog name is not a catalog relation") + func catalogNamedCallIsData() { + #expect(Self.isData("UPDATE orders SET pid = pg_backend_pid() WHERE id = 1")) + } + + @Test("A request form with no data reading is refused, and its own reading decides") + func requestFormsAnswerForThemselves() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + let tagged: SQLStatementClassifier.RequestClassification = { statement, _ in + statement.hasPrefix("REQ ") ? .write : nil + } + #expect(!SQLStatementClassifier.isDataStatement( + "REQ CreateTable", databaseTypeId: "PostgreSQL", readings: readings, request: tagged + )) + #expect(SQLStatementClassifier.isDataStatement( + "REQ PutItem", databaseTypeId: "PostgreSQL", readings: readings, request: tagged, + requestIsData: { statement, _ in statement.hasPrefix("REQ ") ? statement.hasSuffix("PutItem") : nil } + )) + } + + private static let surrealRowChanges: Set = ["CREATE", "INSERT", "UPDATE", "UPSERT", "DELETE", "RELATE"] + + private static func isSurrealData(_ sql: String) -> Bool { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "SurrealDB", declared: nil, session: nil) + return SQLStatementClassifier.isDataStatement( + sql, databaseTypeId: "SurrealDB", readings: readings, rowChanges: surrealRowChanges + ) + } + + // Measured on SurrealDB 2.7.0 and 3.3.5. + @Test("SurrealQL record writes are data", arguments: [ + "SELECT * FROM person LIMIT 10", + "CREATE person:one SET name = 'a'", + "CREATE person CONTENT { name: 'a' }", + "INSERT INTO person { name: 'b' }", + "RELATE person:one->knows->person:two", + "UPDATE person:one SET name = 'c'", + "UPSERT person:three SET name = 'd'", + "DELETE person:three", + "INFO FOR DB", + "SELECT * FROM person WHERE seen = time::now()", + "SELECT sleep FROM person", + ]) + func surrealRecordWritesAreData(sql: String) { + #expect(Self.isSurrealData(sql)) + } + + @Test("SurrealQL definitions, sessions, transactions, nested statements and stored bodies are not", arguments: [ + "DEFINE TABLE person SCHEMAFULL", + "DEFINE USER bob ON DATABASE PASSWORD 'x' ROLES OWNER", + "ACCESS acc ON DATABASE GRANT FOR USER bob", + "ALTER TABLE person COMMENT 'x'", + "REBUILD INDEX i ON person", + "USE NS a DB b", + "LET $x = 1", + "BEGIN TRANSACTION", + "OPTION IMPORT", + "SLEEP 10s", + "RETURN (DEFINE USER x ON DATABASE PASSWORD 'p' ROLES OWNER)", + "SELECT * FROM person WHERE name = (DEFINE TABLE t)", + "SELECT fn::mk() FROM ONLY person:one", + "RETURN api::invoke('/x')", + ]) + func surrealDefinitionsAreNot(sql: String) { + #expect(!Self.isSurrealData(sql)) + } + + @Test("A CQL row change in a system keyspace is not data, and a read of one is", arguments: ["Cassandra", "ScyllaDB"]) + func cassandraSystemKeyspaces(engine: String) { + for sql in [ + "UPDATE system_auth.roles SET is_superuser = true WHERE role = 'app'", + "INSERT INTO system_distributed.partition_denylist (ks_name, table_name, key) VALUES ('ks', 't', 0x00)", + "DELETE FROM system_views.roles_cache_keys WHERE role = 'app'", + "UPDATE system.config SET value = 'false' WHERE name = 'enable_cql_config_updates'", + "BEGIN BATCH UPDATE system_auth.roles SET can_login = true WHERE role = 'x'; APPLY BATCH", + ] { + #expect(!Self.isData(sql, engine)) + } + #expect(Self.isData("SELECT * FROM system_auth.roles", engine)) + #expect(Self.isData("UPDATE ks.users SET name = 'b' WHERE id = 1", engine)) + #expect(SQLStatementClassifier.isSystemScope(database: "system_auth", schema: nil, databaseTypeId: engine)) + #expect(!SQLStatementClassifier.isSystemScope(database: "shop", schema: nil, databaseTypeId: engine)) + } + + @Test("ScyllaDB's audit keyspace holds its audit log, so a row change there is not data; on Cassandra it is a user's") + func scyllaAuditKeyspace() { + for sql in [ + "DELETE FROM audit.audit_log WHERE date = '2026-11-13' AND node = '10.0.0.1'", + "UPDATE \"audit\".audit_log SET operation = 'x' WHERE date = '2026-11-13' AND node = '10.0.0.1'", + "INSERT INTO AUDIT.audit_log (date, node) VALUES ('2026-11-13', '10.0.0.1')", + ] { + #expect(!Self.isData(sql, "ScyllaDB")) + #expect(Self.isData(sql, "Cassandra")) + } + #expect(Self.isData("SELECT * FROM audit.audit_log", "ScyllaDB")) + #expect(SQLStatementClassifier.isSystemScope(database: "audit", schema: nil, databaseTypeId: "ScyllaDB")) + #expect(!SQLStatementClassifier.isSystemScope(database: "audit", schema: nil, databaseTypeId: "Cassandra")) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDelimiterDirectiveTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDelimiterDirectiveTests.swift new file mode 100644 index 0000000000..4495d5c4c9 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDelimiterDirectiveTests.swift @@ -0,0 +1,132 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// The editor reads a DELIMITER line as the mysql client does: it runs nothing and changes what ends a statement. +struct SQLDelimiterDirectiveTests { + private static let mySQL = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil).execution + + private func sent(_ sql: String) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: Self.mySQL).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.mySQL) + } + + private func delimiter(of line: String) -> String? { + let text = line as NSString + return SQLDelimiterDirective.line(startingAt: 0, in: text, length: text.length)?.delimiter + } + + @Test("A routine between DELIMITER lines is one statement, and the lines run nothing") + func routineBetweenDelimiterLines() { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nSELECT 3;" + #expect(sent(sql) == ["CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END", "SELECT 3"]) + #expect(atCursor(sql, on: "SELECT 2") == "CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END") + #expect(atCursor(sql, on: "DELIMITER ;").isEmpty) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.mySQL).count == 2) + } + + @Test("Run Current sends a delimited routine on its own, without the delimiter") + func runCurrentLeavesTheDelimiterOut() { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; END//\nDELIMITER ;" + let statements = SQLStatementScanner.navigableStatements(in: sql, grammar: Self.mySQL) + #expect(statements.count == 1) + let fragment = (sql as NSString).substring(with: statements[0].contentRange) + #expect(fragment == "CREATE PROCEDURE p() BEGIN SELECT 1; END") + #expect(sent(fragment) == [fragment]) + } + + @Test("mysqldump's routine, trigger and event forms each arrive whole", arguments: [ + ( + "DELIMITER ;;\nCREATE DEFINER=`root`@`localhost` PROCEDURE `proc1`(IN p INT)\nBEGIN SELECT p; SELECT 'a;;b'; END\n;;\nDELIMITER ;\n", + "CREATE DEFINER=`root`@`localhost` PROCEDURE `proc1`(IN p INT)\nBEGIN SELECT p; SELECT 'a;;b'; END" + ), + ( + "DELIMITER ;;\nCREATE DEFINER=`root`@`localhost` PROCEDURE `p`()\nBEGIN\n SELECT 1;\nEND ;;\nDELIMITER ;\n", + "CREATE DEFINER=`root`@`localhost` PROCEDURE `p`()\nBEGIN\n SELECT 1;\nEND" + ), + ( + "DELIMITER ;;\n/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER trg1 BEFORE INSERT ON t1 FOR EACH ROW BEGIN SET NEW.note = 'x;y'; INSERT INTO t2 VALUES (NEW.id); END \n*/;;\nDELIMITER ;\n", + "/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER trg1 BEFORE INSERT ON t1 FOR EACH ROW BEGIN SET NEW.note = 'x;y'; INSERT INTO t2 VALUES (NEW.id); END \n*/" + ), + ( + "DELIMITER ;;\n/*!50106 CREATE*/ /*!50106 EVENT `ev1` ON SCHEDULE EVERY 1 DAY DO BEGIN INSERT INTO t2 VALUES (1); INSERT INTO t2 VALUES (2); END \n*/ ;;\nDELIMITER ;\n", + "/*!50106 CREATE*/ /*!50106 EVENT `ev1` ON SCHEDULE EVERY 1 DAY DO BEGIN INSERT INTO t2 VALUES (1); INSERT INTO t2 VALUES (2); END \n*/" + ), + ]) + func mysqldumpForms(sql: String, expected: String) { + #expect(sent(sql) == [expected]) + } + + @Test("The delimiter ends a statement anywhere in code, inside a word too, matched case for case") + func delimiterMatching() { + #expect(sent("DELIMITER $$\nCREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END$$\nDELIMITER ;\nSELECT f();") == [ + "CREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END", "SELECT f()", + ]) + #expect(sent("DELIMITER X\nCREATE TABLE ix (a INT, INDEX(a))X\nDELIMITER ;\nSELECT 1;") == [ + "CREATE TABLE ix (a INT, INDE", "(a))", "SELECT 1", + ]) + #expect(sent("DELIMITER GO\nSELECT 'GO' go\nGO") == ["SELECT 'GO' go"]) + #expect(sent("DELIMITER //\nSELECT '//', `a//b` -- //\n# //\n/* // */ FROM t//") == [ + "SELECT '//', `a//b` -- //\n# //\n/* // */ FROM t", + ]) + } + + @Test("A ; the server would split at still ends a statement under a custom delimiter") + func semicolonStillSplitsForTheGate() { + #expect(sent("DELIMITER //\nSELECT 1; DROP TABLE t//") == ["SELECT 1", "DROP TABLE t"]) + let readings = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify( + "DELIMITER //\nSELECT 1; DROP TABLE t//", databaseTypeId: "MySQL", readings: readings + ).tier == .destructive) + } + + @Test("A DELIMITER line counts only as the first code on a line where no statement is pending") + func whereTheDirectiveCounts() { + #expect(sent(" delimiter //\nSELECT 1//\n-- note\nDeLiMiTeR ;\nSELECT 2;") == ["SELECT 1", "SELECT 2"]) + #expect(sent("SELECT 1\nDELIMITER //\nSELECT 2;") == ["SELECT 1\nDELIMITER //\nSELECT 2"]) + #expect(sent("SELECT 1; DELIMITER //\nSELECT 2;") == ["SELECT 1", "DELIMITER //\nSELECT 2"]) + #expect(sent("/* c */ DELIMITER //\nSELECT 2;") == ["/* c */ DELIMITER //\nSELECT 2"]) + #expect(sent("DELIMITER//\nSELECT 2;") == ["DELIMITER//\nSELECT 2"]) + #expect(sent("SELECT 'a\nDELIMITER //\n';") == ["SELECT 'a\nDELIMITER //\n'"]) + #expect(sent("DELIMITER //\r\nSELECT 1//\r\nDELIMITER ;\r\nSELECT 2;") == ["SELECT 1", "SELECT 2"]) + } + + @Test("The argument is read as mysql's get_arg reads it", arguments: [ + ("DELIMITER //", "//"), + ("delimiter $$ ", "$$"), + ("DELIMITER //\t", "//\t"), + ("DELIMITER //;", "//;"), + ("DELIMITER // ;", "//"), + ("DELIMITER ;;", ";;"), + ("DELIMITER // SELECT 99", "//"), + ("DELIMITER // -- set", "//"), + ("DELIMITER 'a b'", "a b"), + ("DELIMITER \"$$\"", "$$"), + ("DELIMITER `|`", "|"), + ("DELIMITER '$$", "$$"), + ("DELIMITER a\\b", "ab"), + ("DELIMITER //\r", "//"), + ]) + func argument(line: String, expected: String) { + #expect(delimiter(of: line) == expected) + } + + @Test("An empty argument or one holding a backslash is refused and the delimiter stays", arguments: [ + "DELIMITER", "DELIMITER ", "DELIMITER ''", "DELIMITER \\\\", "DELIMITER 'a\\b'", + ]) + func refusedArgument(line: String) { + #expect(delimiter(of: line) == nil) + #expect(sent("DELIMITER //\n\(line)\nSELECT 1//") == ["SELECT 1"]) + } + + @Test("Engines without the mysql client read DELIMITER as a word", arguments: ["PostgreSQL", "SQL Server", "SQLite"]) + func otherEngines(typeId: String) { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "DELIMITER //\nSELECT 1;", grammar: grammar) + #expect(statements.map(\.sql) == ["DELIMITER //\nSELECT 1"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLFileNamedAsTableTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLFileNamedAsTableTests.swift index dc69bc15ba..4b9532357e 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLFileNamedAsTableTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLFileNamedAsTableTests.swift @@ -66,6 +66,30 @@ struct SQLFileNamedAsTableTests { #expect(namesFile(sql)) } + @Test("A table function DuckDB does not keep inside the database counts as one", arguments: [ + "SELECT * FROM quack_serve('x')", + "SELECT * FROM odbc_query('DSN=x', 'DROP TABLE t')", + "SELECT * FROM notes, cloudformation_create_stack('x')", + "SELECT * FROM notes JOIN avro_metadata('x') ON true", + "SELECT * FROM notes, LATERAL quack_query('x')", + "SELECT * FROM (FROM quack_serve('x'))", + "SELECT * FROM system.main.quack_serve('x')", + "SELECT * FROM \"quack_serve\"('x')", + ]) + func unknownTableFunctions(sql: String) { + #expect(namesFile(sql)) + } + + @Test("A dotted name ending in a file extension is a file in the working folder", arguments: [ + "SELECT * FROM finances.csv", + "SELECT * FROM notes.\"csv\"", + "SELECT * FROM backup.duckdb", + "SELECT * FROM notes, data.parquet", + ]) + func dottedFileNames(sql: String) { + #expect(namesFile(sql)) + } + @Test("A FROM that runs on from a number or an operator still lists tables", arguments: [ "SELECT 1FROM 'payroll.csv'", "SELECT 1e5FROM 'payroll.csv'", @@ -122,6 +146,11 @@ struct SQLFileNamedAsTableTests { "COMMENT ON TABLE notes IS 'a.csv'", "CREATE TABLE notes (path VARCHAR DEFAULT 'a.csv')", "SELECT _from, _desc, 'a.csv' FROM notes", + "SELECT * FROM main.range(1, 10)", + "SELECT * FROM duckdb_tables()", + "PIVOT notes ON year USING sum(salary)", + "SELECT * FROM notes JOIN tags ON f(tags.id) = notes.id", + "SELECT * FROM notes.csv_rows", ]) func valuesAndTableNames(sql: String) { #expect(!namesFile(sql)) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift index d855aeb433..04cfded890 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift @@ -115,6 +115,79 @@ struct SQLNonCodeSpanTests { SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil).execution } + private static func statements(_ sql: String, _ engine: String) -> Int { + SQLStatementScanner.executableStatements(in: sql, grammar: execution(engine)).count + } + + private static func tier(_ sql: String, _ engine: String) -> StatementTier { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + return SQLStatementClassifier.classify(sql, databaseTypeId: engine, readings: readings).tier + } + + // Measured on PostgreSQL 17.11: with a domain named after each glued name, the server ran the DROP. + @Test( + "A prefix glued to a name by $ or by any non-ASCII unit is part of the name on PostgreSQL", + arguments: ["$", "\u{E9}", "\u{00A0}", "\u{3000}", "\u{200B}", "\u{2060}", "\u{FEFF}", "\u{2019}", "\u{FF0C}"] + ) + func prefixAfterANameOnPostgreSQL(glue: String) { + let sql = "SELECT x\(glue)e'\\'; DROP TABLE t; --'" + for engine in ["PostgreSQL", "CockroachDB", "Redshift"] { + #expect(Self.statements(sql, engine) == 2, "\(engine)") + #expect(Self.tier(sql, engine) == .destructive, "\(engine)") + } + #expect(Self.statements("SELECT x\(glue)b'\\'; DROP TABLE t; --'", "CockroachDB") == 2) + } + + // Measured on DuckDB 1.5.4 and 2.0.0: these spaces end `varchar` and the DROP ran; every other unit kept it a name. + @Test("DuckDB ends a name at the non-ASCII spaces it blanks before lexing and at no other non-ASCII unit") + func prefixAfterANameOnDuckDB() { + let blanks: [UInt16] = [0x00A0, 0x2000, 0x2005, 0x200A, 0x200B, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF] + for unit in blanks { + let space = String(utf16CodeUnits: [unit], count: 1) + let sql = "SELECT varchar\(space)E'\\''; DROP TABLE t; --'" + #expect(Self.statements(sql, "DuckDB") == 2, "U+\(String(unit, radix: 16))") + #expect(Self.tier(sql, "DuckDB") == .destructive, "U+\(String(unit, radix: 16))") + #expect(Self.statements(sql, "PostgreSQL") == 1, "U+\(String(unit, radix: 16))") + } + for glue in ["$", "\u{E9}", "\u{200C}", "\u{2019}", "\u{FF0C}", "\u{2028}"] { + #expect(Self.statements("SELECT x\(glue)e'\\'; DROP TABLE t; --'", "DuckDB") == 2, "\(glue)") + #expect(Self.statements("SELECT varchar\(glue)E'\\''; DROP TABLE t; --'", "DuckDB") == 1, "\(glue)") + } + } + + // Measured on DuckDB 2.0.0: after these spaces `$$` opened a dollar quote and the DROP ran. + @Test("DuckDB opens a dollar quote after a space it blanks, and PostgreSQL keeps the $ in the name") + func dollarQuoteAfterABlankOnDuckDB() { + for space in ["\u{00A0}", "\u{200B}", "\u{3000}"] { + let sql = "SELECT 1 AS a,\(space)$$'$$; DROP TABLE t; SELECT 'x'" + #expect(Self.statements(sql, "DuckDB") == 3) + #expect(Self.tier(sql, "DuckDB") == .destructive) + #expect(Self.statements(sql, "PostgreSQL") == 1) + } + #expect(Self.statements("SELECT 1 AS a,\u{2019}$$'$$; DROP TABLE t; SELECT 'x'", "DuckDB") == 1) + } + + // Measured on DuckDB 1.5.4 and 2.0.0: its pre-lexing pass kept the space in each text, so the `$$` stayed in the + // name and the DROP ran. + @Test("A gate reads DuckDB both ways, since it keeps a non-ASCII space where its pre-lexing pass sees a quote", arguments: [ + "SELECT 1 AS a$$, 2 AS b\u{200B}$$; DROP TABLE t; --$$", + "/* ' */ SELECT 1 AS a\u{200B}$$; DROP TABLE t; --$$", + "SELECT E'\\'' AS c, 1 AS a\u{200B}$$; DROP TABLE t; --$$", + ]) + func duckDBKeepsSomeSpaces(sql: String) { + #expect(Self.tier(sql, "DuckDB") == .destructive) + } + + @Test("A q'...' glued to a name by $ or # is part of the name where both continue one") + func alternativeQuoteAfterAName() { + for engine in ["Oracle", "Dameng"] { + #expect(end("x$q'[it's]'", at: 2, Self.execution(engine)) == nil, "\(engine)") + #expect(end("x#q'[it's]'", at: 2, Self.execution(engine)) == nil, "\(engine)") + #expect(Self.statements("SELECT x$q'['; DROP TABLE t; --]' FROM dual", engine) == 2, "\(engine)") + } + #expect(end("x#e'\\'' y", at: 2, Self.execution("PostgreSQL")) == 7) + } + @Test("A string after a line break continues a PostgreSQL E'...' with its escapes, as quotecontinue reads it") func escapeStringContinuesAfterALineBreak() { let postgreSQL = Self.execution("PostgreSQL") @@ -125,11 +198,36 @@ struct SQLNonCodeSpanTests { #expect(end("E'x' -- c\n'y' z", postgreSQL) == 13) #expect(end("E'x'\n-- c\n'y' z", postgreSQL) == 13) #expect(end("E'x'\n'y'\n\n'z' w", postgreSQL) == 13) - #expect(end("E'x'\u{00A0}\n\u{3000}'y' z", postgreSQL) == 10) #expect(end("E'x'\u{200C}\n'y' z", postgreSQL) == 4) #expect(SQLNonCodeSpan.span(at: 0, in: "E'x'\r\n'y'\n'z'" as NSString, grammar: postgreSQL)?.newlines == 2) } + // Measured: PostgreSQL 17.11 answers a syntax error at the second string, and DuckDB 1.5.4 returns `xy`. + @Test("A non-ASCII space between the parts of a continued string is a name on PostgreSQL and a blank on DuckDB 1.5") + func nonASCIISpaceBeforeAContinuedString() { + let text = "E'x'\u{00A0}\n\u{3000}'y' z" + for engine in ["PostgreSQL", "CockroachDB", "Redshift", "KingbaseES"] { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + for grammar in readings.all where grammar.contains(.escapeStringPrefix) { + #expect(end(text, grammar) == 4, "\(engine)") + #expect(end("E'x'\n\u{00A0}'y' z", grammar) == 4, "\(engine)") + } + } + let duckDB15 = Self.execution("DuckDB").union(.escapeStringContinuations) + #expect(end(text, duckDB15) == 10) + #expect(end(text, duckDB15.subtracting(.nonASCIISpacesEndNames)) == 4) + } + + // Measured on PostgreSQL 17.11: psql ran the statement after `'\'`. + @Test("Every gate reading sees the statement after a string a non-ASCII space keeps from continuing") + func nonASCIISpaceBeforeAContinuedStringHidesNothing() { + let sql = "SELECT E'a'\u{00A0}\n'\\'; DROP TABLE t; --'" + for engine in ["PostgreSQL", "CockroachDB", "Redshift", "KingbaseES", "DuckDB", "Nonesuch"] { + #expect(Self.tier(sql, engine) == .destructive, "\(engine)") + } + #expect(Self.statements(sql, "PostgreSQL") == 2) + } + @Test("An E'...' ends at its quote without a line break, across a block comment, or on DuckDB") func escapeStringEndsAtItsQuote() { let postgreSQL = Self.execution("PostgreSQL") diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOutsideReachNamesTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOutsideReachNamesTests.swift new file mode 100644 index 0000000000..11a06fa769 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOutsideReachNamesTests.swift @@ -0,0 +1,35 @@ +import Foundation +@testable import TableProSQLGrammar +import Testing + +// The core keeps its own copy of these names for its MCP server, and both test suites compare theirs with one file. +struct SQLOutsideReachNamesTests { + private static let sharedNames = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("core/crates/core/tests/fixtures/outside_reach.json") + + @Test("Each list names what the core lists") + func listsMatchTheSharedNames() throws { + let data = try Data(contentsOf: Self.sharedNames) + let shared = try JSONDecoder().decode([String: [String]].self, from: data) + .mapValues { Set($0.map { $0.uppercased() }) } + let duckDB = SQLStatementTiering.duckDB + let clickHouse = SQLStatementTiering.clickHouse + let listed: [String: Set] = [ + "filesystem_word_runs": Set(SQLStatementTiering.filesystemWordRuns.map { $0.joined(separator: " ") }), + "filesystem_names": SQLStatementTiering.filesystemNames, + "filesystem_or_code_keywords": SQLStatementTiering.filesystemOrCodeKeywords, + "filesystem_functions": SQLStatementTiering.filesystemFunctions, + "duckdb_functions": SQLStatementTiering.functionsReachingOutside[duckDB] ?? [], + "duckdb_families": Set(SQLStatementTiering.functionFamiliesReachingOutside[duckDB] ?? []), + "clickhouse_functions": SQLStatementTiering.functionsReachingOutside[clickHouse] ?? [], + "clickhouse_families": Set(SQLStatementTiering.functionFamiliesReachingOutside[clickHouse] ?? []), + ] + #expect(listed == shared) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift new file mode 100644 index 0000000000..2f9cf1257c --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift @@ -0,0 +1,77 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLParenthesizedSemicolonTests { + private static func grammar(_ typeId: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + } + + private func sent(_ sql: String, _ grammar: SQLLexicalGrammar) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: grammar).map(\.sql) + } + + @Test("A PostgreSQL rule's multi-command action list stays one statement", arguments: [ + "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2))", + "CREATE OR REPLACE RULE r AS ON UPDATE TO t WHERE (NEW.a > 0) DO INSTEAD (UPDATE a SET n = 1; NOTIFY t)", + "CREATE RULE r AS ON DELETE TO t DO (DELETE FROM a; DELETE FROM b)", + ]) + func ruleActionStaysWhole(rule: String) { + let grammar = Self.grammar("PostgreSQL") + let sql = rule + ";\nSELECT 9;" + + #expect(sent(sql, grammar) == [rule, "SELECT 9"]) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: grammar).count == 2) + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("A rule whose action is one command ends at its ;") + func ruleWithoutAnActionList() { + let grammar = Self.grammar("PostgreSQL") + + #expect(sent("CREATE RULE r AS ON INSERT TO t DO ALSO INSERT INTO log VALUES (NEW.id);\nSELECT 2;", grammar).count == 2) + #expect(sent("CREATE RULE r AS ON INSERT TO t DO INSTEAD NOTHING;\nSELECT 2;", grammar).count == 2) + } + + @Test("A stray ( ends at its ; and leaves every statement after it on its own", arguments: [ + "PostgreSQL", "MySQL", "SQLite", "SQL Server", "DuckDB", "ClickHouse", + ]) + func strayParenthesisEndsAtItsSemicolon(typeId: String) { + let grammar = Self.grammar(typeId) + let sql = "SELECT 1;\nSELECT count(id FROM a;\nSELECT 2;\nSELECT 3;" + let caret = (sql as NSString).range(of: "SELECT 3").location + + #expect(sent(sql, grammar) == ["SELECT 1", "SELECT count(id FROM a", "SELECT 2", "SELECT 3"]) + #expect(SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: caret, grammar: grammar) == "SELECT 3") + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("A rule's action list left open holds the rest of the text, which the gate reads as more than one statement") + func openActionListLeavesTheTextInsideABlock() { + let grammar = Self.grammar("PostgreSQL") + let sql = "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1);\nDROP TABLE users;" + + #expect(sent(sql, grammar).count == 1) + #expect(SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("The gate tiers each command a rule's action list holds") + func gateTiersThePieces() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + + #expect(SQLStatementClassifier.deletesEveryRow( + "CREATE RULE r AS ON INSERT TO t DO ALSO (SELECT 1; DELETE FROM audit)", + databaseTypeId: "PostgreSQL", + readings: readings + )) + #expect(SQLStatementClassifier.classify( + "SELECT (1; DROP TABLE users; SELECT 1)", databaseTypeId: "PostgreSQL", readings: readings + ).tier == .destructive) + } + + @Test("Inside a routine body a parenthesis one statement leaves open does not hide the body's END") + func bodyStatementParenthesisDoesNotHideTheEnd() { + let sql = "CREATE PROCEDURE p()\nBEGIN\n SELECT (1;\n SELECT 2;\nEND;\nSELECT 3;" + #expect(sent(sql, Self.grammar("MySQL")).last == "SELECT 3") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift new file mode 100644 index 0000000000..523cb672a4 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift @@ -0,0 +1,197 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// A psql script's COPY rows and pg_dump's \restrict lines are never SQL to the editor. +struct SQLPsqlScriptTests { + private static let postgreSQL = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + + private func sent(_ sql: String, typeId: String = "PostgreSQL") -> [String] { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + return SQLStatementScanner.executableStatements(in: sql, grammar: grammar).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.postgreSQL.execution) + } + + private static let dump = """ + COPY public.t (a, b) FROM stdin; + 1\tit's; here + 2\t-- not a comment + 3\t$$ and /* nothing */ + \\. + + + SELECT 2; + """ + + @Test("The rows after COPY FROM STDIN are no statements, and the COPY is sent without them") + func rowsAreNotStatements() { + #expect(sent(Self.dump) == ["COPY public.t (a, b) FROM stdin", "SELECT 2"]) + #expect(sent("COPY t FROM stdin;\nx\tDROP TABLE users;\n") == ["COPY t FROM stdin"]) + #expect(sent("COPY t FROM stdin;", typeId: "CockroachDB") == ["COPY t FROM stdin"]) + } + + @Test("Run Current anywhere in the rows stands for the COPY") + func caretInTheRows() { + let sql = "SELECT 0;\n\(Self.dump)" + #expect(atCursor(sql, on: "2\t--") == "COPY public.t (a, b) FROM stdin") + #expect(atCursor(sql, on: "\\.") == "COPY public.t (a, b) FROM stdin") + #expect(atCursor(sql, on: "SELECT 2") == "SELECT 2") + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.postgreSQL.execution).count == 3) + } + + @Test("Run Current sends the COPY's whole text, which again reads as the COPY alone", arguments: [ + Self.dump, "COPY t FROM stdin; DROP TABLE users;\n", + ]) + func runCurrentFragment(sql: String) { + let statements = SQLStatementScanner.navigableStatements(in: sql, grammar: Self.postgreSQL.execution) + let fragment = (sql as NSString).substring(with: statements[0].contentRange) + #expect(sent(fragment) == sent(sql).prefix(1).map { $0 }) + #expect(sent(fragment).count == 1) + } + + @Test("The rows end at the first line holding only \\.", arguments: [ + ("COPY t FROM stdin;\n\\. \n\\.\nSELECT 7;", ["COPY t FROM stdin", "SELECT 7"]), + ("COPY t FROM stdin;\n x\\.\n\\.\r\nSELECT 7;", ["COPY t FROM stdin", "SELECT 7"]), + ("COPY t FROM STDIN WITH (FORMAT csv);\n1,\"a;b\"\n\\.", ["COPY t FROM STDIN WITH (FORMAT csv)"]), + ("COPY t FROM stdin;\n1\ta\nSELECT 7;", ["COPY t FROM stdin"]), + ("COPY t FROM stdin; SELECT 8;\n1\ta\n\\.\nSELECT 7;", ["COPY t FROM stdin", "SELECT 7"]), + ("COPY t FROM stdin; DROP TABLE users;", ["COPY t FROM stdin"]), + ("COPY t FROM stdin; DROP TABLE users;\n", ["COPY t FROM stdin"]), + ]) + func endOfRows(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Only psql's COPY FROM STDIN takes rows from the script", arguments: [ + "COPY t FROM 'stdin';\nSELECT 1;", + "COPY (SELECT 1) TO STDOUT;\nSELECT 1;", + "COPY t TO STDOUT;\nSELECT 1;", + "SELECT 1 FROM stdin;\nSELECT 1;", + ]) + func noRows(sql: String) { + #expect(sent(sql).count == 2) + } + + @Test("Every form psql reads rows for takes them", arguments: [ + "COPY t FROM STDIN;", "copy t from stdin;", "COPY BINARY t FROM STDIN;", "COPY db.sch.t (a) FROM STDOUT;", + "COPY t FROM stdin WHERE a > 1;", "COPY \"t\" FROM stdin;", "COPY t (a, b, c, d, e, f, g) FROM STDIN;", + ]) + func rowForms(header: String) { + #expect(sent("\(header)\nSELECT 1;\n\\.\nSELECT 2;").count == 2) + } + + @Test("pg_dump's \\restrict and \\unrestrict lines run nothing") + func restrictLines() { + let sql = "\\restrict abc123\n\nSET statement_timeout = 0;\nSELECT 6;\n\\unrestrict abc123\n" + #expect(sent(sql) == ["SET statement_timeout = 0", "SELECT 6"]) + #expect(sent("SELECT 1\n\\restrict abc;") == ["SELECT 1\n\\restrict abc"]) + #expect(sent("\\connect db\nSELECT 1;") == ["\\connect db\nSELECT 1"]) + #expect(sent("\\RESTRICT abc\nSELECT 1;") == ["\\RESTRICT abc\nSELECT 1"]) + } + + @Test("The gate reads the COPY alone, and an engine TablePro does not know reads the rows as text") + func gate() { + #expect(SQLStatementClassifier.classify(Self.dump, databaseTypeId: "PostgreSQL", readings: Self.postgreSQL).tier + != .safe) + let unknown = SQLLexicalReadings.resolve(databaseTypeId: "Unknown", declared: nil, session: nil) + #expect(!unknown.all.contains { $0.contains(.psqlScripts) }) + let rows = "COPY t FROM stdin;\nx\tDROP TABLE users;\n" + #expect(SQLStatementScanner.executableStatements(in: rows, grammar: unknown.execution).count == 2) + } + + private func copyData(_ sql: String) -> String { + let text = sql as NSString + let semicolon = text.range(of: ";").location + let rows = SQLPsqlScript.copyRows(afterSemicolonAt: semicolon, in: text, length: text.length) + return text.substring(with: SQLPsqlScript.copyData(inRows: rows, of: text)) + } + + @Test("A COPY's data is the lines after its ; line, up to the \\. line, which is never sent", arguments: [ + ("COPY t FROM stdin;\n1\tone\n2\ttwo\n\\.\nSELECT 2;", "1\tone\n2\ttwo\n"), + ("COPY t FROM stdin;\r\n1\tone\r\n2\ttwo\r\n\\.\r\nSELECT 2;", "1\tone\r\n2\ttwo\r\n"), + ("COPY t FROM stdin;\n1\tone\n2\ttwo\n", "1\tone\n2\ttwo\n"), + ("COPY t FROM stdin;\n1\tone\n2\ttwo", "1\tone\n2\ttwo"), + ("COPY t FROM stdin;\n1\tone\n\\.", "1\tone\n"), + ("COPY t FROM stdin;\n1\tone\n\\.\r", "1\tone\n"), + ("COPY t FROM stdin;\n\\. \n\\.\n", "\\. \n"), + ("COPY t FROM stdin;", ""), + ("COPY t FROM stdin;\n\\.\nSELECT 2;", ""), + ("COPY t FROM stdin; SELECT 8;\n1\ta\n\\.\nSELECT 7;", "1\ta\n"), + ("COPY t FROM stdin; DROP TABLE users;", ""), + ("COPY t FROM stdin; DROP TABLE users;\n", ""), + ]) + func copyDataSpan(sql: String, expected: String) { + #expect(copyData(sql) == expected) + } + + @Test("A scanned COPY carries its data, which offset(by:) keeps, and no other statement carries any") + func executableStatementCarriesCopyData() throws { + let sql = "SELECT 1;\nCOPY t (a, b) FROM stdin;\n1\tone\n2\ttwo\n\\.\nSELECT 2;" + let statements = SQLStatementScanner.executableStatements(in: sql, grammar: Self.postgreSQL.execution) + #expect(statements.map { $0.sql } == ["SELECT 1", "COPY t (a, b) FROM stdin", "SELECT 2"]) + #expect(statements.map { $0.copyData } == [nil, "1\tone\n2\ttwo\n", nil]) + + let copy = try #require(statements.dropFirst().first) + let moved = copy.offset(by: 40) + #expect(moved.copyData == "1\tone\n2\ttwo\n") + #expect(moved.range.location == copy.range.location + 40) + #expect(moved.segmentEnd == copy.segmentEnd + 40) + } + + @Test("A COPY with no rows carries empty data and still runs as a COPY", arguments: [ + "COPY t FROM stdin;", "COPY t FROM stdin;\n\\.\n", "COPY t FROM stdin; DROP TABLE users;\n", + ]) + func copyWithoutRows(sql: String) { + let statements = SQLStatementScanner.executableStatements(in: sql, grammar: Self.postgreSQL.execution) + #expect(statements.map { $0.sql } == ["COPY t FROM stdin"]) + #expect(statements.map { $0.copyData } == [""]) + } + + @Test("Run Current's text and a split resumed after a statement give the COPY the same data") + func copyDataInFragmentsAndResumedSplits() throws { + let sql = "SET search_path = public;\r\nCOPY t FROM stdin;\r\n1\tone\r\n2\t\\N\r\n\\.\r\nSELECT 2;" + let grammar = Self.postgreSQL.execution + let statements = SQLStatementScanner.executableStatements(in: sql, grammar: grammar) + let set = try #require(statements.first) + + let resumed = SQLStatementScanner.executableStatements( + in: sql, after: set, through: (sql as NSString).length, grammar: grammar + ) + #expect(resumed.map { $0.copyData } == ["1\tone\r\n2\t\\N\r\n", nil]) + + let navigable = SQLStatementScanner.navigableStatements(in: sql, grammar: grammar) + let fragment = (sql as NSString).substring(with: navigable[1].contentRange) + let rerun = SQLStatementScanner.executableStatements(in: fragment, grammar: grammar) + #expect(rerun.map { $0.copyData } == ["1\tone\r\n2\t\\N\r\n"]) + } + + @Test("Run Current's text keeps a last row's trailing tab when no \\. line ends the rows") + func copyContentKeepsTheLastRowAsWritten() throws { + let sql = "SELECT 1;\nCOPY t (a, b) FROM stdin;\n1\tone\n2\t\n" + let grammar = Self.postgreSQL.execution + let navigable = SQLStatementScanner.navigableStatements(in: sql, grammar: grammar) + let fragment = (sql as NSString).substring(with: navigable[1].contentRange) + let rerun = SQLStatementScanner.executableStatements(in: fragment, grammar: grammar) + #expect(rerun.map { $0.copyData } == ["1\tone\n2\t\n"]) + } + + @Test("An engine that does not read psql scripts carries no copy data") + func noCopyDataOutsidePsqlScripts() { + let mysql = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "COPY t FROM stdin;\n1\n", grammar: mysql) + #expect(statements.count == 2) + #expect(statements.allSatisfy { $0.copyData == nil }) + } + + @Test("The text psql holds as no SQL is reported for the formatter") + func scriptText() { + let sql = "\\restrict k\nCOPY t FROM stdin;\n1\ta\n\\.\nSELECT 1;" + let ranges = SQLStatementScanner.psqlScriptText(in: sql, grammar: Self.postgreSQL.execution) + #expect(ranges.map { (sql as NSString).substring(with: $0) } == ["\\restrict k", "COPY t FROM stdin;\n1\ta\n\\."]) + #expect(SQLStatementScanner.psqlScriptText(in: sql, grammar: SQLLexicalProfile.everyKnownReading[0]).isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLReadOnlyFunctionCallsTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLReadOnlyFunctionCallsTests.swift new file mode 100644 index 0000000000..af75fb77a9 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLReadOnlyFunctionCallsTests.swift @@ -0,0 +1,90 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLReadOnlyFunctionCallsTests { + private static func callsUnlisted(_ sql: String, _ engine: String, types: Set = []) -> Bool { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + return SQLReadOnlyFunctionCalls.callsUnlistedFunction(sql, databaseTypeId: engine, readings: readings, typeNames: types) + } + + @Test("A user function, a qualified or quoted name and a call in a later statement are unlisted", arguments: [ + "SELECT purge_orders()", + "SELECT app.count(id) FROM orders", + "SELECT `purge_orders`() FROM orders", + "SELECT 1;\nSELECT purge_orders()", + ]) + func userCallsAreUnlisted(sql: String) { + #expect(Self.callsUnlisted(sql, "MySQL")) + } + + @Test("Built-ins, a plain EXPLAIN and the engine's own types are listed") + func builtinsAreListed() { + #expect(!Self.callsUnlisted("SELECT COUNT(*), LOWER(name), DATE_FORMAT(at, '%Y') FROM orders", "MySQL")) + #expect(!Self.callsUnlisted("EXPLAIN SELECT purge_orders()", "PostgreSQL")) + #expect(!Self.callsUnlisted("SELECT total::numeric(10, 2) FROM orders", "PostgreSQL", types: ["numeric"])) + #expect(Self.callsUnlisted("SELECT total::numeric(10, 2) FROM orders", "PostgreSQL")) + } + + @Test("The engines whose functions a query can call to write are the type ids the apps use") + func writingEnginesAreTypeIds() { + #expect(SQLReadOnlyFunctionCalls.queryFunctionsCanWrite == [ + "Oracle", "Dameng", "MySQL", "MariaDB", "OceanBase", "PostgreSQL", "CockroachDB", "PGlite", "KingbaseES", + "SurrealDB", + ]) + #expect(SQLReadOnlyFunctionCalls.builtins(forDatabaseTypeId: "Oracle").contains("NVL")) + #expect(!SQLReadOnlyFunctionCalls.builtins(forDatabaseTypeId: "MySQL").contains("NVL")) + } + + // Measured on Oracle 23ai with user functions of each name (apple/scripts/probes/check-match-recognize-calls.sh): + // inside MEASURES and DEFINE none of them runs, outside the clause each does. + @Test("MATCH_RECOGNIZE's navigation and row pattern are the clause's own on Oracle", arguments: [ + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES PREV(x.purge) AS p ALL ROWS PER MATCH " + + "PATTERN (x+) DEFINE x AS PREV(x.purge) IS NULL OR x.purge < PREV(x.purge, 1))", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES NEXT(x.purge) AS n ONE ROW PER MATCH " + + "PATTERN (x) DEFINE x AS NEXT(x.purge) IS NOT NULL)", + "SELECT * FROM orders MATCH_RECOGNIZE (PARTITION BY id ORDER BY id MEASURES FIRST(x.purge) AS f, " + + "FINAL LAST(x.purge) AS l, RUNNING COUNT(*) AS c ONE ROW PER MATCH PATTERN (x+) DEFINE x AS FIRST(x.purge) > 0)", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES CLASSIFIER() AS c, MATCH_NUMBER() AS m " + + "ALL ROWS PER MATCH PATTERN (^ PERMUTE(x, y) {- z -} $) DEFINE x AS 1 = 1, y AS 1 = 1, z AS 1 = 1)", + "SELECT * FROM (SELECT id FROM orders) MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS ABS(PREV(x.id)) > 0) mr", + ]) + func rowPatternNavigationIsListed(sql: String) { + #expect(!Self.callsUnlisted(sql, "Oracle")) + } + + @Test("Outside MATCH_RECOGNIZE, and in a query nested in it, those names are a user's functions", arguments: [ + "SELECT prev(1) FROM dual", + "SELECT first(1), last(1) FROM dual", + "SELECT classifier(), match_number() FROM dual", + "SELECT match_recognize(1) FROM dual", + "SELECT pattern(1) FROM dual", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS x.purge > (SELECT prev(1) FROM dual))", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES wipe() AS w PATTERN (x) DEFINE x AS 1 = 1)", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS pk.prev(x.id) > 0)", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS \"PREV\"(x.id) > 0)", + "SELECT * FROM orders MATCH_RECOGNIZE (PARTITION BY prev(purge) ORDER BY id PATTERN (x) DEFINE x AS 1 = 1)", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES pattern(wipe()) AS m PATTERN (x) DEFINE x AS 1 = 1)", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id MEASURES PREV(x.id) AS p PATTERN (x))", + "SELECT * FROM orders MATCH_RECOGNIZE (DEFINE x AS PREV(x.id) > 0 PATTERN (x))", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS x.purge > " + + "((SELECT 1 FROM dual WHERE 1 = 0) UNION ALL SELECT prev(1) FROM dual))", + "SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id PATTERN (x) DEFINE x AS " + + "EXISTS ((SELECT 1 FROM dual WHERE 1 = 0) UNION ALL SELECT prev(1) FROM dual))", + ]) + func rowPatternNamesElsewhereAreUnlisted(sql: String) { + #expect(Self.callsUnlisted(sql, "Oracle")) + } + + @Test("An engine without MATCH_RECOGNIZE reads PREV as a user function") + func rowPatternIsOracleOnly() { + #expect(Self.callsUnlisted("SELECT * FROM t MATCH_RECOGNIZE (PATTERN (x) DEFINE x AS PREV(x.a) > 0)", "PostgreSQL")) + } + + @Test("On SurrealDB only a stored body is unlisted: its built-ins are namespaced") + func surrealStoredBodies() { + #expect(Self.callsUnlisted("SELECT fn::mk() FROM ONLY person:one", "SurrealDB")) + #expect(Self.callsUnlisted("RETURN api::invoke('/x')", "SurrealDB")) + #expect(!Self.callsUnlisted("SELECT time::now(), string::lowercase(name), count() FROM person", "SurrealDB")) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift index aed32c07b6..e748f5bf38 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift @@ -80,6 +80,8 @@ struct SQLRoutineBodyNameTests { "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT 1 AS x$case, 2 AS \u{E9}case; END", "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC CREATE FUNCTION g() RETURNS int BEGIN ATOMIC SELECT 1; END; END", "CREATE FUNCTION f() RETURNS void BEGIN ATOMIC /* nothing */ ; -- still nothing\n END", + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; END", + "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC CREATE OR REPLACE FUNCTION g() RETURNS int BEGIN ATOMIC SELECT begin atomic FROM t; END; SELECT begin atomic FROM t; END", ]) func atomicBodies(function: String) { #expect(sent(function + "; DROP TABLE users", "PostgreSQL") == [function, "DROP TABLE users"]) @@ -95,6 +97,7 @@ struct SQLRoutineBodyNameTests { "CREATE PROCEDURE p(begin int, \"case\" int) BEGIN SELECT 1; END", "CREATE PROCEDURE p() BEGIN SELECT t.begin, t.case FROM t; END", "CREATE PROCEDURE p() BEGIN SELECT (SELECT begin FROM t); END", + "CREATE PROCEDURE p() BEGIN SELECT CASE WHEN closed THEN begin ELSE NULL END FROM shifts; SELECT 2; END", "CREATE EVENT TRIGGER begin ON ddl_command_start EXECUTE FUNCTION f()", ]) func namesOnOtherEngines(first: String) { @@ -130,11 +133,202 @@ struct SQLRoutineBodyNameTests { "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLSTATE '23000' BEGIN SELECT 1; END; SELECT 2; END", "CREATE PROCEDURE p() BEGIN inner_block: BEGIN SELECT 1; END inner_block; SELECT 2; END", "CREATE PROCEDURE db.p() outer_block: BEGIN SELECT (CASE WHEN 1 THEN 2 END); SELECT 2; END outer_block", + "CREATE FUNCTION f() RETURNS VARCHAR(10) CHARACTER SET utf8mb4 DETERMINISTIC BEGIN SET @a = 1; RETURN 'x'; END", + "CREATE FUNCTION f() RETURNS VARCHAR(10) CHAR SET utf8mb4 DETERMINISTIC BEGIN SET @a = 1; RETURN 'x'; END", + "CREATE FUNCTION f() RETURNS SET('a', 'b') DETERMINISTIC BEGIN SET @a = 1; RETURN 'a'; END", + "CREATE TRIGGER tr AFTER DELETE ON t FOR EACH ROW BEGIN DELETE FROM log; INSERT INTO log VALUES (1); END", + "CREATE TRIGGER tr BEFORE UPDATE ON t FOR EACH ROW FOLLOWS other BEGIN UPDATE log SET n = 1; SELECT 1; END", ]) func routineHeaders(routine: String) { #expect(sent(routine + ";\nSELECT 3", "MySQL") == [routine, "SELECT 3"]) } + @Test("A trigger's events are its header, so the BEGIN after them still opens its body", arguments: [ + "CREATE TRIGGER tr AFTER INSERT OR UPDATE ON t FOR EACH ROW BEGIN SELECT 1; SELECT 2; END", + "CREATE TRIGGER tr INSTEAD OF DELETE ON v FOR EACH ROW BEGIN SELECT 1; SELECT 2; END", + "CREATE TRIGGER tr ON t FOR INSERT, UPDATE AS BEGIN SELECT 1; SELECT 2; END", + "CREATE TRIGGER tr DELETE ON t BEGIN DELETE FROM log; INSERT INTO log VALUES (1); END", + "CREATE TRIGGER tr ON t AFTER INSERT AS IF UPDATE(c) BEGIN SELECT 1; SELECT 2; END", + ]) + func triggerEventsAreHeader(trigger: String) { + #expect(sent(trigger + ";\nSELECT 3", "Nonesuch") == [trigger, "SELECT 3"]) + } + + @Test("A begin column in a routine whose body is one statement opens no block", arguments: [ + "CREATE PROCEDURE latest_shift() SELECT id FROM shifts ORDER BY begin DESC LIMIT 1", + "CREATE EVENT purge ON SCHEDULE EVERY 1 DAY DO DELETE FROM sessions ORDER BY begin LIMIT 1000", + "CREATE PROCEDURE p() UPDATE shifts SET closed = 1 WHERE begin IS NULL", + "CREATE PROCEDURE p() INSERT INTO log SELECT begin FROM shifts", + "CREATE PROCEDURE p() REPLACE INTO log SELECT begin AS b FROM shifts", + "CREATE DEFINER=root@localhost PROCEDURE p() COMMENT 'x' READS SQL DATA SELECT begin end_at FROM shifts", + "CREATE TRIGGER tr AFTER INSERT ON shifts FOR EACH ROW INSERT INTO log SELECT begin FROM shifts ORDER BY begin DESC", + "CREATE FUNCTION open_since(begin DATETIME) RETURNS BOOL DETERMINISTIC RETURN begin IS NULL", + "CREATE PROCEDURE p() SELECT CASE WHEN closed THEN begin ELSE NULL END AS s FROM shifts", + "CREATE PROCEDURE p() UPDATE (shifts s JOIN x ON x.id = s.id) SET s.closed = 1 WHERE begin IS NULL", + ]) + func singleStatementBodies(routine: String) { + for typeId in ["MySQL", "MariaDB", "Nonesuch"] { + #expect( + sent(routine + ";\nCALL latest_shift();\nDROP TABLE users", typeId) + == [routine, "CALL latest_shift()", "DROP TABLE users"], + "\(typeId)" + ) + } + } + + // Measured on MariaDB 13.0.2: each routine was stored with its whole body. + @Test("On MySQL a SET body is the routine's one statement, so a begin column in it opens no block", arguments: [ + "CREATE PROCEDURE p() SET @x = begin IS NULL", + "CREATE PROCEDURE p(begin INT) SET @half = begin DIV 2", + "CREATE DEFINER=root@localhost PROCEDURE p() SQL SECURITY INVOKER SET @x = begin IS NULL", + "CREATE EVENT cleanup ON SCHEDULE EVERY 1 DAY DO SET @open = begin IS NULL", + "ALTER EVENT cleanup DO SET @open = begin IS NULL", + ]) + func setBodiesOnMySQL(routine: String) { + for typeId in ["MySQL", "MariaDB", "OceanBase"] { + #expect( + sent(routine + ";\nCALL p();\nDROP TABLE users", typeId) == [routine, "CALL p()", "DROP TABLE users"], + "\(typeId)" + ) + } + } + + // Measured on MariaDB 13.0.2: each routine was stored with its whole body. + @Test("On MySQL an IF, WHILE, LOOP or REPEAT body holds its statements, and a begin column in it opens no block", arguments: [ + "CREATE PROCEDURE p() IF 1 THEN SELECT begin FROM shifts; SELECT 2; END IF", + "CREATE PROCEDURE p(begin INT) IF begin IS NULL THEN SELECT 1; END IF", + "CREATE PROCEDURE p() IF (SELECT COUNT(*) FROM shifts WHERE begin IS NULL) > 0 THEN SELECT 1; END IF", + "CREATE PROCEDURE p() IF @a THEN SELECT 1; ELSEIF @b THEN SELECT begin FROM t; ELSE IF @c THEN SELECT 3; END IF; END IF", + "CREATE PROCEDURE p() IF 1 THEN BEGIN SELECT begin FROM t; SELECT 2; END; END IF", + "CREATE PROCEDURE p() IF 1 THEN SELECT CASE WHEN 1 THEN IF(1, 2, 3) ELSE REPEAT('a', 2) END; SELECT begin FROM t; END IF", + "CREATE PROCEDURE p() CASE 1 WHEN 1 THEN IF 2 THEN SELECT begin FROM t; END IF; ELSE SELECT 2; END CASE", + "CREATE PROCEDURE p() WHILE @i < 3 DO SET @i = @i + 1; SELECT begin FROM shifts; END WHILE", + "CREATE PROCEDURE p() lbl: LOOP SELECT begin FROM shifts; LEAVE lbl; END LOOP lbl", + "CREATE PROCEDURE p() REPEAT SET @i = @i + 1; SELECT begin FROM shifts; UNTIL @i > 3 END REPEAT", + "CREATE PROCEDURE p() WHILE 1 DO REPEAT SET @s = REPEAT('x', 2); UNTIL 1 END REPEAT; SELECT begin FROM t; END WHILE", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND IF 1 THEN SET @d = 1; END IF; SELECT begin FROM t; END", + "CREATE TRIGGER tr BEFORE UPDATE ON shifts FOR EACH ROW IF NEW.closed THEN SET NEW.id = 0; SELECT begin FROM shifts INTO @b; END IF", + "CREATE EVENT cleanup ON SCHEDULE EVERY 1 DAY DO IF 1 THEN DELETE FROM shifts WHERE begin IS NULL; SELECT 1; END IF", + "CREATE EVENT cleanup ON SCHEDULE AT IF(1, NOW(), NOW()) + INTERVAL 1 DAY DO BEGIN SELECT begin FROM t; SELECT 2; END", + "CREATE PROCEDURE p() DO REPEAT('a', 2)", + "CREATE PROCEDURE p() IF 1 THEN SELECT CASE WHEN closed THEN begin ELSE NULL END FROM shifts; END IF", + "CREATE PROCEDURE p() IF CASE WHEN @a THEN IF(@b, 1, 0) END THEN SELECT 1; SELECT 2; END IF", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND SET @d = CASE WHEN IF(@a, 1, 0) THEN 1 END; SELECT 1; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND CASE @x WHEN 1 THEN BEGIN SELECT 1; END; ELSE SELECT 2; END CASE; SELECT 3; END", + "CREATE PROCEDURE p() BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION SELECT begin INTO @b FROM t; SELECT 1; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR 1062, SQLSTATE VALUE '23000', NOT FOUND SET @d = CASE WHEN @a THEN begin ELSE 0 END; SELECT 1; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLWARNING lbl: BEGIN SELECT 1; END lbl; SELECT 2; END", + "CREATE PROCEDURE p(begin INT) DO CASE WHEN @a THEN begin ELSE 0 END", + "CREATE PROCEDURE p(begin INT) BEGIN DO CASE WHEN @a THEN begin ELSE 0 END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN FOR i IN 1..3 DO BEGIN SELECT i; SELECT 2; END; END FOR; SELECT 3; END", + "CREATE PROCEDURE p() BEGIN DECLARE handler CURSOR FOR SELECT begin FROM t; SELECT 1; END", + ]) + func compoundBodiesOnMySQL(routine: String) { + for typeId in ["MySQL", "MariaDB", "OceanBase"] { + #expect( + sent(routine + ";\nCALL p();\nDROP TABLE users", typeId) == [routine, "CALL p()", "DROP TABLE users"], + "\(typeId)" + ) + } + } + + @Test("A handler's own BEGIN ... END body still opens on every engine with the usual body", arguments: [ + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLSTATE '23000' BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION, my_condition BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR 1062 BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE EXIT HANDLER FOR 1062, 1213 BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLSTATE VALUE '23000' BEGIN SELECT 1; END; SELECT 2; END", + ]) + func handlerBodies(routine: String) { + for typeId in ["MySQL", "Snowflake", "BigQuery", "Nonesuch"] { + #expect(sent(routine + ";\nSELECT 3", typeId) == [routine, "SELECT 3"], "\(typeId)") + } + } + + // Measured on MariaDB 13.0.2 with `sql_mode = ORACLE`. + @Test("MariaDB's Oracle mode WHILE ... LOOP closes at its END LOOP") + func oracleModeWhileLoop() { + let routine = "CREATE PROCEDURE p AS BEGIN WHILE 1 < 3 LOOP SELECT begin FROM t; EXIT; END LOOP; END" + #expect(sent(routine + ";\nDROP TABLE users", "MariaDB") == [routine, "DROP TABLE users"]) + } + + // Measured on PostgreSQL 17.11: the function was stored and the DROP after it ran in the same simple query. + @Test("A begin column with the alias atomic inside a BEGIN ATOMIC body hides nothing from the gate") + func beginAtomicAliasHidesNothing() { + let sql = "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; " + + "END; DROP TABLE users; END" + for typeId in ["PostgreSQL", "Redshift", "CockroachDB", "Nonesuch"] { + let readings = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: typeId, readings: readings).tier == .destructive) + } + } + + @Test("Every gate reading sees the statement after a MySQL SET or IF body") + func mySQLGatesSeeTheStatementAfterTheBody() { + for body in ["SET @x = begin IS NULL", "IF 1 THEN SELECT begin FROM t; END IF", "LOOP SELECT begin FROM t; END LOOP"] { + let sql = "CREATE PROCEDURE p() \(body);\nDROP TABLE audit_tmp;" + for typeId in ["MySQL", "MariaDB"] { + let readings = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: typeId, readings: readings).tier == .destructive) + for grammar in readings.all { + #expect(SQLStatementScanner.allStatements(in: sql, grammar: grammar).last == "DROP TABLE audit_tmp") + } + } + } + } + + @Test("An engine TablePro does not know reads IF and WHILE as T-SQL does, so the statement after one stays its own", arguments: [ + "IF @x = 1 SELECT 1; DROP TABLE users", + "IF @x = 1 SELECT 1; ELSE SELECT 2; DROP TABLE users", + "WHILE @i < 3 SET @i = @i + 1; DROP TABLE users", + "CREATE PROCEDURE p AS IF @x = 1 SELECT 1; DROP TABLE users", + "CREATE PROCEDURE p() IF a THEN SELECT 1; DROP TABLE users", + "CREATE PROCEDURE p AS WHILE @i < 3 SET @i = @i + 1; DROP TABLE users", + "CREATE PROCEDURE p() WHILE 1 DO SELECT 1; DROP TABLE users", + "CREATE PROCEDURE p() LOOP SELECT 1; DROP TABLE users", + ]) + func unknownEnginesKeepTSQLControlFlow(sql: String) { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Nonesuch", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "Nonesuch", readings: readings).tier == .destructive) + #expect(readings.all.allSatisfy { !SQLStatementScanner.endsInsideOpenBlock(sql, grammar: $0) }) + } + + @Test("A body that is a control statement keeps the blocks in it, and header clauses stay header", arguments: [ + "CREATE PROCEDURE p AS IF @x = 1 DELETE FROM t ELSE BEGIN SELECT 1; SELECT 2; END", + "CREATE PROCEDURE p() LANGUAGE SQL SET OPTION DBGVIEW = *SOURCE BEGIN SELECT 1; SELECT 2; END", + "CREATE PROCEDURE p() LANGUAGE SQL COMMIT ON RETURN NO BEGIN SELECT 1; SELECT 2; END", + "CREATE PROCEDURE p AS SET NOCOUNT ON BEGIN SELECT 1; SELECT 2; END", + ]) + func controlStatementBodies(routine: String) { + #expect(sent(routine + ";\nSELECT 3", "Nonesuch") == [routine, "SELECT 3"]) + } + + @Test("A RETURN body's CASE holds no block, so the statements after the function stay their own") + func returnBodyWithACase() { + let function = "CREATE FUNCTION f(begin int) RETURNS int RETURN CASE WHEN begin > 0 THEN begin ELSE 0 END" + #expect(sent(function + "; DROP TABLE users; END", "Nonesuch") == [function, "DROP TABLE users", "END"]) + } + + @Test("A RETURN that names the type before AS or IS still opens the block after it", arguments: [ + "CREATE FUNCTION f RETURN INT AS BEGIN SELECT 1; RETURN 2; END", + "CREATE OR REPLACE FUNCTION f(a INT) RETURN INT DETERMINISTIC IS BEGIN SELECT 1; RETURN a; END", + "CREATE FUNCTION f RETURN INT AS BEGIN IF a THEN BEGIN SELECT 1; END; END IF; RETURN 1; END", + ]) + func returnTypeHeaders(function: String) { + for typeId in ["MariaDB", "Nonesuch"] { + #expect(sent(function + ";\nSELECT 3", typeId) == [function, "SELECT 3"], "\(typeId)") + } + } + + @Test("A one-statement event body with a begin column no longer hides the DROP after it from the gate") + func singleStatementBodyTiersTheNextStatement() { + let sql = "CREATE EVENT purge ON SCHEDULE EVERY 1 DAY DO DELETE FROM sessions ORDER BY begin LIMIT 1000;\n" + + "DROP TABLE audit_tmp;" + let readings = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "MySQL", readings: readings).tier == .destructive) + } + @Test("A block keyword glued to a name character is part of the name", arguments: [ "CREATE TABLE lookup (id INTEGER, upper$case TEXT)", "CREATE TABLE \u{E9}case (id int)", @@ -183,6 +377,10 @@ struct SQLRoutineBodyNameTests { ("MySQL", "CREATE PROCEDURE p() BEGIN SELECT 1; END", false), ("MySQL", "CREATE PROCEDURE p() BEGIN SELECT 1; END;", false), ("MySQL", "CREATE PROCEDURE p() SELECT 1 AS case; DROP TABLE users", true), + ("MySQL", "CREATE PROCEDURE p() IF 1 THEN SELECT 1; DROP TABLE users", true), + ("MySQL", "CREATE PROCEDURE p() WHILE 1 DO SELECT 1; DROP TABLE users", true), + ("MySQL", "CREATE PROCEDURE p() IF 1 THEN SELECT 1; END IF", false), + ("MySQL", "CREATE PROCEDURE p() DO IF(1, 2, 3); DROP TABLE users", false), ("Snowflake", "DECLARE c CURSOR FOR SELECT 1 AS case; DROP TABLE users", true), ("PostgreSQL", "CREATE FUNCTION f() RETURNS int BEGIN ATOMIC SELECT 1; DROP TABLE users", true), ("PostgreSQL", "CREATE TABLE begin(a int); DROP TABLE users", false), @@ -199,13 +397,18 @@ struct SQLRoutineBodyNameTests { @Test("An engine TablePro does not know keeps the usual BEGIN ... END body") func unknownEnginesKeepTheUsualBody() { #expect(SQLLexicalProfile.everyKnownReading.allSatisfy { !$0.hasSingleBodyForm }) + #expect(SQLLexicalProfile.everyKnownReading.allSatisfy { !$0.contains(.compoundStatementBodies) }) let procedure = "CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END" #expect(sent(procedure + "; SELECT 3", "Nonesuch") == [procedure, "SELECT 3"]) } @Test("A reading keeps the form of body its engine has, whatever the text holds") func relevanceKeepsTheBodyForm() { - for (typeId, fact) in [("Redshift", SQLLexicalGrammar.atomicRoutineBodies), ("PostgreSQL", .atomicRoutineBodies)] { + let bodyForms: [(String, SQLLexicalGrammar)] = [ + ("Redshift", .atomicRoutineBodies), ("PostgreSQL", .atomicRoutineBodies), + ("MySQL", .compoundStatementBodies), ("MariaDB", .compoundStatementBodies), + ] + for (typeId, fact) in bodyForms { let readings = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) #expect(readings.distinct(for: "CREATE TABLE begin(a int)").allSatisfy { $0.contains(fact) }, "\(typeId)") } diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift new file mode 100644 index 0000000000..20c3f6bb9d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift @@ -0,0 +1,409 @@ +import Foundation +@testable import TableProSQLGrammar +import Testing + +struct SQLSessionStateClassifierTests { + private static func grammar(_ engine: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil).execution + } + + private static let engines: [SQLSessionDialect: String] = [ + .postgres: "PostgreSQL", .mySQL: "MySQL", .sqlServer: "SQL Server", .oracle: "Oracle", .sqlite: "SQLite", + .duckDB: "DuckDB", + ] + + private func effect(_ sql: String, _ dialect: SQLSessionDialect) -> SQLSessionStateEffect { + SQLSessionStateClassifier.effect(of: sql, dialect: dialect, grammar: Self.grammar(Self.engines[dialect] ?? "")) + } + + private func creates(_ sql: String, _ dialect: SQLSessionDialect) -> SQLSessionStateKinds { + effect(sql, dialect).creates + } + + // MARK: - PostgreSQL + + @Test("PostgreSQL temporary objects, by marker, by pg_temp and by SELECT INTO", arguments: [ + "CREATE TEMP TABLE t (id int)", + "create temporary table t (id int)", + "CREATE GLOBAL TEMPORARY TABLE t (id int)", + "CREATE OR REPLACE TEMP VIEW v AS SELECT 1", + "CREATE TEMP SEQUENCE s", + "CREATE TEMP TABLE t (id int) ON COMMIT DELETE ROWS", + "CREATE TABLE pg_temp.t (id int)", + "CREATE TABLE IF NOT EXISTS pg_temp.t (id int)", + "CREATE FUNCTION pg_temp.f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql", + "SELECT * INTO TEMP t FROM orders", + "SELECT * INTO TEMPORARY TABLE t FROM orders", + ]) + func postgresTemporaryObjects(sql: String) { + #expect(creates(sql, .postgres) == .temporaryObjects) + } + + @Test("PostgreSQL tables that outlive the session or end with the transaction are not session state", arguments: [ + "CREATE TABLE t (id int)", + "CREATE TABLE temp (id int)", + "CREATE UNLOGGED TABLE t (id int)", + "CREATE TEMP TABLE t (id int) ON COMMIT DROP", + "SELECT * INTO archive FROM orders", + ]) + func postgresLastingTables(sql: String) { + #expect(creates(sql, .postgres).isEmpty) + } + + @Test("PostgreSQL SET, SET SESSION and a session set_config are session settings", arguments: [ + "SET search_path TO app", + "SET SESSION statement_timeout = 0", + "SET ROLE reporting", + "SET SESSION AUTHORIZATION alice", + "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + "SET app.key = 'secret'", + "SELECT set_config('app.user', '42', false)", + "SELECT pg_catalog.set_config('app.user', '42', false)", + ]) + func postgresSettings(sql: String) { + #expect(creates(sql, .postgres) == .sessionSettings) + } + + @Test("PostgreSQL settings that last one transaction are not session state", arguments: [ + "SET LOCAL search_path TO app", + "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", + "SET CONSTRAINTS ALL DEFERRED", + "SELECT set_config('app.user', '42', true)", + ]) + func postgresTransactionSettings(sql: String) { + #expect(creates(sql, .postgres).isEmpty) + } + + @Test("PostgreSQL prepared statements, held cursors, session advisory locks and LISTEN") + func postgresOtherKinds() { + #expect(creates("PREPARE q (int) AS SELECT $1", .postgres) == .preparedStatements) + #expect(creates("PREPARE TRANSACTION 'tx1'", .postgres).isEmpty) + #expect(creates("DECLARE c CURSOR WITH HOLD FOR SELECT 1", .postgres) == .heldCursors) + #expect(creates("DECLARE c CURSOR FOR SELECT 1", .postgres).isEmpty) + #expect(creates("DECLARE c CURSOR WITHOUT HOLD FOR SELECT 1", .postgres).isEmpty) + #expect(creates("SELECT pg_advisory_lock(1)", .postgres) == .sessionLocks) + #expect(creates("SELECT pg_try_advisory_lock_shared(1, 2)", .postgres) == .sessionLocks) + #expect(creates("SELECT pg_advisory_xact_lock(1)", .postgres).isEmpty) + #expect(creates("LISTEN jobs", .postgres) == .listeners) + } + + @Test("DISCARD ALL clears every kind; DISCARD TEMP, DEALLOCATE ALL and UNLISTEN * clear their own") + func postgresClears() { + #expect(effect("DISCARD ALL", .postgres).clears == .all) + #expect(effect("DISCARD TEMP", .postgres).clears == .temporaryObjects) + #expect(effect("discard temporary", .postgres).clears == .temporaryObjects) + #expect(effect("DEALLOCATE ALL", .postgres).clears == .preparedStatements) + #expect(effect("DEALLOCATE PREPARE ALL", .postgres).clears == .preparedStatements) + #expect(effect("UNLISTEN *", .postgres).clears == .listeners) + #expect(effect("UNLISTEN jobs", .postgres) == .none) + #expect(effect("RESET search_path", .postgres) == .none) + #expect(effect("RESET ALL", .postgres) == .none) + #expect(effect("DISCARD PLANS", .postgres) == .none) + } + + @Test("RESET ALL keeps the role and the session authorization, so it clears no settings") + func resetAllKeepsTheRole() { + let grammar = Self.grammar("PostgreSQL") + let effect = SQLSessionStateClassifier.effect( + ofStatements: ["SET ROLE app_admin", "RESET ALL"], dialect: .postgres, grammar: grammar + ) + #expect(effect.applied(to: []) == .sessionSettings) + let discarded = SQLSessionStateClassifier.effect( + ofStatements: ["SET ROLE app_admin", "DISCARD ALL"], dialect: .postgres, grammar: grammar + ) + #expect(discarded.applied(to: []).isEmpty) + } + + @Test("Statements apply in order, so what a later one clears goes and what it sets up stays") + func statementsApplyInOrder() { + let grammar = Self.grammar("PostgreSQL") + let cleared = SQLSessionStateClassifier.effect( + ofStatements: ["CREATE TEMP TABLE t (id int)", "DISCARD ALL"], dialect: .postgres, grammar: grammar + ) + #expect(cleared.applied(to: []).isEmpty) + let reset = SQLSessionStateClassifier.effect( + ofStatements: ["DISCARD ALL", "SET search_path TO app"], dialect: .postgres, grammar: grammar + ) + #expect(reset.applied(to: [.temporaryObjects, .listeners]) == .sessionSettings) + let partly = SQLSessionStateClassifier.effect( + ofStatements: ["CREATE TEMP TABLE t (id int); SET search_path TO app", "DISCARD TEMP"], + dialect: .postgres, + grammar: grammar + ) + #expect(partly.applied(to: []) == .sessionSettings) + } + + @Test("Text inside a comment, a literal, a DO body or a procedure call never counts", arguments: [ + "SELECT 'SET search_path TO x'", + "-- SET search_path TO x\nSELECT 1", + "SELECT 1 /* CREATE TEMP TABLE t (id int) */", + "DO $$ BEGIN CREATE TEMP TABLE t (id int); END $$", + "CALL set_up_session()", + "SELECT $$pg_advisory_lock(1)$$", + ]) + func postgresUnseenText(sql: String) { + #expect(effect(sql, .postgres) == .none) + } + + @Test("The effect carries kinds only, never a value the statement held") + func effectHoldsNoText() { + let noted = effect("SET app.key = 'secret'", .postgres) + #expect(noted.creates == .sessionSettings) + #expect(!String(describing: noted).contains("secret")) + #expect(!String(describing: noted).contains("app.key")) + } + + @Test("A statement equal to a startup command is skipped, one that differs still counts") + func startupStatementsAreSkipped() { + let grammar = Self.grammar("PostgreSQL") + let skipped = SQLSessionStateClassifier.effect( + ofStatements: ["SET search_path TO app;", "CREATE TEMP TABLE t (id int)"], + dialect: .postgres, + grammar: grammar, + skipping: [" SET search_path TO app ;\n"] + ) + #expect(skipped.creates == .temporaryObjects) + let differs = SQLSessionStateClassifier.effect( + ofStatements: ["SET search_path TO other"], + dialect: .postgres, + grammar: grammar, + skipping: ["SET search_path TO app"] + ) + #expect(differs.creates == .sessionSettings) + } + + // MARK: - MySQL and MariaDB + + @Test("MySQL temporary tables", arguments: [ + "CREATE TEMPORARY TABLE t (id int)", + "CREATE OR REPLACE TEMPORARY TABLE t (id int)", + "CREATE TEMPORARY TABLE IF NOT EXISTS t SELECT 1", + ]) + func mySQLTemporaryTables(sql: String) { + #expect(creates(sql, .mySQL) == .temporaryObjects) + } + + @Test("MySQL session system variables, NAMES, CHARACTER SET, SESSION TRANSACTION and ROLE", arguments: [ + "SET sql_mode = ''", + "SET autocommit = 0", + "SET SESSION sql_mode = ''", + "SET LOCAL sql_mode = ''", + "SET @@sql_mode = ''", + "SET @@SESSION.sql_mode = ''", + "SET NAMES utf8mb4", + "SET CHARACTER SET utf8mb4", + "SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED", + "SET ROLE reporting", + "SET GLOBAL max_connections = 500, SESSION sql_mode = ''", + "SET GLOBAL max_connections = 500, @@SESSION.sql_mode = ''", + "/*!40101 SET NAMES utf8mb4 */", + ]) + func mySQLSettings(sql: String) { + #expect(creates(sql, .mySQL) == .sessionSettings) + } + + @Test("MySQL global settings, account changes and next-transaction settings are not session state", arguments: [ + "SET GLOBAL max_connections = 500", + "SET GLOBAL max_connections = 500, wait_timeout = 60", + "SET PERSIST max_connections = 500", + "SET PERSIST_ONLY max_connections = 500", + "SET @@GLOBAL.max_connections = 500", + "SET PASSWORD = 'x'", + "SET PASSWORD FOR app = 'x'", + "SET DEFAULT ROLE ALL TO app", + "SET TRANSACTION ISOLATION LEVEL READ COMMITTED", + "SET GLOBAL TRANSACTION READ ONLY", + "SET STATEMENT max_statement_time = 1 FOR SELECT 1", + "CREATE TABLE t (id int)", + ]) + func mySQLLastingChanges(sql: String) { + #expect(creates(sql, .mySQL).isEmpty) + } + + @Test("MySQL user variables are set by SET, := and INTO", arguments: [ + "SET @id = 42", + "SELECT @id := 42", + "SELECT id INTO @id FROM t LIMIT 1", + "SELECT id FROM t LIMIT 1 INTO @id", + "SELECT 1 INTO@id", + "SELECT@id:=1", + "SELECT @id /* note */ := 1", + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE */", + ]) + func mySQLUserVariables(sql: String) { + #expect(creates(sql, .mySQL) == .userVariables) + } + + @Test("MySQL prepared statements and session locks, and what only reads") + func mySQLOtherKinds() { + #expect(creates("PREPARE s FROM 'SELECT 1'", .mySQL) == .preparedStatements) + #expect(creates("SELECT GET_LOCK('job', 10)", .mySQL) == .sessionLocks) + #expect(creates("LOCK TABLES t WRITE", .mySQL) == .sessionLocks) + #expect(creates("SET @got = GET_LOCK('job', 10)", .mySQL) == [.userVariables, .sessionLocks]) + #expect(creates("SELECT @id", .mySQL).isEmpty) + #expect(creates("SELECT @@version", .mySQL).isEmpty) + #expect(creates("SELECT 'SET NAMES utf8mb4'", .mySQL).isEmpty) + #expect(creates("# SET NAMES latin1\nSELECT 1", .mySQL).isEmpty) + #expect(creates("CALL set_up_session()", .mySQL).isEmpty) + } + + // MARK: - SQL Server + + @Test("SQL Server # and ## tables, SELECT INTO a # table and a temporary procedure", arguments: [ + "CREATE TABLE #orders (id int)", + "CREATE TABLE ##orders (id int)", + "SELECT * INTO #orders FROM dbo.orders", + "CREATE PROCEDURE #p AS SELECT 1", + "SELECT 1\nCREATE TABLE #t (id int)", + "CREATE TABLE [#orders] (id int)", + "CREATE TABLE \"#orders\" (id int)", + "SELECT * INTO [#orders] FROM dbo.orders", + "CREATE TABLE #tëmp (id int)", + ]) + func sqlServerTemporaryObjects(sql: String) { + #expect(creates(sql, .sqlServer) == .temporaryObjects) + } + + @Test("SQL Server SET options, EXECUTE AS and session context", arguments: [ + "SET NOCOUNT ON", + "SET XACT_ABORT ON", + "SET TRANSACTION ISOLATION LEVEL SNAPSHOT", + "SELECT 1 SET NOCOUNT ON", + "EXECUTE AS USER = 'auditor'", + "EXEC sp_set_session_context 'tenant', 42", + "EXEC sys.sp_set_session_context @key = N'tenant', @value = 42", + "sp_set_session_context 'tenant', 42", + "EXEC sp_setapprole 'reporting', 'secret'", + "EXEC sys.sp_setapprole @rolename = 'reporting', @password = 'secret'", + "sp_setapprole 'reporting', 'secret'", + ]) + func sqlServerSettings(sql: String) { + #expect(creates(sql, .sqlServer) == .sessionSettings) + } + + @Test("SQL Server variables, procedure calls, dynamic SQL and permanent tables are not session state", arguments: [ + "SET @n = 1", + "DECLARE @n int; SET @n = 1", + "UPDATE t SET a = 1", + "EXEC dbo.rebuild", + "EXEC('SET NOCOUNT ON')", + "CREATE TABLE orders (id int)", + "SELECT * INTO archive FROM orders", + "INSERT INTO #orders VALUES (1)", + ]) + func sqlServerNotSessionState(sql: String) { + #expect(creates(sql, .sqlServer).isEmpty) + } + + @Test("SQL Server global cursors and application locks") + func sqlServerCursorsAndLocks() { + #expect(creates("DECLARE c CURSOR FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c CURSOR GLOBAL FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c INSENSITIVE CURSOR FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c CURSOR LOCAL FOR SELECT 1", .sqlServer).isEmpty) + #expect(creates("DECLARE @c CURSOR", .sqlServer).isEmpty) + #expect(creates("EXEC sp_getapplock @Resource = 'job', @LockMode = 'Exclusive', @LockOwner = 'Session'", .sqlServer) + == .sessionLocks) + #expect(creates("EXEC @rc = sp_getapplock 'job', 'Exclusive', 'Session'", .sqlServer) == .sessionLocks) + } + + // MARK: - Oracle, SQLite and DuckDB + + @Test("Oracle ALTER SESSION, SET ROLE and a private temporary table that keeps its definition") + func oracleKinds() { + #expect(creates("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'", .oracle) == .sessionSettings) + #expect(creates("ALTER SESSION SET CURRENT_SCHEMA = app", .oracle) == .sessionSettings) + #expect(creates("ALTER SESSION ENABLE PARALLEL DML", .oracle) == .sessionSettings) + #expect(creates("SET ROLE reporting", .oracle) == .sessionSettings) + #expect(creates("SET TRANSACTION READ ONLY", .oracle).isEmpty) + #expect( + creates("CREATE PRIVATE TEMPORARY TABLE ora$ptt_t (id NUMBER) ON COMMIT PRESERVE DEFINITION", .oracle) + == .temporaryObjects + ) + #expect(creates("CREATE PRIVATE TEMPORARY TABLE ora$ptt_t (id NUMBER) ON COMMIT DROP DEFINITION", .oracle).isEmpty) + #expect(creates("CREATE GLOBAL TEMPORARY TABLE t (id NUMBER) ON COMMIT PRESERVE ROWS", .oracle).isEmpty) + #expect(creates("BEGIN DBMS_SESSION.SET_ROLE('reporting'); END;", .oracle).isEmpty) + } + + @Test("SQLite and DuckDB temporary objects, by marker and by the temp schema") + func embeddedTemporaryObjects() { + #expect(creates("CREATE TEMP TABLE t (id INTEGER)", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TEMPORARY VIEW v AS SELECT 1", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TABLE temp.t (id INTEGER)", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TABLE t (id INTEGER)", .sqlite).isEmpty) + #expect(creates("CREATE TABLE temp (id INTEGER)", .sqlite).isEmpty) + #expect(creates("CREATE TEMP TABLE t (id INTEGER)", .duckDB) == .temporaryObjects) + #expect(creates("CREATE OR REPLACE TEMP MACRO m(a) AS a + 1", .duckDB) == .temporaryObjects) + #expect(creates("CREATE TABLE temp.t (id INTEGER)", .duckDB) == .temporaryObjects) + } + + @Test("DuckDB SET without GLOBAL is a session setting, SET VARIABLE a variable") + func duckDBSettings() { + #expect(creates("SET threads = 4", .duckDB) == .sessionSettings) + #expect(creates("SET SESSION threads = 4", .duckDB) == .sessionSettings) + #expect(creates("SET VARIABLE x = 42", .duckDB) == .userVariables) + #expect(creates("SET GLOBAL threads = 4", .duckDB).isEmpty) + } + + // MARK: - Effects + + @Test("A later clear removes what an earlier statement set up, a later setup survives an earlier clear") + func effectComposition() { + let setUp = SQLSessionStateEffect(creates: .temporaryObjects) + let discard = SQLSessionStateEffect(clears: .all) + #expect(setUp.followed(by: discard) == SQLSessionStateEffect(creates: [], clears: .all)) + let settings = SQLSessionStateEffect(creates: .sessionSettings) + #expect(discard.followed(by: settings).applied(to: .userVariables) == .sessionSettings) + } + + @Test("A statement longer than the classified head is read only to there") + func longStatementIsReadToItsHead() { + let values = String(repeating: "a", count: 100_000) + #expect(creates("INSERT INTO t VALUES ('\(values)')", .mySQL).isEmpty) + #expect(creates("SET @id = 42, @blob = '\(values)'", .mySQL) == .userVariables) + } + + @Test("A text is cut to its head before it is split, so a setup past it in the same text is not read") + func textIsCutBeforeItIsSplit() { + let values = String(repeating: "a", count: 70_000) + #expect(creates("SET NOCOUNT ON\nINSERT INTO t VALUES ('\(values)')", .sqlServer) == .sessionSettings) + #expect(creates("INSERT INTO t VALUES ('\(values)'); SET NOCOUNT ON", .sqlServer).isEmpty) + let grammar = Self.grammar("SQL Server") + let separate = SQLSessionStateClassifier.effect( + ofStatements: ["INSERT INTO t VALUES ('\(values)')", "SET NOCOUNT ON"], dialect: .sqlServer, grammar: grammar + ) + #expect(separate.creates == .sessionSettings) + } + + @Test("The quick check passes over data words and addresses", arguments: [ + ("SELECT * FROM t ORDER BY id LIMIT 10 OFFSET 20", SQLSessionDialect.postgres), + ("INSERT INTO t VALUES ('template', 'asset', 'block', 'clock')", .postgres), + ("INSERT INTO users VALUES ('ana@example.com', 'bo.li@example.org')", .mySQL), + ("INSERT INTO t VALUES ('reset', 'offset', 'blocked')", .mySQL), + ("INSERT INTO t VALUES ('Order#5', 'a#b')", .sqlServer), + ("SELECT * FROM sessions WHERE role_id = 1", .oracle), + ("SELECT pg_advisory_xact_lock(1)", .postgres), + ]) + func quickCheckSkipsData(sql: String, dialect: SQLSessionDialect) { + #expect(!SQLSessionStateQuickCheck.mayChangeSession(sql, dialect: dialect)) + } + + @Test("The quick check keeps every form a rule reads, however it is spelled", arguments: [ + ("set search_path to app", SQLSessionDialect.postgres), + ("SELECT * INTO pg_temp_3.t FROM orders", .postgres), + ("SELECT 1SET", .postgres), + ("SELECT 1 INTO@v", .mySQL), + ("SELECT 1 INTO /* v */ @v", .mySQL), + ("SELECT@v/* note */:=1", .mySQL), + ("SELECT @`v` := 1", .mySQL), + ("/*!40101 SET NAMES utf8mb4 */", .mySQL), + ("CREATE TABLE [#t] (id int)", .sqlServer), + ("SELECT * INTO ##t FROM orders", .sqlServer), + ("sp_setapprole 'reporting', 'secret'", .sqlServer), + ("ALTER SESSION SET CURRENT_SCHEMA = app", .oracle), + ("CREATE TABLE temp.t (id INTEGER)", .sqlite), + ]) + func quickCheckKeepsEveryRule(sql: String, dialect: SQLSessionDialect) { + #expect(SQLSessionStateQuickCheck.mayChangeSession(sql, dialect: dialect)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementClassifierTests.swift index 215e428eef..46a0a0ce16 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementClassifierTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementClassifierTests.swift @@ -31,6 +31,26 @@ struct SQLStatementClassifierTests { #expect(Self.deletesEveryRow(sql, "DuckDB")) } + // Read-Only refuses a write on both apps before the driver sees it, so none of these reaches the server. + @Test("A statement that turns a transaction or the session read-write is a write", arguments: [ + ("PostgreSQL", "SET TRANSACTION READ WRITE"), + ("PostgreSQL", "SET LOCAL transaction_read_only = off"), + ("PostgreSQL", "SET SESSION CHARACTERISTICS AS TRANSACTION READ WRITE"), + ("PostgreSQL", "SET default_transaction_read_only = off"), + ("PostgreSQL", "BEGIN READ WRITE"), + ("PostgreSQL", "START TRANSACTION READ WRITE"), + ("PostgreSQL", "SELECT set_config('default_transaction_read_only', 'off', false)"), + ("MySQL", "SET SESSION TRANSACTION READ WRITE"), + ("MySQL", "SET TRANSACTION READ WRITE"), + ("MySQL", "SET @@session.transaction_read_only = 0"), + ("MySQL", "START TRANSACTION READ WRITE"), + ("MariaDB", "SET tx_read_only = 0"), + ("MariaDB", "/*!50000 SET SESSION TRANSACTION READ WRITE */"), + ]) + func turningReadOnlyOffIsAWrite(engine: String, sql: String) { + #expect(Self.tier(sql, engine) == .write) + } + @Test("A CockroachDB b'' string continued after a line break hides no statement") func continuedEscapeBytesHidesNoStatement() { #expect(Self.deletesEveryRow("SELECT b'a'\n'\\''; COMMIT; DELETE FROM orders; --'", "CockroachDB")) @@ -53,6 +73,15 @@ struct SQLStatementClassifierTests { #expect(Self.classify("SELECT * FROM `payroll.csv`", "DuckDB").reachesFilesystemOrExecutesCode) } + @Test("INSERT OVERWRITE empties the table before it inserts, and into a directory writes files") + func insertOverwriteIsDestructive() { + #expect(Self.tier("INSERT OVERWRITE INTO orders SELECT * FROM staged", "Snowflake") == .destructive) + #expect(Self.tier("INSERT OVERWRITE TABLE orders SELECT * FROM staged", "Snowflake") == .destructive) + #expect(Self.classify("INSERT OVERWRITE DIRECTORY '/tmp/out' SELECT * FROM orders", "Snowflake") + .reachesFilesystemOrExecutesCode) + #expect(Self.tier("INSERT INTO orders (overwrite) VALUES (1)", "Snowflake") == .write) + } + @Test("T-SQL calls the procedure a batch opens with when no statement begins with that word", arguments: [ "SHOW 'x'", "HELP 'x'", "INFO 'x'", "show tables", "EXPLAIN SELECT 1", "DESCRIBE orders", ]) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift new file mode 100644 index 0000000000..270d9f37f8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift @@ -0,0 +1,293 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct CursorIndexFixture: Sendable, CustomStringConvertible { + let typeId: String + let text: String + + var description: String { + typeId + } +} + +private let cursorIndexFixtures = [ + CursorIndexFixture(typeId: "PostgreSQL", text: """ + SELECT 1; + + -- a note; with a semicolon + SELECT 'a;b'; + CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql; + /* c; */ SELECT E'x' + 'y';; + CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)); + \\restrict k1 + COPY t (a, b) FROM stdin; + 1\tit's; here + 2\t$$ -- x + \\. + SELECT 2 + """), + CursorIndexFixture(typeId: "MySQL", text: """ + CREATE PROCEDURE p() + BEGIN + SELECT 1; + SELECT 2; + END; + SELECT 3;; + # hash; comment + SELECT `a;b` FROM t; + DELIMITER // + CREATE PROCEDURE q() BEGIN SELECT 1; END// + SELECT 'x//'; SELECT 4// + DELIMITER ;; + CREATE TRIGGER g BEFORE INSERT ON t FOR EACH ROW BEGIN SET NEW.a = 1; END ;; + DELIMITER ; + SELECT /*! 1; */ 2 + """), + CursorIndexFixture(typeId: "SQL Server", text: """ + SELECT 1 + GO + SELECT 2; + GO 3 + MERGE t USING s ON 1 = 1 WHEN MATCHED THEN DELETE; + GO + + GO + SELECT [a;b] + GO + IF @x = 1 + BEGIN + SELECT 1; + SELECT CASE WHEN @x = 1 THEN 2 END; + END + ELSE BEGIN TRY SELECT 3; END TRY BEGIN CATCH SELECT 4; END CATCH; + SELECT (5; 6); + SET XACT_ABORT ON + BEGIN TRY SELECT 7; END TRY BEGIN CATCH THROW; END CATCH + IF @y = 1 SELECT 8; -- then + ELSE IF @z = 1 SELECT 9; ELSE SELECT 10; + DROP TABLE IF EXISTS t;; SELECT 11 + IF @w = 1 SELECT 12;; ELSE SELECT 13 + """), + CursorIndexFixture(typeId: "Oracle", text: """ + BEGIN + NULL; + END; + / + SELECT 1 FROM dual + / + SELECT q'[a;b]' FROM dual; + / + CREATE OR REPLACE PROCEDURE p IS + BEGIN + NULL; + END; + / + """), + CursorIndexFixture(typeId: "Cassandra", text: """ + BEGIN BATCH + INSERT INTO t (a) VALUES (1); + INSERT INTO t (a) VALUES (2); + APPLY BATCH; + SELECT * FROM t; + """), +] + +private let cursorIndexInsertions = [ + ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", "(", ")", + "CASE ", "END ", "ELSE ", "IF ", "DELIMITER ", "//", "\\.", "COPY t FROM stdin;\n", "\\restrict x\n", +] + +private let longScriptRow = "SELECT id, name FROM users WHERE active = true;\n" + +struct SQLStatementCursorIndexTests { + private static func grammar(_ typeId: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + } + + private struct Resolved: Equatable, CustomStringConvertible { + let offset: Int + let sql: String + let hasContent: Bool + let terminator: SQLStatementTerminator + let acceptsBindParameters: Bool + let contentRange: NSRange + + init(_ statement: SQLStatementScanner.LocatedStatement) { + offset = statement.offset + sql = statement.sql + hasContent = statement.hasContent + terminator = statement.terminator + acceptsBindParameters = statement.acceptsBindParameters + contentRange = statement.contentRange + } + + var description: String { + "\(offset) \(hasContent) \(sql.debugDescription)" + } + } + + private static func fresh(_ text: String, _ cursor: Int, _ grammar: SQLLexicalGrammar) -> Resolved { + Resolved(SQLStatementScanner.locatedStatementAtCursor(in: text, cursorPosition: cursor, grammar: grammar)) + } + + @Test("Every caret resolves to what a scan from the top resolves to", arguments: cursorIndexFixtures) + func everyCaretMatchesAFreshScan(_ fixture: CursorIndexFixture) { + let grammar = Self.grammar(fixture.typeId) + let text = fixture.text + let length = (text as NSString).length + var index = SQLStatementCursorIndex(grammar: grammar, checkpointSpacing: 1) + var generator = SplitMix64(seed: 7) + let shuffled = Array(0...length).shuffled(using: &generator) + + for cursor in Array(0...length) + Array((0...length).reversed()) + shuffled { + #expect(Resolved(index.statementAtCursor(cursor, in: text)) == Self.fresh(text, cursor, grammar), "caret \(cursor)") + } + #expect(index.checkpointCount > 0) + } + + @Test("After any edit every caret still resolves to what a fresh scan resolves to", arguments: cursorIndexFixtures) + func editsKeepEveryAnswer(_ fixture: CursorIndexFixture) { + let grammar = Self.grammar(fixture.typeId) + var text = fixture.text + var index = SQLStatementCursorIndex(grammar: grammar, checkpointSpacing: 1) + var generator = SplitMix64(seed: 42) + + for step in 0..<80 { + let length = (text as NSString).length + for cursor in Array(0...length).shuffled(using: &generator).prefix(24) { + _ = index.statementAtCursor(cursor, in: text) + } + + let mutable = NSMutableString(string: text) + if length > 0, Bool.random(using: &generator) { + let start = Int.random(in: 0..= length - 2 * rowLength) + + let moves = 1_000 + for step in 0.. UInt64 { + state &+= 0x9E37_79B9_7F4A_7C15 + var value = state + value = (value ^ (value >> 30)) &* 0xBF58_476D_1CE4_E5B9 + value = (value ^ (value >> 27)) &* 0x94D0_49BB_1331_11EB + return value ^ (value >> 31) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SurrealRecordDeleteTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SurrealRecordDeleteTests.swift new file mode 100644 index 0000000000..81bcf85059 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SurrealRecordDeleteTests.swift @@ -0,0 +1,72 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// Record id and DELETE forms from SurrealDB 2.3 and 3.0's parser (`parse_record_id_key`, `parse_delete_stmt`). +struct SurrealRecordDeleteTests { + private static func deletesEveryRow(_ sql: String, _ engine: String = "SurrealDB") -> Bool { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + return SQLStatementClassifier.deletesEveryRow(sql, databaseTypeId: engine, readings: readings) + } + + @Test("A SurrealQL DELETE of one record by its id does not delete every row", arguments: [ + "DELETE person:tobie", + "DELETE person:100", + "delete person:tobie;", + "DELETE ONLY person:tobie RETURN BEFORE", + "DELETE FROM person:tobie", + "DELETE person:-7", + "DELETE person:+7", + "DELETE person:⟨tobie smith⟩", + "DELETE person:`8424486b-85b2-4448-ac8d-5d51083391c7`", + "DELETE `Artykuł`:100", + #"DELETE person:u"0190d7e0-5f2e-7b1c-9a3e-2f6d2c8b4a10""#, + "DELETE weather:['London', d'2025-02-13T05:00:00Z']", + "DELETE k:{ a: 1, b: [2, 3] }", + "DELETE person:tobie TIMEOUT 5s", + "DELETE person:tobie WITH NOINDEX", + "DELETE person:tobie PARALLEL", + "DELETE person:tobie EXPLAIN FULL", + "/* one */ DELETE /* record */ person:tobie -- gone", + "SELECT * FROM (DELETE person:tobie)", + ]) + func oneRecordById(sql: String) { + #expect(!Self.deletesEveryRow(sql)) + } + + @Test("A SurrealQL DELETE of a table, a range, a parameter, a query or several records deletes every row", arguments: [ + "DELETE person", + "DELETE ONLY person", + "DELETE person:1..10", + "DELETE person:1..", + "DELETE person:..", + "DELETE person:..=5", + "DELETE person:1>..5", + "DELETE temperature:['London', NONE]..=['London', ..]", + "DELETE person:['a', ..]", + "DELETE $records", + "DELETE (SELECT VALUE id FROM person WHERE age > 20 LIMIT 2)", + "DELETE type::record('person', 1)", + "DELETE person:tobie->bought", + "DELETE person:tobie.friends", + "DELETE person:tobie, person:jaime", + "DELETE person:rand()", + #"DELETE person:"tobie""#, + #"DELETE r"person:tobie""#, + "DELETE person: tobie", + "DELETE person :tobie", + "DELETE person:tobie RETURN DELETE person", + "DELETE person:tobie; DELETE person", + "SELECT * FROM (DELETE person)", + ]) + func manyRecords(sql: String) { + #expect(Self.deletesEveryRow(sql)) + } + + @Test("Only SurrealDB reads a record id, and a driver's row change is read as before") + func otherEnginesReadAsBefore() { + #expect(Self.deletesEveryRow("DELETE person:tobie", "MySQL")) + let surrealQL = SQLLexicalReadings.resolve(databaseTypeId: "SurrealDB", declared: nil, session: nil).execution + #expect(SQLUnfilteredWrite.writes(in: "DELETE person:tobie", grammar: surrealQL).first == .delete) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift new file mode 100644 index 0000000000..cd56a2f527 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift @@ -0,0 +1,210 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// T-SQL reserves BEGIN, so a BEGIN ... END block holds its `;`s wherever it stands, not only in a routine's body. +struct TSQLControlFlowBlockTests { + private static let sqlServer = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + .execution + + private static let merge = "MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN MATCHED THEN DELETE" + + private func sent(_ sql: String) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: Self.sqlServer).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.sqlServer) + } + + @Test("A block an IF, a WHILE or a statement of its own opens stays one statement", arguments: [ + ("IF 1=1 BEGIN SELECT 1; SELECT 2; END; SELECT 3", ["IF 1=1 BEGIN SELECT 1; SELECT 2; END", "SELECT 3"]), + ( + "WHILE @i < 3\nBEGIN\n SET @i += 1;\n PRINT @i;\nEND;\nSELECT @i;", + ["WHILE @i < 3\nBEGIN\n SET @i += 1;\n PRINT @i;\nEND", "SELECT @i"] + ), + ("BEGIN\n SELECT 1;\n SELECT 2;\nEND;\nSELECT 3;", ["BEGIN\n SELECT 1;\n SELECT 2;\nEND", "SELECT 3"]), + ( + "IF @a = 1 BEGIN IF @b = 1 BEGIN SELECT 1; END ELSE BEGIN SELECT 2; END; SELECT 3; END; SELECT 4", + ["IF @a = 1 BEGIN IF @b = 1 BEGIN SELECT 1; END ELSE BEGIN SELECT 2; END; SELECT 3; END", "SELECT 4"] + ), + ( + "BEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_MESSAGE();\nEND CATCH;\nSELECT 9;", + ["BEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_MESSAGE();\nEND CATCH", "SELECT 9"] + ), + ]) + func blockHoldsItsSemicolons(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Run Current anywhere in a conditional block runs the whole block") + func caretInsideTheBlockRunsTheBlock() { + let block = "IF @env = 'dev'\nBEGIN\n DELETE FROM a;\n DELETE FROM b;\nEND" + let sql = "SELECT 0;\n\(block);\nSELECT 9;" + + #expect(atCursor(sql, on: "IF @env") == block) + #expect(atCursor(sql, on: "DELETE FROM b") == block) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.sqlServer).count == 3) + } + + @Test("An END is never followed by more of its block, so the IF after it is the next statement") + func endBeforeTheNextStatementClosesTheBlock() { + let sql = "IF @a = 1 BEGIN SELECT 1; END\nIF @b = 1 BEGIN SELECT 2; END;\nSELECT 3;" + + #expect(sent(sql) == ["IF @a = 1 BEGIN SELECT 1; END\nIF @b = 1 BEGIN SELECT 2; END", "SELECT 3"]) + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: Self.sqlServer)) + } + + @Test("A transaction's BEGIN is a statement, not a block", arguments: ["TRAN", "TRANSACTION", "DISTRIBUTED TRANSACTION"]) + func transactionBeginIsAStatement(opener: String) { + #expect(sent("BEGIN \(opener);\nUPDATE t SET a = 1;\nCOMMIT;") == ["BEGIN \(opener)", "UPDATE t SET a = 1", "COMMIT"]) + } + + @Test("Service Broker's BEGIN DIALOG, BEGIN CONVERSATION TIMER and END CONVERSATION are statements") + func serviceBrokerStatementsAreNotBlocks() { + let script = """ + BEGIN DIALOG CONVERSATION @h FROM SERVICE [s] TO SERVICE 't' ON CONTRACT [c]; + BEGIN CONVERSATION TIMER (@h) TIMEOUT = 60; + END CONVERSATION @h; + SELECT 1; + """ + #expect(sent(script).count == 4) + + let inBlock = "IF @x = 1 BEGIN END CONVERSATION @h; SELECT 1; END; SELECT 2" + #expect(sent(inBlock) == ["IF @x = 1 BEGIN END CONVERSATION @h; SELECT 1; END", "SELECT 2"]) + } + + @Test("A CASE expression closes at its own END, and one left open at a ; does not hold the next statement") + func caseExpressions() { + #expect(sent("SELECT CASE WHEN a = 1 THEN 'x' END AS c FROM t;\nSELECT 2;").count == 2) + #expect(sent("IF @x = 1 BEGIN SELECT CASE WHEN @x = 1 THEN 1 END; SELECT 2; END; SELECT 3").count == 2) + #expect(sent("SELECT CASE WHEN a = 1 THEN 1;\nSELECT 2;") == ["SELECT CASE WHEN a = 1 THEN 1", "SELECT 2"]) + } + + @Test("A MERGE inside a block keeps its ; and the block's own ; is a separator") + func mergeInsideABlock() { + let block = "IF 1 = 1 BEGIN \(Self.merge); END" + #expect(sent(block + ";\nSELECT 1;") == [block, "SELECT 1"]) + } + + @Test("A block left open runs to its GO line and no further") + func goLineEndsAnOpenBlock() { + let sql = "IF 1 = 1 BEGIN SELECT 1;\nGO\nSELECT 2" + #expect(sent(sql) == ["IF 1 = 1 BEGIN SELECT 1", "SELECT 2"]) + #expect(SQLStatementScanner.endsInsideOpenBlock("IF 1 = 1 BEGIN SELECT 1;", grammar: Self.sqlServer)) + } + + @Test("The gate still tiers every statement a block holds") + func gateTiersEachStatementInsideTheBlock() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + let sql = "IF @env = 'dev' BEGIN SELECT 1; DROP TABLE dbo.t; END" + + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "SQL Server", readings: readings).tier == .destructive) + #expect(SQLStatementClassifier.deletesEveryRow( + "IF 1 = 1 BEGIN DELETE FROM dbo.t; END", databaseTypeId: "SQL Server", readings: readings + )) + } + + @Test("A TRY block after a statement written without a ; is still one block") + func tryBlockAfterAnUnterminatedStatement() { + let script = "SET XACT_ABORT ON\nBEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n THROW;\nEND CATCH;\nSELECT 1;" + #expect(sent(script).count == 2) + #expect(sent(script).last == "SELECT 1") + } + + @Test("A begin where a name stands opens no block", arguments: [ + ("CREATE TABLE begin(a int); DROP TABLE users", 2), + ("INSERT INTO begin VALUES (1); DROP TABLE users", 2), + ("DO BEGIN SELECT 1 FROM DUMMY; SELECT 2 FROM DUMMY; END;", 3), + ]) + func nameInBeginsPlaceOpensNothing(sql: String, count: Int) { + #expect(sent(sql).count == count) + } + + @Test("A block holding more than one statement is told apart from one statement", arguments: [ + ("IF x BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", true), + ("BEGIN MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN MATCHED THEN DELETE; SELECT 1; END", true), + ("IF x BEGIN SELECT 1; WITH c AS (SELECT id FROM t) DELETE FROM c; END", true), + ("IF x BEGIN SELECT 1; DISABLE TRIGGER trg ON t; END", true), + ("BEGIN TRY SELECT 1; THROW 50000, 'x', 1; END TRY BEGIN CATCH SELECT 2; END CATCH", true), + ("IF x BEGIN SELECT 1; done: SELECT 2; END", true), + ("IF x BEGIN SELECT 1; (SELECT 2); END", true), + ("IF x BEGIN SELECT 1; END CONVERSATION @h; END", true), + ("IF x SELECT 1", false), + ("IF x BEGIN SELECT 1; END", false), + ("IF x BEGIN SELECT 1;; END", false), + ("BEGIN IF x SELECT 1; ELSE SELECT 2; END", false), + ("BEGIN TRY SELECT 1; END TRY BEGIN CATCH THROW; END CATCH", false), + ("CREATE PROCEDURE p AS BEGIN SELECT 1; SELECT 2; END", false), + ]) + func blockHoldingSeveralStatements(sql: String, holdsSeveral: Bool) { + #expect(SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: Self.sqlServer) == holdsSeveral) + let postgreSQL = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil).execution + #expect(!SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: postgreSQL)) + } + + @Test("A ; that ends an IF's body before its ELSE stays inside the IF", arguments: [ + ( + "IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday')\n SELECT 'Weekend';\nELSE\n SELECT 'Weekday';", + ["IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday')\n SELECT 'Weekend';\nELSE\n SELECT 'Weekday'"] + ), + ("IF @a = 1 SELECT 1; -- one\nELSE SELECT 2; SELECT 3", ["IF @a = 1 SELECT 1; -- one\nELSE SELECT 2", "SELECT 3"]), + ( + "IF @a = 1 IF @b = 1 SELECT 1; ELSE SELECT 2; ELSE SELECT 3; SELECT 4", + ["IF @a = 1 IF @b = 1 SELECT 1; ELSE SELECT 2; ELSE SELECT 3", "SELECT 4"] + ), + ( + "IF @a = 1 SELECT 1; ELSE IF @b = 1 SELECT 2; ELSE SELECT 3; SELECT 4", + ["IF @a = 1 SELECT 1; ELSE IF @b = 1 SELECT 2; ELSE SELECT 3", "SELECT 4"] + ), + ("WHILE @i < 3 IF @i = 1 BREAK; ELSE SET @i += 1; SELECT @i", ["WHILE @i < 3 IF @i = 1 BREAK; ELSE SET @i += 1", "SELECT @i"]), + ("IF @a = 1 BEGIN SELECT 1; END; ELSE SELECT 2; SELECT 3", ["IF @a = 1 BEGIN SELECT 1; END; ELSE SELECT 2", "SELECT 3"]), + ("IF @a = 1 SELECT 1;; ELSE SELECT 2", ["IF @a = 1 SELECT 1;; ELSE SELECT 2"]), + ("IF @a = 1 \(Self.merge); ELSE SELECT 2;", ["IF @a = 1 \(Self.merge); ELSE SELECT 2"]), + ("IF @a = 1 SELECT 1; ELSE \(Self.merge);", ["IF @a = 1 SELECT 1; ELSE \(Self.merge);"]), + ]) + func semicolonBeforeElse(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + #expect(!SQLUnterminatedStatements.blockHoldsSeveralStatements(expected[0], grammar: Self.sqlServer)) + } + + @Test("A ; after an IF's body ends the IF when no ELSE follows", arguments: [ + ("IF @a = 1 SELECT 1; SELECT 2;", ["IF @a = 1 SELECT 1", "SELECT 2"]), + ("IF @a = 1 SELECT 1;\n", ["IF @a = 1 SELECT 1"]), + ("IF @a = 1 SELECT 1;; SELECT 2", ["IF @a = 1 SELECT 1", "SELECT 2"]), + ("IF @a = 1 SELECT 1; [ELSE]", ["IF @a = 1 SELECT 1", "[ELSE]"]), + ("IF @a = 1 SELECT 1; ELSEWHERE", ["IF @a = 1 SELECT 1", "ELSEWHERE"]), + ("IF @a = 1 SELECT 1 ELSE SELECT 2; ELSE SELECT 3", ["IF @a = 1 SELECT 1 ELSE SELECT 2", "ELSE SELECT 3"]), + ("SELECT 1; ELSE SELECT 2", ["SELECT 1", "ELSE SELECT 2"]), + ("DROP TABLE IF EXISTS t; SELECT 1", ["DROP TABLE IF EXISTS t", "SELECT 1"]), + ( + "IF @a = 1 SELECT CASE WHEN @b = 1 THEN 1 ELSE 2 END; SELECT 3", + ["IF @a = 1 SELECT CASE WHEN @b = 1 THEN 1 ELSE 2 END", "SELECT 3"] + ), + ]) + func semicolonWithoutElse(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Run Current anywhere in an IF ... ELSE runs the whole IF, and the gate tiers both branches") + func caretOnTheElseRunsTheIf() { + let statement = "IF @env = 'dev'\n SELECT 1;\nELSE\n DROP TABLE dbo.t" + let sql = "SELECT 0;\n\(statement);\nSELECT 9;" + + #expect(atCursor(sql, on: "ELSE") == statement) + #expect(atCursor(sql, on: "DROP") == statement) + #expect(atCursor(sql, on: "SELECT 9") == "SELECT 9") + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.sqlServer).count == 3) + + let readings = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(statement, databaseTypeId: "SQL Server", readings: readings).tier == .destructive) + } + + @Test("Other engines still read BEGIN outside a routine as a word", arguments: ["PostgreSQL", "MySQL", "SQLite"]) + func otherEnginesAreUnchanged(typeId: String) { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "BEGIN\nDELETE FROM t;\nSELECT 1;", grammar: grammar) + #expect(statements.map { $0.sql } == ["BEGIN\nDELETE FROM t", "SELECT 1"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/RelayPollStateTests.swift b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/RelayPollStateTests.swift index 97fa773461..c211586af2 100644 --- a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/RelayPollStateTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/RelayPollStateTests.swift @@ -2,89 +2,11 @@ // RelayPollStateTests.swift // TableProSSHTransportTests // -// Tests for relayFDState, the poll revents classifier that decides whether a -// relay fd is idle, readable, draining before teardown, or fatally errored. -// import Foundation @testable import TableProSSHTransport import Testing -@Suite("RelayPollState") -struct RelayPollStateTests { - @Test("No events is idle") - func idle() { - #expect(relayFDState(0) == .idle) - } - - @Test("POLLIN alone is readable") - func readable() { - #expect(relayFDState(Int16(POLLIN)) == .readable) - } - - @Test("POLLHUP drains then stops") - func hangup() { - #expect(relayFDState(Int16(POLLHUP)) == .drainThenStop) - } - - @Test("POLLIN with POLLHUP drains then stops") - func hangupWithBufferedData() { - #expect(relayFDState(Int16(POLLIN) | Int16(POLLHUP)) == .drainThenStop) - } - - @Test("POLLERR is fatal") - func error() { - #expect(relayFDState(Int16(POLLERR)) == .stop) - } - - @Test("POLLNVAL is fatal") - func invalid() { - #expect(relayFDState(Int16(POLLNVAL)) == .stop) - } - - @Test("Fatal errors win over readable data") - func errorWithData() { - #expect(relayFDState(Int16(POLLIN) | Int16(POLLERR)) == .stop) - } - - @Test("Fatal errors win over hangup") - func errorWithHangup() { - #expect(relayFDState(Int16(POLLHUP) | Int16(POLLERR)) == .stop) - } -} - -@Suite("transportPollOutcome") -struct TransportPollOutcomeTests { - @Test("An expired wait is not readiness") - func timeoutIsNotReady() { - #expect(transportPollOutcome(revents: 0, requestedEvents: Int16(POLLIN)) == .timedOut) - } - - @Test("The requested direction becoming ready is readiness") - func requestedDirectionReady() { - #expect(transportPollOutcome(revents: Int16(POLLIN), requestedEvents: Int16(POLLIN)) == .ready) - #expect(transportPollOutcome(revents: Int16(POLLOUT), requestedEvents: Int16(POLLOUT)) == .ready) - } - - @Test("Another direction becoming ready is not the requested one") - func otherDirectionIsNotReady() { - #expect(transportPollOutcome(revents: Int16(POLLIN), requestedEvents: Int16(POLLOUT)) == .timedOut) - } - - @Test("Hangup and errors end the wait") - func hangupEndsWait() { - #expect(transportPollOutcome(revents: Int16(POLLHUP), requestedEvents: Int16(POLLOUT)) == .hangup) - #expect(transportPollOutcome(revents: Int16(POLLERR), requestedEvents: Int16(POLLOUT)) == .hangup) - #expect(transportPollOutcome(revents: Int16(POLLNVAL), requestedEvents: Int16(POLLOUT)) == .hangup) - } - - @Test("Hangup wins over the requested direction") - func hangupWinsOverReady() { - #expect(transportPollOutcome(revents: Int16(POLLOUT) | Int16(POLLHUP), requestedEvents: Int16(POLLOUT)) == .hangup) - } -} - -@Suite("pollReady") struct PollReadyTests { @Test("A directionless wait yields and retries instead of spinning") func directionlessYields() { @@ -113,3 +35,36 @@ struct PollReadyTests { #expect(pollReady(fd: pair.a, directions: .inbound, timeoutMs: 50) == false) } } + +struct TransportRoomTests { + @Test("A socket with free buffer has room") + func roomWhenEmpty() { + let pair = SocketPair() + defer { pair.close() } + + #expect(transportRoom(pair.a) == .room) + } + + @Test("A socket whose buffer is full has no room") + func fullWhenFilled() { + let pair = SocketPair() + defer { pair.close() } + + let flags = fcntl(pair.a, F_GETFL, 0) + _ = fcntl(pair.a, F_SETFL, flags | O_NONBLOCK) + var chunk = [UInt8](repeating: 0x46, count: 4_096) + while Darwin.send(pair.a, &chunk, chunk.count, 0) > 0 {} + + #expect(transportRoom(pair.a) == .full) + } + + @Test("A socket whose peer closed is gone") + func goneWhenPeerClosed() { + let pair = SocketPair() + defer { pair.close() } + + pair.closeB() + + #expect(transportRoom(pair.a) == .gone) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHChannelRelayTests.swift b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHChannelRelayTests.swift deleted file mode 100644 index 3077f8ca2d..0000000000 --- a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHChannelRelayTests.swift +++ /dev/null @@ -1,598 +0,0 @@ -// -// SSHChannelRelayTests.swift -// TableProSSHTransportTests -// -// Tests for SSHChannelRelay termination behaviour. The relay runs over real -// socketpairs with a scripted channel so the regression case (a closed peer fd -// must terminate the loop instead of busy-spinning) is exercised end to end. -// - -import Foundation -import Testing - -@testable import TableProSSHTransport - -@Suite("SSHChannelRelay") -struct SSHChannelRelayTests { - @Test("Cancellation via isActive stops the relay") - func cancelled() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .wouldBlock), - isActive: { false } - ) - - #expect(result == .cancelled) - } - - @Test("Transport hangup terminates instead of spinning") - func transportHangup() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - transport.closeB() - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .wouldBlock) - ) - - #expect(result == .transportHangup) - } - - @Test("Transport half-close at EOF terminates instead of spinning") - func transportHalfCloseEOF() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - shutdown(transport.b, SHUT_WR) - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .wouldBlock) - ) - - #expect(result == .transportHangup) - } - - @Test("Local hangup terminates instead of spinning") - func localHangup() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - local.closeB() - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .wouldBlock) - ) - - #expect(result == .localClosed) - } - - @Test("Channel close terminates the relay") - func channelClosed() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .closed) - ) - - #expect(result == .channelClosed) - } - - @Test("A transport failure on read ends as a transport hangup, not a channel close") - func readTransportFailureIsHangup() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - var dummy: UInt8 = 1 - _ = Darwin.send(transport.b, &dummy, 1, 0) - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .transportFailed) - ) - - #expect(result == .transportHangup) - } - - @Test("A transport failure on read after the SSH server hung up ends as a transport hangup") - func readTransportFailureAfterHangupIsHangup() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - transport.closeB() - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .transportFailed) - ) - - #expect(result == .transportHangup) - } - - @Test("A transport failure on write ends as a transport hangup, not a channel close") - func writeTransportFailureIsHangup() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let payload = Data("world".utf8) - payload.withUnsafeBytes { raw in - _ = Darwin.send(local.b, raw.baseAddress, raw.count, 0) - } - - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: FakeChannelIO(fallback: .wouldBlock, writeResult: .transportFailed) - ) - - #expect(result == .transportHangup) - } - - @Test("Channel data for a client that already closed ends the relay as a local close") - func channelDataForClosedClientIsLocalClose() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - local.closeB() - var dummy: UInt8 = 1 - _ = Darwin.send(transport.b, &dummy, 1, 0) - - let payload = Data("rows".utf8) - let io = FakeChannelIO(actions: [.data(payload)], fallback: .wouldBlock) - let result = runRelay(localFD: local.a, transportFD: transport.a, io: io) - - #expect(result == .localClosed) - #expect(io.readCount == 1) - #expect(io.deliveredCount == payload.count) - } - - @Test("Channel data is forwarded to the local socket") - func forwardsChannelData() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let payload = Data("hello".utf8) - var dummy: UInt8 = 1 - _ = Darwin.send(transport.b, &dummy, 1, 0) - - let io = FakeChannelIO(actions: [.data(payload)], fallback: .closed) - let result = runRelay(localFD: local.a, transportFD: transport.a, io: io) - - #expect(result == .channelClosed) - - var received = [UInt8](repeating: 0, count: payload.count) - let count = recv(local.b, &received, received.count, 0) - #expect(count == payload.count) - #expect(Data(received) == payload) - } - - @Test("Local data is forwarded to the channel") - func forwardsLocalData() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let payload = Data("world".utf8) - payload.withUnsafeBytes { raw in - _ = Darwin.send(local.b, raw.baseAddress, raw.count, 0) - } - - let io = FakeChannelIO(fallback: .wouldBlock) - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: io, - isActive: io.activeUntilWritten(payload.count) - ) - - #expect(result == .cancelled) - #expect(io.written == payload) - } - - @Test("Channel data counts as received") - func countsChannelDataAsReceived() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let payload = Data("hello".utf8) - var dummy: UInt8 = 1 - _ = Darwin.send(transport.b, &dummy, 1, 0) - - let counter = RecordingByteObserver() - let io = FakeChannelIO(actions: [.data(payload)], fallback: .closed) - let result = runRelay(localFD: local.a, transportFD: transport.a, io: io, byteCounter: counter) - - #expect(result == .channelClosed) - #expect(counter.received == payload.count) - #expect(counter.sent == 0) - } - - @Test("Local data counts as sent") - func countsLocalDataAsSent() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let payload = Data("world".utf8) - payload.withUnsafeBytes { raw in - _ = Darwin.send(local.b, raw.baseAddress, raw.count, 0) - } - - let counter = RecordingByteObserver() - let io = FakeChannelIO(fallback: .wouldBlock) - let result = runRelay( - localFD: local.a, - transportFD: transport.a, - io: io, - isActive: io.activeUntilWritten(payload.count), - byteCounter: counter - ) - - #expect(result == .cancelled) - #expect(counter.received == 0) - #expect(counter.sent == payload.count) - } - - @Test("A relay with no counter still runs") - func runsWithoutACounter() { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let payload = Data("hello".utf8) - var dummy: UInt8 = 1 - _ = Darwin.send(transport.b, &dummy, 1, 0) - - let io = FakeChannelIO(actions: [.data(payload)], fallback: .closed) - let result = runRelay(localFD: local.a, transportFD: transport.a, io: io) - - #expect(result == .channelClosed) - } - - @Test("Concurrent relays sharing one transport each receive their own channel data") - func concurrentRelaysShareTransport() { - let transport = SocketPair() - let first = SocketPair() - let second = SocketPair() - defer { transport.close(); first.close(); second.close() } - - var dummy: UInt8 = 1 - _ = Darwin.send(transport.b, &dummy, 1, 0) - - let firstPayload = Data("first".utf8) - let secondPayload = Data("second".utf8) - let firstIO = FakeChannelIO(actions: [.data(firstPayload)], fallback: .closed) - let secondIO = FakeChannelIO(actions: [.data(secondPayload)], fallback: .closed) - - let group = DispatchGroup() - let firstBox = ResultBox() - let secondBox = ResultBox() - - for (localFD, io, box) in [(first.a, firstIO, firstBox), (second.a, secondIO, secondBox)] { - group.enter() - runRelayOnDedicatedThread( - localFD: localFD, - transportFD: transport.a, - io: io, - isActive: { true }, - box: box - ) { group.leave() } - } - - #expect(group.wait(timeout: .now() + 5) == .success) - #expect(firstBox.value == .channelClosed) - #expect(secondBox.value == .channelClosed) - #expect(readPayload(from: first.b, count: firstPayload.count) == firstPayload) - #expect(readPayload(from: second.b, count: secondPayload.count) == secondPayload) - } - - private func readPayload(from fd: Int32, count: Int) -> Data { - var buffer = [UInt8](repeating: 0, count: count) - let received = recv(fd, &buffer, count, 0) - guard received > 0 else { return Data() } - return Data(buffer.prefix(received)) - } - - private func runRelay( - localFD: Int32, - transportFD: Int32, - io: FakeChannelIO, - isActive: @escaping @Sendable () -> Bool = { true }, - byteCounter: (any RelayByteObserver)? = nil, - timeout: Double = 3 - ) -> RelayTermination? { - let box = ResultBox() - let semaphore = DispatchSemaphore(value: 0) - runRelayOnDedicatedThread( - localFD: localFD, - transportFD: transportFD, - io: io, - isActive: isActive, - box: box, - byteCounter: byteCounter - ) { semaphore.signal() } - _ = semaphore.wait(timeout: .now() + timeout) - return box.value - } -} - -/// The lost wakeup and the drain that follows it, both against a transport fd that is never -/// readable. libssh2 decrypts whole SSH packets into its own buffer, so a second channel on the -/// session holds bytes the transport has already given up: `poll` reports nothing, forever. -/// -/// Measured against a real sshd with two channels and 98,304 queued bytes: reading only when the -/// transport polls readable delivered 0 bytes in 60,178ms; pumping on the poll timeout too -/// delivered all of it in 1,504ms with one buffer per timeout; draining until the channel answers -/// EAGAIN delivered all of it in 502ms. -@Suite("SSHChannelRelay backlog drain") -struct SSHChannelRelayBacklogTests { - fileprivate static let bufferSize = 32_768 - fileprivate static let queuedBuffers = 3 - - @Test("A backlog reaches the local socket although the transport never polls readable") - func deliversBacklogWithoutATransportWakeup() throws { - let run = try runBacklogRelay() - - #expect(run.delivered == Self.bufferSize * Self.queuedBuffers) - } - - @Test("One poll round drains the whole backlog instead of one buffer per timeout") - func drainsInOnePollRound() throws { - let run = try runBacklogRelay() - - #expect(run.reads == Self.queuedBuffers + 1) - } - - @Test("A backlog past the per-round cap still lands inside one poll interval") - func drainsPastTheReadCapWithoutWaiting() throws { - let run = try runBacklogRelay(buffers: 20) - - #expect(run.delivered == Self.bufferSize * 20) - #expect(run.reads == 21) - #expect(run.elapsed < 1.0) - } - - private struct BacklogRun { - let delivered: Int - let reads: Int - let elapsed: TimeInterval - } - - /// Drains the local socket on a reader thread so a 96KB backlog cannot fill the socket - /// buffer and stall the relay inside `send`. The relay stops once the channel has handed - /// over every queued byte, which is what the two shapes take different numbers of poll - /// rounds to reach. - private func runBacklogRelay(buffers: Int = SSHChannelRelayBacklogTests.queuedBuffers) throws -> BacklogRun { - let local = SocketPair() - let transport = SocketPair() - defer { local.close(); transport.close() } - - let total = Self.bufferSize * buffers - let io = FakeChannelIO( - actions: Array(repeating: .data(Data(repeating: 0x41, count: Self.bufferSize)), count: buffers), - fallback: .wouldBlock - ) - - let drained = ByteTally() - let reader = Thread { - var buffer = [UInt8](repeating: 0, count: Self.bufferSize) - while drained.value < total { - let count = recv(local.b, &buffer, buffer.count, 0) - if count <= 0 { return } - drained.add(count) - } - } - reader.stackSize = 512 * 1_024 - reader.start() - - let box = ResultBox() - let semaphore = DispatchSemaphore(value: 0) - let started = Date() - runRelayOnDedicatedThread( - localFD: local.a, - transportFD: transport.a, - io: io, - isActive: { io.deliveredCount < total }, - box: box - ) { semaphore.signal() } - - #expect(semaphore.wait(timeout: .now() + 10) == .success) - let elapsed = Date().timeIntervalSince(started) - _ = drained.waitUntil(total, timeout: 2) - return BacklogRun(delivered: drained.value, reads: io.readCount, elapsed: elapsed) - } -} - -/// Runs a relay on a thread of its own rather than borrowing from the shared global -/// queue. The relay loop blocks its thread for as long as it runs, and the test suite -/// runs in parallel, so relays sharing the global queue's bounded pool can starve each -/// other and time out on machines under load. -internal func runRelayOnDedicatedThread( - localFD: Int32, - transportFD: Int32, - io: any SSHChannelIO, - isActive: @escaping @Sendable () -> Bool, - box: ResultBox, - byteCounter: (any RelayByteObserver)? = nil, - onFinish: @escaping @Sendable () -> Void -) { - let thread = Thread { - let relay = SSHChannelRelay( - localFD: localFD, - transportFD: transportFD, - channelIO: io, - bufferSize: 32_768, - isActive: isActive, - byteCounter: byteCounter - ) - box.value = relay.run() - onFinish() - } - thread.stackSize = 512 * 1_024 - thread.start() -} - -internal final class ResultBox: @unchecked Sendable { - var value: RelayTermination? -} - -internal final class ByteTally: @unchecked Sendable { - private let lock = NSLock() - private var count = 0 - - var value: Int { - lock.lock() - defer { lock.unlock() } - return count - } - - func add(_ amount: Int) { - lock.lock() - count += amount - lock.unlock() - } - - func waitUntil(_ target: Int, timeout: TimeInterval) -> Bool { - let deadline = Date().addingTimeInterval(timeout) - while value < target { - if Date() >= deadline { return false } - usleep(2_000) - } - return true - } -} - -internal final class RecordingByteObserver: RelayByteObserver, @unchecked Sendable { - private let lock = NSLock() - private var receivedBytes = 0 - private var sentBytes = 0 - - var received: Int { - lock.lock() - defer { lock.unlock() } - return receivedBytes - } - - var sent: Int { - lock.lock() - defer { lock.unlock() } - return sentBytes - } - - func recordReceived(_ count: Int) { - lock.lock() - receivedBytes += count - lock.unlock() - } - - func recordSent(_ count: Int) { - lock.lock() - sentBytes += count - lock.unlock() - } -} - -internal final class FakeChannelIO: SSHChannelIO, @unchecked Sendable { - enum Action { - case data(Data) - case wouldBlock - case closed - case transportFailed - } - - private let lock = NSLock() - private var actions: [Action] - private let fallback: Action - private let writeResult: ChannelWriteResult? - private var writtenBuffer = Data() - private var reads = 0 - private var delivered = 0 - - init(actions: [Action] = [], fallback: Action = .wouldBlock, writeResult: ChannelWriteResult? = nil) { - self.actions = actions - self.fallback = fallback - self.writeResult = writeResult - } - - var written: Data { - lock.lock() - defer { lock.unlock() } - return writtenBuffer - } - - var readCount: Int { - lock.lock() - defer { lock.unlock() } - return reads - } - - var deliveredCount: Int { - lock.lock() - defer { lock.unlock() } - return delivered - } - - func activeUntilWritten(_ target: Int) -> @Sendable () -> Bool { - { [weak self] in (self?.written.count ?? target) < target } - } - - func read(into buffer: UnsafeMutablePointer, count: Int) -> ChannelReadResult { - lock.lock() - defer { lock.unlock() } - reads += 1 - let action = actions.isEmpty ? fallback : actions.removeFirst() - switch action { - case .data(let data): - let length = min(data.count, count) - buffer.withMemoryRebound(to: UInt8.self, capacity: length) { destination in - _ = data.copyBytes(to: UnsafeMutableBufferPointer(start: destination, count: length)) - } - delivered += length - return .bytes(length) - case .wouldBlock: - return .wouldBlock - case .closed: - return .closed - case .transportFailed: - return .transportFailed - } - } - - func write(_ buffer: UnsafePointer, count: Int) -> ChannelWriteResult { - lock.lock() - defer { lock.unlock() } - if let writeResult { return writeResult } - buffer.withMemoryRebound(to: UInt8.self, capacity: count) { source in - writtenBuffer.append(contentsOf: UnsafeBufferPointer(start: source, count: count)) - } - return .bytes(count) - } - - func blockDirections() -> RelayDirections { [] } -} diff --git a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHListenerAcceptorTests.swift b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHListenerAcceptorTests.swift new file mode 100644 index 0000000000..17a5e1236a --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHListenerAcceptorTests.swift @@ -0,0 +1,233 @@ +// +// SSHListenerAcceptorTests.swift +// TableProSSHTransportTests +// + +import Foundation +import os +import TableProSSHTransport +import TableProTestSupport +import Testing + +struct SSHListenerAcceptorTests { + @Test("A client that connects is reported once, with a blocking descriptor") + func reportsAConnectedClientOnce() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let clients = AcceptedClients() + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { clients.record($0) } + acceptor.start() + + let peer = try listener.connect() + defer { Darwin.close(peer) } + + #expect(await Self.finishes { await clients.arrived(1) }) + acceptor.cancel() + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + + let accepted = clients.descriptors + defer { accepted.forEach { _ = Darwin.close($0) } } + #expect(accepted.count == 1) + #expect(accepted.allSatisfy { Self.isBlocking($0) }) + } + + @Test("Two clients that connect back to back are both reported") + func reportsBackToBackClients() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let clients = AcceptedClients() + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { clients.record($0) } + acceptor.start() + + let first = try listener.connect() + defer { Darwin.close(first) } + let second = try listener.connect() + defer { Darwin.close(second) } + + #expect(await Self.finishes { await clients.arrived(2) }) + acceptor.cancel() + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + + let accepted = clients.descriptors + defer { accepted.forEach { _ = Darwin.close($0) } } + #expect(accepted.count == 2) + #expect(Set(accepted).count == 2) + #expect(accepted.allSatisfy { Self.isBlocking($0) }) + } + + @Test("After cancel the completion fires, and a client that connects later is left for someone else") + func cancelStopsAccepting() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let clients = AcceptedClients() + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { clients.record($0) } + acceptor.start() + + acceptor.cancel() + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + + let late = try listener.connect() + defer { Darwin.close(late) } + // Accepting it here proves the cancelled acceptor did not take it and never can. + let taken = try listener.acceptWaiting() + defer { Darwin.close(taken) } + #expect(clients.descriptors.isEmpty) + } + + @Test("Cancelling more than once, from several threads, is safe") + func cancelIsIdempotent() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { _ = Darwin.close($0.descriptor) } + acceptor.start() + + DispatchQueue.concurrentPerform(iterations: 8) { _ in acceptor.cancel() } + acceptor.cancel() + + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + } + + @Test("An acceptor cancelled before it started still completes") + func cancelBeforeStartCompletes() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { _ = Darwin.close($0.descriptor) } + + acceptor.cancel() + + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + } + + private static func acceptQueue() -> DispatchQueue { + DispatchQueue(label: "com.TablePro.tests.ssh.accept") + } + + private static func finishes(_ work: @escaping @Sendable () async -> Void) async -> Bool { + await BoundedCall.result(of: work) != nil + } + + private static func isBlocking(_ descriptor: Int32) -> Bool { + let flags = fcntl(descriptor, F_GETFL, 0) + return flags >= 0 && flags & O_NONBLOCK == 0 + } +} + +private struct SocketFailure: Error, CustomStringConvertible { + let call: String + let code: Int32 + + var description: String { "\(call) failed: \(String(cString: strerror(code)))" } +} + +private struct LoopbackListener { + let descriptor: Int32 + let port: UInt16 + + init() throws { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw SocketFailure(call: "socket", code: errno) } + + var address = Self.loopback(port: 0) + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(fd, $0, socklen_t(MemoryLayout.size)) + } + } + guard bound == 0, listen(fd, 8) == 0 else { + let failure = SocketFailure(call: "bind", code: errno) + Darwin.close(fd) + throw failure + } + + var assigned = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &assigned) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + getsockname(fd, $0, &length) + } + } + guard named == 0 else { + let failure = SocketFailure(call: "getsockname", code: errno) + Darwin.close(fd) + throw failure + } + + descriptor = fd + port = UInt16(bigEndian: assigned.sin_port) + } + + func close() { + Darwin.close(descriptor) + } + + func connect() throws -> Int32 { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw SocketFailure(call: "socket", code: errno) } + + var address = Self.loopback(port: port) + let connected = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(fd, $0, socklen_t(MemoryLayout.size)) + } + } + guard connected == 0 else { + let failure = SocketFailure(call: "connect", code: errno) + Darwin.close(fd) + throw failure + } + return fd + } + + // Returns as soon as a connection is pending; the timeout only bounds a failing run. + func acceptWaiting() throws -> Int32 { + var pollFD = pollfd(fd: descriptor, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, 10_000) == 1 else { throw SocketFailure(call: "poll", code: errno) } + let fd = accept(descriptor, nil, nil) + guard fd >= 0 else { throw SocketFailure(call: "accept", code: errno) } + return fd + } + + private static func loopback(port: UInt16) -> sockaddr_in { + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_port = port.bigEndian + address.sin_addr.s_addr = inet_addr("127.0.0.1") + return address + } +} + +private final class AcceptedClients: Sendable { + private struct State { + var clients: [SSHAcceptedClient] = [] + var waiters: [(count: Int, continuation: CheckedContinuation)] = [] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + + var descriptors: [Int32] { + state.withLock { $0.clients.map { $0.descriptor } } + } + + func record(_ client: SSHAcceptedClient) { + let ready = state.withLock { state -> [CheckedContinuation] in + state.clients.append(client) + let total = state.clients.count + let due = state.waiters.filter { $0.count <= total }.map { $0.continuation } + state.waiters.removeAll { $0.count <= total } + return due + } + ready.forEach { $0.resume() } + } + + func arrived(_ count: Int) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + let isReady = state.withLock { state -> Bool in + guard state.clients.count < count else { return true } + state.waiters.append((count, continuation)) + return false + } + if isReady { continuation.resume() } + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHSessionPumpTests.swift b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHSessionPumpTests.swift new file mode 100644 index 0000000000..aa99d89314 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHSessionPumpTests.swift @@ -0,0 +1,923 @@ +// +// SSHSessionPumpTests.swift +// TableProSSHTransportTests +// +// The pump runs over real socketpairs with scripted channels, so a closed peer, a full socket and +// a quiet transport behave the way a database client and an SSH server would make them behave. +// + +import Foundation +import os +import TableProTestSupport +import Testing + +@testable import TableProSSHTransport + +struct SSHSessionPumpTests { + @Test("Stopping the pump ends its relays as cancelled and closes their sockets") + func stopCancels() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .wouldBlock)) + fixture.pump.stop() + + try await waitUntil { ended.value != nil } + #expect(ended.value == .cancelled) + #expect(readsEndOfFile(local.b)) + } + + @Test("A relay added after stop ends as cancelled and its socket is closed") + func relayAfterStopCancels() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + fixture.pump.stop() + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .wouldBlock)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .cancelled) + #expect(readsEndOfFile(local.b)) + } + + @Test("Once stopped, every relay has ended and closed its socket") + func waitUntilStoppedCoversRelays() async throws { + let fixture = PumpFixture() + let first = SocketPair() + let second = SocketPair() + defer { first.close(); second.close(); fixture.close() } + + let firstEnded = fixture.relay(first, io: FakeChannelIO(fallback: .wouldBlock)) + let secondEnded = fixture.relay(second, io: FakeChannelIO(fallback: .wouldBlock)) + fixture.pump.stop() + await fixture.pump.waitUntilStopped() + + #expect(firstEnded.value == .cancelled) + #expect(secondEnded.value == .cancelled) + #expect(readsEndOfFile(first.b)) + #expect(readsEndOfFile(second.b)) + } + + @Test("Transport hangup ends the relay instead of spinning") + func transportHangup() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + fixture.transport.closeB() + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .wouldBlock)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .transportHangup) + } + + @Test("Transport half-close at EOF ends the relay instead of spinning") + func transportHalfCloseEOF() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + shutdown(fixture.transport.b, SHUT_WR) + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .wouldBlock)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .transportHangup) + } + + @Test("Local hangup ends the relay instead of spinning") + func localHangup() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + local.closeB() + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .wouldBlock)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .localClosed) + } + + @Test("Channel close ends the relay") + func channelClosed() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .closed)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .channelClosed) + } + + @Test("A transport failure on read ends as a transport hangup, not a channel close") + func readTransportFailureIsHangup() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .transportFailed)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .transportHangup) + } + + @Test("A transport failure on read after the SSH server hung up ends as a transport hangup") + func readTransportFailureAfterHangupIsHangup() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + fixture.transport.closeB() + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .transportFailed)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .transportHangup) + } + + @Test("A transport failure on write ends as a transport hangup, not a channel close") + func writeTransportFailureIsHangup() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + send(Data("world".utf8), to: local.b) + let ended = fixture.relay(local, io: FakeChannelIO(fallback: .wouldBlock, writeFallback: .transportFailed)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .transportHangup) + } + + @Test("Channel data for a client that already closed ends the relay as a local close") + func channelDataForClosedClientIsLocalClose() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + local.closeB() + let payload = Data("rows".utf8) + let io = FakeChannelIO(actions: [.data(payload)], fallback: .wouldBlock) + let ended = fixture.relay(local, io: io) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .localClosed) + #expect(io.readCount == 1) + #expect(io.deliveredCount == payload.count) + } + + @Test("Channel data is forwarded to the local socket") + func forwardsChannelData() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let payload = Data("hello".utf8) + let ended = fixture.relay(local, io: FakeChannelIO(actions: [.data(payload)], fallback: .closed)) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .channelClosed) + #expect(receive(from: local.b, count: payload.count) == payload) + } + + @Test("Local data is forwarded to the channel") + func forwardsLocalData() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let payload = Data("world".utf8) + send(payload, to: local.b) + let io = FakeChannelIO(fallback: .wouldBlock) + _ = fixture.relay(local, io: io) + + try await waitUntil { io.written.count >= payload.count } + #expect(io.written == payload) + } + + @Test("Channel data counts as received and local data as sent") + func countsBothDirections() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let incoming = Data("hello".utf8) + let outgoing = Data("world!".utf8) + send(outgoing, to: local.b) + let counter = RecordingByteObserver() + let io = FakeChannelIO(actions: [.data(incoming)], fallback: .wouldBlock) + _ = fixture.relay(local, io: io, counter: counter) + + try await waitUntil { counter.received == incoming.count && counter.sent == outgoing.count } + #expect(io.written == outgoing) + } + + @Test("Relays sharing one session each receive their own channel data") + func relaysShareOneSession() async throws { + let fixture = PumpFixture() + let first = SocketPair() + let second = SocketPair() + defer { first.close(); second.close(); fixture.close() } + + let firstPayload = Data("first".utf8) + let secondPayload = Data("second".utf8) + let firstEnded = fixture.relay(first, io: FakeChannelIO(actions: [.data(firstPayload)], fallback: .closed)) + let secondEnded = fixture.relay(second, io: FakeChannelIO(actions: [.data(secondPayload)], fallback: .closed)) + + try await waitUntil { firstEnded.value != nil && secondEnded.value != nil } + #expect(firstEnded.value == .channelClosed) + #expect(secondEnded.value == .channelClosed) + #expect(receive(from: first.b, count: firstPayload.count) == firstPayload) + #expect(receive(from: second.b, count: secondPayload.count) == secondPayload) + } + + // The lost wakeup: libssh2 drains every packet on the transport into its own queue whenever any + // channel reads, so data for a channel served earlier in a pass sits there with the socket quiet. + @Test("Data another channel's read queued reaches its client without a transport event") + func deliversDataAnotherReadQueued() async throws { + let fixture = PumpFixture() + let early = SocketPair() + let late = SocketPair() + defer { early.close(); late.close(); fixture.close() } + + let earlyIO = FakeChannelIO(fallback: .wouldBlock) + let queuedForEarly = Data("queued".utf8) + let lateIO = FakeChannelIO(actions: [.data(Data("late".utf8))], fallback: .wouldBlock) { reads in + if reads == 1 { earlyIO.enqueue(.data(queuedForEarly)) } + } + _ = fixture.relay(early, io: earlyIO) + try await waitUntil { earlyIO.readCount >= 1 } + _ = fixture.relay(late, io: lateIO) + + try await waitUntil(timeout: .seconds(2)) { earlyIO.deliveredCount == queuedForEarly.count } + #expect(receive(from: early.b, count: queuedForEarly.count) == queuedForEarly) + } + + @Test("An idle relay makes no channel call") + func idleRelayMakesNoCall() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let io = FakeChannelIO(fallback: .wouldBlock) + _ = fixture.relay(local, io: io) + try await waitUntil { io.readCount >= 1 } + let settled = io.readCount + + // The previous relay read the channel on a 500 ms timer, so this window would have caught one. + try await Task.sleep(for: .milliseconds(700)) + #expect(io.readCount == settled) + #expect(io.writeCount == 0) + } + + @Test("A call on the session relays whatever it queued") + func performRelaysWhatItQueued() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let io = FakeChannelIO(fallback: .wouldBlock) + _ = fixture.relay(local, io: io) + try await waitUntil { io.readCount >= 1 } + + let payload = Data("drained by an open".utf8) + fixture.pump.perform { io.enqueue(.data(payload)) } + + #expect(io.deliveredCount == payload.count) + #expect(receive(from: local.b, count: payload.count) == payload) + } + + // The SQLite agent's heartbeats travel client to channel while its rows wait for a slow reader. + @Test("Local data keeps reaching the channel while the client is not reading") + func localDataFlowsWhileClientDoesNotRead() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let chunk = Data(repeating: 0x52, count: 32_768) + let io = FakeChannelIO(actions: Array(repeating: .data(chunk), count: 64), fallback: .wouldBlock) + _ = fixture.relay(local, io: io) + try await waitUntil { io.readCount >= 1 } + + let heartbeat = Data("heartbeat".utf8) + send(heartbeat, to: local.b) + + try await waitUntil(timeout: .seconds(2)) { io.written == heartbeat } + #expect(io.deliveredCount < chunk.count * 64) + } + + @Test("A write waiting on the transport resumes once the transport can take data") + func writeResumesWhenTransportWritable() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + fillSendBuffer(fixture.transport.a) + let payload = Data("query".utf8) + send(payload, to: local.b) + let io = FakeChannelIO(fallback: .wouldBlock, writeResults: [.wouldBlock], writeBlocks: .outbound) + _ = fixture.relay(local, io: io) + try await waitUntil { io.writeCount >= 1 } + #expect(io.written.isEmpty) + + drain(fixture.transport.b) + + try await waitUntil(timeout: .seconds(2)) { io.written == payload } + } + + // Freeing a channel with a packet half sent would leave libssh2 refusing every later send. + @Test("A relay whose write is in flight ends only once that write finishes") + func inFlightWriteFinishesBeforeEnd() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + send(Data("x".utf8), to: local.b) + let io = FakeChannelIO( + actions: [.wouldBlock], + fallback: .closed, + writeResults: [.wouldBlock, .wouldBlock, .bytes(1)], + writeBlocks: .outbound + ) + let ended = fixture.relay(local, io: io) + try await waitUntil { io.writeCount >= 1 } + send(Data([1]), to: fixture.transport.b) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .channelClosed) + #expect(io.writeCount == 3) + } + + // A read can owe its window adjust, which only another read on that channel sends. + @Test("A relay whose read waits to send ends only once that read goes through") + func readOwingASendFinishesBeforeEnd() async throws { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + local.closeB() + let io = FakeChannelIO(actions: [.wouldBlock, .wouldBlock], fallback: .closed, readBlocks: .outbound) + let ended = fixture.relay(local, io: io) + + try await waitUntil { ended.value != nil } + #expect(ended.value == .localClosed) + #expect(io.readCount == 3) + } + + // libssh2 takes a window adjust off the transport whichever channel reads, and the writer waits on no socket. + @Test("A write waiting on the window goes out once another channel's read opens it") + func writeGoesOutWhenAnotherReadOpensItsWindow() async throws { + let fixture = PumpFixture() + let writer = SocketPair() + let reader = SocketPair() + defer { writer.close(); reader.close(); fixture.close() } + + let payload = Data("query".utf8) + send(payload, to: writer.b) + let writerIO = FakeChannelIO(fallback: .wouldBlock, sendWindow: false) + _ = fixture.relay(writer, io: writerIO) + try await waitUntil { writerIO.writeCount >= 1 } + #expect(writerIO.written.isEmpty) + + let readerIO = FakeChannelIO(actions: [.data(Data("rows".utf8))], fallback: .wouldBlock) { reads in + if reads == 1 { writerIO.openWindow() } + } + _ = fixture.relay(reader, io: readerIO) + + try await waitUntil(timeout: .seconds(2)) { writerIO.written == payload } + } + + // libssh2 sends one packet at a time per session, so an upload that always has one on its way would refuse + // every other channel's write until it ends. + @Test("A message gets through beside an upload limited by the socket") + func messageGetsThroughBesideUpload() async throws { + let fixture = PumpFixture() + let uploader = SocketPair() + let messenger = SocketPair() + defer { uploader.close(); messenger.close(); fixture.close() } + + let session = OnePacketSession(transportFD: fixture.transport.a) + let drain = SlowDrain(fixture.transport.b) + defer { drain.stop() } + let uploaderIO = session.channel() + let messengerIO = session.channel() + let upload = Data(repeating: 0x55, count: 16 << 20) + enlargeBuffers(uploader) + let peer = uploader.b + Thread { _ = upload.withUnsafeBytes { Darwin.send(peer, $0.baseAddress, $0.count, MSG_NOSIGNAL) } }.start() + _ = fixture.relay(uploader, io: uploaderIO) + try await waitUntil { uploaderIO.writtenCount > 0 } + + let message = Data("hello".utf8) + send(message, to: messenger.b) + _ = fixture.relay(messenger, io: messengerIO) + + try await waitUntil(timeout: .seconds(2)) { messengerIO.writtenCount == message.count } + #expect(uploaderIO.writtenCount < upload.count) + } + + @Test("An outside call waiting to send gets a gap beside an upload limited by the socket") + func outsideCallGetsAGapBesideUpload() async throws { + let fixture = PumpFixture() + let uploader = SocketPair() + defer { uploader.close(); fixture.close() } + + let session = OnePacketSession(transportFD: fixture.transport.a) + let drain = SlowDrain(fixture.transport.b) + defer { drain.stop() } + let uploaderIO = session.channel() + let upload = Data(repeating: 0x55, count: 16 << 20) + enlargeBuffers(uploader) + let peer = uploader.b + Thread { _ = upload.withUnsafeBytes { Darwin.send(peer, $0.baseAddress, $0.count, MSG_NOSIGNAL) } }.start() + _ = fixture.relay(uploader, io: uploaderIO) + try await waitUntil { session.isSending } + + let turn = SSHSendTurn(pump: fixture.pump) + turn.waitsToSend(true) + defer { turn.end() } + let pump = fixture.pump + try await waitUntil(timeout: .seconds(2)) { pump.perform { !session.isSending } } + #expect(uploaderIO.writtenCount < upload.count) + } + + // A write that finds no window clears libssh2's outbound bit while another channel's packet is on its way. + @Test("An upload keeps going beside a channel whose window is closed") + func uploadKeepsGoingBesideAClosedWindow() async throws { + let fixture = PumpFixture() + let uploader = SocketPair() + let waiter = SocketPair() + defer { uploader.close(); waiter.close(); fixture.close() } + + let session = OnePacketSession(transportFD: fixture.transport.a) + let drain = SlowDrain(fixture.transport.b) + defer { drain.stop() } + let uploaderIO = session.channel() + let waiterIO = session.channel(windowClosed: true) + let upload = Data(repeating: 0x55, count: 16 << 20) + enlargeBuffers(uploader) + let peer = uploader.b + Thread { _ = upload.withUnsafeBytes { Darwin.send(peer, $0.baseAddress, $0.count, MSG_NOSIGNAL) } }.start() + _ = fixture.relay(uploader, io: uploaderIO) + try await waitUntil { session.isSending } + + send(Data("waits".utf8), to: waiter.b) + _ = fixture.relay(waiter, io: waiterIO) + try await waitUntil { waiterIO.writeCount >= 2 } + let drainedBefore = drain.drained + + // Progress through the transport, at whatever pace the drain thread gets on a loaded machine: a stalled + // pump lets through only what already sat in the socket. + try await waitUntil(timeout: .seconds(15)) { drain.drained >= drainedBefore + (256 << 10) } + #expect(uploaderIO.writtenCount < upload.count) + } +} + +// libssh2 decrypts whole SSH packets into its own buffer, so a channel can hold more than one +// buffer's worth while the transport reports nothing new. +// +// Measured against a real sshd with two channels and 98,304 queued bytes: reading only when the +// transport polls readable delivered 0 bytes in 60,178ms; pumping on a 500ms poll timeout too +// delivered all of it in 1,504ms with one buffer per timeout; draining until the channel answers +// EAGAIN delivered all of it in 502ms. +struct SSHSessionPumpBacklogTests { + fileprivate static let bufferSize = 32_768 + fileprivate static let queuedBuffers = 3 + + @Test("A backlog reaches the local socket although the transport never turns readable") + func deliversBacklogWithoutATransportWakeup() async throws { + let run = try await runBacklog() + + #expect(run.delivered == Self.bufferSize * Self.queuedBuffers) + } + + @Test("One pass drains the whole backlog instead of one buffer per wakeup") + func drainsInOnePass() async throws { + let run = try await runBacklog() + + #expect(run.reads == Self.queuedBuffers + 1) + } + + @Test("A backlog past the per-pass cap still lands at once") + func drainsPastTheReadCapWithoutWaiting() async throws { + let run = try await runBacklog(buffers: 20) + + #expect(run.delivered == Self.bufferSize * 20) + // The pass the cap scheduled can run after a local event's pass took the last buffer, and read once more. + #expect((21 ... 22).contains(run.reads)) + #expect(run.elapsed < 1.0) + } + + private struct BacklogRun { + let delivered: Int + let reads: Int + let elapsed: TimeInterval + } + + // Drains the local socket on a reader thread so a backlog bigger than the socket buffer + // cannot stall the relay on a full socket. + private func runBacklog(buffers: Int = SSHSessionPumpBacklogTests.queuedBuffers) async throws -> BacklogRun { + let fixture = PumpFixture() + let local = SocketPair() + defer { local.close(); fixture.close() } + + let total = Self.bufferSize * buffers + let io = FakeChannelIO( + actions: Array(repeating: .data(Data(repeating: 0x41, count: Self.bufferSize)), count: buffers), + fallback: .wouldBlock + ) + + let drained = ByteTally() + let peer = local.b + let reader = Thread { + var buffer = [UInt8](repeating: 0, count: Self.bufferSize) + while drained.value < total { + let count = recv(peer, &buffer, buffer.count, 0) + if count <= 0 { return } + drained.add(count) + } + } + reader.stackSize = 512 * 1_024 + reader.start() + + let started = Date() + _ = fixture.relay(local, io: io) + try await waitUntil { drained.value >= total && io.readCount > buffers } + let elapsed = Date().timeIntervalSince(started) + return BacklogRun(delivered: drained.value, reads: io.readCount, elapsed: elapsed) + } +} + +// MARK: - Support + +internal final class PumpFixture: @unchecked Sendable { + let transport = SocketPair() + let pump: SSHSessionPump + + init() { + pump = SSHSessionPump(transportFD: transport.a, queue: DispatchQueue(label: "com.TablePro.tests.ssh.pump")) + } + + func relay(_ local: SocketPair, io: any SSHChannelIO, counter: (any RelayByteObserver)? = nil) -> EndRecorder { + let ended = EndRecorder() + pump.relay(localFD: local.takeA(), channel: io, bufferSize: 32_768, byteCounter: counter) { termination in + ended.record(termination) + } + return ended + } + + // The transport closes only after no source of the pump can fire on it, without holding a test thread. + func close() { + pump.stop() + pump.whenStopped { [self] in transport.close() } + } +} + +internal final class EndRecorder: @unchecked Sendable { + private let lock = NSLock() + private var termination: RelayTermination? + + var value: RelayTermination? { + lock.withLock { termination } + } + + func record(_ termination: RelayTermination) { + lock.withLock { + if self.termination == nil { self.termination = termination } + } + } +} + +internal final class ByteTally: @unchecked Sendable { + private let lock = NSLock() + private var count = 0 + + var value: Int { + lock.withLock { count } + } + + func add(_ amount: Int) { + lock.withLock { count += amount } + } +} + +internal final class RecordingByteObserver: RelayByteObserver, @unchecked Sendable { + private let lock = NSLock() + private var receivedBytes = 0 + private var sentBytes = 0 + + var received: Int { + lock.withLock { receivedBytes } + } + + var sent: Int { + lock.withLock { sentBytes } + } + + func recordReceived(_ count: Int) { + lock.withLock { receivedBytes += count } + } + + func recordSent(_ count: Int) { + lock.withLock { sentBytes += count } + } +} + +internal final class FakeChannelIO: SSHChannelIO, @unchecked Sendable { + enum Action { + case data(Data) + case wouldBlock + case closed + case transportFailed + } + + private let lock = NSLock() + private var actions: [Action] + private let fallback: Action + private var writeResults: [ChannelWriteResult] + private let writeFallback: ChannelWriteResult? + private let readBlocks: RelayDirections + private let writeBlocks: RelayDirections + private var sendWindow: Bool + private let onRead: (@Sendable (Int) -> Void)? + private var lastBlock: RelayDirections = [] + private var writtenBuffer = Data() + private var reads = 0 + private var writes = 0 + private var delivered = 0 + + // The directions answer for the last call that would block, as libssh2's do. + init( + actions: [Action] = [], + fallback: Action = .wouldBlock, + writeResults: [ChannelWriteResult] = [], + writeFallback: ChannelWriteResult? = nil, + readBlocks: RelayDirections = [], + writeBlocks: RelayDirections = [], + sendWindow: Bool = true, + onRead: (@Sendable (Int) -> Void)? = nil + ) { + self.actions = actions + self.fallback = fallback + self.writeResults = writeResults + self.writeFallback = writeFallback + self.readBlocks = readBlocks + self.writeBlocks = writeBlocks + self.sendWindow = sendWindow + self.onRead = onRead + } + + var written: Data { lock.withLock { writtenBuffer } } + var readCount: Int { lock.withLock { reads } } + var writeCount: Int { lock.withLock { writes } } + var deliveredCount: Int { lock.withLock { delivered } } + + func enqueue(_ action: Action) { + lock.withLock { actions.append(action) } + } + + func openWindow() { + lock.withLock { sendWindow = true } + } + + func read(into buffer: UnsafeMutablePointer, count: Int) -> ChannelReadResult { + let (action, readNumber) = lock.withLock { () -> (Action, Int) in + reads += 1 + let action = actions.isEmpty ? fallback : actions.removeFirst() + if case .wouldBlock = action { lastBlock = readBlocks } else { lastBlock = [] } + return (action, reads) + } + onRead?(readNumber) + switch action { + case .data(let data): + let length = min(data.count, count) + buffer.withMemoryRebound(to: UInt8.self, capacity: length) { destination in + _ = data.copyBytes(to: UnsafeMutableBufferPointer(start: destination, count: length)) + } + lock.withLock { delivered += length } + return .bytes(length) + case .wouldBlock: + return .wouldBlock + case .closed: + return .closed + case .transportFailed: + return .transportFailed + } + } + + func write(_ buffer: UnsafePointer, count: Int) -> ChannelWriteResult { + lock.withLock { + writes += 1 + guard sendWindow else { + lastBlock = .inbound + return .wouldBlock + } + let result = writeResults.isEmpty ? (writeFallback ?? .bytes(count)) : writeResults.removeFirst() + lastBlock = result == .wouldBlock ? writeBlocks : [] + guard case .bytes(let accepted) = result else { return result } + let length = min(accepted, count) + buffer.withMemoryRebound(to: UInt8.self, capacity: length) { source in + writtenBuffer.append(contentsOf: UnsafeBufferPointer(start: source, count: length)) + } + return .bytes(length) + } + } + + func blockDirections() -> RelayDirections { + lock.withLock { lastBlock } + } + + func hasQueuedInput() -> Bool { + lock.withLock { !actions.isEmpty } + } + + func hasSendWindow() -> Bool { + lock.withLock { sendWindow } + } +} + +private func send(_ data: Data, to descriptor: Int32) { + data.withUnsafeBytes { raw in + _ = Darwin.send(descriptor, raw.baseAddress, raw.count, 0) + } +} + +private func receive(from descriptor: Int32, count: Int) -> Data { + var buffer = [UInt8](repeating: 0, count: count) + var received = 0 + while received < count { + let got = buffer.withUnsafeMutableBytes { raw in + recv(descriptor, raw.baseAddress?.advanced(by: received), count - received, 0) + } + if got <= 0 { break } + received += got + } + return Data(buffer.prefix(received)) +} + +private func readsEndOfFile(_ descriptor: Int32) -> Bool { + var pollFD = pollfd(fd: descriptor, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, 2_000) > 0 else { return false } + var byte: UInt8 = 0 + return recv(descriptor, &byte, 1, 0) == 0 +} + +// The client side feeds the relay whole 32 KB reads, so the 8 KB transport is what the upload waits on. +private func enlargeBuffers(_ pair: SocketPair) { + var size: Int32 = 1 << 20 + for descriptor in [pair.a, pair.b] { + setsockopt(descriptor, SOL_SOCKET, SO_SNDBUF, &size, socklen_t(MemoryLayout.size)) + setsockopt(descriptor, SOL_SOCKET, SO_RCVBUF, &size, socklen_t(MemoryLayout.size)) + } +} + +private func fillSendBuffer(_ descriptor: Int32) { + let flags = fcntl(descriptor, F_GETFL, 0) + _ = fcntl(descriptor, F_SETFL, flags | O_NONBLOCK) + var chunk = [UInt8](repeating: 0x46, count: 4_096) + while Darwin.send(descriptor, &chunk, chunk.count, 0) > 0 {} +} + +private func drain(_ descriptor: Int32) { + var buffer = [UInt8](repeating: 0, count: 65_536) + while recv(descriptor, &buffer, buffer.count, MSG_DONTWAIT) > 0 {} +} + +// A session that, like libssh2, keeps one packet at a time on its way and refuses every other channel's write +// until the channel that started it sends the rest. Packets go into the real transport socket, so its writability +// is what a write source sees. The outbound bit follows libssh2's: set when a send would block, left alone by a +// partial retry, cleared once the packet is out, and cleared outright by a write that finds no window. +internal final class OnePacketSession: @unchecked Sendable { + private let lock = NSLock() + private let transportFD: Int32 + private var owner: ObjectIdentifier? + private var outbound = false + private var remaining = 0 + private var payload = 0 + private let filler = [UInt8](repeating: 0x50, count: 32_768) + + // Non-blocking as libssh2 leaves its socket: Darwin does not honor MSG_DONTWAIT on a send to a Unix socket. + init(transportFD: Int32) { + self.transportFD = transportFD + let flags = fcntl(transportFD, F_GETFL, 0) + _ = fcntl(transportFD, F_SETFL, flags | O_NONBLOCK) + } + + var isSending: Bool { + lock.withLock { owner != nil } + } + + var outboundBit: Bool { + lock.withLock { outbound } + } + + func channel(windowClosed: Bool = false) -> OnePacketChannel { + OnePacketChannel(session: self, windowClosed: windowClosed) + } + + func writeWithNoWindow() -> ChannelWriteResult { + lock.withLock { + outbound = false + return .wouldBlock + } + } + + func write(from channel: OnePacketChannel, count: Int) -> ChannelWriteResult { + lock.withLock { + let id = ObjectIdentifier(channel) + if let owner { + guard owner == id else { return .wouldBlock } + let sent = push(remaining) + if sent == 0 { outbound = true } + remaining -= sent + guard remaining == 0 else { return .wouldBlock } + self.owner = nil + outbound = false + return .bytes(payload) + } + let packet = min(count, filler.count) + let sent = push(packet) + guard sent == packet else { + owner = id + outbound = true + remaining = packet - sent + payload = packet + return .wouldBlock + } + outbound = false + return .bytes(packet) + } + } + + private func push(_ count: Int) -> Int { + let sent = filler.withUnsafeBytes { Darwin.send(transportFD, $0.baseAddress, count, 0) } + return max(0, sent) + } +} + +internal final class OnePacketChannel: SSHChannelIO, @unchecked Sendable { + private let session: OnePacketSession + private let windowClosed: Bool + private let lock = NSLock() + private var written = 0 + private var writes = 0 + + init(session: OnePacketSession, windowClosed: Bool) { + self.session = session + self.windowClosed = windowClosed + } + + var writtenCount: Int { lock.withLock { written } } + var writeCount: Int { lock.withLock { writes } } + + func read(into buffer: UnsafeMutablePointer, count: Int) -> ChannelReadResult { .wouldBlock } + + func write(_ buffer: UnsafePointer, count: Int) -> ChannelWriteResult { + lock.withLock { writes += 1 } + if windowClosed { return session.writeWithNoWindow() } + let result = session.write(from: self, count: count) + if case .bytes(let accepted) = result { + lock.withLock { written += accepted } + } + return result + } + + func blockDirections() -> RelayDirections { session.outboundBit ? .outbound : [] } + + func hasQueuedInput() -> Bool { false } + + func hasSendWindow() -> Bool { !windowClosed } +} + +// Reads the far end of the transport at about 4 MB/s, so an upload fills the socket and waits on it. It reads a +// descriptor of its own, so it never reads a number the test has closed and the kernel handed to another test. +internal final class SlowDrain: @unchecked Sendable { + private let stopped = OSAllocatedUnfairLock(initialState: false) + private let tally = ByteTally() + + var drained: Int { tally.value } + + init(_ descriptor: Int32) { + let stopped = self.stopped + let tally = self.tally + let own = dup(descriptor) + let thread = Thread { + defer { Darwin.close(own) } + var buffer = [UInt8](repeating: 0, count: 16_384) + while !stopped.withLock({ $0 }) { + let count = recv(own, &buffer, buffer.count, MSG_DONTWAIT) + if count <= 0 { + usleep(1_000) + continue + } + tally.add(count) + usleep(4_000) + } + } + thread.start() + } + + func stop() { + stopped.withLock { $0 = true } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/TestSupport/SocketPair.swift b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/TestSupport/SocketPair.swift index d9b4f1efef..1956f527fd 100644 --- a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/TestSupport/SocketPair.swift +++ b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/TestSupport/SocketPair.swift @@ -9,9 +9,9 @@ import Foundation -/// A connected pair of local socket fds, used to drive relay and channel-open tests over -/// real sockets rather than fakes, so closing behaviour is observed the way a database -/// client would observe it. +// A connected pair of local socket fds, used to drive relay and channel-open tests over +// real sockets rather than fakes, so closing behaviour is observed the way a database +// client would observe it. internal final class SocketPair { private(set) var a: Int32 private(set) var b: Int32 @@ -29,6 +29,13 @@ internal final class SocketPair { func closeB() { Self.close(&b) } + // Hands `a` to an owner that closes it, so `close()` leaves it alone. + func takeA() -> Int32 { + let descriptor = a + a = -1 + return descriptor + } + func close() { Self.close(&a) Self.close(&b) diff --git a/apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeBrowserCallbackTests.swift b/apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeBrowserCallbackTests.swift new file mode 100644 index 0000000000..c87b161fa8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeBrowserCallbackTests.swift @@ -0,0 +1,34 @@ +import Foundation +import TableProSnowflakeCore +import Testing + +struct SnowflakeBrowserCallbackTests { + private static func head(_ target: String, method: String = "GET") -> Data { + Data("\(method) \(target) HTTP/1.1\r\nHost: localhost:49152\r\n\r\n".utf8) + } + + @Test("The identity provider's redirect yields its SAML token") + func redirectYieldsTheToken() { + #expect(SnowflakeBrowserCallback.parse(requestHead: Self.head("/?token=ver%3A1-hint%3A2")) == .token("ver:1-hint:2")) + } + + @Test("A request that is not the token redirect is ignored", arguments: [ + "/favicon.ico", + "/", + "/?token=", + "/?token=a&token=b", + "/?error=access_denied", + "/other?token=abc" + ]) + func strayRequestIsIgnored(target: String) { + #expect(SnowflakeBrowserCallback.parse(requestHead: Self.head(target)) == .ignore) + } + + @Test("Only a complete GET head in origin form is read") + func otherRequestShapesAreIgnored() { + #expect(SnowflakeBrowserCallback.parse(requestHead: Self.head("/?token=abc", method: "POST")) == .ignore) + #expect(SnowflakeBrowserCallback.parse(requestHead: Data("GET /?token=abc HTTP/1.1\r\nHost: loc".utf8)) == .ignore) + #expect(SnowflakeBrowserCallback.parse(requestHead: Data("GET http://localhost/?token=abc HTTP/1.1\r\n\r\n".utf8)) == .ignore) + #expect(SnowflakeBrowserCallback.parse(requestHead: Data()) == .ignore) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSyncTests/PullAcknowledgementTests.swift b/apple/Packages/TableProCore/Tests/TableProSyncTests/PullAcknowledgementTests.swift new file mode 100644 index 0000000000..9b241e4775 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSyncTests/PullAcknowledgementTests.swift @@ -0,0 +1,164 @@ +import CloudKit +import Foundation +import Testing + +@testable import TableProSyncTransport + +struct PullAcknowledgementTests { + private static let zoneID = CKRecordZone.ID(zoneName: "TableProSync", ownerName: CKCurrentUserDefaultName) + + private static func id(_ name: String) -> CKRecord.ID { + CKRecord.ID(recordName: name, zoneID: zoneID) + } + + private static func page( + from start: String?, + changed: [String] = [], + deleted: [String] = [], + unread: [String] = [] + ) -> PullAcknowledgement.Page { + PullAcknowledgement.Page( + startToken: start, + changed: Set(changed.map { id($0) }), + deleted: Set(deleted.map { id($0) }), + unread: unread.map { id($0) } + ) + } + + @Test("A pull that read every record acknowledges its final token") + func everyRecordRead() { + let settled = PullAcknowledgement.settle( + [Self.page(from: "t0", changed: ["a"]), Self.page(from: "t1", deleted: ["b"])], + finalToken: "t2" + ) + + #expect(settled.token == "t2") + #expect(settled.unread.isEmpty) + } + + @Test("A record CloudKit could not read holds the token at the start of its page") + func unreadRecordHoldsTheToken() { + let settled = PullAcknowledgement.settle( + [ + Self.page(from: "t0", changed: ["a"]), + Self.page(from: "t1", changed: ["c"], unread: ["b"]), + Self.page(from: "t2", changed: ["d"]) + ], + finalToken: "t3" + ) + + #expect(settled.token == "t1") + #expect(settled.unread == [Self.id("b")]) + } + + @Test("A full fetch whose first page could not read a record acknowledges no token") + func fullFetchKeepsNoToken() { + let settled = PullAcknowledgement.settle([Self.page(from: nil, unread: ["b"])], finalToken: "t1") + + #expect(settled.token == nil) + #expect(settled.unread == [Self.id("b")]) + } + + @Test("A record a later page changed or deleted no longer holds the token") + func laterPageReleasesTheRecord() { + let changedLater = PullAcknowledgement.settle( + [Self.page(from: "t0", unread: ["b"]), Self.page(from: "t1", changed: ["b"])], + finalToken: "t2" + ) + let deletedLater = PullAcknowledgement.settle( + [Self.page(from: "t0", unread: ["b"]), Self.page(from: "t1", deleted: ["b"])], + finalToken: "t2" + ) + + #expect(changedLater.token == "t2") + #expect(changedLater.unread.isEmpty) + #expect(deletedLater.token == "t2") + #expect(deletedLater.unread.isEmpty) + } + + @Test("A record read before a later page failed on it still holds the token") + func earlierReadDoesNotRelease() { + let settled = PullAcknowledgement.settle( + [Self.page(from: "t0", changed: ["b"]), Self.page(from: "t1", unread: ["b"])], + finalToken: "t2" + ) + + #expect(settled.token == "t1") + #expect(settled.unread == [Self.id("b")]) + } + + @Test("A record fetched again by ID no longer holds the token") + func refetchedRecordReleasesTheToken() { + let pages = [Self.page(from: "t0", unread: ["b"]), Self.page(from: "t1", unread: ["c"])] + + let partly = PullAcknowledgement.settle(pages, finalToken: "t2", resolved: [Self.id("b")]) + let fully = PullAcknowledgement.settle(pages, finalToken: "t2", resolved: [Self.id("b"), Self.id("c")]) + + #expect(partly.token == "t1") + #expect(partly.unread == [Self.id("c")]) + #expect(fully.token == "t2") + #expect(fully.unread.isEmpty) + } + + @Test("A record two pages failed on is listed once, and the earlier page holds the token") + func repeatedFailureIsListedOnce() { + let settled = PullAcknowledgement.settle( + [Self.page(from: "t0", unread: ["b"]), Self.page(from: "t1", changed: ["a"]), Self.page(from: "t2", unread: ["b"])], + finalToken: "t3" + ) + + #expect(settled.token == "t0") + #expect(settled.unread == [Self.id("b")]) + } + + @Test("A record unread for the held number of pulls in a row is released, and the token moves past it") + func recordIsReleasedAfterTheHeldPulls() { + var ledger = UnreadRecordLedger() + let pages = [Self.page(from: "t0", changed: ["a"]), Self.page(from: "t1", unread: ["b"])] + + for _ in 1 ..< UnreadRecordLedger.heldPulls { + let held = PullAcknowledgement.settle(pages, finalToken: "t2", resolved: ledger.released) + #expect(held.token == "t1") + let releasing = ledger.countPull(holding: held.unread) + #expect(releasing.isEmpty) + } + let last = PullAcknowledgement.settle(pages, finalToken: "t2", resolved: ledger.released) + let releasing = ledger.countPull(holding: last.unread) + #expect(releasing == [Self.id("b")]) + + let released = PullAcknowledgement.settle(pages, finalToken: "t2", resolved: ledger.released) + #expect(released.token == "t2") + #expect(released.unread.isEmpty) + #expect(ledger.released == [Self.id("b")]) + } + + @Test("A pull that reads the record starts its count over") + func readingStartsTheCountOver() { + var ledger = UnreadRecordLedger() + + var released: Set = [] + for _ in 1 ..< UnreadRecordLedger.heldPulls { + released.formUnion(ledger.countPull(holding: [Self.id("b")])) + } + released.formUnion(ledger.countPull(holding: [])) + for _ in 1 ..< UnreadRecordLedger.heldPulls { + released.formUnion(ledger.countPull(holding: [Self.id("b")])) + } + + #expect(released.isEmpty) + #expect(ledger.released.isEmpty) + } + + @Test("A released record that reads by ID, or is gone, is forgotten") + func releasedRecordIsForgottenOnceResolved() { + var ledger = UnreadRecordLedger() + for _ in 0 ..< UnreadRecordLedger.heldPulls { + _ = ledger.countPull(holding: [Self.id("b"), Self.id("c")]) + } + #expect(ledger.released == [Self.id("b"), Self.id("c")]) + + ledger.forget([Self.id("b")]) + + #expect(ledger.released == [Self.id("c")]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncAdmissionTests.swift b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncAdmissionTests.swift index dff84ae622..1a7f1bad80 100644 --- a/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncAdmissionTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncAdmissionTests.swift @@ -10,7 +10,7 @@ struct SyncAdmissionTests { private var earlier: Date { now.addingTimeInterval(-1) } private static let everyTrigger: [SyncTrigger] = [ - .launch, .activation, .localChange, .accountChange, .networkRestored, .scheduledRetry, .userRequest + .launch, .activation, .remoteChange, .localChange, .accountChange, .networkRestored, .scheduledRetry, .userRequest ] @Test("Nothing in the way admits every trigger", arguments: everyTrigger) @@ -27,6 +27,7 @@ struct SyncAdmissionTests { #expect(SyncAdmission.decide(for: .localChange, after: error, nextAttempt: later, now: now) == .none) #expect(SyncAdmission.decide(for: .activation, after: error, nextAttempt: later, now: now) == .downloadOnly) #expect(SyncAdmission.decide(for: .networkRestored, after: error, nextAttempt: later, now: now) == .downloadOnly) + #expect(SyncAdmission.decide(for: .remoteChange, after: error, nextAttempt: later, now: now) == .downloadOnly) } @Test("Full storage lets the person, the timer and an account change try the upload") @@ -83,6 +84,7 @@ struct SyncAdmissionTests { for error in [SyncError?.none, .blocked(.storageFull), .blocked(.dataDeletedFromICloud)] { #expect(SyncAdmission.decide(for: .activation, after: error, nextAttempt: nil, throttledUntil: later, now: now) == .none) #expect(SyncAdmission.decide(for: .scheduledRetry, after: error, nextAttempt: nil, throttledUntil: later, now: now) == .none) + #expect(SyncAdmission.decide(for: .remoteChange, after: error, nextAttempt: nil, throttledUntil: later, now: now) == .none) #expect(SyncAdmission.decide(for: .userRequest, after: error, nextAttempt: nil, throttledUntil: later, now: now) != .none) } #expect(SyncAdmission.decide(for: .activation, after: nil, nextAttempt: nil, throttledUntil: earlier, now: now) == .full) diff --git a/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift index dfad98f70a..730bf05925 100644 --- a/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncMetadataStorageTests.swift @@ -179,6 +179,16 @@ struct SyncMetadataStorageTests { #expect(storage.lastAccountId == "account") } + @Test("The subscribed account round-trips and a full reset forgets it") + func subscribedAccountIdRoundTrips() { + let storage = makeStorage() + #expect(storage.subscribedAccountId == nil) + storage.subscribedAccountId = "account" + #expect(storage.subscribedAccountId == "account") + storage.clearAll() + #expect(storage.subscribedAccountId == nil) + } + @Test("The first account seen on a device that never synced is recorded and nothing queued is dropped") func firstAccountIsRecorded() { let defaults = ScratchDefaults.make() ?? .standard diff --git a/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncPushChannelTests.swift b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncPushChannelTests.swift new file mode 100644 index 0000000000..897cb195f4 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncPushChannelTests.swift @@ -0,0 +1,166 @@ +import CloudKit +import Foundation +import Testing + +import TableProSyncTransport + +@MainActor +struct SyncPushChannelTests { + private let metadata: SyncMetadataStorage + private let calls = RegistrarCalls() + + init() { + metadata = SyncMetadataStorage(userDefaults: ScratchDefaults.make() ?? .standard) + metadata.lastAccountId = "account-a" + } + + private func makeChannel() -> SyncPushChannel { + let calls = calls + return SyncPushChannel( + registrar: SyncPushRegistrar(register: { calls.registrations += 1 }, unregister: { calls.unregistrations += 1 }), + metadata: metadata + ) + } + + @Test("Registering twice asks once, and nothing is taken back that was never asked for") + func registrationIsAskedOnce() { + let channel = makeChannel() + channel.unregister() + #expect(calls.unregistrations == 0) + + channel.register() + channel.register() + #expect(calls.registrations == 1) + + channel.unregister() + #expect(calls.unregistrations == 1) + } + + @Test("Push is live only with a device registration and this account's subscription") + func liveNeedsRegistrationAndSubscription() async { + let channel = makeChannel() + channel.registrationDidFinish(registered: true) + #expect(!channel.isLive) + + channel.register() + channel.registrationDidFinish(registered: true) + #expect(!channel.isLive) + + await channel.saveSubscriptionIfNeeded {} + #expect(channel.isLive) + + metadata.lastAccountId = "account-b" + #expect(!channel.isLive) + } + + @Test("A failed registration or turning sync off ends push") + func failedRegistrationEndsPush() async { + let channel = makeChannel() + channel.register() + await channel.saveSubscriptionIfNeeded {} + channel.registrationDidFinish(registered: false) + #expect(!channel.isLive) + + channel.registrationDidFinish(registered: true) + #expect(channel.isLive) + channel.unregister() + #expect(!channel.isLive) + } + + @Test("Unannounced changes end push and save the subscription again, until the next notice") + func unannouncedChangesEndPush() async { + let channel = makeChannel() + #expect(!channel.activationFoundUnannouncedChanges(noticesSince: channel.noticesReceived)) + + channel.register() + channel.registrationDidFinish(registered: true) + await channel.saveSubscriptionIfNeeded {} + #expect(channel.activationFoundUnannouncedChanges(noticesSince: channel.noticesReceived)) + #expect(!channel.isLive) + #expect(metadata.subscribedAccountId == nil) + + var saves = 0 + await channel.saveSubscriptionIfNeeded { saves += 1 } + #expect(saves == 1) + #expect(!channel.isLive) + + channel.noticeArrived() + #expect(channel.isLive) + } + + @Test("Changes an activation brings are announced when a notice came since the last download began") + func noticeSinceTheLastDownloadAnnouncesChanges() async { + let channel = makeChannel() + channel.register() + channel.registrationDidFinish(registered: true) + await channel.saveSubscriptionIfNeeded {} + let atLastDownload = channel.noticesReceived + + channel.noticeArrived() + + #expect(!channel.activationFoundUnannouncedChanges(noticesSince: atLastDownload)) + #expect(channel.isLive) + } + + @Test("The subscription is saved once per account") + func savedOncePerAccount() async { + let channel = makeChannel() + var saves = 0 + await channel.saveSubscriptionIfNeeded { saves += 1 } + await channel.saveSubscriptionIfNeeded { saves += 1 } + #expect(saves == 1) + #expect(metadata.subscribedAccountId == "account-a") + + metadata.lastAccountId = "account-b" + await channel.saveSubscriptionIfNeeded { saves += 1 } + #expect(saves == 2) + #expect(metadata.subscribedAccountId == "account-b") + + channel.forgetSubscription() + await channel.saveSubscriptionIfNeeded { saves += 1 } + #expect(saves == 3) + } + + @Test("A save that fails for a reason that can pass is tried again on the next run", + arguments: [CKError.Code.networkFailure, .accountTemporarilyUnavailable, .internalError, .serviceUnavailable]) + func passingFailureRetries(_ code: CKError.Code) async { + let channel = makeChannel() + var saves = 0 + await channel.saveSubscriptionIfNeeded { + saves += 1 + throw CKError(code) + } + #expect(metadata.subscribedAccountId == nil) + + await channel.saveSubscriptionIfNeeded { saves += 1 } + #expect(saves == 2) + #expect(metadata.subscribedAccountId == "account-a") + } + + @Test("A save CloudKit refuses is not sent again for that account, but is for the next one") + func refusedSaveIsNotRepeated() async { + let channel = makeChannel() + var saves = 0 + let refuse: () async throws -> Void = { + saves += 1 + throw CKError(.permissionFailure) + } + await channel.saveSubscriptionIfNeeded(refuse) + await channel.saveSubscriptionIfNeeded(refuse) + #expect(saves == 1) + + metadata.lastAccountId = "account-b" + await channel.saveSubscriptionIfNeeded(refuse) + #expect(saves == 2) + + channel.forgetSubscription() + await channel.saveSubscriptionIfNeeded(refuse) + #expect(saves == 3) + } +} + +@MainActor +private final class RegistrarCalls { + var registrations = 0 + var unregistrations = 0 +} diff --git a/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncPushSubscriptionTests.swift b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncPushSubscriptionTests.swift new file mode 100644 index 0000000000..2d5250eaf7 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSyncTests/SyncPushSubscriptionTests.swift @@ -0,0 +1,60 @@ +import CloudKit +import Foundation +import Testing + +import TableProSyncTransport + +struct SyncPushSubscriptionTests { + // The shape CloudKit sends for a database subscription, read back by CloudKit's own parser. + static func payload( + subscriptionID: String = SyncPushSubscription.id, + container: String = CloudKitSyncEngine.defaultContainerID + ) -> [AnyHashable: Any] { + [ + "aps": ["content-available": 1], + "ck": [ + "ce": 2, + "cid": container, + "nid": UUID().uuidString, + "met": ["dbs": 1, "sid": subscriptionID] + ] as [String: Any] + ] + } + + @Test("The subscription covers the whole private database and is silent") + func subscriptionIsSilent() throws { + let subscription = SyncPushSubscription.make() + #expect(subscription.subscriptionID == SyncPushSubscription.id) + #expect(subscription.recordType == nil) + let info = try #require(subscription.notificationInfo) + #expect(info.shouldSendContentAvailable) + #expect(info.alertBody == nil) + #expect(info.alertLocalizationKey == nil) + #expect(info.title == nil) + #expect(info.soundName == nil) + #expect(!info.shouldBadge) + } + + @Test("A notice from the subscription announces a change") + func ownNoticeAnnouncesChange() { + #expect(SyncPushSubscription.announcesChange(Self.payload())) + } + + @Test("Another subscription, another container or another kind of push announces nothing") + func otherPushesAnnounceNothing() { + #expect(!SyncPushSubscription.announcesChange(Self.payload(subscriptionID: "other"))) + #expect(!SyncPushSubscription.announcesChange(Self.payload(container: "iCloud.com.example"))) + #expect(!SyncPushSubscription.announcesChange(["aps": ["content-available": 1]])) + #expect(!SyncPushSubscription.announcesChange(["aps": ["alert": "Hello"]])) + let zoneNotice: [AnyHashable: Any] = [ + "aps": ["content-available": 1], + "ck": [ + "ce": 2, + "cid": CloudKitSyncEngine.defaultContainerID, + "nid": UUID().uuidString, + "fet": ["dbs": 1, "sid": SyncPushSubscription.id, "zid": CloudKitSyncEngine.zoneName, "zoid": "_owner"] + ] as [String: Any] + ] + #expect(!SyncPushSubscription.announcesChange(zoneNotice)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift new file mode 100644 index 0000000000..47418aa408 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift @@ -0,0 +1,288 @@ +#if os(macOS) +import Foundation +import Security +import TableProTestSupport +import TableProTLSClientIdentity +import TableProTLSTestFixtures +import Testing + +private final class FixtureTrustSource: SystemTrustSource { + struct Entry { + let certificate: SecCertificate + let settings: [SystemTrustSetting]? + var unreadable = false + } + + var domains: [SystemTrustDomain: [Entry]] = [:] + var unreadableDomains: Set = [] + var anchors: [SecCertificate] = [] + private(set) var userDomainReads = 0 + + func add(_ certificate: SecCertificate, to domain: SystemTrustDomain, _ settings: [SystemTrustSetting]?) { + domains[domain, default: []].append(Entry(certificate: certificate, settings: settings)) + } + + func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] { + if domain == .user { userDomainReads += 1 } + guard !unreadableDomains.contains(domain) else { throw SystemTrustAnchorsError.settingsUnreadable(-1) } + return domains[domain, default: []].map(\.certificate) + } + + func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? { + let der = SecCertificateCopyData(certificate) as Data + guard let entry = domains[domain, default: []].first(where: { SecCertificateCopyData($0.certificate) as Data == der }) + else { return nil } + guard !entry.unreadable else { throw SystemTrustAnchorsError.settingsUnreadable(-1) } + return entry.settings + } + + func anchorCertificates() throws -> [SecCertificate] { + anchors + } +} + +struct SystemTrustAnchorsTests { + private let rootA: SecCertificate + private let rootB: SecCertificate + + init() throws { + rootA = try #require(SecCertificateCreateWithData(nil, TLSTestFixtures.der(TLSTestFixtures.caCertificate) as CFData)) + rootB = try #require( + SecCertificateCreateWithData(nil, TLSTestFixtures.der(TLSTestFixtures.serverNameCACertificate) as CFData) + ) + } + + private func ders(_ certificates: [SecCertificate]) -> Set { + Set(certificates.map { SecCertificateCopyData($0) as Data }) + } + + private func exported(_ source: FixtureTrustSource) throws -> Set { + ders(try SystemTrustAnchors.trustedRoots(from: source)) + } + + @Test("A certificate whose trust settings are empty is trusted as a root") + func emptySettingsTrustTheRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A user Deny removes a root the system trusts") + func aUserDenyRemovesASystemRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootB])) + } + + @Test("A root trusted in the admin domain joins the system roots") + func anAdminTrustAddsARoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .admin, [SystemTrustSetting(result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA, rootB])) + } + + @Test("User settings override admin settings in both directions") + func userSettingsOverrideAdminSettings() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .admin, [SystemTrustSetting(result: .deny)]) + source.add(rootA, to: .user, [SystemTrustSetting(result: .trustRoot)]) + source.add(rootB, to: .admin, [SystemTrustSetting(result: .trustRoot)]) + source.add(rootB, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A setting for another policy or for one application does not apply") + func anEntryForAnotherPolicyDoesNotApply() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(appliesToSSL: false, result: .deny)]) + source.add(rootB, to: .user, [SystemTrustSetting(namesApplication: true, result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A trust limited to one hostname is left out, because a PEM file cannot carry the limit") + func aHostnameScopedTrustIsNotExported() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .user, [SystemTrustSetting(hostname: "db.example.com", result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A Deny limited to one hostname removes the root for every host") + func aHostnameScopedDenyRemovesTheRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [ + SystemTrustSetting(hostname: "db.example.com", result: .deny), + SystemTrustSetting(result: .trustRoot) + ]) + + #expect(try exported(source).isEmpty) + } + + @Test("Unspecified moves on to the next entry, then to the next domain") + func unspecifiedFallsThroughToTheNextDomain() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(result: .unspecified)]) + source.add(rootB, to: .user, [SystemTrustSetting(result: .unspecified)]) + + #expect(try exported(source) == ders([rootA])) + + source.add(rootB, to: .admin, [SystemTrustSetting(result: .unspecified), SystemTrustSetting()]) + #expect(try exported(source) == ders([rootA, rootB])) + } + + @Test("A settings array the domain cannot read counts as a Deny") + func unreadableSettingsFailClosed() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.domains[.user] = [FixtureTrustSource.Entry(certificate: rootA, settings: [], unreadable: true)] + + #expect(try exported(source).isEmpty) + } + + @Test("A user or admin domain that cannot be read fails the build") + func anUnreadableUserDomainIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.unreadableDomains = [.admin] + + #expect(throws: SystemTrustAnchorsError.settingsUnreadable(-1)) { + try SystemTrustAnchors.trustedRoots(from: source) + } + } + + @Test("The system anchors stand in when the system domain cannot be read, and a user Deny still wins") + func theSystemAnchorsAreTheFallback() throws { + let source = FixtureTrustSource() + source.unreadableDomains = [.system] + source.anchors = [rootA, rootB] + source.add(rootB, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootA])) + + source.unreadableDomains = [] + #expect(try exported(source) == ders([rootA])) + } + + @Test("No trusted root is an error, never an empty file") + func noAnchorsIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, [SystemTrustSetting(result: .deny)]) + + #expect(throws: SystemTrustAnchorsError.noTrustedRoots) { + try SystemTrustAnchors.pem(from: source) + } + } + + @Test("The PEM holds one block per trusted root") + func thePEMHoldsEveryRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .system, []) + + let pem = try #require(String(bytes: SystemTrustAnchors.pem(from: source), encoding: .utf8)) + + #expect(pem.components(separatedBy: "-----BEGIN CERTIFICATE-----").count == 3) + } + + @Test("This Mac's own trust settings export as PEM") + func theRealStoreExportsPEM() throws { + let source = SecuritySystemTrustSource() + + let roots = try SystemTrustAnchors.trustedRoots(from: source) + let pem = try #require(String(bytes: SystemTrustAnchors.pem(from: source), encoding: .utf8)) + + #expect(!roots.isEmpty) + #expect(pem.hasPrefix("-----BEGIN CERTIFICATE-----")) + } + + @Test("A trust settings entry is read for its policy, hostname, application and result") + func readsATrustSettingsEntry() { + let scoped = SecuritySystemTrustSource.setting(from: [ + kSecTrustSettingsPolicy as String: SecPolicyCreateSSL(true, nil), + kSecTrustSettingsPolicyString as String: "db.example.com", + kSecTrustSettingsResult as String: NSNumber(value: 3) + ]) + #expect(scoped == SystemTrustSetting(hostname: "db.example.com", result: .deny)) + + let otherPolicy = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsPolicy as String: SecPolicyCreateBasicX509()]) + #expect(otherPolicy == SystemTrustSetting(appliesToSSL: false)) + + let application = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsApplication as String: "app"]) + #expect(application == SystemTrustSetting(namesApplication: true)) + } + + @Test("A result value Apple does not define counts as a Deny", arguments: [0, 9]) + func anUnknownResultCountsAsADeny(value: Int) { + let setting = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsResult as String: NSNumber(value: value)]) + #expect(setting.result == .deny) + } + + @Test("The bytes are reused for a minute, then rebuilt, and a failed build is not kept") + func theCacheRebuildsAfterAMinute() throws { + let source = FixtureTrustSource() + let clock = ManualClock() + let cache = SystemTrustAnchorCache(source: source, clock: clock) + + #expect(throws: SystemTrustAnchorsError.noTrustedRoots) { try cache.pem() } + source.add(rootA, to: .system, []) + let first = try cache.pem() + #expect(source.userDomainReads == 2) + + clock.advance(by: .seconds(59)) + #expect(try cache.pem() == first) + #expect(source.userDomainReads == 2) + + clock.advance(by: .seconds(1)) + _ = try cache.pem() + #expect(source.userDomainReads == 3) + } + + @Test("Each use gets its own owner-only file in an owner-only folder, removed with its folder") + func aWrittenFileIsOwnerOnly() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + let cache = SystemTrustAnchorCache(source: source, clock: ManualClock()) + let parent = FileManager.default.temporaryDirectory.appendingPathComponent("trust-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: parent, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: parent) } + + let file = try cache.writeFile(in: parent) + let other = try cache.writeFile(in: parent) + let folder = file.url.deletingLastPathComponent() + + #expect(file.url != other.url) + #expect(try Data(contentsOf: file.url) == cache.pem()) + let folderMode = try FileManager.default.attributesOfItem(atPath: folder.path)[.posixPermissions] as? NSNumber + let fileMode = try FileManager.default.attributesOfItem(atPath: file.url.path)[.posixPermissions] as? NSNumber + #expect(folderMode?.int16Value == 0o700) + #expect(fileMode?.int16Value == 0o600) + + file.remove() + #expect(!FileManager.default.fileExists(atPath: folder.path)) + #expect(FileManager.default.fileExists(atPath: other.url.path)) + } + + @Test("A file that cannot be written is an error and leaves nothing behind") + func anUnwritableFolderIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + let cache = SystemTrustAnchorCache(source: source, clock: ManualClock()) + let missing = FileManager.default.temporaryDirectory.appendingPathComponent("missing-\(UUID().uuidString)") + + #expect(throws: SystemTrustAnchorsError.fileUnwritable) { try cache.writeFile(in: missing) } + #expect(!FileManager.default.fileExists(atPath: missing.path)) + } +} +#endif diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift index 76bf57dbd6..46f6d66fc2 100644 --- a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift @@ -19,6 +19,12 @@ struct TeradataReplyDecodingTests { return bytes } + private static func execute(_ reply: [UInt8]) throws -> TeradataResultSet { + let transport = StoppableTransport(replies: [LanMessage(kind: .start, body: reply).encoded()]) + let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + return try TeradataConnection(config: config, transport: transport).execute("SELECT c FROM t") + } + @Test("A result description whose CHAR column has length -1 fails the reply instead of crashing") func negativeColumnLengthFailsTheReply() throws { let reply = Self.reply(charLength: [UInt8](repeating: 0xFF, count: 8), record: [0x00, 0x78]) @@ -26,13 +32,13 @@ struct TeradataReplyDecodingTests { let parcels = try Parcel.decodeAll(reply) let flavors = parcels.map(\.flavor) #expect(flavors == [ParcelFlavor.prepInfoX.rawValue, ParcelFlavor.record.rawValue, ParcelFlavor.endRequest.rawValue]) - #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try TeradataResultParser.parse(parcels) } + #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try Self.execute(reply) } } @Test("A CHAR column of the declared length still decodes") func charColumnDecodes() throws { let reply = Self.reply(charLength: [0, 0, 0, 0, 0, 0, 0, 3], record: [0x00, 0x61, 0x62, 0x63]) - let result = try TeradataResultParser.parse(try Parcel.decodeAll(reply)) + let result = try Self.execute(reply) #expect(result.rows == [[TeradataValue.text("abc")]]) } diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift new file mode 100644 index 0000000000..76202549d8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift @@ -0,0 +1,272 @@ +import Foundation +@testable import TableProTeradataCore +import Testing + +// Replies are read in order; a hook keyed by a reply's index runs just before that reply is read, which is where a +// request is waiting on the server when Stop is pressed. +final class StoppableTransport: TeradataTransport { + private var replies: [[UInt8]] + private var stream: [UInt8] = [] + private var repliesRead = 0 + var beforeReply: [Int: () -> Void] = [:] + private(set) var sent: [LanMessage] = [] + private(set) var shutDown = false + private(set) var closed = false + + var sentKinds: [UInt8] { + sent.map { $0.kind } + } + + init(replies: [[UInt8]]) { + self.replies = replies + } + + func send(_ bytes: [UInt8]) throws { + if shutDown || closed { throw TeradataWireError.cancelled } + sent.append(LanMessage.decode( + header: Array(bytes.prefix(LanMessage.headerLength)), + body: Array(bytes.dropFirst(LanMessage.headerLength)) + )) + } + + func receive(_ count: Int) throws -> [UInt8] { + if stream.isEmpty { + beforeReply.removeValue(forKey: repliesRead)?() + if shutDown || closed { throw TeradataWireError.cancelled } + guard !replies.isEmpty else { throw TeradataWireError.truncated("no reply") } + stream = replies.removeFirst() + repliesRead += 1 + } + guard stream.count >= count else { throw TeradataWireError.truncated("closed") } + defer { stream.removeFirst(count) } + return Array(stream.prefix(count)) + } + + func finishConnecting() {} + + func cancel() { + shutDown = true + } + + func close() { + closed = true + } +} + +// A server that never answers: a receive waits until the transport is shut down, which the stop's deadline does from +// its own queue. +private final class SilentTransport: TeradataTransport, @unchecked Sendable { + private let lock = NSLock() + private let shutDown = DispatchSemaphore(value: 0) + private var sent: [UInt8] = [] + private var shut = false + private var received = false + var onFirstReceive: (() -> Void)? + + var sentKinds: [UInt8] { lock.withLock { sent } } + var wasShutDown: Bool { lock.withLock { shut } } + + func send(_ bytes: [UInt8]) throws { + try lock.withLock { + if shut { throw TeradataWireError.cancelled } + sent.append(bytes[2]) + } + } + + func receive(_ count: Int) throws -> [UInt8] { + let first = lock.withLock { () -> Bool in + defer { received = true } + return !received + } + if first { onFirstReceive?() } + _ = shutDown.wait(timeout: .now() + 10) + throw TeradataWireError.cancelled + } + + func finishConnecting() {} + + func cancel() { + lock.withLock { shut = true } + shutDown.signal() + } + + func close() { + cancel() + } +} + +struct TeradataStopTests { + private static let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + private static let ansiConfig = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", transactionMode: .ansi + ) + private static let start = MessageKind.start.rawValue + private static let cont = MessageKind.cont.rawValue + private static let abort = MessageKind.abort.rawValue + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + private static func failure(code: Int, _ message: String) -> Parcel { + let text = Array(message.utf8) + let body = [UInt8](repeating: 0, count: 8) + + [UInt8(code >> 8), UInt8(code & 0xFF), UInt8(text.count >> 8), UInt8(text.count & 0xFF)] + text + return Parcel(.failure, body: body) + } + + private static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + private static func thrown(_ body: () throws -> Void) -> Error? { + do { + try body() + return nil + } catch { + return error + } + } + + private static func isCancelled(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .cancelled = wire else { return false } + return true + } + + private static func isRolledBack(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .rolledBackByStop = wire else { return false } + return true + } + + @Test("A Stop outside a session set to ANSI reads as a cancel") + func stopOutsideANSIModeIsACancel() throws { + let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) + let transport = StoppableTransport(replies: [aborted]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } + + guard let wire = error as? TeradataWireError, case .cancelled = wire else { + Issue.record("Expected cancelled, got \(String(describing: error))") + return + } + } + + @Test("Stop while the server works on a request aborts it on the session, which stays logged on and reports the rollback") + func stopAbortsTheRequestAndKeepsTheSession() throws { + let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) + let transport = StoppableTransport(replies: [aborted, Self.finished]) + let connection = TeradataConnection(config: Self.ansiConfig, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } + + #expect(Self.isRolledBack(error)) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(transport.sent.last?.requestNumber == transport.sent.first?.requestNumber) + #expect(transport.sent.last?.body == TeradataMessages.abortParcel().encoded()) + #expect(!transport.shutDown) + #expect(!transport.closed) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.abort, Self.start]) + } + + @Test("A request that failed on its own before the abort landed keeps its own error") + func failureOtherThanTheAbortIsReported() throws { + let missing = Self.reply([Self.failure(code: 3_807, "Object 'sales.orderz' does not exist.")]) + let transport = StoppableTransport(replies: [missing]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orderz") } + + guard let wire = error as? TeradataWireError, case .server(let code, _) = wire else { + Issue.record("expected the server's error, got \(String(describing: error))") + return + } + #expect(code == 3_807) + #expect(connection.isConnected) + } + + // An export Stop lands here: the abort does not take effect once the server has started answering. + @Test("Stop while a long answer is being fetched ends the output and keeps the session") + func stopDuringFetchEndsTheOutput() throws { + let batch = Self.reply([Parcel(.success)]) + let ended = Self.reply([Parcel(.endRequest)]) + let transport = StoppableTransport(replies: [batch, batch, ended, Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[1] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!transport.closed) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont, Self.start]) + } + + @Test("An output the server keeps sending after the Cancel parcel gives the session up") + func unendedOutputGivesTheSessionUp() throws { + let batch = Self.reply([Parcel(.success)]) + let transport = StoppableTransport(replies: [batch, batch, batch]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[1] = { connection.cancel() } + + #expect(Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } != nil) + #expect(transport.closed) + #expect(connection.hasLostConnection) + } + + @Test("Stop in a running call before its request goes out refuses the request and keeps the session") + func stopBeforeTheRequestIsSentRefusesIt() throws { + let transport = StoppableTransport(replies: [Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + connection.beginOperation() + connection.cancel() + let error = Self.thrown { _ = try connection.execute("DELETE FROM sales.orders") } + connection.endOperation() + + #expect(Self.isCancelled(error)) + #expect(transport.sent.isEmpty) + #expect(connection.isConnected) + + connection.beginOperation() + _ = try connection.execute("SELECT 1") + connection.endOperation() + #expect(transport.sentKinds == [Self.start]) + } + + @Test("A stopped request the server never answers gives the session up after the grace", .timeLimit(.minutes(1))) + func unansweredStopGivesTheSessionUp() throws { + let transport = SilentTransport() + let connection = TeradataConnection(config: Self.config, transport: transport, stopGrace: 0.05) + transport.onFirstReceive = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(transport.wasShutDown) + #expect(connection.hasLostConnection) + } + + @Test("Stop with no request running sends nothing and keeps the session") + func stopWhileIdleDoesNothing() throws { + let transport = StoppableTransport(replies: [Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + connection.cancel() + _ = try connection.execute("SELECT 1") + + #expect(transport.sentKinds == [Self.start]) + #expect(!transport.shutDown) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift new file mode 100644 index 0000000000..96e3f43823 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift @@ -0,0 +1,287 @@ +@testable import TableProTeradataCore +import Testing + +struct TeradataStreamTests { + private static let start = MessageKind.start.rawValue + private static let cont = MessageKind.cont.rawValue + private static let abort = MessageKind.abort.rawValue + private static let integer: UInt16 = 496 + private static let char: UInt16 = 452 + + private static let numbers = prepInfoX([(integer, 4, "n")]) + private static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + private static func u16(_ value: Int) -> [UInt8] { + [UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + // A PrepInfoX parcel laid out as in TeradataReplyDecodingTests. + private static func prepInfoX(_ columns: [(type: UInt16, length: Int, name: String)]) -> Parcel { + var body = [UInt8](repeating: 0, count: 8) + u16(0) + u16(columns.count) + for column in columns { + let name = Array(column.name.utf8) + body += u16(Int(column.type)) + body += (0..<8).map { UInt8(truncatingIfNeeded: column.length >> (8 * (7 - $0))) } + body += [0, 0] + body += u16(name.count) + name + body += u16(0) + u16(0) + } + return Parcel(.prepInfoX, body: body) + } + + private static func number(_ value: Int32) -> Parcel { + let bits = UInt32(bitPattern: value) + return Parcel(.record, body: [0x00] + (0..<4).map { UInt8(truncatingIfNeeded: bits >> (8 * (3 - $0))) }) + } + + private static func code(_ text: String) -> Parcel { + Parcel(.record, body: [0x00] + Array(text.utf8)) + } + + private static func failure(code: Int, _ message: String) -> Parcel { + let text = Array(message.utf8) + return Parcel(.failure, body: [UInt8](repeating: 0, count: 8) + u16(code) + u16(text.count) + text) + } + + private static func thrown(_ body: () throws -> Void) -> Error? { + do { + try body() + return nil + } catch { + return error + } + } + + private static func isCancelled(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .cancelled = wire else { return false } + return true + } + + private static func connection( + _ replies: [[UInt8]], + transactionMode: TeradataTransactionMode = .default + ) -> (TeradataConnection, StoppableTransport) { + let transport = StoppableTransport(replies: replies) + let config = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", transactionMode: transactionMode + ) + return (TeradataConnection(config: config, transport: transport), transport) + } + + @Test("Each reply's rows reach the consumer before the next Continue goes out") + func rowsArriveReplyByReply() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2)]), + Self.reply([Self.number(3)]), + Self.reply([Self.number(4), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + var batches: [[[TeradataValue]]] = [] + var sentBeforeEachBatch: [Int] = [] + + try connection.stream("SELECT n FROM sales.orders", onColumns: { headers.append($0.map { $0.name }) }, onRows: { rows in + batches.append(rows) + sentBeforeEachBatch.append(transport.sent.count) + return true + }) + + #expect(headers == [["n"]]) + #expect(batches == [[[.integer(1)], [.integer(2)]], [[.integer(3)]], [[.integer(4)]]]) + #expect(sentBeforeEachBatch == [1, 2, 3]) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.dropFirst().allSatisfy { $0.body == TeradataMessages.responseParcel().encoded() }) + } + + @Test("A SELECT that matches no row still describes its columns") + func emptyResultStillHasColumns() throws { + let (connection, _) = Self.connection([ + Self.reply([Parcel(.success), Self.numbers, Parcel(.endStatement), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + var batches = 0 + + try connection.stream("SELECT n FROM sales.orders WHERE 1 = 0", onColumns: { headers.append($0.map { $0.name }) }, onRows: { _ in + batches += 1 + return true + }) + + #expect(headers == [["n"]]) + #expect(batches == 0) + } + + @Test("A statement without a result set still answers, with no columns") + func statementWithoutResultHasNoColumns() throws { + let (connection, _) = Self.connection([ + Self.reply([Parcel(.success), Parcel(.endStatement), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + + try connection.stream("DELETE FROM sales.orders", onColumns: { headers.append($0.map { $0.name }) }, onRows: { _ in true }) + + #expect(headers == [[]]) + } + + @Test("A consumer that wants no more rows ends the output and keeps the session") + func consumerEndsTheOutput() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + var batches = 0 + + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { _ in + batches += 1 + return false + }) + + #expect(batches == 1) + #expect(transport.sentKinds == [Self.start, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("A failure after some rows throws the server's error and keeps the session") + func failureAfterRowsThrows() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Self.failure(code: 2_646, "No more spool space in dbc.")]), + Self.finished, + ]) + var batches: [[[TeradataValue]]] = [] + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { rows in + batches.append(rows) + return true + }) + } + + var code: Int? + if case .server(let failed, _)? = error as? TeradataWireError { code = failed } + #expect(code == 2_646) + #expect(batches == [[[.integer(1)]]]) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("Stop between replies ends the output, and the stream ends as cancelled") + func stopBetweenRepliesEndsTheStream() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + var batches = 0 + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { _ in + batches += 1 + connection.cancel() + return true + }) + } + + #expect(Self.isCancelled(error)) + #expect(batches == 1) + #expect(transport.sentKinds == [Self.start, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("A later statement's records are never read against the first statement's columns") + func laterStatementKeepsItsOwnColumns() throws { + let answer = Self.reply([ + Self.numbers, Self.number(1), Parcel(.endStatement), + Self.prepInfoX([(Self.char, 3, "code")]), Self.code("abc"), Parcel(.endStatement), + Self.numbers, Self.number(2), Parcel(.endStatement), + Parcel(.endRequest), + ]) + let sql = "SELECT n FROM sales.a; SELECT code FROM sales.b; SELECT n FROM sales.c" + + let result = try Self.connection([answer]).0.execute(sql) + let names = result.columns.map { $0.name } + #expect(names == ["n"]) + #expect(result.rows == [[.integer(1)], [.integer(2)]]) + + var headers: [[String]] = [] + var streamed: [[TeradataValue]] = [] + try Self.connection([answer]).0.stream(sql, onColumns: { headers.append($0.map { $0.name }) }, onRows: { rows in + streamed += rows + return true + }) + #expect(headers == [["n"]]) + #expect(streamed == result.rows) + } + + @Test("Stop before the first reply aborts an ANSI stream's request, and the stream reports the rollback") + func abortedStreamReportsTheRollback() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]), + Self.finished, + ], transactionMode: .ansi) + transport.beforeReply[0] = { connection.cancel() } + var headers = 0 + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in headers += 1 }, onRows: { _ in true }) + } + + var rolledBack = false + if case .rolledBackByStop? = error as? TeradataWireError { rolledBack = true } + #expect(rolledBack) + #expect(headers == 0) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + } + + @Test("A row cap ends the output at the reply that passes it and marks the result cut") + func rowCapEndsTheOutput() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2)]), + Self.reply([Self.number(3), Self.number(4)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + + let capped = try connection.execute("SELECT n FROM sales.orders", rowCap: 3) + + #expect(capped.isTruncated) + #expect(capped.resultSet.rows == [[.integer(1)], [.integer(2)], [.integer(3)]]) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont, Self.start]) + } + + @Test("A result with exactly as many rows as the cap is not cut") + func resultAtTheCapIsComplete() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2), Parcel(.endRequest)]), + ]) + + let capped = try connection.execute("SELECT n FROM sales.orders", rowCap: 2) + + #expect(!capped.isTruncated) + #expect(capped.resultSet.rows == [[.integer(1)], [.integer(2)]]) + #expect(transport.sentKinds == [Self.start]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataTransactionTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataTransactionTests.swift new file mode 100644 index 0000000000..b842c9060a --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataTransactionTests.swift @@ -0,0 +1,565 @@ +import Foundation +@testable import TableProTeradataCore +import Testing + +private enum TeradataWire { + static let varchar: UInt16 = 449 + static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + static func u16(_ value: Int) -> [UInt8] { + [UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + static func prepInfoX(_ columns: [(type: UInt16, length: Int, name: String)]) -> Parcel { + var body = [UInt8](repeating: 0, count: 8) + u16(0) + u16(columns.count) + for column in columns { + let name = Array(column.name.utf8) + body += u16(Int(column.type)) + body += (0..<8).map { UInt8(truncatingIfNeeded: column.length >> (8 * (7 - $0))) } + body += [0, 0] + body += u16(name.count) + name + body += u16(0) + u16(0) + } + return Parcel(.prepInfoX, body: body) + } + + // One row of VARCHAR columns, each named and holding the text given. + static func textRow(_ cells: [(name: String, value: String)]) -> [UInt8] { + let header = prepInfoX(cells.map { (type: varchar, length: 64, name: $0.name) }) + var record = [UInt8](repeating: 0, count: (cells.count + 7) / 8) + for cell in cells { + let text = Array(cell.value.utf8) + record += u16(text.count) + text + } + return reply([Parcel(.success), header, Parcel(.record, body: record), Parcel(.endRequest)]) + } + + static func failure(code: Int, _ message: String = "refused") -> [UInt8] { + let text = Array(message.utf8) + return reply([Parcel(.failure, body: [UInt8](repeating: 0, count: 8) + u16(code) + u16(text.count) + text)]) + } + + static func connection( + _ replies: [[UInt8]], + mode: TeradataTransactionMode, + database: String? = nil + ) -> (TeradataConnection, StoppableTransport) { + let transport = StoppableTransport(replies: replies) + let config = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", database: database, transactionMode: mode + ) + return (TeradataConnection(config: config, transport: transport), transport) + } + + static func sentSQL(_ transport: StoppableTransport) throws -> [String] { + try transport.sent.compactMap { message in + try Parcel.decodeAll(message.body) + .first { $0.flavor == ParcelFlavor.indicRequest.rawValue } + .flatMap { String(bytes: $0.body, encoding: .utf8) } + } + } + + static func serverCode(of body: () throws -> Void) -> Int? { + do { + try body() + return nil + } catch TeradataWireError.server(let code, _) { + return code + } catch { + return nil + } + } +} + +struct TeradataStatementKindTests { + @Test( + "Reads leave the transaction as it was", + arguments: [ + "SELECT 1", + "sel * from sales.t", + "HELP SESSION", + "SHOW TABLE sales.t", + "EXPLAIN DELETE FROM sales.t", + "(SELECT 1) UNION (SELECT 2)", + "/* nightly */ SELECT 1", + "-- nightly\nSELECT 1", + "LOCKING ROW FOR ACCESS SELECT * FROM sales.t", + "LOCK TABLE sales.t FOR READ OVERRIDE SELECT * FROM sales.t", + "LOCKING TABLE a FOR ACCESS LOCKING TABLE b FOR ACCESS SELECT * FROM a, b", + "WITH c (n) AS (SELECT 1) SELECT * FROM c", + ] + ) + func reads(statement: String) { + #expect(TeradataStatementKind.kinds(in: statement) == [.read]) + } + + @Test( + "Anything that is not a known read or transaction statement counts as a change", + arguments: [ + "INSERT INTO sales.t VALUES (1)", + "ins sales.t (1)", + "UPDATE sales.t SET a = 1", + "DELETE sales.t ALL", + "MERGE INTO sales.t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE SET a = s.a", + "CREATE TABLE sales.t (a INTEGER)", + "DATABASE sales", + "SET SESSION DATEFORM = ANSIDATE", + "CALL sales.p()", + "EXEC sales.m", + "BEGIN QUERY LOGGING ON ALL", + "END LOGGING ON ALL ON TABLE sales.t", + "USING (a INTEGER) INSERT INTO sales.t VALUES (:a)", + "LOCKING TABLE sales.t FOR WRITE UPDATE sales.t SET a = 1", + "LOCKING TABLE sales.t FOR WRITE", + "WITH c (n) AS (SELECT 1) INSERT INTO sales.t SELECT n FROM c", + "'not a statement'", + "COLLECT STATISTICS COLUMN (a) ON sales.t", + "SELECT AND CONSUME TOP 1 * FROM sales.q", + "sel and consume top 1 * from sales.q", + ] + ) + func changes(statement: String) { + #expect(TeradataStatementKind.kinds(in: statement) == [.change]) + } + + @Test("Transaction statements are told apart, in either spelling") + func transactionStatements() { + #expect(TeradataStatementKind.kinds(in: "BT") == [.begin]) + #expect(TeradataStatementKind.kinds(in: "begin transaction") == [.begin]) + #expect(TeradataStatementKind.kinds(in: "ET") == [.end]) + #expect(TeradataStatementKind.kinds(in: "END TRANSACTION") == [.end]) + #expect(TeradataStatementKind.kinds(in: "COMMIT") == [.commit]) + #expect(TeradataStatementKind.kinds(in: "COMMIT WORK") == [.commit]) + #expect(TeradataStatementKind.kinds(in: "ROLLBACK WORK") == [.rollback]) + #expect(TeradataStatementKind.kinds(in: "ABORT 'stopped by hand'") == [.rollback]) + } + + @Test("A request is split at each semicolon outside quotes and comments") + func requestSplitting() { + #expect(TeradataStatementKind.kinds(in: "INSERT INTO t VALUES (1); COMMIT;") == [.change, .commit]) + #expect(TeradataStatementKind.kinds(in: "SELECT 'a; COMMIT'") == [.read]) + #expect(TeradataStatementKind.kinds(in: "SELECT 'it''s; COMMIT' FROM t") == [.read]) + #expect(TeradataStatementKind.kinds(in: "SELECT \"COMMIT;\" FROM t") == [.read]) + #expect(TeradataStatementKind.kinds(in: "SELECT 1 -- ; COMMIT\n") == [.read]) + #expect(TeradataStatementKind.kinds(in: "/* ; COMMIT */ SELECT 1") == [.read]) + #expect(TeradataStatementKind.kinds(in: " ; ; ").isEmpty) + } +} + +struct TeradataTransactionTrackerTests { + @Test("In ANSI a change opens the transaction, a read does not, and COMMIT closes it") + func ansiChangesOpenTheTransaction() { + var tracker = TeradataTransactionTracker(mode: .ansi) + tracker.ran([.read]) + #expect(tracker.state == .idle) + tracker.ran([.change]) + #expect(tracker.state == .open) + tracker.ran([.read]) + #expect(tracker.state == .open) + tracker.ran([.commit]) + #expect(tracker.state == .idle) + } + + @Test("In ANSI a rollback's own failure answer closes the transaction and any other failure keeps it") + func ansiFailures() { + var tracker = TeradataTransactionTracker(mode: .ansi) + tracker.ran([.change]) + tracker.failed([.change], code: 3_807) + #expect(tracker.state == .open) + tracker.failed([.change], code: 3_514) + #expect(tracker.state == .open) + tracker.failed([.rollback], code: 3_514) + #expect(tracker.state == .idle) + + tracker.ran([.change]) + tracker.failed([.rollback], code: 3_513) + #expect(tracker.state == .idle) + + tracker.ran([.change]) + tracker.failed([.read], code: TeradataConnection.userAbortCode) + #expect(tracker.state == .idle) + } + + @Test("A rollback that answered success had a condition that held nothing, so the transaction stays open") + func conditionalRollbackKeepsTheTransaction() { + var ansi = TeradataTransactionTracker(mode: .ansi) + ansi.ran([.change, .rollback]) + #expect(ansi.state == .open) + + var tera = TeradataTransactionTracker(mode: .tera) + tera.ran([.begin, .rollback]) + #expect(tera.state == .open) + } + + @Test("In Teradata mode only an explicit transaction is open, and the outermost pair decides") + func teradataModeTracksTheOutermostPair() { + var tracker = TeradataTransactionTracker(mode: .tera) + tracker.ran([.change]) + #expect(tracker.state == .idle) + #expect(!tracker.abortRollsBackTransaction) + tracker.ran([.begin]) + #expect(tracker.state == .open) + #expect(tracker.abortRollsBackTransaction) + tracker.ran([.begin]) + tracker.ran([.end]) + #expect(tracker.state == .open) + tracker.ran([.end]) + #expect(tracker.state == .idle) + tracker.ran([.end]) + #expect(tracker.state == .idle) + tracker.ran([.begin]) + #expect(tracker.state == .open) + } + + @Test("In Teradata mode any failure rolls the explicit transaction back") + func teradataFailureEndsTheTransaction() { + var tracker = TeradataTransactionTracker(mode: .tera) + tracker.ran([.begin, .begin, .change]) + tracker.failed([.change], code: 3_706) + #expect(tracker.state == .idle) + } + + @Test("A session whose mode could not be read reports nothing it cannot know") + func unresolvedModeIsUnknown() { + var tracker = TeradataTransactionTracker(mode: .default) + #expect(tracker.state == .unknown) + tracker.ran([.begin]) + #expect(tracker.state == .unknown) + #expect(tracker.abortRollsBackTransaction) + } +} + +struct TeradataSessionModeProbeTests { + private static func result(_ cells: [(name: String, value: String)]) -> TeradataResultSet { + TeradataResultSet( + columns: cells.map { TeradataColumn(name: $0.name, typeCode: TeradataWire.varchar, dataLength: 64) }, + rows: [cells.map { .text($0.value) }], + activityCount: 1 + ) + } + + @Test("HELP SESSION's Transaction Semantics column is found by its name, wherever it is") + func helpSessionColumnByName() { + let ansi = Self.result([("User Name", "DBC"), ("Transaction Semantics", "ANSI"), ("Current DateForm", "IntegerDate")]) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: ansi) == .ansi) + + let teradata = Self.result([("Transaction Semantics", "Teradata"), ("User Name", "ANSI")]) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: teradata) == .tera) + + let spelled = Self.result([("TRANSACTION_SEMANTICS", "teradata ")]) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: spelled) == .tera) + } + + @Test("A HELP SESSION answer without a known value says nothing") + func helpSessionWithoutTheAnswer() { + #expect(TeradataSessionModeProbe.mode(fromHelpSession: Self.result([("User Name", "ANSI")])) == nil) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: Self.result([("Transaction Semantics", "Other")])) == nil) + let empty = TeradataResultSet( + columns: [TeradataColumn(name: "Transaction Semantics", typeCode: TeradataWire.varchar, dataLength: 8)], + rows: [], + activityCount: 0 + ) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: empty) == nil) + } + + @Test("DBC.SessionInfoV codes the mode as A or T") + func sessionInfoCodes() { + #expect(TeradataSessionModeProbe.mode(fromSessionInfo: Self.result([("Transaction_Mode", "A")])) == .ansi) + #expect(TeradataSessionModeProbe.mode(fromSessionInfo: Self.result([("Transaction_Mode", "T")])) == .tera) + #expect(TeradataSessionModeProbe.mode(fromSessionInfo: Self.result([("Transaction_Mode", "D")])) == nil) + } +} + +struct TeradataSessionSetupTests { + private static let ansiSession = TeradataWire.textRow([("User Name", "DBC"), ("Transaction Semantics", "ANSI")]) + + @Test("DEFAULT asks HELP SESSION, and an ANSI session commits what its setup opened") + func defaultReadsHelpSession() throws { + let (connection, transport) = TeradataWire.connection( + [Self.ansiSession, TeradataWire.finished, TeradataWire.finished], mode: .default, database: "sales" + ) + + try connection.prepareSession() + + #expect(try TeradataWire.sentSQL(transport) == [ + "HELP SESSION", TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK", + ]) + #expect(connection.sessionTransactionMode == .ansi) + #expect(connection.sessionModeSource == .helpSession) + #expect(connection.transactionState == .idle) + } + + @Test("A HELP SESSION the server refuses falls back to DBC.SessionInfoV") + func refusedHelpSessionFallsBackToTheView() throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.failure(code: 3_523), TeradataWire.textRow([("Transaction_Mode", "T")])], mode: .default + ) + + try connection.prepareSession() + + #expect(try TeradataWire.sentSQL(transport) == ["HELP SESSION", TeradataSessionModeProbe.sessionInfo]) + #expect(connection.sessionTransactionMode == .tera) + #expect(connection.sessionModeSource == .sessionInfoView) + #expect(connection.transactionState == .idle) + } + + @Test("A HELP SESSION reply the decoder cannot read falls back to DBC.SessionInfoV") + func unreadableHelpSessionFallsBackToTheView() throws { + let unreadable = TeradataWire.reply([ + TeradataWire.prepInfoX([(605, 20, "Spool")]), Parcel(.record, body: [0x00, 0x01]), Parcel(.endRequest), + ]) + let (connection, _) = TeradataWire.connection( + [unreadable, TeradataWire.textRow([("Transaction_Mode", "A")]), TeradataWire.finished], mode: .default + ) + + try connection.prepareSession() + + #expect(connection.sessionTransactionMode == .ansi) + #expect(connection.sessionModeSource == .sessionInfoView) + } + + @Test("When neither read answers, the session keeps Teradata semantics and its state is unknown") + func unresolvedModeKeepsTodaysBehaviour() throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.failure(code: 3_523), TeradataWire.failure(code: 3_523), TeradataWire.finished], mode: .default + ) + + try connection.prepareSession() + #expect(connection.sessionTransactionMode == .default) + #expect(connection.sessionModeSource == .unresolved) + #expect(connection.transactionState == .unknown) + + try connection.beginTransaction() + #expect(try TeradataWire.sentSQL(transport).last == "BEGIN TRANSACTION") + } + + @Test("An explicit mode is trusted without asking") + func explicitModeIsTrusted() throws { + let (ansi, ansiTransport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished], mode: .ansi, database: "sales" + ) + try ansi.prepareSession() + #expect(try TeradataWire.sentSQL(ansiTransport) == [TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK"]) + #expect(ansi.sessionModeSource == .configured) + + let (tera, teraTransport) = TeradataWire.connection([], mode: .tera) + try tera.prepareSession() + #expect(teraTransport.sent.isEmpty) + #expect(tera.sessionTransactionMode == .tera) + #expect(tera.transactionState == .idle) + } +} + +struct TeradataTransactionStatementTests { + @Test("ANSI begins with nothing, commits with COMMIT WORK, and reports a change until then") + func ansiTransaction() throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished, TeradataWire.finished], mode: .ansi + ) + + try connection.beginTransaction() + #expect(transport.sent.isEmpty) + _ = try connection.execute("SELECT * FROM sales.t") + #expect(connection.transactionState == .idle) + _ = try connection.execute("UPDATE sales.t SET a = 1") + #expect(connection.transactionState == .open) + try connection.commitTransaction() + + #expect(try TeradataWire.sentSQL(transport) == ["SELECT * FROM sales.t", "UPDATE sales.t SET a = 1", "COMMIT WORK"]) + #expect(connection.transactionState == .idle) + } + + @Test("Teradata mode begins with BEGIN TRANSACTION and commits with END TRANSACTION") + func teradataTransaction() throws { + let (connection, transport) = TeradataWire.connection([TeradataWire.finished, TeradataWire.finished], mode: .tera) + + try connection.beginTransaction() + #expect(connection.transactionState == .open) + try connection.commitTransaction() + + #expect(try TeradataWire.sentSQL(transport) == ["BEGIN TRANSACTION", "END TRANSACTION"]) + #expect(connection.transactionState == .idle) + } + + @Test( + "A rollback answered with 3513 or 3514 succeeded", + arguments: [TeradataTransactionMode.ansi, .tera], + [3_513, 3_514] + ) + func rollbackAnswerIsSuccess(mode: TeradataTransactionMode, code: Int) throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.failure(code: code, "User-generated transaction ABORT.")], mode: mode + ) + _ = try connection.execute(mode == .ansi ? "DELETE FROM sales.t" : "BT") + #expect(connection.transactionState == .open) + + try connection.rollbackTransaction() + + #expect(try TeradataWire.sentSQL(transport).last == (mode == .ansi ? "ROLLBACK WORK" : "ROLLBACK")) + #expect(connection.transactionState == .idle) + } + + @Test("Any other failure from a rollback still throws") + func otherRollbackFailureThrows() { + let (connection, _) = TeradataWire.connection([TeradataWire.failure(code: 2_631, "Transaction ABORTed due to deadlock.")], mode: .ansi) + #expect(TeradataWire.serverCode { try connection.rollbackTransaction() } == 2_631) + } + + @Test("The rollback answer code is success only for a rollback") + func rollbackCodeFromAnotherStatementThrows() { + let (connection, _) = TeradataWire.connection([TeradataWire.failure(code: 3_514)], mode: .ansi) + #expect(TeradataWire.serverCode { _ = try connection.execute("UPDATE sales.t SET a = 1") } == 3_514) + } + + @Test("In Teradata mode a failure inside BT ends the transaction") + func teradataFailureInsideTransaction() throws { + let (connection, _) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.failure(code: 3_706, "Syntax error")], mode: .tera + ) + try connection.beginTransaction() + #expect(TeradataWire.serverCode { _ = try connection.execute("INSRT INTO sales.t VALUES (1)") } == 3_706) + #expect(connection.transactionState == .idle) + } + + @Test("Stop inside BT reports that the server rolled the transaction back") + func stopInsideExplicitTransactionRollsBack() throws { + let aborted = TeradataWire.failure(code: TeradataConnection.userAbortCode, "The transaction was aborted by the user.") + let (connection, transport) = TeradataWire.connection([TeradataWire.finished, aborted], mode: .tera) + try connection.beginTransaction() + transport.beforeReply[1] = { connection.cancel() } + + do { + _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") + Issue.record("expected the Stop to throw") + } catch TeradataWireError.rolledBackByStop { + #expect(connection.transactionState == .idle) + } + } + + @Test("A database switch commits at once in ANSI and never joins an open transaction") + func databaseSwitch() throws { + let (ansi, ansiTransport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished, TeradataWire.finished], mode: .ansi + ) + try ansi.switchDatabase(to: "sales") + #expect(try TeradataWire.sentSQL(ansiTransport) == [TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK"]) + #expect(ansi.transactionState == .idle) + + _ = try ansi.execute("UPDATE sales.t SET a = 1") + #expect(throws: TeradataWireError.self) { try ansi.switchDatabase(to: "hr") } + #expect(try TeradataWire.sentSQL(ansiTransport).count == 3) + + let (tera, teraTransport) = TeradataWire.connection([TeradataWire.finished, TeradataWire.finished], mode: .tera) + try tera.switchDatabase(to: "sales") + try tera.beginTransaction() + #expect(throws: TeradataWireError.self) { try tera.switchDatabase(to: "hr") } + #expect(try TeradataWire.sentSQL(teraTransport) == [TeradataSchemaQueries.setDatabase("sales"), "BEGIN TRANSACTION"]) + } + + @Test("Inside an open transaction a switch to the database the session is on sends nothing") + func switchToTheCurrentDatabaseInsideATransaction() throws { + let (ansi, ansiTransport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished, TeradataWire.finished], mode: .ansi + ) + try ansi.switchDatabase(to: "sales") + _ = try ansi.execute("UPDATE sales.t SET a = 1") + + try ansi.switchDatabase(to: "SALES") + + #expect(try TeradataWire.sentSQL(ansiTransport) == [ + TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK", "UPDATE sales.t SET a = 1", + ]) + #expect(ansi.transactionState == .open) + + let (tera, teraTransport) = TeradataWire.connection([TeradataWire.finished, TeradataWire.finished], mode: .tera) + try tera.switchDatabase(to: "sales") + try tera.beginTransaction() + try tera.switchDatabase(to: "sales") + #expect(try TeradataWire.sentSQL(teraTransport) == [TeradataSchemaQueries.setDatabase("sales"), "BEGIN TRANSACTION"]) + } + + @Test( + "A request that moves the default database leaves the next switch inside a transaction refused", + arguments: ["DATABASE hr", "SET SESSION DATABASE hr", "SS DATABASE hr", "EXEC sales.refresh", "CALL sales.load()"] + ) + func userMovedDatabaseIsNotTrusted(_ statement: String) throws { + let (ansi, ansiTransport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished, TeradataWire.finished], mode: .ansi + ) + try ansi.switchDatabase(to: "sales") + _ = try ansi.execute(statement) + + #expect(throws: TeradataWireError.self) { try ansi.switchDatabase(to: "sales") } + #expect(try TeradataWire.sentSQL(ansiTransport).count == 3) + } +} + +struct TeradataDefaultDatabaseStatementTests { + @Test( + "Statements that can move the default database", + arguments: [ + ("DATABASE hr", true), + ("database \"hr\";", true), + ("SET SESSION DATABASE hr", true), + ("ss database hr", true), + ("EXEC sales.refresh", true), + ("CALL sales.load()", true), + ("SELECT 1; DATABASE hr", true), + ("SET SESSION DATEFORM = ANSIDATE", false), + ("SELECT DATABASE", false), + ("CREATE DATABASE hr FROM dbc AS PERM = 0", false), + ("COMMENT ON DATABASE hr AS 'x'", false), + ] + ) + func mayMoveDefaultDatabase(_ request: String, _ moves: Bool) { + #expect(TeradataStatementKind.mayMoveDefaultDatabase(request) == moves) + } +} + +struct TeradataUnreadableColumnTests { + private static func decodingError(type: UInt16, name: String) -> TeradataWireError? { + let reply = TeradataWire.reply([ + TeradataWire.prepInfoX([(type, 20, name)]), Parcel(.record, body: [0x00, 0x01]), Parcel(.endRequest), + ]) + let (connection, _) = TeradataWire.connection([reply], mode: .tera) + do { + _ = try connection.execute("SELECT \(name) FROM sales.t") + return nil + } catch { + return error as? TeradataWireError + } + } + + @Test("A NUMBER column names itself and says to cast it to DECIMAL or VARCHAR") + func numberColumn() throws { + let error = try #require(Self.decodingError(type: 605, name: "amount")) + guard case .unreadableColumn(let name, let typeCode) = error else { + Issue.record("expected an unreadable column, got \(error)") + return + } + #expect(name == "amount") + #expect(typeCode == 605) + #expect(error.description.contains("\"amount\"")) + #expect(error.description.contains("NUMBER")) + #expect(error.description.contains("CAST it to DECIMAL or VARCHAR")) + } + + @Test("Another unreadable type names itself and says to cast it to VARCHAR") + func periodColumn() throws { + let error = try #require(Self.decodingError(type: 833, name: "span")) + #expect(error.description.contains("\"span\"")) + #expect(error.description.contains("PERIOD(DATE)")) + #expect(error.description.contains("CAST it to VARCHAR")) + #expect(!error.description.contains("DECIMAL")) + } + + @Test("A type code with no known name is given by its number") + func unknownTypeCode() throws { + let error = try #require(Self.decodingError(type: 1_234, name: "odd")) + #expect(error.description.contains("type 1234")) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeImportTests.swift b/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeImportTests.swift index c0d16fd667..5462490d66 100644 --- a/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeImportTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeImportTests.swift @@ -169,13 +169,38 @@ struct VSCodeThemeImportTests { {"colors": { "editor.background": "#202020", "editor.foreground": "#E0E0E0" - }, "tokenColors": [{"scope": "constant.language.boolean", "settings": {"foreground": "#E0E0E0"}}]} + }, "tokenColors": [ + {"scope": "constant.language.boolean", "settings": {"foreground": "#E0E0E0"}}, + {"scope": "string", "settings": {"foreground": "#E0E0E0"}} + ]} """ ) #expect(theme.hex(.gridBoolTrue) == nil) #expect(theme.hex(.gridBoolFalse) == nil) + #expect(theme.hex(.gridString) == nil) #expect(theme.value(for: .gridBoolTrue) == .themed(ThemeColor(red: 0xE0, green: 0xE0, blue: 0xE0))) + #expect(theme.value(for: .gridString) == .themed(ThemeColor(red: 0xE0, green: 0xE0, blue: 0xE0))) + } + + @Test("String and number rules color the grid's strings and numbers, and the other value types follow the text") + func stringAndNumberRulesColorGridValues() throws { + let theme = try Self.theme( + """ + {"colors": {"editor.background": "#1E1E1E", "editor.foreground": "#D4D4D4"}, "tokenColors": [ + {"scope": "string", "settings": {"foreground": "#CE9178"}}, + {"scope": "constant.numeric", "settings": {"foreground": "#B5CEA8"}} + ]} + """ + ) + let text = ThemeColor(red: 0xD4, green: 0xD4, blue: 0xD4) + + #expect(theme.hex(.gridString) == "#CE9178") + #expect(theme.hex(.gridNumber) == "#B5CEA8") + for slot in [ThemeSlot.gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum] { + #expect(theme.hex(slot) == nil, "\(slot)") + #expect(theme.value(for: slot) == .themed(text), "\(slot)") + } } @Test("A VS Code theme's table header, list selection and focus colors import as set, even where they match the grid") diff --git a/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeMappingTests.swift b/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeMappingTests.swift index 99a31ad1a2..06d97e6691 100644 --- a/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeMappingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProThemeTests/Import/VSCodeThemeMappingTests.swift @@ -83,6 +83,9 @@ struct VSCodeThemeMappingTests { case .gridNullValue: return "#707070" case .gridBoolTrue: return "#C00009" case .gridBoolFalse: return "#C0000A" + case .gridString: return "#C00002" + case .gridNumber: return "#C00003" + case .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: return nil case .gridRowNumber: return "#808080" // 0xFF scaled by 0.3 and by 0.5, rounded. case .gridModified: return "#E0C0004D" @@ -143,6 +146,9 @@ struct VSCodeThemeMappingTests { case .gridNullValue: return "#40404080" case .gridBoolTrue: return nil case .gridBoolFalse: return nil + case .gridString: return nil + case .gridNumber: return nil + case .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: return nil case .gridRowNumber: return nil case .gridModified: return nil case .gridInserted: return "#00AA0033" diff --git a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileFormatTests.swift b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileFormatTests.swift index 25e0faeb27..b8c0d77846 100644 --- a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileFormatTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileFormatTests.swift @@ -53,11 +53,95 @@ struct ThemeFileFormatTests { } } + private static let valueSlots: [ThemeSlot] = [.gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum] + + private static func encodedSchema(_ theme: ThemeDefinition) throws -> Int? { + let object = try JSONSerialization.jsonObject(with: try ThemeFileFormat.encode(theme)) + return (object as? [String: Any])?["schema"] as? Int + } + // MARK: - What loads - @Test("The current format is 3") - func currentFormatIsThree() { - #expect(ThemeSchema.current == 3) + @Test("The current format is 4, and format 3 still reads") + func currentFormatIsFour() { + #expect(ThemeSchema.current == 4) + #expect(ThemeSchema.oldestReadable == 3) + } + + @Test("A format 3 file and a format 4 file both load", arguments: [3, 4]) + func readableFormatsLoad(schema: Int) throws { + var root = Self.metadata(schema: schema) + Self.insert("#FF0000", at: ["editor", "syntax", "keyword"], into: &root) + Self.insert("#00FF00", at: ["dataGrid", "boolTrue"], into: &root) + + let colors = try Self.decode(root).overrides.mapValues(\.hex) + + #expect(colors == [.syntaxKeyword: "#FF0000", .gridBoolTrue: "#00FF00"]) + } + + @Test("A format 4 file loads its value colors") + func formatFourLoadsValueColors() throws { + var root = Self.metadata(schema: 4) + for slot in Self.valueSlots { + Self.insert("#ABCDEF", at: slot.rawValue.components(separatedBy: "."), into: &root) + } + + let theme = try Self.decode(root) + + #expect(Set(theme.overrides.keys) == Set(Self.valueSlots)) + } + + // A format 3 reader refuses these keys, so a file that claims format 3 cannot hold them. + @Test("A format 3 file holding a format 4 key is refused with the format the key needs") + func newerKeyInOlderFileIsRefused() throws { + #expect( + try Self.refusal(Self.withColor("#F1FA8C", at: ["dataGrid", "string"])) + == .keysNeedNewerFormat(["dataGrid.string"], format: 4) + ) + + var root = Self.metadata(schema: 3) + Self.insert("#F8F8F2", at: ["dataGrid", "text"], into: &root) + for slot in Self.valueSlots { + Self.insert("#ABCDEF", at: slot.rawValue.components(separatedBy: "."), into: &root) + } + let paths = Self.valueSlots.map(\.rawValue).sorted() + #expect(try Self.refusal(root) == .keysNeedNewerFormat(paths, format: 4)) + } + + @Test("A key TablePro does not use outranks a key that needs a newer format") + func unknownKeyOutranksANewerKey() throws { + var root = Self.withColor("#F1FA8C", at: ["dataGrid", "string"]) + Self.insert("#000000", at: ["panel", "text"], into: &root) + + #expect(try Self.refusal(root) == .unknownKeys(["panel.text"])) + } + + @Test("The encoder writes format 3 for a theme with no value color, and format 4 once it has one") + func encoderWritesTheOldestFormatThatHoldsTheTheme() throws { + let theme = ThemeDefinition( + id: "user.values", + name: "Values", + author: "", + appearance: .dark, + overrides: [.gridText: try ThemeColor(validating: "#F8F8F2"), .gridBoolTrue: try ThemeColor(validating: "#50FA7B")] + ) + #expect(try Self.encodedSchema(theme) == 3) + #expect(try Self.encodedSchema(.defaultDark) == 3) + + for slot in Self.valueSlots { + var withValue = theme + withValue.overrides[slot] = try ThemeColor(validating: "#F1FA8C") + #expect(try Self.encodedSchema(withValue) == 4, "\(slot.rawValue)") + #expect(try ThemeFileFormat.decode(try ThemeFileFormat.encode(withValue)) == withValue, "\(slot.rawValue)") + } + } + + @Test("A format 4 file with no value color is saved back as format 3") + func formatFourWithoutValueColorsSavesAsThree() throws { + var root = Self.metadata(schema: 4) + Self.insert("#282A36", at: ["dataGrid", "background"], into: &root) + + #expect(try Self.encodedSchema(try Self.decode(root)) == 3) } @Test("A file with only metadata loads and sets nothing") @@ -85,9 +169,9 @@ struct ThemeFileFormatTests { #expect(colors == [.syntaxKeyword: "#FF0000", .gridAlternateRow: "#00000080"]) } - @Test("Every slot path is accepted") + @Test("Every slot path is accepted in the current format") func everySlotPathLoads() throws { - var root = Self.metadata() + var root = Self.metadata(schema: ThemeSchema.current) for slot in ThemeSlot.allCases { Self.insert("#123456", at: slot.rawValue.components(separatedBy: "."), into: &root) } @@ -271,10 +355,10 @@ struct ThemeFileFormatTests { } } - @Test("Format 2 and a file with no format are refused") + @Test("Format 2, format 5 and a file with no format are refused") func otherFormatsAreRefused() throws { #expect(try Self.refusal(Self.metadata(schema: 2)) == .schemaTooOld(found: 2, supported: 3)) - #expect(try Self.refusal(Self.metadata(schema: 4)) == .schemaTooNew(found: 4, supported: 3)) + #expect(try Self.refusal(Self.metadata(schema: 5)) == .schemaTooNew(found: 5, supported: 4)) #expect(try Self.refusal(Self.metadata(schema: "3")) == .missingSchema) var unversioned = Self.metadata() @@ -479,8 +563,9 @@ struct ThemeFileFormatTests { func errorMessages() { let errors: [ThemeLoadError] = [ .malformedJSON(detail: nil), .missingSchema, .schemaTooOld(found: 2, supported: 3), - .schemaTooNew(found: 4, supported: 3), .missingField("name"), .invalidName, .invalidAuthor, - .invalidColor("#XYZ"), .unknownKeys(["a.b"]), .duplicateKey("a.b"), .reservedIdentifier("tablepro.x"), + .schemaTooNew(found: 5, supported: 4), .missingField("name"), .invalidName, .invalidAuthor, + .invalidColor("#XYZ"), .unknownKeys(["a.b"]), .keysNeedNewerFormat(["a.b"], format: 4), + .duplicateKey("a.b"), .reservedIdentifier("tablepro.x"), .invalidIdentifier("x/y"), .duplicateIdentifier("user.a", existingFile: "a.json"), .changedOnDisk("a.json"), .deletedOnDisk("a.json"), .nameTaken("Ocean"), .fileTooLarge(limit: ThemeFileImporter.maximumFileSize), .notATheme, @@ -489,6 +574,18 @@ struct ThemeFileFormatTests { #expect(!(error.errorDescription ?? "").isEmpty, "\(error)") } #expect(ThemeLoadError.malformedJSON(detail: nil).errorDescription == "The file is not valid JSON.") + #expect( + ThemeLoadError.schemaTooOld(found: 2, supported: 3).errorDescription + == "Theme format 2 is no longer supported. This version of TablePro reads format 3 and later." + ) + #expect( + ThemeLoadError.schemaTooNew(found: 5, supported: 4).errorDescription + == "Theme format 5 was made for a newer TablePro. This version reads format 4 and earlier." + ) + #expect( + ThemeLoadError.keysNeedNewerFormat(["dataGrid.string"], format: 4).errorDescription + == "These theme keys need theme format 4: \"dataGrid.string\"." + ) #expect(ThemeLoadError.deletedOnDisk("Ocean.json").errorDescription == "Ocean.json was deleted outside TablePro.") #expect(ThemeLoadError.nameTaken("Ocean").errorDescription == "A theme named “Ocean” already exists.") #expect(ThemeLoadError.notATheme.errorDescription == "This file is not a TablePro theme or a VS Code color theme.") diff --git a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileImporterTests.swift b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileImporterTests.swift index ccec91c0d0..50f972ab17 100644 --- a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileImporterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeFileImporterTests.swift @@ -118,7 +118,16 @@ struct ThemeFileImporterTests { defer { folder.remove() } let url = try folder.write("{\"version\": 1, \"id\": \"user.a\", \"name\": \"x\"}", as: "old.json") - #expect(Self.refusal(importing: url) == .schemaTooOld(found: 1, supported: ThemeSchema.current)) + #expect(Self.refusal(importing: url) == .schemaTooOld(found: 1, supported: ThemeSchema.oldestReadable)) + } + + @Test("A TablePro theme in a newer format names the newest format this version reads") + func newerFormatKeepsItsError() throws { + let folder = try Folder() + defer { folder.remove() } + let url = try folder.write("{\"schema\": 5, \"id\": \"user.a\", \"name\": \"x\", \"appearance\": \"dark\"}", as: "new.json") + + #expect(Self.refusal(importing: url) == .schemaTooNew(found: 5, supported: ThemeSchema.current)) } @Test("A missing file throws the file system's error") diff --git a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotFallbackTests.swift b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotFallbackTests.swift index 6822fb722c..064034cb8f 100644 --- a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotFallbackTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotFallbackTests.swift @@ -54,6 +54,13 @@ struct ThemeSlotFallbackTests { .gridNullValue: .system, .gridBoolTrue: .slot(.gridText), .gridBoolFalse: .slot(.gridText), + .gridString: .slot(.gridText), + .gridNumber: .slot(.gridText), + .gridDateTime: .slot(.gridText), + .gridBinary: .slot(.gridText), + .gridJSON: .slot(.gridText), + .gridUUID: .slot(.gridText), + .gridEnum: .slot(.gridText), .gridRowNumber: .system, .gridModified: .system, .gridInserted: .system, @@ -189,11 +196,14 @@ struct ThemeSlotFallbackTests { // MARK: - Slot fallbacks - @Test("Bools follow themed grid text, and stay native without it") + @Test("Bools and value types follow themed grid text, and stay native without it") func gridTextFollowers() throws { let themed = try Self.theme(.dark, [.gridText: "#F8F8F2"]) let text = Self.color("#F8F8F2") - for slot in [ThemeSlot.gridBoolTrue, .gridBoolFalse] { + let followers: [ThemeSlot] = [ + .gridBoolTrue, .gridBoolFalse, .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum, + ] + for slot in followers { #expect(themed.value(for: slot) == .themed(text), "\(slot.rawValue)") #expect(ThemeDefinition.defaultDark.value(for: slot) == .system(slot), "\(slot.rawValue)") } @@ -201,6 +211,10 @@ struct ThemeSlotFallbackTests { let ownBool = try Self.theme(.dark, [.gridText: "#F8F8F2", .gridBoolTrue: "#50FA7B"]) #expect(ownBool.value(for: .gridBoolTrue) == .themed(Self.color("#50FA7B"))) #expect(ownBool.value(for: .gridBoolFalse) == .themed(text)) + + let ownString = try Self.theme(.dark, [.gridText: "#F8F8F2", .gridString: "#F1FA8C"]) + #expect(ownString.value(for: .gridString) == .themed(Self.color("#F1FA8C"))) + #expect(ownString.value(for: .gridNumber) == .themed(text)) } @Test("Operator falls back to the editor text, themed or native") diff --git a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotTests.swift b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotTests.swift index a097ffd52c..fdfa516c54 100644 --- a/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProThemeTests/ThemeSlotTests.swift @@ -23,9 +23,36 @@ struct ThemeSlotTests { } } - @Test("The registry holds 35 slots") + @Test("The registry holds 42 slots") func slotCount() { - #expect(ThemeSlot.allCases.count == 35) + #expect(ThemeSlot.allCases.count == 42) + } + + @Test("The value slots sit under the data grid key") + func valueSlots() { + let expected: [String: ThemeSlot] = [ + "dataGrid.string": .gridString, + "dataGrid.number": .gridNumber, + "dataGrid.dateTime": .gridDateTime, + "dataGrid.binary": .gridBinary, + "dataGrid.json": .gridJSON, + "dataGrid.uuid": .gridUUID, + "dataGrid.enum": .gridEnum, + ] + for (path, slot) in expected { + #expect(ThemeSlot(rawValue: path) == slot, "\(path)") + #expect(slot.group == .dataGrid, "\(path)") + #expect(slot.introducedInSchema == 4, "\(path)") + } + } + + @Test("Every slot came with format 3 or a later one this build reads, and the newest is the current format") + func slotsCarryTheirFormat() { + let formats = ThemeSlot.allCases.map(\.introducedInSchema) + #expect(formats.allSatisfy { (ThemeSchema.oldestReadable...ThemeSchema.current).contains($0) }) + #expect(formats.max() == ThemeSchema.current) + let format3 = ThemeSlot.allCases.filter { $0.introducedInSchema == ThemeSchema.oldestReadable } + #expect(format3.count == 35) } @Test("Every slot belongs to exactly one group, listed in registry order") @@ -44,16 +71,20 @@ struct ThemeSlotTests { #expect(sectioned.count == ThemeSlot.allCases.count) } - @Test("The editor sections split the data grid into rows, selection and changes") + @Test("The editor sections split the data grid into rows, values, selection and changes") func sectionsSplitTheGrid() { - #expect(ThemeSlotSection.allCases == [.window, .editor, .syntax, .dataGrid, .selection, .changes]) + #expect(ThemeSlotSection.allCases == [.window, .editor, .syntax, .dataGrid, .values, .selection, .changes]) #expect(ThemeSlotSection.window.slots == [.windowBackdrop]) #expect(ThemeSlotSection.editor.slots == ThemeSlotGroup.editor.slots) #expect(ThemeSlotSection.syntax.slots == ThemeSlotGroup.syntax.slots) #expect(ThemeSlotSection.dataGrid.slots == [ - .gridBackground, .gridText, .gridAlternateRow, .gridHeaderBackground, .gridHeaderText, - .gridLine, .gridNullValue, .gridBoolTrue, .gridBoolFalse, .gridRowNumber, + .gridBackground, .gridText, .gridAlternateRow, .gridHeaderBackground, .gridHeaderText, .gridLine, .gridRowNumber, + ]) + #expect(ThemeSlotSection.values.slots == [ + .gridNullValue, .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum, ]) + #expect(ThemeSlotSection.values.label == "Values") #expect(ThemeSlotSection.selection.slots == [.gridSelection, .gridSelectionText, .gridInactiveSelection, .gridFocusBorder]) #expect(ThemeSlotSection.changes.slots == [.gridModified, .gridInserted, .gridDeleted, .gridDeletedText]) } @@ -96,6 +127,11 @@ struct ThemeSlotTests { #expect(ThemeSlot.gridSelection.accessibilityLabel == "Data Grid Selection") #expect(ThemeSlot.gridSelectionText.accessibilityLabel == "Data Grid Selected Text") #expect(ThemeSlot.windowBackdrop.accessibilityLabel == "Window Background") + #expect(ThemeSlot.gridString.label == "String") + #expect(ThemeSlot.gridString.accessibilityLabel == "Data Grid String") + #expect(ThemeSlot.gridDateTime.label == "Date & Time") + #expect(ThemeSlot.gridDateTime.accessibilityLabel == "Data Grid Date & Time") + #expect(ThemeSlot.gridEnum.accessibilityLabel == "Data Grid Enum") } @Test("Only the fills under everything else require an opaque color") diff --git a/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoHeartbeatTests.swift b/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoHeartbeatTests.swift new file mode 100644 index 0000000000..f2875b5003 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTrinoCoreTests/TrinoHeartbeatTests.swift @@ -0,0 +1,181 @@ +import Foundation +import TableProTrinoCore +import Testing + +// A coordinator that abandons the query once `clientTimeout` passes without a request, on a clock the test moves. +private final class IdleCoordinator: TrinoTransport, @unchecked Sendable { + static let base = "https://trino.example:8443/v1/statement/executing/q1/s" + static let clientTimeout: Duration = .seconds(300) + + private let lock = NSLock() + private let headStatus: Int + private var now: Duration = .zero + private var lastContact: Duration = .zero + private var abandoned = false + private var recorded: [TrinoHTTPRequest] = [] + let deletes = AsyncStream.makeStream() + + init(headStatus: Int) { + self.headStatus = headStatus + } + + var requests: [TrinoHTTPRequest] { + lock.withLock { recorded } + } + + func advance(by interval: Duration) { + lock.withLock { now += interval } + } + + func send(_ request: TrinoHTTPRequest) async throws -> TrinoHTTPResponse { + let (status, body) = lock.withLock { () -> (Int, String) in + recorded.append(request) + if now - lastContact > Self.clientTimeout { abandoned = true } + lastContact = now + if abandoned { + return (200, #"{"id":"q1","error":{"message":"Query was abandoned","errorName":"ABANDONED_QUERY"}}"#) + } + switch request.method { + case .post: + return (200, #"{"id":"q1","nextUri":"\#(Self.base)/1"}"#) + case .head: + return (headStatus, "") + case .delete: + deletes.continuation.yield() + return (204, "") + case .get: + return (200, Self.page(Int(request.url.lastPathComponent) ?? 0)) + } + } + return TrinoHTTPResponse(statusCode: status, headers: TrinoHeaderFields([:]), body: Data(body.utf8)) + } + + func cancelAll() {} + + private static func page(_ token: Int) -> String { + let columns = #"[{"name":"n","type":"integer"}]"# + let next = token < 3 ? #","nextUri":"\#(base)/\#(token + 1)""# : "" + return #"{"id":"q1","columns":\#(columns),"data":[[\#(token)]]\#(next)}"# + } +} + +struct TrinoHeartbeatTests { + private func client(_ transport: IdleCoordinator) -> TrinoStatementClient { + let config = TrinoClientConfig( + host: "trino.example", + port: 8_443, + useTLS: true, + user: "analyst", + auth: .basic(password: "secret") + ) + return TrinoStatementClient(transport: transport, config: config, session: TrinoSessionState()) + } + + // Holds the first page for 20 minutes, sending a heartbeat every 30 seconds as a paced reader does. + private func read( + _ transport: IdleCoordinator, + heartbeat: TrinoHeartbeat, + sendsHeartbeats: Bool = true + ) async throws -> (rows: [String], requestsWhileHeld: [TrinoHTTPRequest]) { + var rows: [String] = [] + var requestsWhileHeld: [TrinoHTTPRequest] = [] + try await client(transport).executeStreamed( + "SELECT n FROM t", + heartbeat: heartbeat, + onColumns: { _ in }, + onPage: { page, keepAlive in + rows += page.compactMap { row -> String? in + guard case .text(let text)? = row.first else { return nil } + return text + } + guard rows.count == 1 else { return } + let before = transport.requests.count + for _ in 0..<40 { + transport.advance(by: .seconds(30)) + if sendsHeartbeats { try await keepAlive() } + } + requestsWhileHeld = Array(transport.requests.dropFirst(before)) + } + ) + return (rows, requestsWhileHeld) + } + + @Test("A held page outlives the client timeout on HEAD heartbeats, and no later page is fetched while held") + func headHeartbeatsKeepTheQuery() async throws { + let transport = IdleCoordinator(headStatus: 200) + let (rows, held) = try await read(transport, heartbeat: .head) + #expect(rows == ["1", "2", "3"]) + #expect(held.count == 40) + #expect(held.allSatisfy { $0.method == .head && $0.url.absoluteString == "\(IdleCoordinator.base)/2" }) + } + + @Test("A server without the HEAD heartbeat gets the held page asked for again, and its rows are not read twice") + func refusedHeadFallsBackToTheHeldPage() async throws { + let transport = IdleCoordinator(headStatus: 405) + let (rows, held) = try await read(transport, heartbeat: .head) + #expect(rows == ["1", "2", "3"]) + #expect(held.first?.method == .head) + #expect(held.dropFirst().allSatisfy { $0.method == .get && $0.url.absoluteString == "\(IdleCoordinator.base)/1" }) + #expect(held.filter { $0.method == .head }.count == 1) + } + + @Test("A HEAD that a proxy refuses falls back to the held page for that beat and is tried again on the next") + func refusedHeadBeatFallsBackOnce() async throws { + let transport = IdleCoordinator(headStatus: 403) + let (rows, held) = try await read(transport, heartbeat: .head) + #expect(rows == ["1", "2", "3"]) + #expect(held.count == 80) + #expect(held.enumerated().allSatisfy { index, request in + index.isMultiple(of: 2) + ? request.method == .head && request.url.absoluteString == "\(IdleCoordinator.base)/2" + : request.method == .get && request.url.absoluteString == "\(IdleCoordinator.base)/1" + }) + } + + @Test("Older Trino and Presto get the held page asked for again") + func repeatLastPageHeartbeat() async throws { + let transport = IdleCoordinator(headStatus: 200) + let (rows, held) = try await read(transport, heartbeat: .repeatLastPage) + #expect(rows == ["1", "2", "3"]) + #expect(held.count == 40) + #expect(held.allSatisfy { $0.method == .get && $0.url.absoluteString == "\(IdleCoordinator.base)/1" }) + } + + @Test("Without heartbeats the coordinator abandons a query held past its client timeout") + func heldPageWithoutHeartbeatsIsAbandoned() async { + let transport = IdleCoordinator(headStatus: 200) + await #expect { + _ = try await read(transport, heartbeat: .head, sendsHeartbeats: false) + } throws: { error in + guard case TrinoError.query(let failure) = error else { return false } + return failure.errorName == "ABANDONED_QUERY" + } + } + + @Test("A reader that stops taking pages ends the query on the coordinator", .timeLimit(.minutes(1))) + func readerThatStopsSendsTheDelete() async { + let transport = IdleCoordinator(headStatus: 200) + await #expect(throws: CancellationError.self) { + try await client(transport).executeStreamed( + "SELECT n FROM t", + heartbeat: .head, + onColumns: { _ in }, + onPage: { _, _ in throw CancellationError() } + ) + } + var deletes = transport.deletes.stream.makeAsyncIterator() + await deletes.next() + #expect(transport.requests.filter { $0.method == .delete }.map(\.url.absoluteString) == ["\(IdleCoordinator.base)/1"]) + } + + @Test("HEAD heartbeats start at Trino 475", arguments: [ + (TrinoServerFlavor.trino, "475" as String?, TrinoHeartbeat.head), + (.trino, "476-e.1", .head), + (.trino, "474", .repeatLastPage), + (.trino, nil, .repeatLastPage), + (.presto, "0.300", .repeatLastPage) + ]) + func heartbeatFollowsTheServer(flavor: TrinoServerFlavor, version: String?, expected: TrinoHeartbeat) { + #expect(TrinoHeartbeat(flavor: flavor, version: version) == expected) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProWeaviateClientTests/WeaviateStatementGeneratorTests.swift b/apple/Packages/TableProCore/Tests/TableProWeaviateClientTests/WeaviateStatementGeneratorTests.swift new file mode 100644 index 0000000000..aa462b71cd --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProWeaviateClientTests/WeaviateStatementGeneratorTests.swift @@ -0,0 +1,37 @@ +import Foundation +import TableProWeaviateClient +import TableProWeaviateCore +import Testing + +struct WeaviateStatementGeneratorTests { + private static let uuid = "36ddd591-2dee-4e7e-a3cc-eb86d30a4303" + + private static func request(_ kind: WeaviateTrackedChange.Kind) -> WeaviateWriteRequest? { + let change = WeaviateTrackedChange( + kind: kind, + uuid: uuid, + values: ["title": "A"], + cellChanges: [WeaviateCellChange(column: "title", newText: "B")] + ) + return WeaviateStatementGenerator.generate( + collection: "Article", + columns: ["title"], + typeNames: ["text"], + changes: [change] + ).requests.first + } + + // `/v1/objects/{id}` deletes the id in every collection that holds it. + @Test("A row's delete and update name its collection in the route") + func rowChangesNameTheCollection() throws { + let delete = try #require(Self.request(.delete)) + #expect(delete.method == "DELETE") + #expect(delete.path == "/v1/objects/Article/\(Self.uuid)") + #expect(delete.query.isEmpty) + + let update = try #require(Self.request(.update)) + #expect(update.method == "PATCH") + #expect(update.path == "/v1/objects/Article/\(Self.uuid)") + #expect(update.query.isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateTunneledTrustTests.swift b/apple/Packages/TableProCore/Tests/TableProWeaviateClientTests/WeaviateTunneledTrustTests.swift similarity index 98% rename from apple/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateTunneledTrustTests.swift rename to apple/Packages/TableProCore/Tests/TableProWeaviateClientTests/WeaviateTunneledTrustTests.swift index 36f2f36215..95468d435c 100644 --- a/apple/Packages/TableProCore/Tests/TableProWeaviateCoreTests/WeaviateTunneledTrustTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProWeaviateClientTests/WeaviateTunneledTrustTests.swift @@ -1,7 +1,7 @@ import Foundation import Security import TableProTLSTestFixtures -@testable import TableProWeaviateCore +@testable import TableProWeaviateClient import Testing struct WeaviateTunneledTrustTests { diff --git a/apple/Packages/TableProCore/Tests/TableProWireIOTests/SpoolFileTests.swift b/apple/Packages/TableProCore/Tests/TableProWireIOTests/SpoolFileTests.swift new file mode 100644 index 0000000000..da198119df --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProWireIOTests/SpoolFileTests.swift @@ -0,0 +1,104 @@ +#if canImport(Darwin) +import Darwin +import Foundation +import TableProWireIO +import Testing + +private func withScratchDirectory(_ body: (URL) async throws -> Void) async throws { + let directory = FileManager.default.temporaryDirectory.appending(path: "spool-tests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + try await body(directory) +} + +// The open descriptors whose file was made with this prefix, named or not: F_GETPATH still names an unlinked file. +private func openFiles(prefix: String) -> [(nlink: UInt16, mode: mode_t)] { + (0.. [UInt8] = { [UInt8](repeating: UInt8($0 % 251), count: 1 + ($0 * 37) % 90_000) } + let appended = DispatchSemaphore(value: 0) + Thread.detachNewThread { + for index in 0.. lower else { return nil } var closeCount = 0 var index: Int? textView.textStorage.mutableString.enumerateSubstrings( - in: reverse ? - NSRange(location: limit, length: from - limit) : - NSRange(location: from, length: limit - from), + in: NSRange(location: lower, length: upper - lower), options: options, using: { substring, range, _, stop in if substring == close { diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UndoHistory.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UndoHistory.swift new file mode 100644 index 0000000000..750cf9647f --- /dev/null +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UndoHistory.swift @@ -0,0 +1,54 @@ +// +// TextViewController+UndoHistory.swift +// TableProEditorKit +// + +import AppKit +import TableProTextEngine + +// Every step's ranges point into `text`, so the history is only good for that exact text. +public struct EditorUndoHistory { + let history: CEUndoManager.History + public let text: String +} + +public extension TextViewController { + func takeUndoHistory() -> EditorUndoHistory? { + guard let textView, let undoManager = textView.editorUndoManager else { return nil } + let history = undoManager.history + guard !history.isEmpty else { return nil } + undoManager.clearStack() + return EditorUndoHistory(history: history, text: textView.string) + } + + @discardableResult + func restoreUndoHistory(_ history: EditorUndoHistory) -> Bool { + // Literal equality: the text can be megabytes, and canonical `==` normalizes all of it. + guard let textView, let undoManager = textView.editorUndoManager, + (history.text as NSString).isEqual(to: textView.string) else { return false } + undoManager.restore(history.history) + return true + } + + // A whole-document change made by code, as one step Undo takes back. Like Replace All it skips the typing filters + // and the suggestion triggers. The selection stays where it was, clamped, as `setText` leaves it. + func replaceDocumentText(_ text: String) { + guard let textView else { return } + guard textView.isEditable else { + setText(text) + return + } + endSnippetSession() + let selection = textView.selectionManager.textSelections.map(\.range) + textView.undoManager?.beginUndoGrouping() + isApplyingUnfilteredEdits = true + textView.replaceCharacters( + in: NSRange(location: 0, length: textView.textStorage.length), + with: text, + skipUpdateSelection: true + ) + isApplyingUnfilteredEdits = false + textView.undoManager?.endUndoGrouping() + textView.selectionManager.setSelectedRanges(selection) + } +} diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UnfilteredEdits.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UnfilteredEdits.swift index 933c02074b..0b13653b05 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UnfilteredEdits.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+UnfilteredEdits.swift @@ -26,4 +26,19 @@ public extension TextViewController { defer { isApplyingUnfilteredEdits = false } textView.replaceCharacters(in: span, with: rebuilt, skipUpdateSelection: true) } + + // A command's replacement, unlike `setText`, keeps the undo history: one undo group, no typing filters, no + // suggestion triggers. The binding gets the text before this returns, so a run right after reads it. + @discardableResult + func replaceTextAsOneEdit(in range: NSRange, with string: String) -> Bool { + guard let textView, textView.isEditable, NSMaxRange(range) <= textView.textStorage.length else { return false } + guard !range.isEmpty || !string.isEmpty else { return true } + textView.undoManager?.beginUndoGrouping() + isApplyingUnfilteredEdits = true + textView.replaceCharacters(in: range, with: string) + isApplyingUnfilteredEdits = false + textView.undoManager?.endUndoGrouping() + flushPendingTextWriteback() + return true + } } diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift index 4662583a2e..86cc92352f 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/LineFoldProviders/LineFoldProvider.swift @@ -54,10 +54,30 @@ public protocol LineFoldProvider: AnyObject { /// The provider owns this string because it is the only part of the folding system the host application supplies, /// so it is the only part that can reach the application's localized strings. func foldPlaceholderLabel(for summary: FoldPlaceholderSummary) -> String + + // A provider that finds every fold in one pass returns that pass here. The calculator runs it on a copy of the + // text off the main thread and skips the per-line walk, which has to stay on the main thread. + func documentFoldScanner() -> LineFoldDocumentScanner? } public extension LineFoldProvider { func foldPlaceholderLabel(for summary: FoldPlaceholderSummary) -> String { summary.previewText } + + func documentFoldScanner() -> LineFoldDocumentScanner? { + nil + } +} + +public struct LineFoldProviderFold: Sendable, Equatable { + public let depth: Int + public let range: Range + + public init(depth: Int, range: Range) { + self.depth = depth + self.range = range + } } + +public typealias LineFoldDocumentScanner = @Sendable (String) -> [LineFoldProviderFold] diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift index 8714dd2723..f393c4578d 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldCalculator.swift @@ -8,33 +8,50 @@ import AppKit import TableProTextEngine -/// `LineFoldCalculator` receives text edits and rebuilds fold regions asynchronously. -/// -/// This is an actor, all methods and modifications happen in isolation in it's async region. All text requests are -/// marked `@MainActor` for safety. +// Each pass carries the document version it read, so the model can drop a result for text that has since changed. +struct LineFoldPass: Sendable { + let storage: LineFoldStorage + let documentVersion: Int +} + +@MainActor +final class LineFoldDocumentVersion { + private(set) var value = 0 + + func advance() { + value &+= 1 + } +} + actor LineFoldCalculator { weak var foldProvider: LineFoldProvider? weak var controller: TextViewController? - var valueStream: AsyncStream + var valueStream: AsyncStream - private var valueStreamContinuation: AsyncStream.Continuation + private var valueStreamContinuation: AsyncStream.Continuation private var textChangedTask: Task? + private let documentVersion: LineFoldDocumentVersion + + private struct PassStart: Sendable { + let documentRange: NSRange + let sizeLimit: Int + let documentVersion: Int + let scanner: LineFoldDocumentScanner? + let text: String? + } - /// Create a new calculator object that listens to a given stream for text changes. - /// - Parameters: - /// - foldProvider: The object to use to calculate fold regions. - /// - controller: The text controller to use for text and attachment fetching. - /// - textChangedStream: A stream of text changes, received as the document is edited. init( foldProvider: LineFoldProvider, controller: TextViewController, + documentVersion: LineFoldDocumentVersion, textChangedStream: AsyncStream ) { // This could be grabbed from the controller, but Swift 6 doesn't like that (concurrency safety) self.foldProvider = foldProvider self.controller = controller - (valueStream, valueStreamContinuation) = AsyncStream.makeStream() + self.documentVersion = documentVersion + (valueStream, valueStreamContinuation) = AsyncStream.makeStream() Task { await listenToTextChanges(textChangedStream: textChangedStream) } } @@ -42,8 +59,6 @@ actor LineFoldCalculator { textChangedTask?.cancel() } - /// Sets up an attached task to listen to values on a stream of text changes. - /// - Parameter textChangedStream: A stream of text changes. private func listenToTextChanges(textChangedStream: AsyncStream) { textChangedTask = Task { for await _ in textChangedStream { @@ -52,18 +67,63 @@ actor LineFoldCalculator { } } - /// Build out the folds for the entire document. - /// - /// For each line in the document, find the indentation level using the ``levelProvider``. At each line, if the - /// indent increases from the previous line, we start a new fold. If it decreases we end the fold we were in. private func buildFoldsForDocument() async { guard let controller = self.controller, let foldProvider = self.foldProvider else { return } - let documentRange = await controller.textView.documentRange - let sizeLimit = await controller.configuration.peripherals.foldingSizeLimit - guard documentRange.length <= sizeLimit else { - valueStreamContinuation.yield(LineFoldStorage(documentLength: documentRange.length)) + let start = await Self.startPass( + controller: controller, + foldProvider: foldProvider, + documentVersion: documentVersion + ) + guard start.documentRange.length <= start.sizeLimit else { + valueStreamContinuation.yield( + LineFoldPass( + storage: LineFoldStorage(documentLength: start.documentRange.length), + documentVersion: start.documentVersion + ) + ) return } + + let folds: [LineFoldStorage.RawFold] + if let scanner = start.scanner, let text = start.text { + folds = scanner(text).map { LineFoldStorage.RawFold(depth: $0.depth, range: $0.range) } + } else { + folds = await walkLines(controller: controller, foldProvider: foldProvider, documentRange: start.documentRange) + } + + await yieldNewStorage( + newFolds: folds, + controller: controller, + documentRange: start.documentRange, + documentVersion: start.documentVersion + ) + } + + // Bridging the storage's mutable string gives an immutable copy, so the scanner can read it from the calculator's + // own executor while the main thread goes on editing. + @MainActor + private static func startPass( + controller: TextViewController, + foldProvider: LineFoldProvider, + documentVersion: LineFoldDocumentVersion + ) -> PassStart { + let documentRange = controller.textView.documentRange + let sizeLimit = controller.configuration.peripherals.foldingSizeLimit + let scanner = documentRange.length <= sizeLimit ? foldProvider.documentFoldScanner() : nil + return PassStart( + documentRange: documentRange, + sizeLimit: sizeLimit, + documentVersion: documentVersion.value, + scanner: scanner, + text: scanner == nil ? nil : controller.textView.textStorage.string + ) + } + + private func walkLines( + controller: TextViewController, + foldProvider: LineFoldProvider, + documentRange: NSRange + ) async -> [LineFoldStorage.RawFold] { var foldCache: [LineFoldStorage.RawFold] = [] // Depth: Open range var openFolds: [Int: LineFoldStorage.RawFold] = [:] @@ -109,19 +169,14 @@ actor LineFoldCalculator { ) ) } - - await yieldNewStorage(newFolds: foldCache, controller: controller, documentRange: documentRange) + return foldCache } - /// Yield a new storage value on the value stream using a new set of folds. - /// - Parameters: - /// - newFolds: The new folds to yield with the storage value. - /// - controller: The text controller used for range and attachment fetching. - /// - documentRange: The total range of the current document. private func yieldNewStorage( newFolds: [LineFoldStorage.RawFold], controller: TextViewController, - documentRange: NSRange + documentRange: NSRange, + documentVersion: Int ) async { let attachments = await controller.textView.layoutManager.attachments .getAttachmentsOverlapping(documentRange) @@ -136,16 +191,17 @@ actor LineFoldCalculator { documentLength: newFolds.max( by: { $0.range.upperBound < $1.range.upperBound } )?.range.upperBound ?? documentRange.length, - folds: newFolds.sorted(by: { $0.range.lowerBound < $1.range.lowerBound }), + // The store keeps the fold set last where two overlap, so of folds that start together the outermost goes + // last and is the one shown, whichever order the provider reported them in. + folds: newFolds.sorted { + $0.range.lowerBound == $1.range.lowerBound ? $0.depth > $1.depth : $0.range.lowerBound < $1.range.lowerBound + }, collapsedRanges: Set(attachments) ) - valueStreamContinuation.yield(storage) + valueStreamContinuation.yield(LineFoldPass(storage: storage, documentVersion: documentVersion)) } - /// Asynchronously gets more line information from the fold provider. - /// Runs on the main thread so all text-related calculations are safe with the main text storage. - /// - /// Has to be an `AsyncSequence` so it can be main actor isolated. + // An `AsyncSequence` so the provider's per-line calls run on the main actor, where the text storage lives. @MainActor struct ChunkedLineIterator: AsyncSequence, AsyncIteratorProtocol { var controller: TextViewController diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift index 6212fa9817..79650179c7 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/LineFolding/Model/LineFoldModel.swift @@ -26,24 +26,36 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { @Published var foldCache = LineFoldStorage(documentLength: 0) private var calculator: LineFoldCalculator + // StatementRunController's threshold and delay: below it a pass costs less than a keystroke, above it a pass on + // every key would keep a core busy while the user types. + static let immediateRecalculationLimit = 64_000 + static let recalculationDelay: Duration = .milliseconds(150) + private var textChangedStream: AsyncStream private var textChangedStreamContinuation: AsyncStream.Continuation private var cacheListenTask: Task? + private let documentVersion: LineFoldDocumentVersion + private let clock: any Clock + private var pendingRecalculation: Task? private var pendingRestoreRanges: Set>? private var placeholderTracker: LineFoldPlaceholderTracker? weak var controller: TextViewController? weak var foldView: NSView? - init(controller: TextViewController, foldView: NSView) { + init(controller: TextViewController, foldView: NSView, clock: any Clock = ContinuousClock()) { self.controller = controller self.foldView = foldView + self.clock = clock (textChangedStream, textChangedStreamContinuation) = AsyncStream.makeStream( bufferingPolicy: .bufferingNewest(1) ) + let documentVersion = LineFoldDocumentVersion() + self.documentVersion = documentVersion self.calculator = LineFoldCalculator( foldProvider: controller.foldProvider, controller: controller, + documentVersion: documentVersion, textChangedStream: textChangedStream ) super.init() @@ -51,9 +63,12 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { let calculator = self.calculator cacheListenTask = Task { @MainActor [weak self, weak foldView] in - for await newFolds in await calculator.valueStream { + for await pass in await calculator.valueStream { guard let self else { return } - self.foldCache = newFolds + // An edit after the pass read the text queued another pass; until it lands `storageUpdated` keeps + // the current folds in step with the text. + guard pass.documentVersion == self.documentVersion.value else { continue } + self.foldCache = pass.storage foldView?.needsDisplay = true self.applyPendingRestoreIfPossible() } @@ -78,6 +93,7 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { gutterHover = nil placeholderHover = nil hoveredFold = nil + documentVersion.advance() foldCache = LineFoldStorage(documentLength: controller?.textView.documentRange.length ?? 0) pendingRestoreRanges = nil refresh() @@ -93,6 +109,8 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { placeholderTracker = nil cacheListenTask?.cancel() cacheListenTask = nil + pendingRecalculation?.cancel() + pendingRecalculation = nil } // MARK: - Collapse @@ -256,11 +274,31 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { ) } - /// Recomputes folds for the whole document. Used to catch up after the ribbon has been hidden and is shown again. func refresh() { + pendingRecalculation?.cancel() + pendingRecalculation = nil textChangedStreamContinuation.yield() } + private func scheduleRecalculation(documentLength: Int) { + guard documentLength > Self.immediateRecalculationLimit else { + refresh() + return + } + pendingRecalculation?.cancel() + let clock = clock + pendingRecalculation = Task { [weak self] in + do { + try await clock.sleep(for: Self.recalculationDelay) + } catch { + return + } + guard let self, !Task.isCancelled else { return } + self.pendingRecalculation = nil + self.textChangedStreamContinuation.yield() + } + } + func textStorage( _ textStorage: NSTextStorage, didProcessEditing editedMask: NSTextStorageEditActions, @@ -270,11 +308,12 @@ class LineFoldModel: NSObject, NSTextStorageDelegate, ObservableObject { guard editedMask.contains(.editedCharacters) else { return } + documentVersion.advance() foldCache.storageUpdated(editedRange: editedRange, changeInLength: delta) // Recalculating folds walks the whole document. Skip it while the ribbon is hidden; `refresh()` rebuilds when // it is shown again. guard foldView?.isHidden != true else { return } - textChangedStreamContinuation.yield() + scheduleRecalculation(documentLength: textStorage.length) } /// Finds the deepest cached fold and depth of the fold for a line number. diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift index b19f9c59c6..1f5b44908c 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift @@ -37,11 +37,12 @@ public extension SourceEditor { init( text: TextAPI, + documentID: AnyHashable? = nil, editorState: Binding, highlightProviders: [any HighlightProviding]?, textCoordinators: [any TextViewCoordinator] ) { - self.textSync = TextBindingSync(text: text, phase: phase) + self.textSync = TextBindingSync(text: text, documentID: documentID, phase: phase) self._editorState = editorState self.highlightProviders = highlightProviders ?? [TreeSitterClient()] self.textCoordinators = textCoordinators diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor.swift index 413bc50586..d42f0647f1 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor.swift @@ -39,7 +39,8 @@ public struct SourceEditor: NSViewControllerRepresentable { foldProvider: LineFoldProvider? = nil, undoManager: CEUndoManager? = nil, coordinators: [any TextViewCoordinator] = [], - completionDelegate: CodeSuggestionDelegate? = nil + completionDelegate: CodeSuggestionDelegate? = nil, + documentID: AnyHashable? = nil ) { self.text = .binding(text) self.language = language @@ -50,6 +51,7 @@ public struct SourceEditor: NSViewControllerRepresentable { self.undoManager = undoManager self.coordinators = coordinators self.completionDelegate = completionDelegate + self.documentID = documentID } /// Initializes a new source editor @@ -96,6 +98,8 @@ public struct SourceEditor: NSViewControllerRepresentable { var undoManager: CEUndoManager? var coordinators: [any TextViewCoordinator] var completionDelegate: CodeSuggestionDelegate? + // While it stays the same, a new binding value is an edit Undo can take back; a new one starts a new history. + var documentID: AnyHashable? public typealias NSViewControllerType = TextViewController @@ -135,6 +139,7 @@ public struct SourceEditor: NSViewControllerRepresentable { public func makeCoordinator() -> Coordinator { Coordinator( text: text, + documentID: documentID, editorState: $state, highlightProviders: highlightProviders, textCoordinators: coordinators @@ -166,7 +171,11 @@ public struct SourceEditor: NSViewControllerRepresentable { coordinator.textSync.text = text if case .binding(let binding) = text { - coordinator.textSync.applyRepresentableText(binding.wrappedValue, controller: controller) + coordinator.textSync.applyRepresentableText( + binding.wrappedValue, + documentID: documentID, + controller: controller + ) } if !coordinator.phase.consumePendingEditorChange() { diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift index 9bf4a5641b..f306bdfee1 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift @@ -20,12 +20,14 @@ final class TextBindingSync { var text: SourceEditor.TextAPI private(set) var lastSyncedText: String? + private(set) var documentID: AnyHashable? private let phase: RepresentableSyncPhase private(set) var writebackTask: Task? - init(text: SourceEditor.TextAPI, phase: RepresentableSyncPhase) { + init(text: SourceEditor.TextAPI, documentID: AnyHashable? = nil, phase: RepresentableSyncPhase) { self.text = text + self.documentID = documentID self.phase = phase if case .binding(let binding) = text { lastSyncedText = binding.wrappedValue @@ -90,27 +92,29 @@ final class TextBindingSync { return newText } - /// Pushes an external binding change down into the text view. The editor's - /// content wins while one of its own edits is still in flight, so user - /// typing is never clobbered by a stale binding snapshot. - /// - /// Routes through `TextViewController.setText` on purpose. The text-view - /// level `replaceCharacters` is the user-edit path: it is gated on - /// `isEditable`, runs mutation filters, and fires suggestion triggers, - /// none of which should happen for a programmatic whole-document - /// replacement. The controller-level call also rebuilds the highlighter - /// for the replacement storage; calling the text view directly leaves - /// tree-sitter state for the old document and highlighting never recovers. - func applyRepresentableText(_ newValue: String, controller: TextViewController) { + // The editor's content wins while one of its own edits is in flight. In the same named document a new value is one + // undoable edit; another document, or an owner that names none, gets `setText`: a new history and a highlighter + // rebuilt for the new storage, even when the text is the same. + func applyRepresentableText( + _ newValue: String, + documentID: AnyHashable? = nil, + controller: TextViewController + ) { guard !phase.isEditorChangePending else { return } + let isSameDocument = documentID == self.documentID + self.documentID = documentID // Compare with NSString literal equality. The text can be multiple megabytes and bridged from NSTextStorage, // so Swift's canonical `!=` walks the whole string through Unicode normalization on every representable update. - if let lastSyncedText, (newValue as NSString).isEqual(to: lastSyncedText) { return } + if isSameDocument, let lastSyncedText, (newValue as NSString).isEqual(to: lastSyncedText) { return } writebackTask?.cancel() writebackTask = nil phase.applyRepresentableValue { - controller.setText(newValue) + if isSameDocument, documentID != nil { + controller.replaceDocumentText(newValue) + } else { + controller.setText(newValue) + } } lastSyncedText = newValue } diff --git a/apple/Packages/TableProEditor/Sources/TableProTextEngine/Utils/CEUndoManager.swift b/apple/Packages/TableProEditor/Sources/TableProTextEngine/Utils/CEUndoManager.swift index 730b98a2b7..aa121c0e98 100644 --- a/apple/Packages/TableProEditor/Sources/TableProTextEngine/Utils/CEUndoManager.swift +++ b/apple/Packages/TableProEditor/Sources/TableProTextEngine/Utils/CEUndoManager.swift @@ -17,16 +17,23 @@ import TextStory /// If needed, the automatic undo grouping can be overridden using the `beginGrouping()` and `endGrouping()` methods. public class CEUndoManager: UndoManager { /// Represents a group of mutations that should be treated as one mutation when undoing/redoing. - private struct UndoGroup { + struct UndoGroup { var mutations: [Mutation] } /// A single undo mutation. - private struct Mutation { + struct Mutation { var mutation: TextMutation var inverse: TextMutation } + public struct History { + var undoStack: [UndoGroup] + var redoStack: [UndoGroup] + + public var isEmpty: Bool { undoStack.isEmpty && redoStack.isEmpty } + } + private var _isUndoing: Bool = false private var _isRedoing: Bool = false @@ -195,6 +202,18 @@ public class CEUndoManager: UndoManager { redoStack.removeAll() } + public var history: History { + History(undoStack: undoStack, redoStack: redoStack) + } + + // The next edit starts its own step instead of joining the restored history's last one. + public func restore(_ history: History) { + undoStack = history.undoStack + redoStack = history.redoStack + isGrouping = false + shouldBreakNextGroup = true + } + // MARK: - Mutations override public func registerUndo(withTarget target: Any, selector: Selector, object anObject: Any?) { diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/ReplaceTextAsOneEditTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/ReplaceTextAsOneEditTests.swift new file mode 100644 index 0000000000..54718ec320 --- /dev/null +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/ReplaceTextAsOneEditTests.swift @@ -0,0 +1,49 @@ +import AppKit +@testable import TableProEditorKit +import TableProTextEngine +import Testing + +@MainActor +struct ReplaceTextAsOneEditTests { + @Test("A command's replacement keeps the undo history and undoes in one step") + func replacementIsUndoable() { + let (window, controller) = Mock.focusedTextViewController(string: "SELECT 1") + defer { window.orderOut(nil) } + let whole = NSRange(location: 0, length: controller.textView.textStorage.length) + + #expect(controller.replaceTextAsOneEdit(in: whole, with: "SELECT * FROM users")) + #expect(controller.textView.string == "SELECT * FROM users") + #expect(controller.textView.undoManager?.canUndo == true) + + controller.textView.undoManager?.undo() + #expect(controller.textView.string == "SELECT 1") + } + + @Test("A command's replacement skips the typing filters a typed edit runs") + func replacementSkipsTypingFilters() { + let typed = Mock.loadedTextViewController(string: "") + typed.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "\t") + #expect(typed.textView.string == " ") + + let controller = Mock.loadedTextViewController(string: "") + #expect(controller.replaceTextAsOneEdit(in: NSRange(location: 0, length: 0), with: "\t")) + #expect(controller.textView.string == "\t") + } + + @Test("An editor that takes no edits refuses, so the caller writes the text itself") + func uneditableEditorRefuses() { + let controller = Mock.loadedTextViewController(string: "SELECT 1") + controller.textView.isEditable = false + + #expect(!controller.replaceTextAsOneEdit(in: NSRange(location: 0, length: 8), with: "SELECT 2")) + #expect(controller.textView.string == "SELECT 1") + } + + @Test("A range past the end of the text is refused") + func rangePastTheTextIsRefused() { + let controller = Mock.loadedTextViewController(string: "SELECT 1") + + #expect(!controller.replaceTextAsOneEdit(in: NSRange(location: 4, length: 20), with: "x")) + #expect(controller.textView.string == "SELECT 1") + } +} diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift index ff714958bf..8542ff6869 100644 --- a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift @@ -330,6 +330,24 @@ final class TextViewControllerTests: XCTestCase { ) } + func test_findClosingPairWithALimitOnTheWrongSideOrPastTheText() { + controller.textView.string = "{ Lorem }" + + XCTAssertNil(controller.findClosingPair("{", "}", from: 5, limit: 2, reverse: false)) + XCTAssertNil(controller.findClosingPair("}", "{", from: 2, limit: 5, reverse: true)) + XCTAssertEqual(controller.findClosingPair("{", "}", from: 1, limit: 10_000, reverse: false), 8) + } + + func test_bracketEmphasisFarPastTheVisibleTextFindsThePair() { + controller.configuration.appearance.bracketPairEmphasis = .flash + controller.setText(String(repeating: "x", count: 60_000) + "{ a }") + + controller.setCursorPositions([CursorPosition(range: NSRange(location: 60_001, length: 0))]) + + XCTAssertEqual(controller.cursorPositions.first?.range.location, 60_001) + XCTAssertEqual(controller.findClosingPair("{", "}", from: 60_001, limit: 60_005, reverse: false), 60_004) + } + // MARK: Set Text func test_setText() { diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/LineFoldingTests/LineFoldDocumentScannerTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/LineFoldingTests/LineFoldDocumentScannerTests.swift new file mode 100644 index 0000000000..ac8acd4ed4 --- /dev/null +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/LineFoldingTests/LineFoldDocumentScannerTests.swift @@ -0,0 +1,265 @@ +import AppKit +import Combine +@testable import TableProEditorKit +import TableProTestSupport +import TableProTextEngine +import Testing + +// A provider with a whole-document scanner is folded off the main thread from a copy of the text, so these hold that +// path to the folds the per-line walk builds, and to the document version the model checks before taking a result. +@MainActor +struct LineFoldDocumentScannerTests { + struct Region: Equatable { + let depth: Int + let range: Range + let startLine: Int + let endLine: Int + } + + struct Fold: Equatable { + let depth: Int + let range: Range + } + + final class ScanLog: @unchecked Sendable { + private let lock = NSLock() + private var started = 0 + private var finished = 0 + private var read: [[UInt16]] = [] + let gate: DispatchSemaphore? + + init(gated: Bool = false) { + gate = gated ? DispatchSemaphore(value: 0) : nil + } + + var startedCount: Int { + lock.withLock { started } + } + + var count: Int { + lock.withLock { finished } + } + + var texts: [[UInt16]] { + lock.withLock { read } + } + + func scan(_ text: String) -> [LineFoldProviderFold] { + lock.withLock { started += 1 } + gate?.wait() + // Read after the wait, so a held pass shows whether its copy moved with the edits made meanwhile. + let snapshot = Array(text.utf16) + defer { lock.withLock { finished += 1; read.append(snapshot) } } + return LineFoldDocumentScannerTests.regions(in: text).map { + LineFoldProviderFold(depth: $0.depth, range: $0.range) + } + } + } + + // Braces nest the way a SQL scanner's frames do: a region that opens and closes on one line folds nothing. + nonisolated static func regions(in text: String) -> [Region] { + var stack: [(start: Int, line: Int)] = [] + var line = 0 + var regions: [Region] = [] + for (offset, unit) in text.utf16.enumerated() { + switch unit { + case 0x0A: + line += 1 + case 0x7B: + stack.append((offset + 1, line)) + case 0x7D: + guard let open = stack.popLast(), open.line != line else { continue } + regions.append( + Region(depth: stack.count + 1, range: open.start.. LineFoldDocumentScanner? { + guard offersScanner else { return nil } + let log = log + return { text in log.scan(text) } + } + + func foldLevelAtLine( + lineNumber: Int, + lineRange: NSRange, + previousDepth: Int, + controller: TextViewController + ) -> [LineFoldProviderLineInfo] { + if lineNumber == 0 { + regions = LineFoldDocumentScannerTests.regions(in: controller.textView.string) + } + let ends = regions.filter { $0.endLine == lineNumber }.sorted { $0.depth > $1.depth } + let starts = regions.filter { $0.startLine == lineNumber }.sorted { $0.depth < $1.depth } + return ends.map { .endFold(rangeEnd: $0.range.upperBound, newDepth: $0.depth - 1) } + + starts.map { .startFold(rangeStart: $0.range.lowerBound, newDepth: $0.depth) } + } + } + + // A statement and its parenthesised body both fold from the end of `CREATE TABLE t (`. + @MainActor + final class SharedStartProvider: LineFoldProvider { + let offersScanner: Bool + private let regions = [ + Region(depth: 1, range: 2..<7, startLine: 0, endLine: 2), + Region(depth: 2, range: 2..<6, startLine: 0, endLine: 2), + ] + + init(offersScanner: Bool) { + self.offersScanner = offersScanner + } + + func documentFoldScanner() -> LineFoldDocumentScanner? { + guard offersScanner else { return nil } + let folds = regions.map { LineFoldProviderFold(depth: $0.depth, range: $0.range) } + return { _ in folds } + } + + func foldLevelAtLine( + lineNumber: Int, + lineRange: NSRange, + previousDepth: Int, + controller: TextViewController + ) -> [LineFoldProviderLineInfo] { + let ends = regions.filter { $0.endLine == lineNumber }.sorted { $0.depth > $1.depth } + let starts = regions.filter { $0.startLine == lineNumber }.sorted { $0.depth < $1.depth } + return ends.map { .endFold(rangeEnd: $0.range.upperBound, newDepth: $0.depth - 1) } + + starts.map { .startFold(rangeStart: $0.range.lowerBound, newDepth: $0.depth) } + } + } + + private static let nestedDocument: String = (0..<40).map { index in + switch index % 4 { + case 0: return "block \(index) {\n inner {\n x\n }\n y\n}" + case 1: return "flat \(index) { same line }" + case 2: return "deep {\n {\n {\n z\n }\n }\n}" + default: return "open {\n a\n} close { b\n}" + } + }.joined(separator: "\n") + + private func makeController(_ text: String, provider: LineFoldProvider) -> TextViewController { + let controller = Mock.textViewController(theme: Mock.theme()) + controller.textView.string = text + controller.textView.frame = NSRect(x: 0, y: 0, width: 1_000, height: 1_000) + controller.textView.updatedViewport(NSRect(x: 0, y: 0, width: 1_000, height: 1_000)) + controller.foldProvider = provider + return controller + } + + private func folds(of model: LineFoldModel, in controller: TextViewController) -> [Fold] { + model.getFolds(in: 0.. [Fold] { + Self.regions(in: text) + .map { Fold(depth: $0.depth, range: $0.range) } + .sorted { $0.range.lowerBound < $1.range.lowerBound } + } + + @Test("The scanner path builds the same folds as the per-line walk") + func scannerMatchesLineWalk() async throws { + let walking = makeController(Self.nestedDocument, provider: BraceFoldProvider(offersScanner: false)) + let scanning = makeController(Self.nestedDocument, provider: BraceFoldProvider(offersScanner: true)) + let walkModel = LineFoldModel(controller: walking, foldView: NSView()) + let scanModel = LineFoldModel(controller: scanning, foldView: NSView()) + + try await waitUntil(timeout: .seconds(60)) { + !folds(of: walkModel, in: walking).isEmpty && !folds(of: scanModel, in: scanning).isEmpty + } + + let walked = folds(of: walkModel, in: walking) + #expect(walked.count > 20) + #expect(folds(of: scanModel, in: scanning) == walked) + #expect(walked == expected(Self.nestedDocument)) + } + + @Test("Of folds that start together, both paths show the outer one") + func sharedStartShowsTheOuterFold() async throws { + let text = "a(\n b\n);\n" + let walking = makeController(text, provider: SharedStartProvider(offersScanner: false)) + let scanning = makeController(text, provider: SharedStartProvider(offersScanner: true)) + let walkModel = LineFoldModel(controller: walking, foldView: NSView()) + let scanModel = LineFoldModel(controller: scanning, foldView: NSView()) + + try await waitUntil(timeout: .seconds(30)) { + !folds(of: walkModel, in: walking).isEmpty && !folds(of: scanModel, in: scanning).isEmpty + } + + #expect(folds(of: walkModel, in: walking) == [Fold(depth: 1, range: 2..<7)]) + #expect(folds(of: scanModel, in: scanning) == [Fold(depth: 1, range: 2..<7)]) + } + + @Test("A pass over text that changed before it landed is dropped") + func stalePassIsDropped() async throws { + let log = ScanLog(gated: true) + // A failed wait must not leave a pool thread parked in the scanner for the rest of the run. + defer { + log.gate?.signal() + log.gate?.signal() + } + let original = "a {\n b\n}\nc {\n d\n}\n" + let controller = makeController(original, provider: BraceFoldProvider(offersScanner: true, log: log)) + let model = LineFoldModel(controller: controller, foldView: NSView()) + + var published: [[Fold]] = [] + let subscription = model.$foldCache.sink { cache in + published.append(cache.folds(in: 0.. LineFoldModel.immediateRecalculationLimit) + let controller = makeController(large, provider: BraceFoldProvider(offersScanner: true, log: log)) + let model = LineFoldModel(controller: controller, foldView: NSView(), clock: clock) + try await waitUntil(timeout: .seconds(20)) { log.count == 1 && !folds(of: model, in: controller).isEmpty } + + controller.textView.textStorage.replaceCharacters(in: NSRange(location: 0, length: 0), with: "z") + await clock.waitUntilSleeping() + #expect(log.count == 1) + + clock.advance(by: LineFoldModel.recalculationDelay) + try await waitUntil(timeout: .seconds(20)) { log.count == 2 } + + let smallLog = ScanLog() + let small = makeController("x {\n y\n}\n", provider: BraceFoldProvider(offersScanner: true, log: smallLog)) + let smallModel = LineFoldModel(controller: small, foldView: NSView(), clock: clock) + try await waitUntil(timeout: .seconds(20)) { smallLog.count == 1 } + small.textView.textStorage.replaceCharacters(in: NSRange(location: 0, length: 0), with: "z") + try await waitUntil(timeout: .seconds(20)) { smallLog.count == 2 } + _ = smallModel + } +} diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorUndoHistoryTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorUndoHistoryTests.swift new file mode 100644 index 0000000000..8af3872bdf --- /dev/null +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorUndoHistoryTests.swift @@ -0,0 +1,181 @@ +// +// SourceEditorUndoHistoryTests.swift +// TableProEditorKitTests +// +// An owner that names its document gets text it sets in that document as an edit Undo takes back, and a new +// document as a new history. A history saved with a document comes back only onto the same text. +// + +import AppKit +import SwiftUI +@testable import TableProEditorKit +import TableProTextEngine +import Testing + +@MainActor +private final class DocumentOwner { + var text: String + var documentID: AnyHashable? + + init(text: String, documentID: AnyHashable?) { + self.text = text + self.documentID = documentID + } + + func editor(_ state: Binding) -> SourceEditor { + SourceEditor( + Binding(get: { self.text }, set: { self.text = $0 }), + language: .default, + configuration: Mock.config(), + state: state, + highlightProviders: [], + documentID: documentID + ) + } +} + +@MainActor +struct SourceEditorUndoHistoryTests { + private static let query = AnyHashable("query") + private static let other = AnyHashable("other") + + private func mount(_ owner: DocumentOwner) -> (SourceEditorHost, NSWindow) { + let host = SourceEditorHost { owner.editor($0) } + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 600, height: 400), + styleMask: [.titled], + backing: .buffered, + defer: false + ) + window.isReleasedWhenClosed = false + window.contentView = host.controller.view + window.contentView?.layoutSubtreeIfNeeded() + return (host, window) + } + + private func show(_ text: String, as documentID: AnyHashable?, owner: DocumentOwner, host: SourceEditorHost) { + owner.text = text + owner.documentID = documentID + host.update { owner.editor($0) } + } + + // The owner's update pass after the editor writes the text back, as SwiftUI runs it. + private func type(_ string: String, at location: Int, owner: DocumentOwner, host: SourceEditorHost) { + host.controller.textView.replaceCharacters(in: NSRange(location: location, length: 0), with: string) + host.update { owner.editor($0) } + } + + private func selection(_ host: SourceEditorHost) -> [NSRange] { + host.controller.textView.selectionManager.textSelections.map(\.range) + } + + @Test("Text the owner sets in the same document is one step Undo takes back") + func sameDocumentChangeIsUndoable() { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + + show("", as: Self.query, owner: owner, host: host) + #expect(host.controller.textView.string == "") + + host.controller.textView.undoManager?.undo() + #expect(host.controller.textView.string == "SELECT 1") + #expect(owner.text == "SELECT 1") + } + + @Test("Text set in the same document skips the typing filters") + func sameDocumentChangeIsUnfiltered() { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + + show("(", as: Self.query, owner: owner, host: host) + + #expect(host.controller.textView.string == "(") + } + + @Test("Text set in the same document leaves the caret where it was") + func sameDocumentChangeKeepsTheCaret() { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + host.controller.setCursorPositions([CursorPosition(range: NSRange(location: 3, length: 0))]) + + show("SELECT 2", as: Self.query, owner: owner, host: host) + + #expect(host.controller.textView.string == "SELECT 2") + #expect(selection(host) == [NSRange(location: 3, length: 0)]) + } + + @Test("Another document starts a new history") + func newDocumentStartsANewHistory() { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + type("0", at: 8, owner: owner, host: host) + #expect(host.controller.textView.undoManager?.canUndo == true) + + show("SELECT 2", as: Self.other, owner: owner, host: host) + + #expect(host.controller.textView.string == "SELECT 2") + #expect(host.controller.textView.undoManager?.canUndo == false) + } + + @Test("Another document with the same text still starts a new history") + func newDocumentWithTheSameTextStartsANewHistory() { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + type("0", at: 8, owner: owner, host: host) + + show("SELECT 10", as: Self.other, owner: owner, host: host) + + #expect(host.controller.textView.undoManager?.canUndo == false) + } + + @Test("An owner that names no document gets a new history on every change, as before") + func unnamedDocumentKeepsReplacingTheText() { + let owner = DocumentOwner(text: "SELECT 1", documentID: nil) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + type("0", at: 8, owner: owner, host: host) + + show("SELECT 2", as: nil, owner: owner, host: host) + + #expect(host.controller.textView.string == "SELECT 2") + #expect(host.controller.textView.undoManager?.canUndo == false) + } + + @Test("A history saved with its document comes back with it") + func savedHistoryComesBackWithItsDocument() throws { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + type("0", at: 8, owner: owner, host: host) + + let saved = try #require(host.controller.takeUndoHistory()) + #expect(host.controller.textView.undoManager?.canUndo == false) + #expect(host.controller.takeUndoHistory() == nil) + show("SELECT 2", as: Self.other, owner: owner, host: host) + show("SELECT 10", as: Self.query, owner: owner, host: host) + + #expect(host.controller.restoreUndoHistory(saved)) + host.controller.textView.undoManager?.undo() + #expect(host.controller.textView.string == "SELECT 1") + } + + @Test("A saved history is not put back on text that changed") + func savedHistoryIsDroppedForChangedText() throws { + let owner = DocumentOwner(text: "SELECT 1", documentID: Self.query) + let (host, window) = mount(owner) + defer { window.orderOut(nil) } + type("0", at: 8, owner: owner, host: host) + + let saved = try #require(host.controller.takeUndoHistory()) + show("SELECT 2", as: Self.other, owner: owner, host: host) + show("SELECT 99", as: Self.query, owner: owner, host: host) + + #expect(!host.controller.restoreUndoHistory(saved)) + #expect(host.controller.textView.undoManager?.canUndo == false) + } +} diff --git a/apple/Packages/TableProEditor/Tests/TableProTextEngineTests/UndoHistoryTests.swift b/apple/Packages/TableProEditor/Tests/TableProTextEngineTests/UndoHistoryTests.swift new file mode 100644 index 0000000000..004d528fc4 --- /dev/null +++ b/apple/Packages/TableProEditor/Tests/TableProTextEngineTests/UndoHistoryTests.swift @@ -0,0 +1,56 @@ +import AppKit +@testable import TableProTextEngine +import Testing + +@MainActor +struct UndoHistoryTests { + private func makeTextView() throws -> (TextView, CEUndoManager) { + let textView = TextView(string: "") + return (textView, try #require(textView.editorUndoManager)) + } + + @Test("A history taken out and put back undoes the edits it recorded") + func undoStackRoundTrip() throws { + let (textView, undoManager) = try makeTextView() + textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "SELECT 1") + textView.replaceCharacters(in: NSRange(location: 7, length: 1), with: "2") + let history = undoManager.history + undoManager.clearStack() + #expect(undoManager.history.isEmpty) + #expect(!undoManager.canUndo) + + undoManager.restore(history) + undoManager.undo() + #expect(textView.string == "SELECT 1") + undoManager.undo() + #expect(textView.string == "") + } + + @Test("A restored history keeps the steps Redo can apply") + func redoStackRoundTrip() throws { + let (textView, undoManager) = try makeTextView() + textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "SELECT 1") + undoManager.undo() + let history = undoManager.history + undoManager.clearStack() + + undoManager.restore(history) + undoManager.redo() + + #expect(textView.string == "SELECT 1") + } + + @Test("Typing after a restore is its own undo step") + func typingAfterRestoreStartsANewStep() throws { + let (textView, undoManager) = try makeTextView() + textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "a") + let history = undoManager.history + undoManager.clearStack() + undoManager.restore(history) + + textView.replaceCharacters(in: NSRange(location: 1, length: 0), with: "b") + undoManager.undo() + + #expect(textView.string == "a") + } +} diff --git a/apple/Packages/TableProMacCore/Package.resolved b/apple/Packages/TableProMacCore/Package.resolved index c787a21fdc..5d1eba5006 100644 --- a/apple/Packages/TableProMacCore/Package.resolved +++ b/apple/Packages/TableProMacCore/Package.resolved @@ -1,6 +1,96 @@ { - "originHash" : "8416d6d6bf8d3efe83e2ee5a577c19c0a4be27502e2168e80ca90b420e98f493", + "originHash" : "29b72a8457d74c9a15f782101f6c1a36838ce85add174125f4a5973f800d250f", "pins" : [ + { + "identity" : "openapikit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattpolzin/OpenAPIKit", + "state" : { + "revision" : "5b72eb9a6b23b1fec439d9a0bf38f115400421f9", + "version" : "6.4.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms", + "state" : { + "revision" : "87e50f483c54e6efd60e885f7f5aa946cee68023", + "version" : "1.2.1" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "6a52f3251125d74daf04fcbd5e6f08a75d074382", + "version" : "1.8.2" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections", + "state" : { + "revision" : "88754449ab766a739610b660e117762115fabc9c", + "version" : "1.7.6" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types", + "state" : { + "revision" : "bff4b6903cdc99dda49649dd52f46c11cfd3ed50", + "version" : "1.8.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0c0290ff6b24942dadb83a929ffaaa1481df04a2", + "version" : "1.1.1" + } + }, + { + "identity" : "swift-openapi-generator", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-openapi-generator", + "state" : { + "revision" : "bf578ce357a0d2aa07af9de41b083cc2c986a498", + "version" : "1.14.1" + } + }, + { + "identity" : "swift-openapi-runtime", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-openapi-runtime", + "state" : { + "revision" : "50ad7976f18fec6c09466d2aebf7be3dc9476cfb", + "version" : "1.13.0" + } + }, + { + "identity" : "swift-openapi-urlsession", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-openapi-urlsession", + "state" : { + "revision" : "c97e0f0187d95542b4a254f5bb4dc67f43bf73c9", + "version" : "1.4.0" + } + }, + { + "identity" : "yams", + "kind" : "remoteSourceControl", + "location" : "https://github.com/jpsim/Yams", + "state" : { + "revision" : "a27b21e0c81c5bf42049b897a62aaf387e80f279", + "version" : "6.2.2" + } + }, { "identity" : "zstd", "kind" : "remoteSourceControl", diff --git a/apple/Packages/TableProMacCore/Package.swift b/apple/Packages/TableProMacCore/Package.swift index 33a1e2a300..016b422475 100644 --- a/apple/Packages/TableProMacCore/Package.swift +++ b/apple/Packages/TableProMacCore/Package.swift @@ -20,6 +20,7 @@ let package = Package( .library(name: "TableProUtilities", targets: ["TableProUtilities"]), .library(name: "TableProSSHConfig", targets: ["TableProSSHConfig"]), .library(name: "TableProMCP", targets: ["TableProMCP"]), + .library(name: "TableProMCPClients", targets: ["TableProMCPClients"]), // Only the tablepro-cli tool links this; the app never does, so ArgumentParser stays out of it. .library(name: "TableProCommandLine", targets: ["TableProCommandLine"]), .library(name: "TableProMCPTestSupport", targets: ["TableProMCPTestSupport"]), @@ -58,7 +59,6 @@ let package = Package( .target( name: "TableProMCP", dependencies: [ - "TableProUtilities", .product(name: "TableProCoreTypes", package: "TableProCore"), .product(name: "TableProLogRedaction", package: "TableProCore"), .product(name: "TableProProcess", package: "TableProCore"), @@ -67,11 +67,15 @@ let package = Package( path: "Sources/TableProMCP", swiftSettings: appModuleSettings ), + .target( + name: "TableProMCPClients", + path: "Sources/TableProMCPClients", + swiftSettings: appModuleSettings + ), .target( name: "TableProCommandLine", dependencies: [ "TableProMCP", - "TableProUtilities", .product(name: "TableProCoreTypes", package: "TableProCore"), .product(name: "TableProProcess", package: "TableProCore"), .product(name: "ArgumentParser", package: "swift-argument-parser") @@ -122,12 +126,18 @@ let package = Package( dependencies: [ "TableProMCP", "TableProMCPTestSupport", - "TableProUtilities", + .product(name: "TableProProcess", package: "TableProCore"), .product(name: "TableProTestSupport", package: "TableProCore") ], path: "Tests/TableProMCPTests", swiftSettings: appModuleSettings ), + .testTarget( + name: "TableProMCPClientsTests", + dependencies: ["TableProMCPClients"], + path: "Tests/TableProMCPClientsTests", + swiftSettings: appModuleSettings + ), .testTarget( name: "TableProCommandLineTests", dependencies: [ diff --git a/apple/Packages/TableProMacCore/Sources/TableProCloudAPI/openapi-generator-config.yaml b/apple/Packages/TableProMacCore/Sources/TableProCloudAPI/openapi-generator-config.yaml index 8e60f914f6..fb565cf1e3 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProCloudAPI/openapi-generator-config.yaml +++ b/apple/Packages/TableProMacCore/Sources/TableProCloudAPI/openapi-generator-config.yaml @@ -3,3 +3,13 @@ generate: - client accessModifier: public namingStrategy: idiomatic +# The operations the app calls, and the schemas they reference. A new call adds its operation id here. +filter: + operations: + - get-me + - list-devices + - register-device + - delete-device + - issue-entitlement + - list-library-changes + - push-library-changes diff --git a/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLICallbackListener.swift b/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLICallbackListener.swift index 289d8c499a..b7e391044a 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLICallbackListener.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLICallbackListener.swift @@ -144,6 +144,9 @@ public final class CLICallbackListener: @unchecked Sendable { case .denied: status = "200 OK" message = "Pairing was declined. You can close this tab." + case .failed: + status = "200 OK" + message = "TablePro could not finish pairing. You can close this tab." case .ignored: status = "404 Not Found" message = "Nothing here." diff --git a/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLILiveEnvironment.swift b/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLILiveEnvironment.swift index 86b19ab287..0e017cdbfd 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLILiveEnvironment.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProCommandLine/CLILiveEnvironment.swift @@ -3,7 +3,6 @@ import Darwin import Foundation import TableProMCP import TableProProcess -import TableProUtilities public extension CLIEnvironment { static func live(console: any CLIConsole) -> CLIEnvironment { @@ -40,13 +39,8 @@ enum CLILive { let handshake: MCPHandshakePayload do { handshake = try await MCPHandshakeAcquirer(logger: logger).acquire() - } catch MCPHandshakeAcquisitionError.serverTurnedOff { - throw CLIFailure( - .refused, - "TablePro's MCP server is turned off, and the tablepro command reaches TablePro through it. Turn it on under Settings > MCP." - ) } catch { - throw CLIFailure(.failure, "TablePro did not start its MCP server: \(error.localizedDescription)") + throw failure(acquiring: error) } do { return try await verified(handshake, logger: logger) @@ -55,6 +49,23 @@ enum CLILive { } } + static func failure(acquiring error: Error) -> CLIFailure { + switch error as? MCPHandshakeAcquisitionError { + case .serverTurnedOff?: + return CLIFailure( + .refused, + "TablePro's MCP server is turned off, and the tablepro command reaches TablePro through it. Turn it on under Settings > MCP." + ) + case .serverFailedToStart?: + return CLIFailure( + .failure, + "TablePro's MCP server is on but could not start. Open Settings > MCP in TablePro to see why, then run the command again." + ) + case .launchFailed?, .timeout?, nil: + return CLIFailure(.failure, "TablePro did not start its MCP server: \(error.localizedDescription)") + } + } + // One read of the handshake, no launch and no polling: status must answer at once. static func findServer(app: CLIHostApp) async -> CLIServerProbe { let logger = MCPOSBridgeLogger(category: "CLI") @@ -104,6 +115,8 @@ enum CLILive { return try await CLIServerClient.exchange(code: code, verifier: request.verifier, socketPath: server.socketPath, serverPID: server.serverPID) case .denied: throw CLIFailure(.refused, "Pairing was declined in TablePro.") + case .failed: + throw CLIFailure(.failure, "TablePro could not issue a token for the tablepro command. TablePro shows why; run the command again.") case .ignored, nil: throw CLIFailure(.refused, "Pairing was not approved in time.") } diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPAuthenticator.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPAuthenticator.swift index c09fb3f0a7..a39571e89f 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPAuthenticator.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPAuthenticator.swift @@ -21,6 +21,16 @@ public enum MCPClientAddress: Sendable, Equatable, Hashable { return host } } + + // For the window only: the audit log stores displayValue, so that one stays English. + public var localizedDisplayValue: String { + switch self { + case .localSocket: + return String(localized: "Local socket") + case .loopback, .remote: + return displayValue + } + } } public protocol MCPAuthenticator: Sendable { diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift index fe6855968c..c5b5fed9ee 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift @@ -28,19 +28,23 @@ public struct MCPPrincipal: Sendable, Equatable, Hashable { public let scopes: Set public let connectionAccess: ConnectionAccess public let metadata: MCPPrincipalMetadata + // Nil means every tool the scopes allow. + public let toolAllowlist: Set? public init( tokenFingerprint: String, tokenId: UUID, scopes: Set, connectionAccess: ConnectionAccess = .all, - metadata: MCPPrincipalMetadata + metadata: MCPPrincipalMetadata, + toolAllowlist: Set? = nil ) { self.tokenFingerprint = tokenFingerprint self.tokenId = tokenId self.scopes = scopes self.connectionAccess = connectionAccess self.metadata = metadata + self.toolAllowlist = toolAllowlist } public static let inAppAssistantTokenId = UUID(uuid: (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xA1, 0x1A)) @@ -69,6 +73,10 @@ public struct MCPPrincipal: Sendable, Equatable, Hashable { scopes.contains(scope) } + public func allowsTool(_ name: String) -> Bool { + toolAllowlist?.contains(name) ?? true + } + public func missingScopes(from required: Set) -> Set { required.subtracting(scopes) } @@ -86,6 +94,7 @@ public struct MCPPrincipal: Sendable, Equatable, Hashable { && lhs.scopes == rhs.scopes && lhs.connectionAccess == rhs.connectionAccess && lhs.metadata == rhs.metadata + && lhs.toolAllowlist == rhs.toolAllowlist } public func hash(into hasher: inout Hasher) { diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/BridgeProxy.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/BridgeProxy.swift index b78370b988..f980f2428c 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/BridgeProxy.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/BridgeProxy.swift @@ -1,6 +1,6 @@ import Foundation import os -import TableProUtilities +import TableProProcess public struct BridgeDiscovery: Sendable { let supportedVersions: [String] @@ -176,6 +176,7 @@ public actor BridgeProxy { private let hostLines: AsyncStream private let logger: any MCPBridgeLogger private let discovery: BridgeDiscovery + private let instanceId = UUID().uuidString private var clientInfo: JsonValue? @@ -248,12 +249,12 @@ public actor BridgeProxy { private func forward(line: Data, peek: BridgeMessagePeek, method: String) async { var body = line - if !peek.declaresProtocolVersion { - if let stamped = BridgeJson.stampMeta(line, fields: metaFields()) { - body = stamped - } else { - logger.log(.warning, "Could not stamp request metadata onto \(method)") - } + // The instance goes on every message, so a cancellation sent without metadata meets the request it names. + let fields = peek.declaresProtocolVersion ? instanceField() : metaFields() + if let stamped = BridgeJson.stampMeta(line, fields: fields) { + body = stamped + } else { + logger.log(.warning, "Could not stamp request metadata onto \(method)") } do { @@ -312,9 +313,15 @@ public actor BridgeProxy { await stdout.write(data) } + // Two sessions of one client send the same clientInfo and the same request ids; this keeps them apart. + private func instanceField() -> Data { + Data("\"\(MCPMetaKeys.bridgeInstance)\":\"\(instanceId)\"".utf8) + } + private func metaFields() -> Data { var fields = Data("\"\(MCPMetaKeys.protocolVersion)\":\"\(MCPProtocolVersion.latest.rawValue)\"".utf8) - fields.append(Data(",\"\(MCPMetaKeys.clientCapabilities)\":{}".utf8)) + fields.append(Data(",\"\(MCPMetaKeys.clientCapabilities)\":{},".utf8)) + fields.append(instanceField()) guard let clientInfo, let encoded = try? JSONEncoder().encode(clientInfo) else { return fields } fields.append(Data(",\"\(MCPMetaKeys.clientInfo)\":".utf8)) fields.append(encoded) diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/Handshake.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/Handshake.swift index 23daf32939..dd81a1479d 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/Handshake.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Bridge/Handshake.swift @@ -7,6 +7,7 @@ public enum MCPHandshakeAcquisitionError: Error, LocalizedError, Equatable { case launchFailed(String) case timeout(lastFailure: String) case serverTurnedOff + case serverFailedToStart public var errorDescription: String? { switch self { @@ -16,6 +17,8 @@ public enum MCPHandshakeAcquisitionError: Error, LocalizedError, Equatable { return "Timed out waiting for a trusted TablePro MCP handshake (\(lastFailure))" case .serverTurnedOff: return "The TablePro MCP server is turned off. Turn it on in TablePro under Settings > MCP." + case .serverFailedToStart: + return "The TablePro MCP server is turned on but failed to start. Open Settings > MCP in TablePro to see why and try again." } } } @@ -72,12 +75,13 @@ public struct MCPHandshakeAcquirer: Sendable { // A forged refusal can only make the bridge give up, so it is read without the handshake's // ownership checks. - private func refusalAnswers(launchRequestedAt: Date) -> Bool { + private func refusalAnswering(launchRequestedAt: Date) -> MCPStartRefusal? { guard let data = try? Data(contentsOf: refusalUrl), - let refusal = try? MCPHandshakeCoding.decoder().decode(MCPStartRefusal.self, from: data) else { - return false + let refusal = try? MCPHandshakeCoding.decoder().decode(MCPStartRefusal.self, from: data), + refusal.answers(launchRequestedAt: launchRequestedAt) else { + return nil } - return refusal.answers(launchRequestedAt: launchRequestedAt) + return refusal } // The copy this helper ships in, not whichever copy owns `tablepro://`. Substitution stays on, so a @@ -115,9 +119,15 @@ public struct MCPHandshakeAcquirer: Sendable { if let payload = attempt.payload { return payload } - if let launchRequestedAt, refusalAnswers(launchRequestedAt: launchRequestedAt) { - logger.log(.error, "TablePro refused to start the MCP server: it is turned off") - throw MCPHandshakeAcquisitionError.serverTurnedOff + if let launchRequestedAt, let refusal = refusalAnswering(launchRequestedAt: launchRequestedAt) { + switch refusal.reason { + case .turnedOff: + logger.log(.error, "TablePro refused to start the MCP server: it is turned off") + throw MCPHandshakeAcquisitionError.serverTurnedOff + case .failedToStart: + logger.log(.error, "TablePro tried to start the MCP server and failed") + throw MCPHandshakeAcquisitionError.serverFailedToStart + } } lastFailure = attempt.failure try? await Task.sleep(for: Self.pollInterval) diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIPairing.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIPairing.swift index 3d1f2d74e1..d46f8a42c3 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIPairing.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/CommandLine/CLIPairing.swift @@ -58,6 +58,8 @@ public struct CLIPairingRequest: Sendable, Equatable { public enum CLIPairingCallback: Sendable, Equatable { case approved(code: String) case denied + // Approved in TablePro, which then could not issue the token. + case failed // Another path, or a `state` from some other request. case ignored @@ -74,6 +76,13 @@ public enum CLIPairingCallback: Sendable, Equatable { if let code = value("code"), !code.isEmpty { return .approved(code: code) } - return value("error") == nil ? .ignored : .denied + switch value("error") { + case nil: + return .ignored + case "access_denied"?, "denied"?: + return .denied + default: + return .failed + } } } diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientActivityLedger.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientActivityLedger.swift index c88480fa5a..87e2a53438 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientActivityLedger.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientActivityLedger.swift @@ -4,29 +4,35 @@ public actor MCPClientActivityLedger { public struct Entry: Sendable, Identifiable, Equatable { public let id: String public let clientName: String + public let clientInfoName: String? public let clientVersion: String? public let tokenId: UUID? public let tokenName: String? public let address: String + public let transport: MCPClientTransport public let firstSeenAt: Date public let lastSeenAt: Date public init( id: String, clientName: String, + clientInfoName: String? = nil, clientVersion: String?, tokenId: UUID?, tokenName: String?, address: String, + transport: MCPClientTransport = .http, firstSeenAt: Date, lastSeenAt: Date ) { self.id = id self.clientName = clientName + self.clientInfoName = clientInfoName self.clientVersion = clientVersion self.tokenId = tokenId self.tokenName = tokenName self.address = address + self.transport = transport self.firstSeenAt = firstSeenAt self.lastSeenAt = lastSeenAt } @@ -57,10 +63,12 @@ public actor MCPClientActivityLedger { entries[identity.id] = Entry( id: identity.id, clientName: identity.clientName, + clientInfoName: identity.clientInfoName, clientVersion: identity.clientVersion, tokenId: identity.tokenId, tokenName: identity.tokenName, - address: identity.addressDisplayValue, + address: identity.address.localizedDisplayValue, + transport: identity.transport, firstSeenAt: existing?.firstSeenAt ?? timestamp, lastSeenAt: timestamp ) diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientIdentity.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientIdentity.swift index 9471fe998f..2495284642 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientIdentity.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPClientIdentity.swift @@ -1,31 +1,43 @@ import CryptoKit import Foundation -import TableProUtilities + +public enum MCPClientTransport: Sendable, Equatable, Hashable { + // The credential every stdio client shares through the tablepro-mcp helper; it rotates on its own. + case stdioBridge + // A credential of its own: a token from Settings or pairing, over TCP or the app's socket. + case http +} public struct MCPClientIdentity: Sendable, Equatable, Hashable, Identifiable { public static let unknownClientName = String(localized: "Unknown client") public let clientName: String + public let clientInfoName: String? public let clientVersion: String? public let tokenId: UUID? public let tokenName: String? public let principalFingerprint: String public let address: MCPClientAddress + public let transport: MCPClientTransport public init( clientName: String, + clientInfoName: String? = nil, clientVersion: String?, tokenId: UUID?, tokenName: String?, principalFingerprint: String, - address: MCPClientAddress + address: MCPClientAddress, + transport: MCPClientTransport = .http ) { self.clientName = clientName + self.clientInfoName = clientInfoName self.clientVersion = clientVersion self.tokenId = tokenId self.tokenName = tokenName self.principalFingerprint = principalFingerprint self.address = address + self.transport = transport } public init(meta: MCPRequestMeta, principal: MCPPrincipal, address: MCPClientAddress) { @@ -34,11 +46,13 @@ public struct MCPClientIdentity: Sendable, Equatable, Hashable, Identifiable { let resolvedVersion = info?.version.nilIfBlank self.init( clientName: resolvedName, + clientInfoName: info?.name.nilIfBlank, clientVersion: resolvedVersion, tokenId: principal.tokenId, tokenName: principal.metadata.label, principalFingerprint: principal.tokenFingerprint, - address: address + address: address, + transport: principal.metadata.isBridgeCredential ? .stdioBridge : .http ) } @@ -47,16 +61,18 @@ public struct MCPClientIdentity: Sendable, Equatable, Hashable, Identifiable { } public var id: String { - let fields = [ - clientName, - clientVersion ?? "", - tokenId?.uuidString ?? "", - principalFingerprint, - address.displayValue - ] + let credential: [String] + switch transport { + case .stdioBridge: + // The bridge credential rotates every 45 minutes, so a stdio client is the client it says it is. + credential = ["stdio-bridge"] + case .http: + credential = [tokenId?.uuidString ?? "", principalFingerprint] + } + let fields = [clientName, clientInfoName ?? "", clientVersion ?? ""] + credential + [address.displayValue] let joined = fields.joined(separator: "\u{1F}") let digest = SHA256.hash(data: Data(joined.utf8)) - return String(digest.hexEncoded.prefix(16)) + return String(digest.map { String(format: "%02x", $0) }.joined().prefix(16)) } } diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Lifecycle/MCPHandshakeFile.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Lifecycle/MCPHandshakeFile.swift index bc736e293e..50b29284c3 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Lifecycle/MCPHandshakeFile.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Lifecycle/MCPHandshakeFile.swift @@ -37,14 +37,39 @@ public struct MCPHandshakePayload: Codable, Sendable, Equatable { } public struct MCPStartRefusal: Codable, Sendable, Equatable { + // A code, never text: the file is read without the handshake's ownership checks, so whatever it + // says reaches the client only as one of the bridge's own messages. + public enum Reason: String, Codable, Sendable { + case turnedOff + case failedToStart + } + public let pid: Int32 // Seconds since 1970 with the fraction kept: an ISO 8601 date drops it, and a refusal from // earlier in the same second would then answer a launch it never saw. public let refusedAt: TimeInterval + public let reason: Reason + + private enum CodingKeys: String, CodingKey { + case pid + case refusedAt + case reason + } - public init(pid: Int32, refusedAt: Date) { + public init(pid: Int32, refusedAt: Date, reason: Reason = .turnedOff) { self.pid = pid self.refusedAt = refusedAt.timeIntervalSince1970 + self.reason = reason + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + pid = try container.decode(Int32.self, forKey: .pid) + refusedAt = try container.decode(TimeInterval.self, forKey: .refusedAt) + // Builds before the reason wrote a refusal only when MCP was off; a reason this build does not + // know still means the server is not coming up. + let raw = (try? container.decodeIfPresent(String.self, forKey: .reason)) ?? nil + reason = raw.map { Reason(rawValue: $0) ?? .failedToStart } ?? .turnedOff } // Only a refusal written after the launch answers it, so one left from an earlier run cannot diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPMetaKeys.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPMetaKeys.swift index d7ce070179..e1f26bc4ce 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPMetaKeys.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPMetaKeys.swift @@ -8,6 +8,7 @@ public enum MCPMetaKeys { public static let logLevel = "io.modelcontextprotocol/logLevel" public static let subscriptionId = "io.modelcontextprotocol/subscriptionId" public static let serverInfo = "io.modelcontextprotocol/serverInfo" + public static let bridgeInstance = "app.tablepro/bridgeInstance" public static let traceParent = "traceparent" public static let traceState = "tracestate" diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPRequestMeta.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPRequestMeta.swift index 6f3e700044..45e38225fa 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPRequestMeta.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Meta/MCPRequestMeta.swift @@ -29,6 +29,15 @@ public struct MCPRequestMeta: Sendable, Equatable { params?["_meta"] } + // The tablepro-mcp process that forwarded the message. Any local client can write it, so only the + // in-flight key reads it, and only for the bridge credential. + public static func bridgeInstance(in params: JsonValue?) -> String? { + guard let value = metaObject(in: params)?[MCPMetaKeys.bridgeInstance]?.stringValue, !value.isEmpty else { + return nil + } + return value + } + public static func declaresModernProtocol(params: JsonValue?) -> Bool { metaObject(in: params)?[MCPMetaKeys.protocolVersion]?.stringValue != nil } diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/MCPInflightRegistry.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/MCPInflightRegistry.swift index 026dc332aa..e1a536de4b 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/MCPInflightRegistry.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/MCPInflightRegistry.swift @@ -1,23 +1,52 @@ import Foundation public struct MCPInflightKey: Sendable, Hashable { + // The bridge credential rotates while its requests run, so a cancellation sent with the renewed + // one has to find a request started with the old one. + public static let bridgeCredentialFingerprint = "stdio-bridge" + public let clientFingerprint: String + // Only for stdio without a bridge instance: stdio clients share one credential, so the sender's identity splits them. + public let clientId: String + // The whole stdio sender, since a request may carry clientInfo its cancellation lacks; Disconnect uses the entry's clientId. + public let bridgeInstance: String public let requestId: JsonRpcId - public init(clientFingerprint: String, requestId: JsonRpcId) { + public init(clientFingerprint: String, clientId: String = "", bridgeInstance: String = "", requestId: JsonRpcId) { self.clientFingerprint = clientFingerprint + self.clientId = clientId + self.bridgeInstance = bridgeInstance self.requestId = requestId } public init(principal: MCPPrincipal, requestId: JsonRpcId) { self.init(clientFingerprint: principal.tokenFingerprint, requestId: requestId) } + + public init(bridgeInstance: String, requestId: JsonRpcId) { + self.init(clientFingerprint: Self.bridgeCredentialFingerprint, bridgeInstance: bridgeInstance, requestId: requestId) + } + + // A token one client holds ignores bridgeInstance: the field only splits requests of one stdio client kind. + public init(client: MCPClientIdentity, bridgeInstance: String? = nil, requestId: JsonRpcId) { + switch client.transport { + case .stdioBridge: + if let bridgeInstance, !bridgeInstance.isEmpty { + self.init(bridgeInstance: bridgeInstance, requestId: requestId) + } else { + self.init(clientFingerprint: Self.bridgeCredentialFingerprint, clientId: client.id, requestId: requestId) + } + case .http: + self.init(clientFingerprint: client.principalFingerprint, requestId: requestId) + } + } } public actor MCPInflightRegistry { private struct Entry { let token: MCPCancellationToken let tokenId: UUID? + let clientId: String? let method: String let startedAt: Date } @@ -31,11 +60,12 @@ public actor MCPInflightRegistry { key: MCPInflightKey, token: MCPCancellationToken, tokenId: UUID?, + clientId: String? = nil, method: String, startedAt: Date ) -> Bool { let displaced = entries[key] != nil - entries[key] = Entry(token: token, tokenId: tokenId, method: method, startedAt: startedAt) + entries[key] = Entry(token: token, tokenId: tokenId, clientId: clientId, method: method, startedAt: startedAt) return !displaced } @@ -56,6 +86,11 @@ public actor MCPInflightRegistry { await cancelMatching(reason: reason) { _, entry in entry.tokenId == tokenId } } + @discardableResult + public func cancelAll(matchingClientId clientId: String, reason: MCPCancellationReason) async -> Int { + await cancelMatching(reason: reason) { _, entry in entry.clientId == clientId } + } + @discardableResult public func cancelAll(matchingFingerprint fingerprint: String, reason: MCPCancellationReason) async -> Int { await cancelMatching(reason: reason) { key, _ in key.clientFingerprint == fingerprint } diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/Tools/MCPToolSchema.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/Tools/MCPToolSchema.swift index 3e8afd4e6e..28d43ff89c 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/Tools/MCPToolSchema.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Protocol/Tools/MCPToolSchema.swift @@ -95,7 +95,13 @@ public extension MCPToolSchema { "row_count": integer(String(localized: "Number of rows returned")), "rows_affected": integer(String(localized: "Rows the statement changed")), "execution_time_ms": number(String(localized: "Server round trip in milliseconds")), - "is_truncated": boolean(String(localized: "Whether the row limit clipped the result")), + "is_truncated": boolean( + String(localized: "Whether the row limit or the answer's size limit left rows out of the result") + ), + "notes": array( + String(localized: "Why rows were left out or values cut, when they were"), + of: stringItem + ), "status_message": string(String(localized: "Driver status message, when the engine sent one")), "database": string(String(localized: "Database the statement ran against")), "schema": string(String(localized: "Schema the statement ran against")) diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Transport/MCPBridgeLogger.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Transport/MCPBridgeLogger.swift index d0f63d9665..60d695d898 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Transport/MCPBridgeLogger.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Transport/MCPBridgeLogger.swift @@ -1,6 +1,6 @@ import Foundation import os -import TableProUtilities +import TableProProcess public enum MCPBridgeLogLevel: String, Sendable { case debug diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONCMemberEditor.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONCMemberEditor.swift new file mode 100644 index 0000000000..da8841f8b2 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONCMemberEditor.swift @@ -0,0 +1,411 @@ +import Foundation + +// Edits one member of a JSON or JSONC file as text. A JSONSerialization round trip drops comments, reorders keys and +// re-indents the whole file, so nothing here writes through it; it only checks each edit afterwards. +public enum JSONCMemberEditor { + public static func member(_ name: String, at keyPath: [String], in text: String) throws -> JSONValue? { + let bytes = Array(text.utf8) + guard let root = try JSONCParser.parse(bytes) else { return nil } + guard let container = try walk(root, keyPath: keyPath) else { return nil } + return try uniqueMember(name, in: container, path: keyPath)?.value.value + } + + public static func upserting(_ name: String, value: JSONValue, at keyPath: [String], in text: String) throws -> String { + let bytes = Array(text.utf8) + let result: String + if let root = try JSONCParser.parse(bytes) { + let document = Document(bytes: bytes, root: root) + result = try document.upserting(name, value: value, at: keyPath) + } else { + result = startingDocument(name, value: value, at: keyPath, after: text) + } + try verify(old: text, new: result, keyPath: keyPath, name: name, expected: value) + return result + } + + // Update: `entry` is rendered from `template`, which says which members are TablePro's to rewrite (MCPEntryUpdate). + public static func updating( + _ name: String, + entry: JSONValue, + template: JSONValue, + at keyPath: [String], + in text: String + ) throws -> String { + switch MCPEntryUpdate.plan(existing: try member(name, at: keyPath, in: text), entry: entry, template: template) { + case .replace(let value): try upserting(name, value: value, at: keyPath, in: text) + case .merge(let value): try merging(name, value: value, at: keyPath, in: text) + } + } + + // Rewrites only the members `value` names inside an existing object entry, so keys the user added to it stay. + public static func merging(_ name: String, value: JSONValue, at keyPath: [String], in text: String) throws -> String { + guard case .object(let wanted) = value, + let existing = try member(name, at: keyPath, in: text), + existing.objectMembers != nil else { + return try upserting(name, value: value, at: keyPath, in: text) + } + var result = text + for part in wanted where existing[part.key] != part.value { + result = try merging(part.key, value: part.value, at: keyPath + [name], in: result) + } + guard try member(name, at: keyPath, in: result) == existing.merging(value) else { + throw MCPConfigEditError.verificationFailed + } + return result + } + + public static func removing(_ name: String, at keyPath: [String], in text: String) throws -> String { + let bytes = Array(text.utf8) + guard let root = try JSONCParser.parse(bytes) else { return text } + let document = Document(bytes: bytes, root: root) + let result = try document.removing(name, at: keyPath) + guard result != text else { return text } + try verify(old: text, new: result, keyPath: keyPath, name: name, expected: nil) + return result + } + + static func walk(_ root: JSONCNode, keyPath: [String]) throws -> JSONCNode? { + guard root.members != nil else { throw MCPConfigEditError.notAnObject([]) } + var container = root + var walked: [String] = [] + for key in keyPath { + walked.append(key) + guard let member = try uniqueMember(key, in: container, path: Array(walked.dropLast())) else { return nil } + guard member.value.members != nil else { throw MCPConfigEditError.notAnObject(walked) } + container = member.value + } + return container + } + + static func uniqueMember(_ key: String, in container: JSONCNode, path: [String]) throws -> JSONCMember? { + let matches = (container.members ?? []).filter { $0.key == key } + guard matches.count <= 1 else { throw MCPConfigEditError.duplicateKey(path + [key]) } + return matches.first + } + + private static func startingDocument(_ name: String, value: JSONValue, at keyPath: [String], after text: String) -> String { + let newline = text.contains("\r\n") ? "\r\n" : "\n" + let style = JSONTextStyle(indentUnit: " ", newline: newline, multiline: true, spaced: true) + let body = style.render(JSONValue.nested(keyPath + [name], leaf: value), indent: "") + newline + guard !text.allSatisfy(\.isWhitespace) else { return body } + let separator = text.utf8.last == 0x0A ? "" : newline + return text + separator + body + } + + private static func verify(old: String, new: String, keyPath: [String], name: String, expected: JSONValue?) throws { + let before = try foundationObject(old) + let after = try foundationObject(new) + let wanted = replacing(name, in: before, at: keyPath, with: expected?.foundationObject) + guard NSDictionary(dictionary: wanted).isEqual(to: after) else { + throw MCPConfigEditError.verificationFailed + } + } + + private static func foundationObject(_ text: String) throws -> [String: Any] { + let bytes = Array(text.utf8) + guard try JSONCParser.parse(bytes) != nil else { return [:] } + let data = Data(JSONCParser.replacingLoneSurrogateEscapes(bytes)) + let object = try? JSONSerialization.jsonObject(with: data, options: [.json5Allowed]) + guard let dictionary = object as? [String: Any] else { throw MCPConfigEditError.invalidSyntax } + return dictionary + } + + private static func replacing(_ name: String, in dictionary: [String: Any], at keyPath: [String], with value: Any?) -> [String: Any] { + var result = dictionary + guard let key = keyPath.first else { + result[name] = value + return result + } + if let child = dictionary[key] as? [String: Any] { + result[key] = replacing(name, in: child, at: Array(keyPath.dropFirst()), with: value) + } else if value != nil { + result[key] = replacing(name, in: [:], at: Array(keyPath.dropFirst()), with: value) + } + return result + } +} + +private struct Document { + let bytes: [UInt8] + let root: JSONCNode + let newline: String + let indentUnit: String + let spaced: Bool + let isMultiline: Bool + + init(bytes: [UInt8], root: JSONCNode) { + self.bytes = bytes + self.root = root + newline = Self.contains(bytes, [0x0D, 0x0A]) ? "\r\n" : "\n" + // An empty root is a fresh file, so it is laid out like a written one rather than kept on one line. + isMultiline = bytes[root.range].contains(0x0A) || root.members?.isEmpty == true + var unit: String? + var spacedSeparator: Bool? + Self.survey(root, bytes: bytes, unit: &unit, spaced: &spacedSeparator) + indentUnit = unit ?? " " + spaced = spacedSeparator ?? true + } + + func upserting(_ name: String, value: JSONValue, at keyPath: [String]) throws -> String { + guard root.members != nil else { throw MCPConfigEditError.notAnObject([]) } + var container = root + for (position, key) in keyPath.enumerated() { + let walked = Array(keyPath.prefix(position)) + guard let member = try JSONCMemberEditor.uniqueMember(key, in: container, path: walked) else { + let rest = Array(keyPath.dropFirst(position + 1)) + [name] + return apply(inserting(key, value: JSONValue.nested(rest, leaf: value), into: container)) + } + guard member.value.members != nil else { throw MCPConfigEditError.notAnObject(walked + [key]) } + container = member.value + } + guard let existing = try JSONCMemberEditor.uniqueMember(name, in: container, path: keyPath) else { + return apply(inserting(name, value: value, into: container)) + } + let style = textStyle(multiline: spansLines(container)) + let rendered = style.render(value, indent: lineIndent(at: existing.keyRange.lowerBound)) + return apply([Splice(range: existing.value.range, text: rendered)]) + } + + func removing(_ name: String, at keyPath: [String]) throws -> String { + guard let container = try JSONCMemberEditor.walk(root, keyPath: keyPath), + let target = try JSONCMemberEditor.uniqueMember(name, in: container, path: keyPath), + let members = container.members, + let position = members.firstIndex(where: { $0.keyRange == target.keyRange }) else { + return utf8Text(bytes) + } + let previous = position > 0 ? members[position - 1] : nil + var splices: [Splice] = [] + let memberEnd = target.commaOffset.map { $0 + 1 } ?? target.value.range.upperBound + if spansLines(container), isFirstOnLine(target.keyRange.lowerBound), let lineEnd = endOfLine(after: memberEnd) { + splices.append(Splice(range: lineStart(of: target.keyRange.lowerBound).. String { + let edited = Array(text.utf8) + guard let root = try? JSONCParser.parse(edited), + let container = try? JSONCMemberEditor.walk(root, keyPath: keyPath), + container.members?.isEmpty == true else { + return text + } + let interior = (container.range.lowerBound + 1)..<(container.range.upperBound - 1) + guard !interior.isEmpty, edited[interior].allSatisfy(Self.isWhitespace) else { return text } + var result = edited + result.removeSubrange(interior) + return utf8Text(result) + } + + private func inserting(_ name: String, value: JSONValue, into container: JSONCNode) -> [Splice] { + let members = container.members ?? [] + let open = container.range.lowerBound + let close = container.range.upperBound - 1 + let multiline = spansLines(container) || (members.isEmpty && isMultiline) + let style = textStyle(multiline: multiline) + let colon = spaced ? ": " : ":" + + guard let last = members.last else { + guard multiline else { + let text = JSONTextStyle.quoted(name) + colon + style.render(value, indent: "") + return [Splice(range: (open + 1).. lineStart(of: open) { + let start = lineStart(of: close) + return [Splice(range: start.. JSONTextStyle { + JSONTextStyle(indentUnit: indentUnit, newline: newline, multiline: multiline, spaced: spaced) + } + + private struct Splice { + let range: Range + let text: String + } + + private func apply(_ splices: [Splice]) -> String { + var result = bytes + for splice in splices.sorted(by: { $0.range.lowerBound > $1.range.lowerBound }) { + result.replaceSubrange(splice.range, with: Array(splice.text.utf8)) + } + return utf8Text(result) + } + + private func spansLines(_ node: JSONCNode) -> Bool { + bytes[node.range].contains(0x0A) + } + + private func lineStart(of offset: Int) -> Int { + var index = offset + while index > 0, bytes[index - 1] != 0x0A { + index -= 1 + } + return index + } + + private func lineIndent(at offset: Int) -> String { + let start = lineStart(of: offset) + var end = start + while end < bytes.count, bytes[end] == 0x20 || bytes[end] == 0x09 { + end += 1 + } + return utf8Text(bytes[start.. Bool { + bytes[lineStart(of: offset).. Int { + var index = offset + while index < bytes.count, bytes[index] == 0x20 || bytes[index] == 0x09 { + index += 1 + } + return index + } + + // The offset of the line break after `offset`, past spaces and comments on the same line; nil when a token follows. + private func endOfLine(after offset: Int) -> Int? { + var index = offset + while index < bytes.count { + switch bytes[index] { + case 0x20, 0x09: + index += 1 + case 0x0D where index + 1 < bytes.count && bytes[index + 1] == 0x0A: + return index + case 0x0A: + return index + case UInt8(ascii: "/") where index + 1 < bytes.count && bytes[index + 1] == UInt8(ascii: "/"): + while index < bytes.count, bytes[index] != 0x0A { + index += 1 + } + if index > offset, bytes[index - 1] == 0x0D { + return index - 1 + } + return index + case UInt8(ascii: "/") where index + 1 < bytes.count && bytes[index + 1] == UInt8(ascii: "*"): + var end = index + 2 + while end + 1 < bytes.count, !(bytes[end] == UInt8(ascii: "*") && bytes[end + 1] == UInt8(ascii: "/")) { + if bytes[end] == 0x0A { return nil } + end += 1 + } + index = end + 2 + default: + return nil + } + } + return bytes.count + } + + private func afterNewline(_ offset: Int) -> Int { + if offset < bytes.count, bytes[offset] == 0x0D { + return min(offset + 2, bytes.count) + } + return min(offset + 1, bytes.count) + } + + private static func isWhitespace(_ byte: UInt8) -> Bool { + byte == 0x20 || byte == 0x09 || byte == 0x0A || byte == 0x0D + } + + private static func contains(_ bytes: [UInt8], _ pair: [UInt8]) -> Bool { + zip(bytes, bytes.dropFirst()).contains { $0.0 == pair[0] && $0.1 == pair[1] } + } + + // The indent unit is the step from a container's line to its first member's line, from the first container that has one. + private static func survey(_ node: JSONCNode, bytes: [UInt8], unit: inout String?, spaced: inout Bool?) { + switch node.content { + case .object(let members): + if let first = members.first { + if spaced == nil { + spaced = bytes[first.keyRange.upperBound.. String? { + func indent(_ offset: Int) -> (start: Int, text: [UInt8])? { + var start = offset + while start > 0, bytes[start - 1] != 0x0A { + start -= 1 + } + var end = start + while end < offset, bytes[end] == 0x20 || bytes[end] == 0x09 { + end += 1 + } + guard end == offset else { return nil } + return (start, Array(bytes[start.. 0, bytes[openStart - 1] != 0x0A { + openStart -= 1 + } + guard openStart != childIndent.start else { return nil } + var openEnd = openStart + while openEnd < bytes.count, bytes[openEnd] == 0x20 || bytes[openEnd] == 0x09 { + openEnd += 1 + } + let outer = Array(bytes[openStart.. outer.count, childIndent.text.starts(with: outer) else { return nil } + return utf8Text(childIndent.text[outer.count...]) + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONCSyntax.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONCSyntax.swift new file mode 100644 index 0000000000..4b4bc1c2e8 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONCSyntax.swift @@ -0,0 +1,347 @@ +import Foundation + +public enum MCPConfigEditError: Error, Sendable, Equatable { + case invalidSyntax + case duplicateKey([String]) + case notAnObject([String]) + case unsupportedShape + case verificationFailed +} + +// The editors splice only at ASCII boundaries of text that arrived as a String, so the bytes stay valid UTF-8. +func utf8Text(_ bytes: Bytes) -> String where Bytes.Element == UInt8 { + String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion +} + +struct JSONCNode { + enum Content { + case object([JSONCMember]) + case array([JSONCNode]) + case scalar(JSONValue) + } + + let content: Content + let range: Range + + var members: [JSONCMember]? { + guard case .object(let members) = content else { return nil } + return members + } + + var value: JSONValue { + switch content { + case .object(let members): return .object(members.map { (key: $0.key, value: $0.value.value) }) + case .array(let items): return .array(items.map(\.value)) + case .scalar(let scalar): return scalar + } + } +} + +struct JSONCMember { + let key: String + let keyRange: Range + let value: JSONCNode + let commaOffset: Int? +} + +// JSON with `//` and `/* */` comments and trailing commas, the dialect VS Code and Zed write. Offsets are UTF-8. +struct JSONCParser { + private let bytes: [UInt8] + private var index = 0 + private var depth = 0 + private static let maximumDepth = 256 + + static func parse(_ bytes: [UInt8]) throws -> JSONCNode? { + var parser = JSONCParser(bytes: bytes) + try parser.skipTrivia() + guard parser.index < bytes.count else { return nil } + let root = try parser.parseValue() + try parser.skipTrivia() + guard parser.index == bytes.count else { throw MCPConfigEditError.invalidSyntax } + return root + } + + // JSONSerialization refuses an unpaired surrogate escape, which Node writes; this parser reads it as U+FFFD, so the check does too. + static func replacingLoneSurrogateEscapes(_ bytes: [UInt8]) -> [UInt8] { + let backslash = UInt8(ascii: "\\") + let replacement = [backslash] + Array("uFFFD".utf8) + var result: [UInt8] = [] + result.reserveCapacity(bytes.count) + var index = 0 + while index < bytes.count { + guard bytes[index] == backslash, index + 1 < bytes.count else { + result.append(bytes[index]) + index += 1 + continue + } + guard bytes[index + 1] == UInt8(ascii: "u"), let unit = hexUnit(bytes, at: index + 2) else { + result.append(contentsOf: bytes[index...(index + 1)]) + index += 2 + continue + } + if (0xD800..<0xDC00).contains(unit), index + 7 < bytes.count, + bytes[index + 6] == backslash, bytes[index + 7] == UInt8(ascii: "u"), + let low = hexUnit(bytes, at: index + 8), (0xDC00..<0xE000).contains(low) { + result.append(contentsOf: bytes[index..<(index + 12)]) + index += 12 + continue + } + if (0xD800..<0xE000).contains(unit) { + result.append(contentsOf: replacement) + } else { + result.append(contentsOf: bytes[index..<(index + 6)]) + } + index += 6 + } + return result + } + + private static func hexUnit(_ bytes: [UInt8], at offset: Int) -> UInt32? { + guard offset + 4 <= bytes.count else { return nil } + return UInt32(utf8Text(bytes[offset..<(offset + 4)]), radix: 16) + } + + private init(bytes: [UInt8]) { + self.bytes = bytes + if bytes.starts(with: [0xEF, 0xBB, 0xBF]) { + index = 3 + } + } + + private var current: UInt8? { + index < bytes.count ? bytes[index] : nil + } + + private mutating func skipTrivia() throws { + while let byte = current { + switch byte { + case 0x20, 0x09, 0x0A, 0x0D: + index += 1 + case UInt8(ascii: "/"): + guard index + 1 < bytes.count else { throw MCPConfigEditError.invalidSyntax } + if bytes[index + 1] == UInt8(ascii: "/") { + while let next = current, next != 0x0A { + index += 1 + } + } else if bytes[index + 1] == UInt8(ascii: "*") { + index += 2 + while true { + guard index + 1 < bytes.count else { throw MCPConfigEditError.invalidSyntax } + if bytes[index] == UInt8(ascii: "*"), bytes[index + 1] == UInt8(ascii: "/") { + index += 2 + break + } + index += 1 + } + } else { + throw MCPConfigEditError.invalidSyntax + } + default: + return + } + } + } + + private mutating func parseValue() throws -> JSONCNode { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + let start = index + switch byte { + case UInt8(ascii: "{"): + return try nested { try $0.parseObject() } + case UInt8(ascii: "["): + return try nested { try $0.parseArray() } + case UInt8(ascii: "\""): + let string = try parseString() + return JSONCNode(content: .scalar(.string(string)), range: start.. JSONCNode) throws -> JSONCNode { + depth += 1 + defer { depth -= 1 } + guard depth <= Self.maximumDepth else { throw MCPConfigEditError.invalidSyntax } + return try body(&self) + } + + private mutating func parseObject() throws -> JSONCNode { + let start = index + index += 1 + var members: [JSONCMember] = [] + while true { + try skipTrivia() + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + if byte == UInt8(ascii: "}") { + index += 1 + return JSONCNode(content: .object(members), range: start.. JSONCNode { + let start = index + index += 1 + var items: [JSONCNode] = [] + while true { + try skipTrivia() + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + if byte == UInt8(ascii: "]") { + index += 1 + return JSONCNode(content: .array(items), range: start.. Double { + let start = index + if current == UInt8(ascii: "-") { + index += 1 + } + guard let first = current, Self.isDigit(first) else { throw MCPConfigEditError.invalidSyntax } + if first == UInt8(ascii: "0") { + index += 1 + } else { + skipDigits() + } + if current == UInt8(ascii: ".") { + index += 1 + guard let digit = current, Self.isDigit(digit) else { throw MCPConfigEditError.invalidSyntax } + skipDigits() + } + if current == UInt8(ascii: "e") || current == UInt8(ascii: "E") { + index += 1 + if current == UInt8(ascii: "+") || current == UInt8(ascii: "-") { + index += 1 + } + guard let digit = current, Self.isDigit(digit) else { throw MCPConfigEditError.invalidSyntax } + skipDigits() + } + let text = utf8Text(bytes[start.. Bool { + byte >= UInt8(ascii: "0") && byte <= UInt8(ascii: "9") + } + + private mutating func parseString() throws -> String { + index += 1 + var buffer: [UInt8] = [] + while true { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + switch byte { + case UInt8(ascii: "\""): + index += 1 + return utf8Text(buffer) + case UInt8(ascii: "\\"): + index += 1 + try appendEscape(to: &buffer) + case 0x00..<0x20: + throw MCPConfigEditError.invalidSyntax + default: + buffer.append(byte) + index += 1 + } + } + } + + private mutating func appendEscape(to buffer: inout [UInt8]) throws { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + index += 1 + switch byte { + case UInt8(ascii: "\""), UInt8(ascii: "\\"), UInt8(ascii: "/"): buffer.append(byte) + case UInt8(ascii: "b"): buffer.append(0x08) + case UInt8(ascii: "f"): buffer.append(0x0C) + case UInt8(ascii: "n"): buffer.append(0x0A) + case UInt8(ascii: "r"): buffer.append(0x0D) + case UInt8(ascii: "t"): buffer.append(0x09) + case UInt8(ascii: "u"): + var scalarValue = try parseHex4() + if (0xD800..<0xDC00).contains(scalarValue), + index + 1 < bytes.count, bytes[index] == UInt8(ascii: "\\"), bytes[index + 1] == UInt8(ascii: "u") { + let saved = index + index += 2 + let low = try parseHex4() + if (0xDC00..<0xE000).contains(low) { + scalarValue = 0x10000 + ((scalarValue - 0xD800) << 10) + (low - 0xDC00) + } else { + index = saved + } + } + let scalar = Unicode.Scalar(scalarValue) ?? "\u{FFFD}" + buffer.append(contentsOf: Array(String(Character(scalar)).utf8)) + default: + throw MCPConfigEditError.invalidSyntax + } + } + + private mutating func parseHex4() throws -> UInt32 { + guard index + 4 <= bytes.count, + let value = UInt32(utf8Text(bytes[index..<(index + 4)]), radix: 16) else { + throw MCPConfigEditError.invalidSyntax + } + index += 4 + return value + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONValue.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONValue.swift new file mode 100644 index 0000000000..011471818f --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/JSONValue.swift @@ -0,0 +1,207 @@ +import Foundation + +public enum JSONValue: Sendable { + case object([(key: String, value: JSONValue)]) + case array([JSONValue]) + case string(String) + case number(Double) + case bool(Bool) + case null + + public subscript(key: String) -> JSONValue? { + guard case .object(let members) = self else { return nil } + return members.first { $0.key == key }?.value + } + + public var stringValue: String? { + guard case .string(let string) = self else { return nil } + return string + } + + public var arrayValue: [JSONValue]? { + guard case .array(let items) = self else { return nil } + return items + } + + public var objectMembers: [(key: String, value: JSONValue)]? { + guard case .object(let members) = self else { return nil } + return members + } + + public var compactText: String { + JSONTextStyle.compact.render(self, indent: "") + } + + public func prettyText(indentUnit: String = " ") -> String { + JSONTextStyle(indentUnit: indentUnit, newline: "\n", multiline: true, spaced: true).render(self, indent: "") + } + + static func nested(_ path: [String], leaf: JSONValue) -> JSONValue { + path.reversed().reduce(leaf) { inner, key in .object([(key: key, value: inner)]) } + } + + // The overlay wins member by member and an object in both merges again, so what the overlay never names stays. + func merging(_ overlay: JSONValue) -> JSONValue { + guard case .object(var members) = self, case .object(let additions) = overlay else { return overlay } + for addition in additions { + if let position = members.firstIndex(where: { $0.key == addition.key }) { + members[position].value = members[position].value.merging(addition.value) + } else { + members.append(addition) + } + } + return .object(members) + } + + var foundationObject: Any { + switch self { + case .object(let members): + var dictionary: [String: Any] = [:] + for member in members where dictionary[member.key] == nil { + dictionary[member.key] = member.value.foundationObject + } + return dictionary + case .array(let items): return items.map(\.foundationObject) + case .string(let string): return string + case .number(let number): return NSNumber(value: number) + case .bool(let bool): return NSNumber(value: bool) + case .null: return NSNull() + } + } +} + +extension JSONValue: Equatable { + // Object members compare as a set, since JSON gives key order no meaning. + public static func == (lhs: JSONValue, rhs: JSONValue) -> Bool { + switch (lhs, rhs) { + case let (.object(left), .object(right)): + guard left.count == right.count else { return false } + return left.allSatisfy { member in + right.contains { $0.key == member.key && $0.value == member.value } + } + case let (.array(left), .array(right)): + return left == right + case let (.string(left), .string(right)): + return left == right + case let (.number(left), .number(right)): + return left == right + case let (.bool(left), .bool(right)): + return left == right + case (.null, .null): + return true + default: + return false + } + } +} + +extension JSONValue: Decodable { + private struct AnyKey: CodingKey { + let stringValue: String + let intValue: Int? = nil + + init(stringValue: String) { + self.stringValue = stringValue + } + + init?(intValue: Int) { + nil + } + } + + // JSONDecoder hands keys back unordered; MCPClientCatalog.decode(_:) parses the bytes itself to keep them. + public init(from decoder: any Decoder) throws { + if let container = try? decoder.container(keyedBy: AnyKey.self) { + self = .object(try container.allKeys.map { key in + (key: key.stringValue, value: try container.decode(JSONValue.self, forKey: key)) + }) + return + } + if var container = try? decoder.unkeyedContainer() { + var items: [JSONValue] = [] + while !container.isAtEnd { + items.append(try container.decode(JSONValue.self)) + } + self = .array(items) + return + } + let container = try decoder.singleValueContainer() + if container.decodeNil() { + self = .null + } else if let bool = try? container.decode(Bool.self) { + self = .bool(bool) + } else if let number = try? container.decode(Double.self) { + self = .number(number) + } else { + self = .string(try container.decode(String.self)) + } + } +} + +struct JSONTextStyle { + var indentUnit: String + var newline: String + var multiline: Bool + var spaced: Bool + + static let compact = JSONTextStyle(indentUnit: "", newline: "", multiline: false, spaced: false) + + func render(_ value: JSONValue, indent: String) -> String { + let inner = multiline ? indent + indentUnit : indent + switch value { + case .object(let members): + guard !members.isEmpty else { return "{}" } + let colon = spaced ? ": " : ":" + return wrap("{", "}", indent: indent, members.map { Self.quoted($0.key) + colon + render($0.value, indent: inner) }) + case .array(let items): + guard !items.isEmpty else { return "[]" } + return wrap("[", "]", indent: indent, items.map { render($0, indent: inner) }) + case .string(let string): + return Self.quoted(string) + case .number(let number): + return Self.numberText(number) + case .bool(let bool): + return bool ? "true" : "false" + case .null: + return "null" + } + } + + private func wrap(_ open: String, _ close: String, indent: String, _ items: [String]) -> String { + guard multiline else { + return open + items.joined(separator: spaced ? ", " : ",") + close + } + let inner = indent + indentUnit + return open + newline + items.map { inner + $0 }.joined(separator: "," + newline) + newline + indent + close + } + + static func quoted(_ string: String) -> String { + var result = "\"" + for scalar in string.unicodeScalars { + switch scalar { + case "\"": result += "\\\"" + case "\\": result += "\\\\" + case "\n": result += "\\n" + case "\r": result += "\\r" + case "\t": result += "\\t" + case "\u{08}": result += "\\b" + case "\u{0C}": result += "\\f" + default: + if scalar.value < 0x20 { + result += String(format: "\\u%04x", scalar.value) + } else { + result.unicodeScalars.append(scalar) + } + } + } + return result + "\"" + } + + static func numberText(_ number: Double) -> String { + guard number.isFinite else { return "null" } + if number == number.rounded(), abs(number) < 1e15 { + return String(Int64(number)) + } + return "\(number)" + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPClientCatalog.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPClientCatalog.swift new file mode 100644 index 0000000000..3f1e9e679f --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPClientCatalog.swift @@ -0,0 +1,347 @@ +import Foundation + +public enum MCPClientPlatform: String, Sendable, CaseIterable { + case macos + case linux +} + +public enum MCPClientCatalogError: Error, Sendable, Equatable { + case unsupportedSchemaVersion(Int) + case invalidField([String]) +} + +public struct MCPClientCatalog: Decodable, Sendable { + public let schemaVersion: Int + public let serverName: String + public let configs: [String: MCPClientConfigSet] + public let clients: [MCPClientDefinition] + + static let supportedSchemaVersion = 1 + + // Parses the bytes itself: JSONDecoder loses the key order the `json` template filter has to keep. + public static func decode(_ data: Data) throws -> MCPClientCatalog { + guard let root = try JSONCParser.parse(Array(data)) else { throw MCPClientCatalogError.invalidField([]) } + return try MCPClientCatalog(json: root.value) + } + + public init(from decoder: any Decoder) throws { + try self.init(json: try JSONValue(from: decoder)) + } + + init(json: JSONValue) throws { + let reader = FieldReader(value: json, path: []) + schemaVersion = try reader.int("schemaVersion") + guard schemaVersion <= Self.supportedSchemaVersion else { + throw MCPClientCatalogError.unsupportedSchemaVersion(schemaVersion) + } + serverName = try reader.string("serverName") + var configs: [String: MCPClientConfigSet] = [:] + for (id, value) in try reader.objectMembers("configs") { + configs[id] = try MCPClientConfigSet(reader: FieldReader(value: value, path: ["configs", id])) + } + self.configs = configs + clients = try reader.array("clients").enumerated().map { position, value in + try MCPClientDefinition(reader: FieldReader(value: value, path: ["clients", String(position)])) + } + } + + public func client(id: String) -> MCPClientDefinition? { + clients.first { $0.id == id } + } + + public func client(matchingClientInfoName name: String) -> MCPClientDefinition? { + clients(matchingClientInfoName: name).first + } + + // ChatGPT and Codex CLI send one clientInfo name, so a session is one of them only when that one alone is detected. + public func client(matchingClientInfoName name: String, detected: Set) -> MCPClientDefinition? { + let candidates = clients(matchingClientInfoName: name) + guard candidates.count > 1 else { return candidates.first } + let found = candidates.filter { detected.contains($0.id) } + return found.count == 1 ? found.first : nil + } + + // Exact names first, then the longest prefix, so "Visual Studio Code - Insiders" is not taken for VS Code. + public func clients(matchingClientInfoName name: String) -> [MCPClientDefinition] { + let lowered = name.lowercased() + let exact = clients.filter { $0.clientInfoNames.contains { $0.lowercased() == lowered } } + guard exact.isEmpty else { return exact } + let lengths = clients.map { client in + client.clientInfoPrefixes.filter { lowered.hasPrefix($0.lowercased()) }.map(\.count).max() ?? 0 + } + guard let longest = lengths.max(), longest > 0 else { return [] } + return zip(clients, lengths).filter { $0.1 == longest }.map(\.0) + } + + public func location(for client: MCPClientDefinition, platform: MCPClientPlatform) -> MCPConfigLocation? { + guard let configID = client.configID else { return nil } + return configs[configID]?.location(for: platform) + } + + public func httpEntry(for client: MCPClientDefinition) -> JSONValue? { + MCPClientPlatform.allCases.lazy + .compactMap { client.spec(for: $0)?.entry } + .first { Self.containsURLPlaceholder($0) } + } + + private static func containsURLPlaceholder(_ value: JSONValue) -> Bool { + switch value { + case .string(let string): return string.contains("{url}") + case .array(let items): return items.contains(where: containsURLPlaceholder) + case .object(let members): return members.contains { containsURLPlaceholder($0.value) } + case .number, .bool, .null: return false + } + } +} + +public struct MCPClientConfigSet: Sendable, Equatable { + public let macos: MCPConfigLocation? + public let linux: MCPConfigLocation? + + public func location(for platform: MCPClientPlatform) -> MCPConfigLocation? { + switch platform { + case .macos: return macos + case .linux: return linux + } + } + + init(reader: FieldReader) throws { + macos = try reader.optional("macos").map(MCPConfigLocation.init(reader:)) + linux = try reader.optional("linux").map(MCPConfigLocation.init(reader:)) + } +} + +public enum MCPConfigFormat: String, Sendable, CaseIterable { + case json + case jsonc + case toml + case yaml +} + +public struct MCPConfigLocation: Sendable, Equatable { + public let path: String + public let format: MCPConfigFormat + public let keyPath: [String] + public let isWritable: Bool + // An older layout of the same file and scope, such as opencode's V1 `mcp.`; read like keyPath, never written. + public let alternateKeyPaths: [[String]] + public let otherScopes: [[String]] + + public init( + path: String, + format: MCPConfigFormat, + keyPath: [String], + isWritable: Bool = true, + alternateKeyPaths: [[String]] = [], + otherScopes: [[String]] = [] + ) { + self.path = path + self.format = format + self.keyPath = keyPath + self.isWritable = isWritable + self.alternateKeyPaths = alternateKeyPaths + self.otherScopes = otherScopes + } + + init(reader: FieldReader) throws { + path = try reader.string("path") + format = try reader.enumeration("format") + keyPath = try reader.strings("keyPath") + isWritable = try reader.optional("writable").map { try $0.bool() } ?? true + alternateKeyPaths = try Self.keyPaths(reader.optional("alternateKeyPaths")) + otherScopes = try Self.keyPaths(reader.optional("otherScopes")) + } + + private static func keyPaths(_ reader: FieldReader?) throws -> [[String]] { + try reader?.elements().map { try $0.stringArray() } ?? [] + } +} + +public struct MCPClientDefinition: Sendable, Equatable, Identifiable { + public let id: String + public let name: String + public let icon: String + public let configID: String? + public let clientInfoNames: [String] + public let clientInfoPrefixes: [String] + public let macos: MCPClientPlatformSpec? + public let linux: MCPClientPlatformSpec? + + public func spec(for platform: MCPClientPlatform) -> MCPClientPlatformSpec? { + switch platform { + case .macos: return macos + case .linux: return linux + } + } + + init(reader: FieldReader) throws { + id = try reader.string("id") + name = try reader.string("name") + icon = try reader.string("icon") + configID = try reader.optional("config").map { try $0.string() } + let clientInfo = reader.optional("clientInfo") + clientInfoNames = try clientInfo?.optional("names").map { try $0.stringArray() } ?? [] + clientInfoPrefixes = try clientInfo?.optional("prefixes").map { try $0.stringArray() } ?? [] + macos = try reader.optional("macos").map(MCPClientPlatformSpec.init(reader:)) + linux = try reader.optional("linux").map(MCPClientPlatformSpec.init(reader:)) + } +} + +public struct MCPClientPlatformSpec: Sendable, Equatable { + public let detect: [MCPClientDetectRule] + public let entry: JSONValue? + public let install: [MCPInstallMethod] + public let applies: MCPClientApplies? + public let unsupported: MCPClientUnsupportedReason? + + init(reader: FieldReader) throws { + detect = try reader.optional("detect")?.elements().map(MCPClientDetectRule.init(reader:)) ?? [] + entry = reader.optional("entry")?.value + install = try reader.optional("install")?.elements().map(MCPInstallMethod.init(reader:)) ?? [] + applies = try reader.optional("applies").map { try $0.enumeration() } + unsupported = try reader.optional("unsupported").map { try $0.enumeration() } + } +} + +public enum MCPClientDetectRule: Sendable, Equatable { + case bundleID(String) + case binary(name: String, paths: [String]) + case path(String) + + init(reader: FieldReader) throws { + if let bundleID = reader.optional("bundleId") { + self = .bundleID(try bundleID.string()) + } else if let binary = reader.optional("binary") { + self = .binary(name: try binary.string(), paths: try reader.optional("paths")?.stringArray() ?? []) + } else if let path = reader.optional("path") { + self = .path(try path.string()) + } else { + throw MCPClientCatalogError.invalidField(reader.path) + } + } +} + +public enum MCPInstallMethod: Sendable, Equatable { + case deeplink(scheme: String, template: String) + case cli(MCPCLIInstall) + case file + + init(reader: FieldReader) throws { + switch try reader.string("kind") { + case "deeplink": + self = .deeplink(scheme: try reader.string("scheme"), template: try reader.string("url")) + case "cli": + self = .cli(MCPCLIInstall( + binary: try reader.string("binary"), + add: try reader.strings("add"), + remove: try reader.optional("remove")?.stringArray(), + update: try reader.optional("update").map { try $0.enumeration() } ?? .add + )) + case "file": + self = .file + default: + throw MCPClientCatalogError.invalidField(reader.path + ["kind"]) + } + } +} + +public struct MCPCLIInstall: Sendable, Equatable { + public let binary: String + public let add: [String] + public let remove: [String]? + public let update: MCPCLIUpdate + + public init(binary: String, add: [String], remove: [String]?, update: MCPCLIUpdate) { + self.binary = binary + self.add = add + self.remove = remove + self.update = update + } +} + +public enum MCPCLIUpdate: String, Sendable { + case add + case removeThenAdd +} + +public enum MCPClientApplies: String, Sendable { + case live + case newSession + case restartApp + case reload +} + +public enum MCPClientUnsupportedReason: String, Sendable { + case remoteOnly + case stdioOnly +} + +struct FieldReader { + let value: JSONValue + let path: [String] + + func optional(_ key: String) -> FieldReader? { + guard let child = value[key], child != .null else { return nil } + return FieldReader(value: child, path: path + [key]) + } + + func field(_ key: String) throws -> FieldReader { + guard let child = optional(key) else { throw MCPClientCatalogError.invalidField(path + [key]) } + return child + } + + func string(_ key: String) throws -> String { + try field(key).string() + } + + func strings(_ key: String) throws -> [String] { + try field(key).stringArray() + } + + func int(_ key: String) throws -> Int { + guard case .number(let number) = try field(key).value, let integer = Int(exactly: number) else { + throw MCPClientCatalogError.invalidField(path + [key]) + } + return integer + } + + func array(_ key: String) throws -> [JSONValue] { + guard let items = try field(key).value.arrayValue else { throw MCPClientCatalogError.invalidField(path + [key]) } + return items + } + + func objectMembers(_ key: String) throws -> [(key: String, value: JSONValue)] { + guard let members = try field(key).value.objectMembers else { throw MCPClientCatalogError.invalidField(path + [key]) } + return members + } + + func enumeration(_ key: String) throws -> Value where Value.RawValue == String { + try field(key).enumeration() + } + + func string() throws -> String { + guard let string = value.stringValue else { throw MCPClientCatalogError.invalidField(path) } + return string + } + + func bool() throws -> Bool { + guard case .bool(let bool) = value else { throw MCPClientCatalogError.invalidField(path) } + return bool + } + + func stringArray() throws -> [String] { + try elements().map { try $0.string() } + } + + func elements() throws -> [FieldReader] { + guard let items = value.arrayValue else { throw MCPClientCatalogError.invalidField(path) } + return items.enumerated().map { FieldReader(value: $1, path: path + [String($0)]) } + } + + func enumeration() throws -> Value where Value.RawValue == String { + guard let raw = value.stringValue, let result = Value(rawValue: raw) else { + throw MCPClientCatalogError.invalidField(path) + } + return result + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPClientStatusEvaluator.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPClientStatusEvaluator.swift new file mode 100644 index 0000000000..3bf7867979 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPClientStatusEvaluator.swift @@ -0,0 +1,181 @@ +import Foundation + +public enum MCPClientStaleReason: Sendable, Equatable { + case commandMissing + case otherCommand + case otherURL + case otherToken + // The keyPath entry is current, but another scope's entry points elsewhere and wins there; `scope` is the key `*` matched. + case shadowed(scope: String) +} + +public enum MCPClientConfigStatus: Sendable, Equatable { + case notConfigured + case current + case stale(MCPClientStaleReason) + case otherScope + case disabled + // A stdio target was expected, but the entry reaches TablePro over HTTP with a token the user chose. + case usesToken + case unreadable + case unknown +} + +public enum MCPClientStatusEvaluator { + public static func status( + configText: String?, + location: MCPConfigLocation?, + serverName: String, + expected: MCPServerTarget, + fileExists: (String) -> Bool + ) -> MCPClientConfigStatus { + guard let location, location.format != .yaml else { return .unknown } + guard let configText else { return .notConfigured } + let entry: JSONValue? + do { + entry = try primaryEntry(serverName, location: location, text: configText) + } catch { + return .unreadable + } + let scoped = location.otherScopes.flatMap { scope in + scopedEntries(serverName, in: scope, text: configText, format: location.format, fileExists: fileExists) + } + guard let entry else { + return scoped.isEmpty ? .notConfigured : .otherScope + } + let status = evaluate(entry, expected: expected, fileExists: fileExists) + guard status == .current else { return status } + let shadowing = scoped.first { scopedEntry in + if case .stale = evaluate(scopedEntry.entry, expected: expected, fileExists: fileExists) { return true } + return false + } + return shadowing.map { .stale(.shadowed(scope: $0.scope)) } ?? .current + } + + private static func evaluate(_ entry: JSONValue, expected: MCPServerTarget, fileExists: (String) -> Bool) -> MCPClientConfigStatus { + // Turned off is the user's choice whatever the entry points at, so it is never offered an Update. + guard !isTurnedOff(entry) else { return .disabled } + switch expected { + case .stdio(let command): + guard let configured = commandValue(entry) else { + return urlValue(entry) == nil ? .stale(.otherCommand) : .usesToken + } + guard configured == command else { + return fileExists(configured) ? .stale(.otherCommand) : .stale(.commandMissing) + } + case let .http(url, authorization): + guard let configured = urlValue(entry) else { + return commandValue(entry) == nil ? .stale(.otherURL) : .stale(.otherCommand) + } + guard sameEndpoint(configured, url) else { return .stale(.otherURL) } + guard let header = authorizationValue(entry), + header.trimmingCharacters(in: .whitespaces) == authorization.trimmingCharacters(in: .whitespaces) else { + return .stale(.otherToken) + } + } + return .current + } + + private static func isTurnedOff(_ entry: JSONValue) -> Bool { + entry["disabled"] == .bool(true) || entry["enabled"] == .bool(false) + } + + private static func primaryEntry(_ name: String, location: MCPConfigLocation, text: String) throws -> JSONValue? { + for keyPath in [location.keyPath] + location.alternateKeyPaths { + if let entry = try member(name, at: keyPath, in: text, format: location.format) { + return entry + } + } + return nil + } + + private static func member(_ name: String, at keyPath: [String], in text: String, format: MCPConfigFormat) throws -> JSONValue? { + switch format { + case .json, .jsonc: return try JSONCMemberEditor.member(name, at: keyPath, in: text) + case .toml: return try TOMLTableEditor.member(name, at: keyPath, in: text) + case .yaml: return nil + } + } + + // Another scope is someone else's layout, so an odd shape there is skipped rather than making the file unreadable. + private static func scopedEntries( + _ name: String, + in scope: [String], + text: String, + format: MCPConfigFormat, + fileExists: (String) -> Bool + ) -> [(scope: String, entry: JSONValue)] { + let label = scope.joined(separator: ".") + guard format == .json || format == .jsonc else { + guard !scope.contains("*"), let entry = try? member(name, at: scope, in: text, format: format) else { return [] } + return [(scope: label, entry: entry)] + } + guard let root = try? JSONCParser.parse(Array(text.utf8))?.value else { return [] } + var candidates: [(scope: String, value: JSONValue)] = [(scope: label, value: root)] + for key in scope { + candidates = candidates.flatMap { candidate -> [(scope: String, value: JSONValue)] in + if key == "*" { + return (candidate.value.objectMembers ?? []) + .filter { !isGoneFolder($0.key, fileExists: fileExists) } + .map { (scope: $0.key, value: $0.value) } + } + return candidate.value[key].map { [(scope: candidate.scope, value: $0)] } ?? [] + } + } + return candidates.compactMap { candidate in + candidate.value[name].map { (scope: candidate.scope, entry: $0) } + } + } + + // Claude Code keeps a project for every folder it ever opened, so one deleted since says nothing about today. + private static func isGoneFolder(_ key: String, fileExists: (String) -> Bool) -> Bool { + key.hasPrefix("/") && !fileExists(key) + } + + private static func commandValue(_ entry: JSONValue) -> String? { + for key in MCPEntryUpdate.commandKeys { + switch entry[key] { + case .string(let command)?: return command + case .array(let parts)?: return parts.first?.stringValue + default: continue + } + } + return nil + } + + private static func urlValue(_ entry: JSONValue) -> String? { + MCPEntryUpdate.urlKeys.lazy.compactMap { entry[$0]?.stringValue }.first + } + + private static func authorizationValue(_ entry: JSONValue) -> String? { + for key in ["headers", "http_headers"] { + if let header = entry[key]?.objectMembers?.first(where: { $0.key.lowercased() == "authorization" }) { + return header.value.stringValue + } + } + return nil + } + + private static let loopbackHosts: Set = ["localhost", "127.0.0.1", "::1", "[::1]"] + + private static func sameEndpoint(_ configured: String, _ expected: String) -> Bool { + guard let left = URLComponents(string: configured), let right = URLComponents(string: expected), + let leftScheme = left.scheme?.lowercased(), let rightScheme = right.scheme?.lowercased() else { + return configured == expected + } + let leftHost = left.host?.lowercased() ?? "" + let rightHost = right.host?.lowercased() ?? "" + let sameHost = leftHost == rightHost || (loopbackHosts.contains(leftHost) && loopbackHosts.contains(rightHost)) + return leftScheme == rightScheme && sameHost + && port(left, scheme: leftScheme) == port(right, scheme: rightScheme) + && trimmedPath(left.path) == trimmedPath(right.path) + } + + private static func port(_ components: URLComponents, scheme: String) -> Int? { + components.port ?? (scheme == "https" ? 443 : scheme == "http" ? 80 : nil) + } + + private static func trimmedPath(_ path: String) -> String { + path.hasSuffix("/") ? String(path.dropLast()) : path + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPConfigFileWriter.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPConfigFileWriter.swift new file mode 100644 index 0000000000..33120f0767 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPConfigFileWriter.swift @@ -0,0 +1,126 @@ +import Foundation + +public struct MCPConfigWriteResult: Sendable, Equatable { + public let backupURL: URL? + public let created: Bool + + public init(backupURL: URL?, created: Bool) { + self.backupURL = backupURL + self.created = created + } +} + +public enum MCPConfigWriteError: Error, Sendable, Equatable { + case changedDuringWrite + case verificationFailed + case io(String) +} + +// FileManager is documented as safe across threads when it has no delegate, and this one never gets one. +public struct MCPConfigFileWriter: @unchecked Sendable { + public static let backupSuffix = ".tablepro-backup" + + private let fileManager: FileManager + + public init(fileManager: FileManager = .default) { + self.fileManager = fileManager + } + + // `Data.write(.atomic)` would turn a symlinked config into a regular file, so the link is resolved and its + // target replaced; replaceItemAt keeps the target's mode and leaves the previous bytes beside it as the backup. + public func update(_ url: URL, transform: (String?) throws -> String) throws -> MCPConfigWriteResult { + let target = try resolvedTarget(url) + var snapshot = try read(target) + for _ in 0..<2 { + let newData = Data(try transform(try snapshot.map(decoded)).utf8) + if newData == snapshot { + return MCPConfigWriteResult(backupURL: nil, created: false) + } + let latest = try read(target) + guard latest == snapshot else { + snapshot = latest + continue + } + let result = snapshot == nil ? try create(newData, at: target) : try replace(target, with: newData) + guard try read(target) == newData else { throw MCPConfigWriteError.verificationFailed } + return result + } + throw MCPConfigWriteError.changedDuringWrite + } + + private func resolvedTarget(_ url: URL) throws -> URL { + var current = url.standardizedFileURL + for _ in 0..<32 { + guard let destination = try? fileManager.destinationOfSymbolicLink(atPath: current.path) else { + return current + } + let next = destination.hasPrefix("/") + ? URL(fileURLWithPath: destination) + : current.deletingLastPathComponent().appendingPathComponent(destination) + current = next.standardizedFileURL + } + throw MCPConfigWriteError.io("symlinkLoop") + } + + private func read(_ url: URL) throws -> Data? { + do { + return try Data(contentsOf: url) + } catch let error as CocoaError where error.code == .fileReadNoSuchFile { + return nil + } catch { + throw Self.ioError(error) + } + } + + private func decoded(_ data: Data) throws -> String { + guard let text = String(data: data, encoding: .utf8) else { throw MCPConfigWriteError.io("notUTF8") } + return text + } + + private func replace(_ target: URL, with data: Data) throws -> MCPConfigWriteResult { + let backupName = target.lastPathComponent + Self.backupSuffix + do { + let directory = try fileManager.url( + for: .itemReplacementDirectory, in: .userDomainMask, appropriateFor: target, create: true + ) + defer { try? fileManager.removeItem(at: directory) } + let temporary = directory.appendingPathComponent(target.lastPathComponent) + try data.write(to: temporary) + _ = try fileManager.replaceItemAt( + target, withItemAt: temporary, backupItemName: backupName, options: [.withoutDeletingBackupItem] + ) + } catch { + throw Self.ioError(error) + } + return MCPConfigWriteResult(backupURL: target.deletingLastPathComponent().appendingPathComponent(backupName), created: false) + } + + // Client configs often hold env secrets, so a new one starts at 0600. + private func create(_ data: Data, at target: URL) throws -> MCPConfigWriteResult { + let parent = target.deletingLastPathComponent() + do { + try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) + let directory = try fileManager.url( + for: .itemReplacementDirectory, in: .userDomainMask, appropriateFor: parent, create: true + ) + defer { try? fileManager.removeItem(at: directory) } + let temporary = directory.appendingPathComponent(target.lastPathComponent) + guard fileManager.createFile(atPath: temporary.path, contents: data, attributes: [.posixPermissions: 0o600]) else { + throw MCPConfigWriteError.io("createFailed") + } + try fileManager.moveItem(at: temporary, to: target) + } catch let error as CocoaError where error.code == .fileWriteFileExists { + throw MCPConfigWriteError.changedDuringWrite + } catch let error as MCPConfigWriteError { + throw error + } catch { + throw Self.ioError(error) + } + return MCPConfigWriteResult(backupURL: nil, created: true) + } + + private static func ioError(_ error: any Error) -> MCPConfigWriteError { + let nsError = error as NSError + return .io("\(nsError.localizedDescription) (\(nsError.domain) \(nsError.code))") + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPEntryUpdate.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPEntryUpdate.swift new file mode 100644 index 0000000000..505780c841 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPEntryUpdate.swift @@ -0,0 +1,89 @@ +import Foundation + +// Update rewrites only how an entry reaches TablePro; other catalog members are added only when missing, never widening a user's limit. +enum MCPEntryUpdate { + static let commandKeys = ["command", "cmd"] + static let urlKeys = ["url", "serverUrl", "httpUrl", "uri"] + + private static let alwaysWritten: Set = ["type", "args"] + private static let placeholders: Set = ["command", "url", "authorization", "name"] + + private enum Transport { + case command + case url + } + + enum Plan: Equatable { + case replace(JSONValue) + case merge(JSONValue) + } + + static func plan(existing: JSONValue?, entry: JSONValue, template: JSONValue) -> Plan { + guard let existing, case .object(let current) = existing, case .object(let wanted) = entry else { return .replace(entry) } + if let had = transport(of: existing), let have = transport(of: entry), had != have { + // A merge would leave the old transport's members behind; the user's choices still carry over. + return .replace(.object(wanted.map { part in + guard !alwaysWritten.contains(part.key), !holdsPlaceholder(template[part.key]), + let mine = existing[part.key] else { return part } + return (key: part.key, value: mine) + })) + } + return .merge(.object(overlay(current, with: wanted, template: template, alwaysWritten: alwaysWritten))) + } + + private static func overlay( + _ current: [(key: String, value: JSONValue)], + with wanted: [(key: String, value: JSONValue)], + template: JSONValue?, + alwaysWritten: Set + ) -> [(key: String, value: JSONValue)] { + var members = current + for part in wanted { + guard let position = members.firstIndex(where: { $0.key == part.key }) else { + members.append(part) + continue + } + let partTemplate = template?[part.key] + if alwaysWritten.contains(part.key) { + members[position].value = part.value + } else if holdsPlaceholder(partTemplate) { + if case .object(let inner) = members[position].value, case .object(let innerWanted) = part.value { + members[position].value = .object(overlay(inner, with: innerWanted, template: partTemplate, alwaysWritten: [])) + } else { + members[position].value = part.value + } + } + } + return members + } + + private static func transport(of entry: JSONValue) -> Transport? { + if commandKeys.contains(where: { entry[$0] != nil }) { return .command } + if urlKeys.contains(where: { entry[$0] != nil }) { return .url } + return nil + } + + static func holdsPlaceholder(_ template: JSONValue?) -> Bool { + switch template { + case .string(let text)?: + return placeholderNames(in: text).contains { placeholders.contains($0) } + case .array(let items)?: + return items.contains { holdsPlaceholder($0) } + case .object(let members)?: + return members.contains { holdsPlaceholder($0.value) } + default: + return false + } + } + + private static func placeholderNames(in text: String) -> [String] { + var names: [String] = [] + var rest = Substring(text) + while let open = rest.firstIndex(of: "{"), let close = rest[open...].firstIndex(of: "}") { + let expression = rest[rest.index(after: open).. String { + var result = "" + var rest = Substring(path) + while let open = rest.range(of: "${"), let close = rest[open.upperBound...].firstIndex(of: "}") { + result += rest[.. String { + let trimmedHome = home.count > 1 && home.hasSuffix("/") ? String(home.dropLast()) : home + if path == "~" { + return trimmedHome + } + if path.hasPrefix("~/") { + return trimmedHome + path.dropFirst() + } + return path + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPTemplateRenderer.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPTemplateRenderer.swift new file mode 100644 index 0000000000..5b7530cebe --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/MCPTemplateRenderer.swift @@ -0,0 +1,173 @@ +import Foundation + +public enum MCPServerTarget: Sendable, Equatable { + case stdio(command: String) + case http(url: String, authorization: String) +} + +public enum MCPTemplateError: Error, Sendable, Equatable { + case missingEntry + case unknownPlaceholder(String) + case unavailablePlaceholder(String) + case unknownFilter(String) + case invalidURL +} + +public struct MCPTemplateRenderer: Sendable { + public let serverName: String + public let target: MCPServerTarget + + public init(serverName: String, target: MCPServerTarget) { + self.serverName = serverName + self.target = target + } + + public func entry(_ spec: MCPClientPlatformSpec) throws -> JSONValue { + guard let template = spec.entry else { throw MCPTemplateError.missingEntry } + return try entry(template: template) + } + + public func entry(template: JSONValue) throws -> JSONValue { + switch template { + case .string(let string): + return .string(try render(string, entry: nil)) + case .array(let items): + return .array(try items.map { try entry(template: $0) }) + case .object(let members): + return .object(try members.map { (key: $0.key, value: try entry(template: $0.value)) }) + case .number, .bool, .null: + return template + } + } + + public func arguments(_ template: [String], entry: JSONValue) throws -> [String] { + try template.map { try render($0, entry: entry) } + } + + public func url(_ template: String, entry: JSONValue) throws -> URL { + guard let url = URL(string: try render(template, entry: entry)) else { throw MCPTemplateError.invalidURL } + return url + } + + public func snippet(for location: MCPConfigLocation?, entry: JSONValue) -> String { + let path = (location?.keyPath ?? []) + [serverName] + let json = JSONValue.nested(path, leaf: entry).prettyText() + switch location?.format { + case .toml: + guard let block = try? TOMLTableEditor.renderBlock(header: path, value: entry, newline: "\n") else { return json } + return String(block.dropLast()) + case .yaml: + return Self.yamlLines(key: path[0], value: JSONValue.nested(Array(path.dropFirst()), leaf: entry), indent: "") + .joined(separator: "\n") + case .json, .jsonc, nil: + return json + } + } + + public func commandLine(binary: String, arguments: [String]) -> String { + ([binary] + arguments).map(Self.shellQuoted).joined(separator: " ") + } + + static func shellQuoted(_ word: String) -> String { + guard !word.isEmpty else { return "''" } + if word.unicodeScalars.allSatisfy(Self.shellSafe.contains) { + return word + } + return "'" + word.replacingOccurrences(of: "'", with: "'\\''", options: .literal) + "'" + } + + private static let shellSafe = CharacterSet( + charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_@%+=:,./-" + ) + + private static let uriUnreserved = CharacterSet( + charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + ) + + private enum Rendered { + case text(String) + case json(JSONValue) + } + + private func render(_ template: String, entry: JSONValue?) throws -> String { + var result = "" + var rest = Substring(template) + while let open = rest.firstIndex(of: "{"), let close = rest[open...].firstIndex(of: "}") { + result += rest[.. String { + let parts = expression.split(separator: "|", omittingEmptySubsequences: false).map(String.init) + var value = try source(parts[0], entry: entry) + for filter in parts.dropFirst() { + value = try apply(filter, to: value) + } + switch value { + case .text(let text): return text + case .json(let json): return json.compactText + } + } + + private func source(_ name: String, entry: JSONValue?) throws -> Rendered { + switch (name, target) { + case ("name", _): + return .text(serverName) + case let ("command", .stdio(command)): + return .text(command) + case let ("url", .http(url, _)): + return .text(url) + case let ("authorization", .http(_, authorization)): + return .text(authorization) + case ("command", _), ("url", _), ("authorization", _): + throw MCPTemplateError.unavailablePlaceholder(name) + case ("entry", _): + guard let entry else { throw MCPTemplateError.unavailablePlaceholder(name) } + return .json(entry) + case ("entryWithName", _): + guard let entry else { throw MCPTemplateError.unavailablePlaceholder(name) } + let others = entry.objectMembers?.filter { $0.key != "name" } ?? [] + return .json(.object([(key: "name", value: .string(serverName))] + others)) + default: + throw MCPTemplateError.unknownPlaceholder(name) + } + } + + private func apply(_ filter: String, to value: Rendered) throws -> Rendered { + switch (filter, value) { + case let ("json", .json(json)): + return .text(json.compactText) + case let ("json", .text(text)): + return .text(JSONTextStyle.quoted(text)) + case let ("base64", .text(text)): + return .text(Data(text.utf8).base64EncodedString()) + case let ("uri", .text(text)): + return .text(text.addingPercentEncoding(withAllowedCharacters: Self.uriUnreserved) ?? text) + default: + throw MCPTemplateError.unknownFilter(filter) + } + } + + private static func yamlLines(key: String, value: JSONValue, indent: String) -> [String] { + let label = indent + yamlKey(key) + ":" + switch value { + case .object(let members) where !members.isEmpty: + return [label] + members.flatMap { yamlLines(key: $0.key, value: $0.value, indent: indent + " ") } + case .array(let items) where !items.isEmpty: + return [label] + items.map { indent + " - " + yamlFlow.render($0, indent: "") } + default: + return [label + " " + yamlFlow.render(value, indent: "")] + } + } + + // A JSON scalar or flow collection is also valid YAML. + private static let yamlFlow = JSONTextStyle(indentUnit: "", newline: "", multiline: false, spaced: true) + + private static func yamlKey(_ key: String) -> String { + guard !key.isEmpty, key.utf8.allSatisfy(TOMLParser.isBareKeyByte) else { return JSONTextStyle.quoted(key) } + return key + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/TOMLSyntax.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/TOMLSyntax.swift new file mode 100644 index 0000000000..ff1008e9d8 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/TOMLSyntax.swift @@ -0,0 +1,461 @@ +import Foundation + +struct TOMLStatement { + enum Kind { + case table(keys: [String], isArray: Bool) + case keyValue(keys: [String], value: JSONValue) + } + + let kind: Kind + let lineStart: Int + // Past the line break that ends the statement, or the end of the text. + let lineEnd: Int +} + +// Enough of TOML 1.0 to find every statement's keys and line range and to read the values a server table holds. +// Dates and times come back as their source text. +struct TOMLParser { + private let bytes: [UInt8] + private var index = 0 + private var depth = 0 + + static func parse(_ bytes: [UInt8]) throws -> [TOMLStatement] { + var parser = TOMLParser(bytes: bytes) + return try parser.statements() + } + + private init(bytes: [UInt8]) { + self.bytes = bytes + if bytes.starts(with: [0xEF, 0xBB, 0xBF]) { + index = 3 + } + } + + private var current: UInt8? { + index < bytes.count ? bytes[index] : nil + } + + private func byte(at offset: Int) -> UInt8? { + offset < bytes.count ? bytes[offset] : nil + } + + private mutating func statements() throws -> [TOMLStatement] { + var result: [TOMLStatement] = [] + while true { + skipBlankLinesAndComments() + guard let byte = current else { return result } + let start = index + var lineStart = start + while lineStart > 0, bytes[lineStart - 1] != 0x0A { + lineStart -= 1 + } + let kind: TOMLStatement.Kind + if byte == UInt8(ascii: "[") { + let isArray = self.byte(at: index + 1) == UInt8(ascii: "[") + index += isArray ? 2 : 1 + skipSpaces() + let keys = try parseKey() + try expect("]") + if isArray { + try expect("]") + } + kind = .table(keys: keys, isArray: isArray) + } else { + let keys = try parseKey() + try expect("=") + skipSpaces() + kind = .keyValue(keys: keys, value: try parseValue()) + } + skipSpaces() + skipComment() + switch current { + case nil: + break + case 0x0A: + index += 1 + case 0x0D where self.byte(at: index + 1) == 0x0A: + index += 2 + default: + throw MCPConfigEditError.invalidSyntax + } + result.append(TOMLStatement(kind: kind, lineStart: lineStart, lineEnd: index)) + } + } + + private mutating func skipSpaces() { + while let byte = current, byte == 0x20 || byte == 0x09 { + index += 1 + } + } + + private mutating func skipComment() { + guard current == UInt8(ascii: "#") else { return } + while let byte = current, byte != 0x0A { + index += 1 + } + if index > 0, bytes[index - 1] == 0x0D { + index -= 1 + } + } + + private mutating func skipBlankLinesAndComments() { + while let byte = current { + switch byte { + case 0x20, 0x09, 0x0A, 0x0D: + index += 1 + case UInt8(ascii: "#"): + while let next = current, next != 0x0A { + index += 1 + } + default: + return + } + } + } + + private mutating func expect(_ character: Character) throws { + skipSpaces() + guard let ascii = character.asciiValue, current == ascii else { throw MCPConfigEditError.invalidSyntax } + index += 1 + } + + private mutating func parseKey() throws -> [String] { + var keys: [String] = [] + while true { + skipSpaces() + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + switch byte { + case UInt8(ascii: "\""): + keys.append(try parseBasicString()) + case UInt8(ascii: "'"): + keys.append(try parseLiteralString()) + default: + let start = index + while let next = current, Self.isBareKeyByte(next) { + index += 1 + } + guard index > start else { throw MCPConfigEditError.invalidSyntax } + keys.append(utf8Text(bytes[start.. Bool { + switch byte { + case UInt8(ascii: "A")...UInt8(ascii: "Z"), UInt8(ascii: "a")...UInt8(ascii: "z"), + UInt8(ascii: "0")...UInt8(ascii: "9"), UInt8(ascii: "_"), UInt8(ascii: "-"): + return true + default: + return false + } + } + + private mutating func parseValue() throws -> JSONValue { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + switch byte { + case UInt8(ascii: "\""): + if self.byte(at: index + 1) == UInt8(ascii: "\""), self.byte(at: index + 2) == UInt8(ascii: "\"") { + return .string(try parseMultilineBasicString()) + } + return .string(try parseBasicString()) + case UInt8(ascii: "'"): + if self.byte(at: index + 1) == UInt8(ascii: "'"), self.byte(at: index + 2) == UInt8(ascii: "'") { + return .string(try parseMultilineLiteralString()) + } + return .string(try parseLiteralString()) + case UInt8(ascii: "["): + return try nested { try $0.parseArray() } + case UInt8(ascii: "{"): + return try nested { try $0.parseInlineTable() } + default: + return try parseBareValue() + } + } + + private mutating func nested(_ body: (inout TOMLParser) throws -> JSONValue) throws -> JSONValue { + depth += 1 + defer { depth -= 1 } + guard depth <= 128 else { throw MCPConfigEditError.invalidSyntax } + return try body(&self) + } + + private mutating func skipValueTrivia() { + while let byte = current { + switch byte { + case 0x20, 0x09, 0x0A, 0x0D: + index += 1 + case UInt8(ascii: "#"): + while let next = current, next != 0x0A { + index += 1 + } + default: + return + } + } + } + + private mutating func parseArray() throws -> JSONValue { + index += 1 + var items: [JSONValue] = [] + while true { + skipValueTrivia() + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + if byte == UInt8(ascii: "]") { + index += 1 + return .array(items) + } + items.append(try parseValue()) + skipValueTrivia() + switch current { + case UInt8(ascii: ","): + index += 1 + case UInt8(ascii: "]"): + index += 1 + return .array(items) + default: + throw MCPConfigEditError.invalidSyntax + } + } + } + + private mutating func parseInlineTable() throws -> JSONValue { + index += 1 + var members: [(key: String, value: JSONValue)] = [] + while true { + skipValueTrivia() + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + if byte == UInt8(ascii: "}") { + index += 1 + return .object(members) + } + let keys = try parseKey() + try expect("=") + skipSpaces() + let value = try parseValue() + try TOMLTree.insert(value, at: keys, into: &members) + skipValueTrivia() + switch current { + case UInt8(ascii: ","): + index += 1 + case UInt8(ascii: "}"): + index += 1 + return .object(members) + default: + throw MCPConfigEditError.invalidSyntax + } + } + } + + private mutating func parseBareValue() throws -> JSONValue { + let start = index + while let byte = current, !Self.endsBareValue(byte) { + index += 1 + } + // A local date and a time may be separated by one space. + if index - start == 10, current == 0x20, let next = self.byte(at: index + 1), Self.isDigit(next), + bytes[start + 4] == UInt8(ascii: "-") { + index += 1 + while let byte = current, !Self.endsBareValue(byte) { + index += 1 + } + } + let text = utf8Text(bytes[start.. Double? { + let cleaned = text.replacingOccurrences(of: "_", with: "") + guard !cleaned.isEmpty, !text.hasPrefix("_"), !text.hasSuffix("_"), !text.contains("__") else { return nil } + for (prefix, radix) in [("0x", 16), ("0o", 8), ("0b", 2)] where cleaned.hasPrefix(prefix) { + return Int64(cleaned.dropFirst(2), radix: radix).map(Double.init) + } + let allowed = Set("0123456789+-.eE") + guard cleaned.allSatisfy({ allowed.contains($0) }), cleaned.contains(where: \.isNumber) else { return nil } + return Double(cleaned) + } + + private static func endsBareValue(_ byte: UInt8) -> Bool { + switch byte { + case 0x20, 0x09, 0x0A, 0x0D, UInt8(ascii: ","), UInt8(ascii: "]"), UInt8(ascii: "}"), UInt8(ascii: "#"): + return true + default: + return false + } + } + + private static func isDigit(_ byte: UInt8) -> Bool { + byte >= UInt8(ascii: "0") && byte <= UInt8(ascii: "9") + } + + private mutating func parseBasicString() throws -> String { + index += 1 + var buffer: [UInt8] = [] + while true { + guard let byte = current, byte != 0x0A else { throw MCPConfigEditError.invalidSyntax } + switch byte { + case UInt8(ascii: "\""): + index += 1 + return utf8Text(buffer) + case UInt8(ascii: "\\"): + index += 1 + try appendEscape(to: &buffer) + default: + buffer.append(byte) + index += 1 + } + } + } + + private mutating func parseMultilineBasicString() throws -> String { + index += 3 + skipLeadingNewline() + var buffer: [UInt8] = [] + while true { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + if byte == UInt8(ascii: "\""), closesMultiline(UInt8(ascii: "\"")) { + return finishMultiline(buffer, quote: UInt8(ascii: "\"")) + } + if byte == UInt8(ascii: "\\") { + index += 1 + if let whitespaceEnd = lineEndingBackslashEnd() { + index = whitespaceEnd + } else { + try appendEscape(to: &buffer) + } + continue + } + buffer.append(byte) + index += 1 + } + } + + private mutating func parseLiteralString() throws -> String { + index += 1 + let start = index + while let byte = current, byte != UInt8(ascii: "'") { + guard byte != 0x0A else { throw MCPConfigEditError.invalidSyntax } + index += 1 + } + guard current != nil else { throw MCPConfigEditError.invalidSyntax } + let text = utf8Text(bytes[start.. String { + index += 3 + skipLeadingNewline() + var buffer: [UInt8] = [] + while true { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + if byte == UInt8(ascii: "'"), closesMultiline(UInt8(ascii: "'")) { + return finishMultiline(buffer, quote: UInt8(ascii: "'")) + } + buffer.append(byte) + index += 1 + } + } + + private mutating func skipLeadingNewline() { + if current == 0x0A { + index += 1 + } else if current == 0x0D, byte(at: index + 1) == 0x0A { + index += 2 + } + } + + private func closesMultiline(_ quote: UInt8) -> Bool { + byte(at: index + 1) == quote && byte(at: index + 2) == quote + } + + // Up to two quotes may sit right before the closing three. + private mutating func finishMultiline(_ buffer: [UInt8], quote: UInt8) -> String { + var run = 0 + while byte(at: index + run) == quote { + run += 1 + } + let extra = min(max(run - 3, 0), 2) + index += extra + 3 + return utf8Text(buffer + Array(repeating: quote, count: extra)) + } + + private func lineEndingBackslashEnd() -> Int? { + var offset = index + while let next = byte(at: offset), next == 0x20 || next == 0x09 { + offset += 1 + } + guard byte(at: offset) == 0x0A || (byte(at: offset) == 0x0D && byte(at: offset + 1) == 0x0A) else { return nil } + while let next = byte(at: offset), next == 0x20 || next == 0x09 || next == 0x0A || next == 0x0D { + offset += 1 + } + return offset + } + + private mutating func appendEscape(to buffer: inout [UInt8]) throws { + guard let byte = current else { throw MCPConfigEditError.invalidSyntax } + index += 1 + switch byte { + case UInt8(ascii: "\""), UInt8(ascii: "\\"): buffer.append(byte) + case UInt8(ascii: "b"): buffer.append(0x08) + case UInt8(ascii: "t"): buffer.append(0x09) + case UInt8(ascii: "n"): buffer.append(0x0A) + case UInt8(ascii: "f"): buffer.append(0x0C) + case UInt8(ascii: "r"): buffer.append(0x0D) + case UInt8(ascii: "e"): buffer.append(0x1B) + case UInt8(ascii: "x"): try appendScalar(hexDigits: 2, to: &buffer) + case UInt8(ascii: "u"): try appendScalar(hexDigits: 4, to: &buffer) + case UInt8(ascii: "U"): try appendScalar(hexDigits: 8, to: &buffer) + default: throw MCPConfigEditError.invalidSyntax + } + } + + private mutating func appendScalar(hexDigits: Int, to buffer: inout [UInt8]) throws { + guard index + hexDigits <= bytes.count, + let value = UInt32(utf8Text(bytes[index..<(index + hexDigits)]), radix: 16), + let scalar = Unicode.Scalar(value) else { + throw MCPConfigEditError.invalidSyntax + } + index += hexDigits + buffer.append(contentsOf: Array(String(Character(scalar)).utf8)) + } +} + +enum TOMLTree { + static func insert(_ value: JSONValue, at keys: [String], into members: inout [(key: String, value: JSONValue)]) throws { + guard let key = keys.first else { throw MCPConfigEditError.invalidSyntax } + let position = members.firstIndex { $0.key == key } + guard keys.count > 1 else { + guard position == nil else { throw MCPConfigEditError.duplicateKey(keys) } + members.append((key: key, value: value)) + return + } + var children: [(key: String, value: JSONValue)] = [] + if let position { + guard case .object(let existing) = members[position].value else { throw MCPConfigEditError.invalidSyntax } + children = existing + } + try insert(value, at: Array(keys.dropFirst()), into: &children) + if let position { + members[position].value = .object(children) + } else { + members.append((key: key, value: .object(children))) + } + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCPClients/TOMLTableEditor.swift b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/TOMLTableEditor.swift new file mode 100644 index 0000000000..7d1d3f3fc3 --- /dev/null +++ b/apple/Packages/TableProMacCore/Sources/TableProMCPClients/TOMLTableEditor.swift @@ -0,0 +1,347 @@ +import Foundation + +// Reads and splices one `[.]` table (and its subtables) of a TOML file such as Codex's config.toml. +// A server defined any other way is refused, since rewriting it would mean rewriting someone else's layout. +public enum TOMLTableEditor { + public static func member(_ name: String, at keyPath: [String], in text: String) throws -> JSONValue? { + let file = try File(text: text, target: keyPath + [name]) + guard file.layout.table != nil || !file.layout.subtables.isEmpty else { return nil } + return try file.serverValue() + } + + public static func upserting(_ name: String, value: JSONValue, at keyPath: [String], in text: String) throws -> String { + let target = keyPath + [name] + let file = try File(text: text, target: target) + guard !file.layout.containerIsValue else { throw MCPConfigEditError.unsupportedShape } + let block = try renderBlock(header: target, value: value, newline: file.newline) + + var splices: [(range: Range, text: String)] = file.layout.subtables.map { (range: file.removalRange(of: $0), text: "") } + if let table = file.layout.table { + let range = file.blockRange(of: table) + let endsWithoutNewline = range.upperBound == file.bytes.count && file.bytes.last != 0x0A + splices.append((range: range, text: endsWithoutNewline ? String(block.dropLast()) : block)) + } + var result = file.applying(splices) + if file.layout.table == nil { + result = appending(block, to: result, newline: file.newline) + } + guard try member(name, at: keyPath, in: result) == value else { throw MCPConfigEditError.verificationFailed } + return result + } + + // Update: `entry` is rendered from `template`, which says which keys are TablePro's to rewrite (MCPEntryUpdate). + public static func updating( + _ name: String, + entry: JSONValue, + template: JSONValue, + at keyPath: [String], + in text: String + ) throws -> String { + switch MCPEntryUpdate.plan(existing: try member(name, at: keyPath, in: text), entry: entry, template: template) { + case .replace(let value): try upserting(name, value: value, at: keyPath, in: text) + case .merge(let value): try merging(name, value: value, at: keyPath, in: text) + } + } + + // Rewrites only the keys `value` names, so the user's other keys, comments and subtables stay; an unchanged key is untouched. + public static func merging(_ name: String, value: JSONValue, at keyPath: [String], in text: String) throws -> String { + let target = keyPath + [name] + let file = try File(text: text, target: target) + guard !file.layout.containerIsValue else { throw MCPConfigEditError.unsupportedShape } + guard case .object(let wanted) = value, file.layout.table != nil || !file.layout.subtables.isEmpty else { + return try upserting(name, value: value, at: keyPath, in: text) + } + let existing = try file.serverValue() + let merged = existing.merging(value) + + var splices: [(range: Range, text: String)] = [] + var appended: [String] = [] + for part in wanted { + guard let newValue = merged[part.key], existing[part.key] != newValue else { continue } + let line = renderKey(part.key) + " = " + (try renderValue(newValue)) + for subtable in file.subtables(under: part.key) { + splices.append((range: file.removalRange(of: subtable), text: "")) + } + let statements = file.tableStatements(for: part.key) + guard let first = statements.first else { + appended.append(line) + continue + } + splices.append((range: file.lineRange(of: first), text: line + file.lineBreak(endingStatement: first))) + splices += statements.dropFirst().map { (range: file.lineRange(of: $0), text: "") } + } + + if let table = file.layout.table, !appended.isEmpty { + let end = file.blockRange(of: table).upperBound + let lead = end > 0 && file.bytes[end - 1] != 0x0A ? file.newline : "" + splices.append((range: end.. String { + var result = text + if !result.isEmpty, result.utf8.last != 0x0A { + result += newline + } + if !result.isEmpty, !endsWithBlankLine(result) { + result += newline + } + return result + block + } + + public static func removing(_ name: String, at keyPath: [String], in text: String) throws -> String { + let file = try File(text: text, target: keyPath + [name]) + let blocks = (file.layout.table.map { [$0] } ?? []) + file.layout.subtables + guard !blocks.isEmpty else { return text } + let result = file.applying(blocks.map { (range: file.removalRange(of: $0), text: "") }) + guard try member(name, at: keyPath, in: result) == nil else { throw MCPConfigEditError.verificationFailed } + return result + } + + // Compared as bytes: Swift treats "\r\n" as one Character, so hasSuffix("\n") misses it. + private static func endsWithBlankLine(_ text: String) -> Bool { + var bytes = Array(text.utf8) + guard bytes.last == 0x0A else { return false } + bytes.removeLast() + if bytes.last == 0x0D { + bytes.removeLast() + } + return bytes.isEmpty || bytes.last == 0x0A + } + + static func renderBlock(header: [String], value: JSONValue, newline: String) throws -> String { + guard case .object(let members) = value else { throw MCPConfigEditError.unsupportedShape } + var lines = ["[" + header.map(renderKey).joined(separator: ".") + "]"] + for member in members { + lines.append(renderKey(member.key) + " = " + (try renderValue(member.value))) + } + return lines.joined(separator: newline) + newline + } + + static func renderKey(_ key: String) -> String { + guard !key.isEmpty, key.utf8.allSatisfy(TOMLParser.isBareKeyByte) else { return quoted(key) } + return key + } + + static func renderValue(_ value: JSONValue) throws -> String { + switch value { + case .string(let string): + return quoted(string) + case .number(let number): + if number.isNaN { return "nan" } + if number.isInfinite { return number < 0 ? "-inf" : "inf" } + return JSONTextStyle.numberText(number) + case .bool(let bool): + return bool ? "true" : "false" + case .null: + throw MCPConfigEditError.unsupportedShape + case .array(let items): + return "[" + (try items.map(renderValue)).joined(separator: ", ") + "]" + case .object(let members): + guard !members.isEmpty else { return "{}" } + let pairs = try members.map { renderKey($0.key) + " = " + (try renderValue($0.value)) } + return "{ " + pairs.joined(separator: ", ") + " }" + } + } + + static func quoted(_ string: String) -> String { + var result = "\"" + for scalar in string.unicodeScalars { + switch scalar { + case "\"": result += "\\\"" + case "\\": result += "\\\\" + case "\u{08}": result += "\\b" + case "\t": result += "\\t" + case "\n": result += "\\n" + case "\u{0C}": result += "\\f" + case "\r": result += "\\r" + default: + if scalar.value < 0x20 || scalar.value == 0x7F { + result += String(format: "\\u%04X", scalar.value) + } else { + result.unicodeScalars.append(scalar) + } + } + } + return result + "\"" + } +} + +private struct ServerLayout { + var table: Int? + var subtables: [Int] = [] + // A key on the way to the server holds a value (usually an inline table), so no [table] can be added under it. + var containerIsValue = false +} + +private struct File { + let bytes: [UInt8] + let statements: [TOMLStatement] + let target: [String] + let layout: ServerLayout + let newline: String + + init(text: String, target: [String]) throws { + bytes = Array(text.utf8) + statements = try TOMLParser.parse(bytes) + self.target = target + newline = text.contains("\r\n") ? "\r\n" : "\n" + layout = try Self.layout(statements, target: target) + } + + private static func layout(_ statements: [TOMLStatement], target: [String]) throws -> ServerLayout { + var layout = ServerLayout() + var table: [String] = [] + for (position, statement) in statements.enumerated() { + switch statement.kind { + case let .table(keys, isArray): + table = keys + if isArray { + if keys.starts(with: target) || target.starts(with: keys) { + throw MCPConfigEditError.unsupportedShape + } + } else if keys == target { + guard layout.table == nil else { throw MCPConfigEditError.duplicateKey(target) } + layout.table = position + } else if keys.starts(with: target) { + layout.subtables.append(position) + } + case let .keyValue(keys, value): + guard !table.starts(with: target) else { continue } + let full = table + keys + if full.starts(with: target) { + throw MCPConfigEditError.unsupportedShape + } + if target.starts(with: full) { + if Self.lookup(value, path: Array(target.dropFirst(full.count))) { + throw MCPConfigEditError.unsupportedShape + } + layout.containerIsValue = true + } + } + } + return layout + } + + private static func lookup(_ value: JSONValue, path: [String]) -> Bool { + var current = value + for key in path { + guard let next = current[key] else { return false } + current = next + } + return true + } + + func serverValue() throws -> JSONValue { + var members: [(key: String, value: JSONValue)] = [] + let headers = ((layout.table.map { [$0] } ?? []) + layout.subtables).sorted() + for header in headers { + guard case let .table(keys, _) = statements[header].kind else { continue } + let prefix = Array(keys.dropFirst(target.count)) + if !prefix.isEmpty { + try ensureObject(at: prefix, in: &members) + } + for statement in statements[(header + 1)...] { + guard case let .keyValue(keys, value) = statement.kind else { break } + try TOMLTree.insert(value, at: prefix + keys, into: &members) + } + } + return .object(members) + } + + private func ensureObject(at path: [String], in members: inout [(key: String, value: JSONValue)]) throws { + guard let key = path.first else { return } + var children: [(key: String, value: JSONValue)] = [] + let position = members.firstIndex { $0.key == key } + if let position { + guard case .object(let existing) = members[position].value else { throw MCPConfigEditError.invalidSyntax } + children = existing + } + try ensureObject(at: Array(path.dropFirst()), in: &children) + if let position { + members[position].value = .object(children) + } else { + members.append((key: key, value: .object(children))) + } + } + + func tableStatements(for key: String) -> [Int] { + guard let table = layout.table else { return [] } + var result: [Int] = [] + var position = table + 1 + while position < statements.count, case let .keyValue(keys, _) = statements[position].kind { + if keys.first == key { + result.append(position) + } + position += 1 + } + return result + } + + func subtables(under key: String) -> [Int] { + layout.subtables.filter { header in + guard case let .table(keys, _) = statements[header].kind else { return false } + return keys.count > target.count && keys[target.count] == key + } + } + + func lineRange(of statement: Int) -> Range { + statements[statement].lineStart.. String { + let end = statements[statement].lineEnd + guard end > 0, bytes[end - 1] == 0x0A else { return "" } + return end > 1 && bytes[end - 2] == 0x0D ? "\r\n" : "\n" + } + + func blockRange(of header: Int) -> Range { + var last = header + while last + 1 < statements.count, case .keyValue = statements[last + 1].kind { + last += 1 + } + return statements[header].lineStart.. Range { + let range = blockRange(of: header) + guard range.lowerBound > 0 else { return range } + var previousStart = range.lowerBound - 1 + while previousStart > 0, bytes[previousStart - 1] != 0x0A { + previousStart -= 1 + } + guard isBlank(previousStart..) -> Bool { + bytes[range].allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0A || $0 == 0x0D } + } + + // At one offset a removal goes before an insertion, or it would take the inserted text with it. + func applying(_ splices: [(range: Range, text: String)]) -> String { + var result = bytes + let ordered = splices.sorted { left, right in + left.range.lowerBound != right.range.lowerBound + ? left.range.lowerBound > right.range.lowerBound + : !left.range.isEmpty && right.range.isEmpty + } + for splice in ordered { + result.replaceSubrange(splice.range, with: Array(splice.text.utf8)) + } + return utf8Text(result) + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift index 89f38cd9e9..2a58e8731b 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift @@ -13,7 +13,8 @@ public enum DownloadedBinary { var hasher = SHA256() do { - while let chunk = try handle.read(upToCount: hashChunkSize), !chunk.isEmpty { + // Each read returns an autoreleased buffer, and the caller's job may not drain its pool until the whole file is read. + while let chunk = try autoreleasepool(invoking: { try handle.read(upToCount: hashChunkSize) }), !chunk.isEmpty { hasher.update(data: chunk) } } catch { @@ -23,11 +24,7 @@ public enum DownloadedBinary { return hasher.finalize().map { String(format: "%02x", $0) }.joined() } - /// `ditto -xk` writes `com.apple.quarantine` onto every file it extracts, not just the - /// directory it extracts into, so clearing the bundle root alone leaves the Mach-O inside - /// `Contents/MacOS` still quarantined. Gatekeeper reads the flag on the file it is about - /// to map, and refuses a quarantined bundle with "library load disallowed by system - /// policy" even inside a process that has `com.apple.security.cs.disable-library-validation`. + // `ditto -xk` quarantines every extracted file, and Gatekeeper refuses to map a quarantined Mach-O even with library validation off. public static func stripQuarantine(at url: URL) { stripQuarantineFromSingleItem(at: url) diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift index 30a7ce249d..4806926b3f 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift @@ -40,6 +40,12 @@ private struct Scan { /// in the same gutter agree about where a statement stops. private var boundaries: any SQLStatementBoundaryTracking + // A `;` that ended the statement while the tracker could still take it back at the next token, and its line. + private var heldEnd: (index: Int, line: Int)? + + // The client delimiter a `DELIMITER` line set, which ends a statement wherever it stands in code. + private var delimiter: SQLClientDelimiter? + init(text: NSString, grammar: SQLLexicalGrammar) { self.text = text self.length = text.length @@ -53,16 +59,33 @@ private struct Scan { while index < length { step() } + settleHeldEnd(word: nil) closeFramesAtEndOfDocument() } + private mutating func settleHeldEnd(word: String?) { + guard let held = heldEnd else { return } + heldEnd = nil + guard !boundaries.continuesStatement(with: word) else { return } + boundaries.reset() + if frames.last?.kind == .statement { + popFrame(.statement, end: held.index, endLine: held.line) + } + } + private mutating func step() { let character = text.character(at: index) if consumeTrivia(character) { return } + if consumeClientLine(character) { return } + if consumeDelimiter() { return } if consumeNonCode() { return } if consumeSlashLine(character) { return } + if heldEnd != nil, character != SqlLexer.semicolon { + let word = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar).text + settleHeldEnd(word: word.isEmpty ? nil : word) + } openStatementIfNeeded() consumeStructure(character) } @@ -161,6 +184,22 @@ private struct Scan { /// frame of their own: when it ends the unit, whatever the fold still holds open ends with it. private mutating func consumeSemicolon() { let endsStatement = boundaries.observeSemicolon() + if heldEnd != nil || (endsStatement && boundaries.awaitsContinuation) { + heldEnd = heldEnd ?? (index, line) + index += 1 + return + } + // The rows after a `COPY ... FROM STDIN` belong to it, so its fold runs through them. + if endsStatement, boundaries.copiesFromStandardInput { + let rows = SQLPsqlScript.copyRows(afterSemicolonAt: index, in: text, length: length) + boundaries.reset() + line += newlines(in: rows) + index = NSMaxRange(rows) + if frames.last?.kind == .statement { + popFrame(.statement, end: index) + } + return + } if endsStatement { boundaries.reset() if grammar.contains(.plsqlBlocks) { @@ -182,7 +221,8 @@ private struct Scan { } private mutating func consumeKeyword(_ character: UInt16) { - let word = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) + let read = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) + let word = read.text.isEmpty ? read : delimiter?.cut(read, readAt: index, in: text, length: length) ?? read guard !word.text.isEmpty else { if !SqlLexer.isWhitespace(character) { boundaries.observeSymbol(character) @@ -211,6 +251,50 @@ private struct Scan { } } + // MARK: - Client lines + + /// A `DELIMITER` line or a psql `\restrict` line, where no statement is open, runs nothing and folds nothing. + private mutating func consumeClientLine(_ character: UInt16) -> Bool { + guard frames.isEmpty else { return false } + if grammar.contains(.delimiterDirective), character | 0x20 == Self.smallD, + let directive = SQLDelimiterDirective.line(at: index, in: text, length: length) { + if let set = directive.delimiter { + delimiter = SQLClientDelimiter(set) + } + index = NSMaxRange(directive.range) + return true + } + if grammar.contains(.psqlScripts), character == SqlLexer.backslash, + let restrict = SQLPsqlScript.restrictLine(at: index, in: text, length: length) { + index = NSMaxRange(restrict) + return true + } + return false + } + + /// A client delimiter ends the statement whatever the fold still holds open, as it does for the mysql client. + private mutating func consumeDelimiter() -> Bool { + guard let delimiter, delimiter.starts(in: text, at: index, length: length) else { return false } + settleHeldEnd(word: nil) + closeFramesThroughStatement(end: index) + if frames.last?.kind == .statement { + popFrame(.statement, end: index) + } + boundaries.reset() + index += delimiter.units.count + return true + } + + private func newlines(in range: NSRange) -> Int { + var count = 0 + for offset in range.location.. Bool { @@ -234,6 +318,8 @@ private struct Scan { let startLine = line if span.kind.isComment { boundaries.observeGap() + } else { + settleHeldEnd(word: nil) } switch span.kind { case .lineComment: diff --git a/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLICallbackListenerTests.swift b/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLICallbackListenerTests.swift index 66bbae8c8b..44a6981bb0 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLICallbackListenerTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLICallbackListenerTests.swift @@ -24,6 +24,21 @@ struct CLICallbackListenerTests { #expect(await listener.waitForCallback(timeout: .seconds(5)) == .approved(code: "c-1")) } + @Test("A pairing TablePro could not finish reaches the command as a failure, not a refusal") + func listenerReportsAFailedPairing() async throws { + let listener = CLICallbackListener(state: "s2") + defer { listener.stop() } + let port = try await listener.start() + + let session = URLSession(configuration: .ephemeral) + let callback = try #require(URL(string: "http://127.0.0.1:\(port)\(CLIPairingRequest.callbackPath)?error=server_error&state=s2")) + let (body, response) = try await session.data(from: callback) + #expect((response as? HTTPURLResponse)?.statusCode == 200) + #expect(String(data: body, encoding: .utf8)?.contains("could not finish pairing") == true) + + #expect(await listener.waitForCallback(timeout: .seconds(5)) == .failed) + } + @Test("A listener nobody calls gives up") func listenerTimeout() async throws { let listener = CLICallbackListener(state: "s1") diff --git a/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLIFailureTests.swift b/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLIFailureTests.swift index 2c42d1c929..90a08a128f 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLIFailureTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProCommandLineTests/CLIFailureTests.swift @@ -34,4 +34,24 @@ struct CLIFailureTests { func rpcErrors(code: Int, expected: CLIExitCode) { #expect(CLIFailure.fromRpcError(code: code, message: "m").exitCode == expected) } + + @Test("A server that is on but failed to start sends the reader to Settings > MCP, said once") + func failedStartNamesSettings() { + let failure = CLILive.failure(acquiring: MCPHandshakeAcquisitionError.serverFailedToStart) + #expect(failure.exitCode == .failure) + #expect(failure.message.contains("Settings > MCP")) + #expect(!failure.message.contains("did not start its MCP server:")) + } + + @Test("A server that is turned off is a refusal") + func turnedOffIsARefusal() { + #expect(CLILive.failure(acquiring: MCPHandshakeAcquisitionError.serverTurnedOff).exitCode == .refused) + } + + @Test("Any other launch failure keeps its own reason") + func otherLaunchFailureKeepsItsReason() { + let failure = CLILive.failure(acquiring: MCPHandshakeAcquisitionError.launchFailed("no app")) + #expect(failure.exitCode == .failure) + #expect(failure.message.contains("no app")) + } } diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/JSONCMemberEditorTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/JSONCMemberEditorTests.swift new file mode 100644 index 0000000000..2821e938cb --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/JSONCMemberEditorTests.swift @@ -0,0 +1,447 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct JSONCMemberEditorTests { + private let entry: JSONValue = .object([(key: "command", value: .string("/b"))]) + + @Test("A member is read through comments and trailing commas") + func readsThroughComments() throws { + let text = """ + // Zed settings + // + // documentation: https://zed.dev/docs/configuring-zed + { + "ui_font_size": 16, + /* servers */ + "context_servers": { + "tablepro": { + "command": "/b", // bridge + "args": [], + }, + }, + } + """ + let member = try JSONCMemberEditor.member("tablepro", at: ["context_servers"], in: text) + #expect(member == .object([(key: "command", value: .string("/b")), (key: "args", value: .array([]))])) + #expect(try JSONCMemberEditor.member("other", at: ["context_servers"], in: text) == nil) + #expect(try JSONCMemberEditor.member("tablepro", at: ["missing"], in: text) == nil) + } + + @Test("Strings that look like comments and escapes are read as strings") + func stringsAndEscapes() throws { + let text = #"{"url": "http://x//y/*z*/", "quote": "a\"b\\cé😀", "mcpServers": {}}"# + #expect(try JSONCMemberEditor.member("url", at: [], in: text) == .string("http://x//y/*z*/")) + #expect(try JSONCMemberEditor.member("quote", at: [], in: text) == .string("a\"b\\c\u{e9}\u{1F600}")) + } + + @Test("Inserting into an existing container keeps comments, order and the file's four-space indent") + func insertKeepsLayout() throws { + let text = """ + { + // Servers I use + "mcpServers": { + "other": { + "command": "/usr/local/bin/other" // keep + } + }, + /* trailing block */ + "theme": "dark" + } + + """ + let result = try JSONCMemberEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcpServers"], in: text) + #expect(result == """ + { + // Servers I use + "mcpServers": { + "other": { + "command": "/usr/local/bin/other" // keep + }, + "tablepro": { + "command": "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp", + "args": [] + } + }, + /* trailing block */ + "theme": "dark" + } + + """) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: result) == text) + } + + @Test("A missing container is created after the last top-level key") + func createsContainer() throws { + let text = """ + { + "coworkUserFilesPath": "/Users/me/Claude", + "preferences": { + "quickEntry": true + } + } + """ + let result = try JSONCMemberEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcpServers"], in: text) + #expect(result == """ + { + "coworkUserFilesPath": "/Users/me/Claude", + "preferences": { + "quickEntry": true + }, + "mcpServers": { + "tablepro": { + "command": "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp", + "args": [] + } + } + } + """) + } + + @Test("Every missing object along a longer key path is created") + func createsNestedPath() throws { + let result = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcp", "servers"], in: "{}\n") + #expect(result == """ + { + "mcp": { + "servers": { + "tablepro": { + "command": "/b" + } + } + } + } + + """) + let partial = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcp", "servers"], in: #"{"mcp": {"a": 1}}"#) + #expect(partial == #"{"mcp": {"a": 1, "servers": {"tablepro": {"command": "/b"}}}}"#) + } + + @Test("An existing member is replaced where it stands") + func replacesInPlace() throws { + let text = """ + { + "mcpServers": { + "tablepro": { + "command": "/old/path", + "args": ["--x"] + }, + "other": { "command": "o" } + } + } + """ + let result = try JSONCMemberEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcpServers"], in: text) + #expect(result == """ + { + "mcpServers": { + "tablepro": { + "command": "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp", + "args": [] + }, + "other": { "command": "o" } + } + } + """) + #expect(try JSONCMemberEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcpServers"], in: result) == result) + } + + @Test("A file with trailing commas and tabs keeps both") + func trailingCommasAndTabs() throws { + let text = [ + "{", + "\t// VS Code MCP", + "\t\"servers\": {", + "\t\t\"other\": {", + "\t\t\t\"type\": \"stdio\",", + "\t\t\t\"command\": \"o\",", + "\t\t},", + "\t},", + "\t\"inputs\": [],", + "}", + ].joined(separator: "\n") + let result = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["servers"], in: text) + #expect(result == [ + "{", + "\t// VS Code MCP", + "\t\"servers\": {", + "\t\t\"other\": {", + "\t\t\t\"type\": \"stdio\",", + "\t\t\t\"command\": \"o\",", + "\t\t},", + "\t\t\"tablepro\": {", + "\t\t\t\"command\": \"/b\"", + "\t\t},", + "\t},", + "\t\"inputs\": [],", + "}", + ].joined(separator: "\n")) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["servers"], in: result) == text) + } + + @Test("CRLF line breaks are kept") + func crlf() throws { + let result = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: "{\r\n \"a\": 1\r\n}\r\n") + #expect(result == "{\r\n \"a\": 1,\r\n \"mcpServers\": {\r\n \"tablepro\": {\r\n \"command\": \"/b\"\r\n }\r\n }\r\n}\r\n") + } + + @Test("Empty, blank and comment-only files count as an empty object") + func emptyFiles() throws { + let expected = """ + { + "mcpServers": { + "tablepro": { + "command": "/b" + } + } + } + + """ + #expect(try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: "") == expected) + #expect(try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: " \n") == expected) + #expect(try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: "// my config") == "// my config\n" + expected) + #expect(try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: "/* a */\n") == "/* a */\n" + expected) + #expect(try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: "// only a comment") == nil) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: "// only a comment") == "// only a comment") + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: "") == "") + } + + @Test("An empty container opens up at the file's indent, after any comment inside it") + func emptyContainers() throws { + let inline = """ + { + "mcpServers": {}, + "x": 1 + } + """ + #expect(try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: inline) == """ + { + "mcpServers": { + "tablepro": { + "command": "/b" + } + }, + "x": 1 + } + """) + let commented = """ + { + "context_servers": { + // none yet + } + } + """ + #expect(try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["context_servers"], in: commented) == """ + { + "context_servers": { + // none yet + "tablepro": { + "command": "/b" + } + } + } + """) + } + + @Test("A minified file stays on one line") + func minified() throws { + let text = #"{"mcpServers":{"other":{"command":"o"}}}"# + let result = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: text) + #expect(result == #"{"mcpServers":{"other":{"command":"o"},"tablepro":{"command":"/b"}}}"#) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: result) == text) + } + + @Test("Removing a middle member takes its line, comma and comment") + func removeMiddle() throws { + let text = """ + { + "mcpServers": { + "a": {}, + "tablepro": { + "command": "/b" + }, // ours + "c": {} + } + } + """ + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: text) == """ + { + "mcpServers": { + "a": {}, + "c": {} + } + } + """) + } + + @Test("Removing the only member leaves an empty container") + func removeOnly() throws { + let text = """ + { + "mcpServers": { + "tablepro": { + "command": "/b" + } + }, + "x": 1 + } + """ + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: text) == """ + { + "mcpServers": {}, + "x": 1 + } + """) + } + + @Test("Removing from one line keeps the separators valid") + func removeInline() throws { + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: #"{"mcpServers": {"a": 1, "tablepro": {"command": "/b"}}}"#) + == #"{"mcpServers": {"a": 1}}"#) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: #"{"mcpServers": {"tablepro": {}, "a": 1}}"#) + == #"{"mcpServers": {"a": 1}}"#) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: #"{"mcpServers": { "tablepro": {} }}"#) + == #"{"mcpServers": {}}"#) + } + + @Test("Removing a member that is not there changes nothing") + func removeAbsent() throws { + let text = "{\n // keep\n \"mcpServers\": {\"a\": 1}\n}\n" + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: text) == text) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["missing"], in: text) == text) + } + + @Test("A duplicate key on the walked path is refused") + func duplicateKeys() { + let duplicateContainer = #"{"mcpServers": {}, "mcpServers": {}}"# + #expect(throws: MCPConfigEditError.duplicateKey(["mcpServers"])) { + try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: duplicateContainer) + } + #expect(throws: MCPConfigEditError.duplicateKey(["mcpServers"])) { + try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: duplicateContainer) + } + #expect(throws: MCPConfigEditError.duplicateKey(["mcpServers"])) { + try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: duplicateContainer) + } + let duplicateMember = #"{"mcpServers": {"tablepro": {}, "tablepro": {}}}"# + #expect(throws: MCPConfigEditError.duplicateKey(["mcpServers", "tablepro"])) { + try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: duplicateMember) + } + #expect(throws: MCPConfigEditError.duplicateKey(["mcp", "servers"])) { + try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcp", "servers"], in: #"{"mcp": {"servers": {}, "servers": {}}}"#) + } + } + + @Test("A non-object on the path is refused") + func notAnObject() { + #expect(throws: MCPConfigEditError.notAnObject(["mcpServers"])) { + try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: #"{"mcpServers": []}"#) + } + #expect(throws: MCPConfigEditError.notAnObject(["mcp", "servers"])) { + try JSONCMemberEditor.member("tablepro", at: ["mcp", "servers"], in: #"{"mcp": {"servers": "x"}}"#) + } + #expect(throws: MCPConfigEditError.notAnObject([])) { + try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: "[]") + } + } + + @Test("Broken syntax is refused", arguments: [ + #"{"a": }"#, #"{"a": 1 /* open"#, #"{"a" 1}"#, "{,}", #"{"a": 1,,}"#, #"{"a": 'x'}"#, #"{"a": 01}"#, "{} {}", "{\"a\": \"\n\"}", + ]) + func invalidSyntax(text: String) { + #expect(throws: MCPConfigEditError.invalidSyntax) { + try JSONCMemberEditor.upserting("tablepro", value: .object([]), at: ["mcpServers"], in: text) + } + } + + @Test("Escapes and non-ASCII text round trip through an edit") + func escapesRoundTrip() throws { + let value: JSONValue = .object([ + (key: "command", value: .string("/Apps/Tab\"le\\Pro é\n\u{1}/bridge")), + (key: "timeout", value: .number(300)), + (key: "ratio", value: .number(0.5)), + (key: "enabled", value: .bool(false)), + (key: "extra", value: .null), + ]) + let text = "{\n \"név\": \"ő\"\n}\n" + let result = try JSONCMemberEditor.upserting("tablepro", value: value, at: ["mcpServers"], in: text) + #expect(try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: result) == value) + #expect(result.hasPrefix("{\n \"név\": \"ő\",\n")) + #expect(result.contains(#""timeout": 300,"#)) + let parsed = try MCPClientsFixtures.parsed(result) + let servers = try #require(parsed["mcpServers"] as? NSDictionary) + let server = try #require(servers["tablepro"] as? NSDictionary) + #expect(server["command"] as? String == "/Apps/Tab\"le\\Pro é\n\u{1}/bridge") + } + + @Test("A duplicate key elsewhere in the file does not block the edit") + func unrelatedDuplicate() throws { + let text = "{\n \"a\": 1,\n \"a\": 2,\n \"mcpServers\": {}\n}" + let result = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: text) + #expect(try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: result) == entry) + } + + @Test("Merging rewrites only the members the entry names and keeps the rest where they stand") + func mergeKeepsUserMembers() throws { + let text = """ + { + "mcpServers": { + "tablepro": { + "command": "/old/path", // moved + "args": ["--x"], + "autoApprove": ["list_connections"], + "env": { "A": "1" }, + "timeout": 30 + } + } + } + """ + let value: JSONValue = .object([ + (key: "command", value: .string("/b")), + (key: "args", value: .array([])), + (key: "env", value: .object([])), + (key: "type", value: .string("stdio")), + ]) + let result = try JSONCMemberEditor.merging("tablepro", value: value, at: ["mcpServers"], in: text) + #expect(result == """ + { + "mcpServers": { + "tablepro": { + "command": "/b", // moved + "args": [], + "autoApprove": ["list_connections"], + "env": { "A": "1" }, + "timeout": 30, + "type": "stdio" + } + } + } + """) + #expect(try JSONCMemberEditor.merging("tablepro", value: value, at: ["mcpServers"], in: result) == result) + } + + @Test("Merging a missing or non-object entry writes the whole value") + func mergeWithoutEntry() throws { + let missing = #"{"mcpServers": {"other": {}}}"# + #expect(try JSONCMemberEditor.merging("tablepro", value: entry, at: ["mcpServers"], in: missing) + == JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: missing)) + #expect(try JSONCMemberEditor.merging("tablepro", value: entry, at: ["mcpServers"], in: #"{"mcpServers": {"tablepro": "x"}}"#) + == #"{"mcpServers": {"tablepro": {"command": "/b"}}}"#) + #expect(try JSONCMemberEditor.merging("tablepro", value: entry, at: ["mcpServers"], in: "") + == JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: "")) + } + + @Test("An unpaired surrogate escape, which Node writes, reads as U+FFFD and survives an edit unchanged") + func loneSurrogates() throws { + func escape(_ hex: String) -> String { "\\" + "u" + hex } + for title in [escape("d83d"), escape("dc00") + "x", escape("d83d") + escape("0041")] { + let text = "{\n \"title\": \"" + title + "\",\n \"mcpServers\": {}\n}\n" + let added = try JSONCMemberEditor.upserting("tablepro", value: entry, at: ["mcpServers"], in: text) + #expect(added.contains("\"title\": \"" + title + "\",")) + #expect(try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: added) == entry) + #expect(try JSONCMemberEditor.member("title", at: [], in: added)?.stringValue?.hasPrefix("\u{FFFD}") == true) + #expect(try JSONCMemberEditor.removing("tablepro", at: ["mcpServers"], in: added) == text) + } + let pair = escape("d83d") + escape("de00") + #expect(try JSONCMemberEditor.member("t", at: [], in: "{\"t\": \"" + pair + "\"}") == .string("\u{1F600}")) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientCatalogTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientCatalogTests.swift new file mode 100644 index 0000000000..40d64110b9 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientCatalogTests.swift @@ -0,0 +1,208 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct MCPClientCatalogTests { + @Test("The bundled catalog decodes") + func decodesBundledCatalog() throws { + let catalog = try MCPClientsFixtures.catalog() + #expect(catalog.schemaVersion == 1) + #expect(catalog.serverName == "tablepro") + #expect(catalog.clients.first?.id == "claude-desktop") + #expect(catalog.clients.count > 10) + } + + @Test("Client ids are unique") + func clientIDsAreUnique() throws { + let ids = try MCPClientsFixtures.catalog().clients.map(\.id) + #expect(Set(ids).count == ids.count) + } + + @Test("Every config id resolves for each platform the client has") + func everyConfigResolves() throws { + let catalog = try MCPClientsFixtures.catalog() + for client in catalog.clients { + guard let configID = client.configID else { continue } + #expect(catalog.configs[configID] != nil, "\(client.id) names \(configID)") + for platform in MCPClientPlatform.allCases where client.spec(for: platform) != nil { + #expect(catalog.location(for: client, platform: platform) != nil, "\(client.id) on \(platform)") + } + } + } + + @Test("Every client renders its entry, arguments and deeplinks on both platforms") + func everyClientRenders() throws { + let catalog = try MCPClientsFixtures.catalog() + var rendered = 0 + for client in catalog.clients { + for platform in MCPClientPlatform.allCases { + guard let spec = client.spec(for: platform) else { continue } + if spec.unsupported != nil { + #expect(spec.install.isEmpty, "\(client.id) on \(platform) is unsupported but installs") + continue + } + let renderer = MCPClientsFixtures.renderer(for: platform) + let entry = try renderer.entry(spec) + #expect(!MCPClientsFixtures.hasPlaceholder(entry.compactText), "\(client.id) on \(platform): \(entry.compactText)") + for method in spec.install { + switch method { + case let .deeplink(scheme, template): + let url = try renderer.url(template, entry: entry) + #expect(url.scheme == scheme) + case .cli(let cli): + let added = try renderer.arguments(cli.add, entry: entry) + #expect(!added.contains { MCPClientsFixtures.hasPlaceholder($0) }) + if let remove = cli.remove { + #expect(try renderer.arguments(remove, entry: entry).contains("tablepro")) + } + case .file: + let location = try #require(catalog.location(for: client, platform: platform)) + #expect(location.isWritable, "\(client.id) on \(platform) offers a file install on a read-only config") + } + } + _ = renderer.snippet(for: catalog.location(for: client, platform: platform), entry: entry) + rendered += 1 + } + } + #expect(rendered > 20) + } + + @Test("Entries keep the catalog's key order") + func entryKeyOrder() throws { + let catalog = try MCPClientsFixtures.catalog() + let spec = try #require(catalog.client(id: "claude-code")?.spec(for: .macos)) + let entry = try MCPClientsFixtures.renderer(for: .macos).entry(spec) + #expect(entry.objectMembers?.map(\.key) == ["type", "command", "args", "env"]) + } + + @Test("Locations decode their defaults and flags") + func locations() throws { + let catalog = try MCPClientsFixtures.catalog() + let claudeCode = try #require(catalog.client(id: "claude-code")) + let location = try #require(catalog.location(for: claudeCode, platform: .macos)) + #expect(location.format == .json) + #expect(location.keyPath == ["mcpServers"]) + #expect(!location.isWritable) + #expect(location.otherScopes == [["projects", "*", "mcpServers"]]) + + let cursor = try #require(catalog.client(id: "cursor")) + let cursorLocation = try #require(catalog.location(for: cursor, platform: .macos)) + #expect(cursorLocation.isWritable) + #expect(cursorLocation.otherScopes.isEmpty) + #expect(cursorLocation.alternateKeyPaths.isEmpty) + #expect(catalog.location(for: try MCPClientsFixtures.client("raycast"), platform: .macos) == nil) + + let opencode = try MCPClientsFixtures.location("opencode") + #expect(opencode.keyPath == ["mcp", "servers"]) + #expect(opencode.alternateKeyPaths == [["mcp"]]) + #expect(opencode.otherScopes.isEmpty) + } + + @Test("Detect rules and install methods decode") + func detectAndInstall() throws { + let catalog = try MCPClientsFixtures.catalog() + let cursor = try #require(catalog.client(id: "cursor")?.spec(for: .macos)) + #expect(cursor.detect == [.bundleID("com.todesktop.230313mzl4w4u92"), .path("~/.cursor")]) + #expect(cursor.install.last == .file) + #expect(cursor.applies == .live) + + let claudeCode = try #require(catalog.client(id: "claude-code")?.spec(for: .macos)) + guard case .cli(let cli)? = claudeCode.install.first else { + Issue.record("claude-code installs through its CLI") + return + } + #expect(cli.binary == "claude") + #expect(cli.update == .removeThenAdd) + #expect(cli.remove == ["mcp", "remove", "--scope", "user", "{name}"]) + + let opencode = try #require(catalog.client(id: "opencode")?.spec(for: .macos)) + guard case .cli(let opencodeCLI)? = opencode.install.first else { + Issue.record("opencode installs through its CLI") + return + } + #expect(opencodeCLI.remove == nil) + #expect(opencodeCLI.update == .add) + + let classic = try #require(catalog.client(id: "chatgpt-classic")) + #expect(classic.configID == nil) + #expect(classic.spec(for: .macos)?.unsupported == .remoteOnly) + #expect(classic.spec(for: .linux) == nil) + } + + @Test("clientInfo names match exactly first, then by the longest prefix, ignoring case") + func clientInfoMatching() throws { + let catalog = try MCPClientsFixtures.catalog() + #expect(catalog.client(matchingClientInfoName: "claude-code")?.id == "claude-code") + #expect(catalog.client(matchingClientInfoName: "CLAUDE-AI")?.id == "claude-desktop") + #expect(catalog.client(matchingClientInfoName: "Visual Studio Code")?.id == "vscode") + #expect(catalog.client(matchingClientInfoName: "Visual-Studio-Code")?.id == "vscode") + #expect(catalog.client(matchingClientInfoName: "Visual Studio Code - Insiders")?.id == "vscode-insiders") + #expect(catalog.client(matchingClientInfoName: "zed")?.id == "zed") + #expect(catalog.client(matchingClientInfoName: "goose-cli")?.id == "goose") + #expect(catalog.client(matchingClientInfoName: "goose-server")?.id == "goose") + #expect(catalog.client(matchingClientInfoName: "something-else") == nil) + #expect(catalog.client(matchingClientInfoName: "") == nil) + } + + @Test("A clientInfo name two clients send is the one detected here, or neither") + func sharedClientInfoName() throws { + let catalog = try MCPClientsFixtures.catalog() + #expect(catalog.clients(matchingClientInfoName: "codex-mcp-client").map { $0.id } == ["chatgpt", "codex"]) + #expect(catalog.clients(matchingClientInfoName: "Visual Studio Code - Insiders").map { $0.id } == ["vscode-insiders"]) + #expect(catalog.client(matchingClientInfoName: "Codex", detected: ["codex"])?.id == "codex") + #expect(catalog.client(matchingClientInfoName: "codex-mcp-client", detected: ["chatgpt", "cursor"])?.id == "chatgpt") + #expect(catalog.client(matchingClientInfoName: "codex-mcp-client", detected: ["chatgpt", "codex"]) == nil) + #expect(catalog.client(matchingClientInfoName: "codex-mcp-client", detected: []) == nil) + #expect(catalog.client(matchingClientInfoName: "claude-code", detected: [])?.id == "claude-code") + } + + @Test("The HTTP entry is the first platform entry that names {url}") + func httpEntry() throws { + let catalog = try MCPClientsFixtures.catalog() + let claudeCode = try #require(catalog.client(id: "claude-code")) + let entry = try #require(catalog.httpEntry(for: claudeCode)) + #expect(entry["type"] == .string("http")) + #expect(entry["url"] == .string("{url}")) + + let rendered = try MCPClientsFixtures.renderer(for: .linux).entry(template: entry) + #expect(rendered["headers"]?["Authorization"] == .string(MCPClientsFixtures.authorization)) + + #expect(catalog.httpEntry(for: try MCPClientsFixtures.client("claude-desktop")) == nil) + #expect(catalog.httpEntry(for: try MCPClientsFixtures.client("chatgpt-classic")) == nil) + #expect(catalog.httpEntry(for: try MCPClientsFixtures.client("raycast")) == nil) + } + + @Test("A newer schema version is refused") + func newerSchemaRefused() { + let data = Data(#"{"schemaVersion": 2, "serverName": "tablepro", "configs": {}, "clients": []}"#.utf8) + #expect(throws: MCPClientCatalogError.unsupportedSchemaVersion(2)) { + try MCPClientCatalog.decode(data) + } + } + + @Test("A missing field names its path") + func missingFieldPath() { + let data = Data(#"{"schemaVersion": 1, "serverName": "tablepro", "configs": {}, "clients": [{"id": "x", "icon": "x"}]}"#.utf8) + #expect(throws: MCPClientCatalogError.invalidField(["clients", "0", "name"])) { + try MCPClientCatalog.decode(data) + } + } + + @Test("An unknown install kind is refused") + func unknownInstallKind() { + let data = Data(#""" + {"schemaVersion": 1, "serverName": "tablepro", "configs": {}, + "clients": [{"id": "x", "name": "X", "icon": "x", "config": null, "macos": {"install": [{"kind": "magic"}]}}]} + """#.utf8) + #expect(throws: MCPClientCatalogError.invalidField(["clients", "0", "macos", "install", "0", "kind"])) { + try MCPClientCatalog.decode(data) + } + } + + @Test("Decoding through JSONDecoder gives the same clients") + func jsonDecoderConformance() throws { + let data = try Data(contentsOf: MCPClientsFixtures.catalogURL) + let decoded = try JSONDecoder().decode(MCPClientCatalog.self, from: data) + #expect(decoded.clients.map(\.id) == (try MCPClientsFixtures.catalog()).clients.map(\.id)) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientStatusEvaluatorTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientStatusEvaluatorTests.swift new file mode 100644 index 0000000000..022d840c62 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientStatusEvaluatorTests.swift @@ -0,0 +1,203 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct MCPClientStatusEvaluatorTests { + private let json = MCPConfigLocation(path: "~/.cursor/mcp.json", format: .json, keyPath: ["mcpServers"]) + private let stdio = MCPServerTarget.stdio(command: MCPClientsFixtures.bridge) + private let http = MCPServerTarget.http(url: MCPClientsFixtures.httpURL, authorization: MCPClientsFixtures.authorization) + + private func status( + _ text: String?, + location: MCPConfigLocation?, + expected: MCPServerTarget, + existing: Set = [] + ) -> MCPClientConfigStatus { + MCPClientStatusEvaluator.status( + configText: text, location: location, serverName: "tablepro", expected: expected, fileExists: { existing.contains($0) } + ) + } + + @Test("No config location is unknown; no file is not configured") + func missingInputs() { + #expect(status("{}", location: nil, expected: stdio) == .unknown) + #expect(status(nil, location: json, expected: stdio) == .notConfigured) + #expect(status("", location: json, expected: stdio) == .notConfigured) + #expect(status(#"{"mcpServers": {"other": {}}}"#, location: json, expected: stdio) == .notConfigured) + } + + @Test("The running bridge's path is current") + func stdioCurrent() { + let text = "{\n // c\n \"mcpServers\": {\"tablepro\": {\"command\": \"\(MCPClientsFixtures.bridge)\", \"args\": []},}\n}" + #expect(status(text, location: json, expected: stdio) == .current) + } + + @Test("An array command is compared by its first element") + func arrayCommand() throws { + let location = try MCPClientsFixtures.location("opencode") + let text = #"{"mcp": {"servers": {"tablepro": {"type": "local", "command": ["\#(MCPClientsFixtures.bridge)", "--x"]}}}}"# + #expect(status(text, location: location, expected: stdio) == .current) + } + + @Test("A command that is gone is stale as missing; another existing one as another command") + func stdioStale() { + let text = #"{"mcpServers": {"tablepro": {"command": "\#(MCPClientsFixtures.betaBridge)"}}}"# + #expect(status(text, location: json, expected: stdio) == .stale(.commandMissing)) + #expect(status(text, location: json, expected: stdio, existing: [MCPClientsFixtures.betaBridge]) == .stale(.otherCommand)) + #expect(status(#"{"mcpServers": {"tablepro": {"type": "stdio"}}}"#, location: json, expected: stdio) == .stale(.otherCommand)) + } + + @Test("An HTTP entry where the bridge is expected is set up with a token, never stale") + func httpEntryUsesToken() throws { + let tokenSheet = #"{"mcpServers": {"tablepro": {"url": "http://127.0.0.1:23508/mcp", "headers": {"Authorization": "Bearer tp_x"}}}}"# + #expect(status(tokenSheet, location: json, expected: stdio) == .usesToken) + let claudeCode = try MCPClientsFixtures.location("claude-code") + let added = #"{"mcpServers": {"tablepro": {"type": "http", "url": "http://localhost:23508/mcp", "headers": {"Authorization": "Bearer tp_x"}}}}"# + #expect(status(added, location: claudeCode, expected: stdio) == .usesToken) + let codex = try MCPClientsFixtures.location("codex") + let table = "[mcp_servers.tablepro]\nurl = \"http://127.0.0.1:23508/mcp\"\nhttp_headers = { Authorization = \"Bearer tp_x\" }\n" + #expect(status(table, location: codex, expected: stdio) == .usesToken) + } + + @Test("A current entry the user turned off reads as disabled") + func disabledEntries() throws { + func entry(_ fields: String) -> String { + #"{"mcpServers": {"tablepro": {"command": "\#(MCPClientsFixtures.bridge)", \#(fields)}}}"# + } + #expect(status(entry(#""disabled": true"#), location: json, expected: stdio) == .disabled) + #expect(status(entry(#""enabled": false"#), location: json, expected: stdio) == .disabled) + #expect(status(entry(#""disabled": false"#), location: json, expected: stdio) == .current) + #expect(status(entry(#""enabled": true"#), location: json, expected: stdio) == .current) + + let codex = try MCPClientsFixtures.location("codex") + let table = "[mcp_servers.tablepro]\ncommand = \"\(MCPClientsFixtures.bridge)\"\nenabled = false\n" + #expect(status(table, location: codex, expected: stdio) == .disabled) + } + + @Test("A turned-off entry reads as disabled whatever it points at, so it is never offered an Update") + func disabledBeforeStale() throws { + let moved = #"{"mcpServers": {"tablepro": {"command": "/gone", "disabled": true}}}"# + #expect(status(moved, location: json, expected: stdio) == .disabled) + let token = #"{"mcpServers": {"tablepro": {"url": "http://127.0.0.1:23508/mcp", "enabled": false}}}"# + #expect(status(token, location: json, expected: stdio) == .disabled) + let codex = try MCPClientsFixtures.location("codex") + let oldToken = "[mcp_servers.tablepro]\nurl = \"\(MCPClientsFixtures.httpURL)\"\n" + + "http_headers = { Authorization = \"Bearer tp_old\" }\nenabled = false\n" + #expect(status(oldToken, location: codex, expected: http) == .disabled) + } + + @Test("A hit only in another scope is otherScope, with * matching any key") + func otherScopes() throws { + let claudeCode = try MCPClientsFixtures.location("claude-code") + let local = #"{"projects": {"/Users/me/app": {"mcpServers": {"tablepro": {"command": "/b"}}}}, "mcpServers": {}}"# + #expect(status(local, location: claudeCode, expected: stdio, existing: ["/Users/me/app"]) == .otherScope) + let bridge = MCPClientsFixtures.bridge + let both = #"{"projects": {"/x": {"mcpServers": {"tablepro": {"command": "\#(bridge)"}}}}, "mcpServers": {"tablepro": {"command": "\#(bridge)"}}}"# + #expect(status(both, location: claudeCode, expected: stdio, existing: ["/x"]) == .current) + #expect(status(#"{"projects": {"/x": {"mcpServers": {}}, "/y": []}}"#, location: claudeCode, expected: stdio) == .notConfigured) + } + + @Test("A project whose folder is gone is not a scope any more") + func goneProjectFolders() throws { + let claudeCode = try MCPClientsFixtures.location("claude-code") + let onlyThere = #"{"projects": {"/Users/me/old": {"mcpServers": {"tablepro": {"command": "/b"}}}}, "mcpServers": {}}"# + #expect(status(onlyThere, location: claudeCode, expected: stdio) == .notConfigured) + let bridge = MCPClientsFixtures.bridge + let shadowedByGone = #"{"projects": {"/Users/me/old": {"mcpServers": {"tablepro": {"command": "/Volumes/Gone/tablepro-mcp"}}}}, "# + + #""mcpServers": {"tablepro": {"command": "\#(bridge)"}}}"# + #expect(status(shadowedByGone, location: claudeCode, expected: stdio) == .current) + #expect(status(shadowedByGone, location: claudeCode, expected: stdio, existing: ["/Users/me/old"]) + == .stale(.shadowed(scope: "/Users/me/old"))) + } + + @Test("A stale entry in another scope shadows a current one there and names it; a token or turned-off one does not") + func shadowingScopes() throws { + let claudeCode = try MCPClientsFixtures.location("claude-code") + func file(project: String) -> String { + #"{"projects": {"/a": {"mcpServers": {}}, "/b": {"mcpServers": {"tablepro": \#(project)}}}, "# + + #""mcpServers": {"tablepro": {"type": "stdio", "command": "\#(MCPClientsFixtures.bridge)"}}}"# + } + let folders: Set = ["/a", "/b"] + #expect(status(file(project: #"{"command": "/Volumes/Gone/tablepro-mcp"}"#), location: claudeCode, expected: stdio, existing: folders) + == .stale(.shadowed(scope: "/b"))) + #expect(status(file(project: #"{"command": "/x"}"#), location: claudeCode, expected: stdio, existing: folders.union(["/x"])) + == .stale(.shadowed(scope: "/b"))) + let token = #"{"type": "http", "url": "http://127.0.0.1:23508/mcp", "headers": {"Authorization": "Bearer tp_x"}}"# + #expect(status(file(project: token), location: claudeCode, expected: stdio, existing: folders) == .current) + let off = #"{"command": "/Volumes/Gone/tablepro-mcp", "disabled": true}"# + #expect(status(file(project: off), location: claudeCode, expected: stdio, existing: folders) == .current) + + let staleUser = #"{"projects": {"/b": {"mcpServers": {"tablepro": {"command": "/x"}}}}, "mcpServers": {"tablepro": {"command": "/y"}}}"# + #expect(status(staleUser, location: claudeCode, expected: stdio, existing: folders) == .stale(.commandMissing)) + } + + @Test("opencode's V1 mcp. reads like its V2 mcp.servers., never as another scope") + func alternateKeyPaths() throws { + let opencode = try MCPClientsFixtures.location("opencode") + let bridge = MCPClientsFixtures.bridge + #expect(status(#"{"mcp": {"tablepro": {"type": "local", "command": ["\#(bridge)"]}}}"#, location: opencode, expected: stdio) == .current) + #expect(status(#"{"mcp": {"tablepro": {"type": "local", "command": ["/gone"]}}}"#, location: opencode, expected: stdio) + == .stale(.commandMissing)) + #expect(status(#"{"mcp": {"servers": {"tablepro": {"command": ["\#(bridge)"]}}}}"#, location: opencode, expected: stdio) == .current) + #expect(status(#"{"mcp": {"servers": {}, "other": {}}}"#, location: opencode, expected: stdio) == .notConfigured) + #expect(status(#"{"mcp": {"tablepro": {"type": "local"}}}"#, location: opencode, expected: stdio) == .stale(.otherCommand)) + } + + @Test("An unpaired surrogate escape, which Node writes, does not make a file unreadable") + func loneSurrogate() { + let escape = "\\" + "ud83d" + let text = #"{"title": ""# + escape + #"", "mcpServers": {"tablepro": {"command": "\#(MCPClientsFixtures.bridge)"}}}"# + #expect(status(text, location: json, expected: stdio) == .current) + } + + @Test("Text that cannot be read safely is unreadable") + func unreadable() { + #expect(status("{", location: json, expected: stdio) == .unreadable) + #expect(status(#"{"mcpServers": {}, "mcpServers": {}}"#, location: json, expected: stdio) == .unreadable) + #expect(status(#"{"mcpServers": []}"#, location: json, expected: stdio) == .unreadable) + let toml = MCPConfigLocation(path: "~/.codex/config.toml", format: .toml, keyPath: ["mcp_servers"]) + #expect(status("[mcp_servers]\ntablepro.command = \"x\"\n", location: toml, expected: stdio) == .unreadable) + } + + @Test("A YAML config is unknown") + func yaml() { + let location = MCPConfigLocation(path: "~/.config/goose/config.yaml", format: .yaml, keyPath: ["extensions"], isWritable: false) + #expect(status("extensions: {}\n", location: location, expected: stdio) == .unknown) + #expect(status(nil, location: location, expected: stdio) == .unknown) + } + + @Test("Codex's TOML table is compared like JSON") + func toml() throws { + let location = try MCPClientsFixtures.location("codex") + #expect(status("[mcp_servers.tablepro]\ncommand = \"\(MCPClientsFixtures.bridge)\"\n", location: location, expected: stdio) == .current) + #expect(status("[mcp_servers.tablepro]\ncommand = \"/gone\"\n", location: location, expected: stdio) == .stale(.commandMissing)) + #expect(status("[mcp_servers.other]\ncommand = \"/o\"\n", location: location, expected: stdio) == .notConfigured) + + let httpTable = """ + [mcp_servers.tablepro] + url = "\(MCPClientsFixtures.httpURL)" + http_headers = { Authorization = "\(MCPClientsFixtures.authorization)" } + """ + #expect(status(httpTable, location: location, expected: http) == .current) + #expect(status(httpTable.replacingOccurrences(of: "tp_test", with: "tp_old"), location: location, expected: http) == .stale(.otherToken)) + #expect(status(httpTable.replacingOccurrences(of: "23508", with: "23509"), location: location, expected: http) == .stale(.otherURL)) + } + + @Test("HTTP entries compare the endpoint and the Authorization header, whatever the key names; a command entry is another command") + func httpEntries() { + func entry(_ fields: String) -> String { + #"{"mcpServers": {"tablepro": {\#(fields)}}}"# + } + let headers = #""headers": {"authorization": "Bearer tp_test"}"# + #expect(status(entry(#""url": "http://127.0.0.1:23508/mcp", \#(headers)"#), location: json, expected: http) == .current) + #expect(status(entry(#""serverUrl": "http://localhost:23508/mcp/", \#(headers)"#), location: json, expected: http) == .current) + #expect(status(entry(#""httpUrl": "http://127.0.0.1:23508/mcp", \#(headers)"#), location: json, expected: http) == .current) + #expect(status(entry(#""url": "https://127.0.0.1:23508/mcp", \#(headers)"#), location: json, expected: http) == .stale(.otherURL)) + #expect(status(entry(#""url": "http://example.com:23508/mcp", \#(headers)"#), location: json, expected: http) == .stale(.otherURL)) + #expect(status(entry(#""command": "/b""#), location: json, expected: http) == .stale(.otherCommand)) + #expect(status(entry(#""type": "http""#), location: json, expected: http) == .stale(.otherURL)) + #expect(status(entry(#""url": "http://127.0.0.1:23508/mcp""#), location: json, expected: http) == .stale(.otherToken)) + #expect(status(entry(#""url": "http://127.0.0.1:23508/mcp", "headers": {"Authorization": "Bearer other"}"#), location: json, expected: http) + == .stale(.otherToken)) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientsFixtures.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientsFixtures.swift new file mode 100644 index 0000000000..fa0a5e6c1e --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPClientsFixtures.swift @@ -0,0 +1,72 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +enum MCPClientsFixtures { + static let bridge = "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp" + static let betaBridge = "/Applications/TablePro Beta.app/Contents/MacOS/tablepro-mcp" + static let httpURL = "http://127.0.0.1:23508/mcp" + static let authorization = "Bearer tp_test" + + // The catalog both apps read lives in the Rust core, six folders above this file. + static let catalogURL: URL = { + var url = URL(fileURLWithPath: #filePath) + for _ in 0..<6 { + url = url.deletingLastPathComponent() + } + return url.appendingPathComponent("core/crates/mcp/clients.json") + }() + + static func catalog() throws -> MCPClientCatalog { + try MCPClientCatalog.decode(Data(contentsOf: catalogURL)) + } + + struct MissingFixture: Error { + let id: String + } + + static func client(_ id: String) throws -> MCPClientDefinition { + guard let client = try catalog().client(id: id) else { throw MissingFixture(id: id) } + return client + } + + static func location(_ id: String, platform: MCPClientPlatform = .macos) throws -> MCPConfigLocation { + let catalog = try catalog() + guard let client = catalog.client(id: id), let location = catalog.location(for: client, platform: platform) else { + throw MissingFixture(id: id) + } + return location + } + + static func target(for platform: MCPClientPlatform) -> MCPServerTarget { + switch platform { + case .macos: return .stdio(command: bridge) + case .linux: return .http(url: httpURL, authorization: authorization) + } + } + + static func renderer(for platform: MCPClientPlatform) -> MCPTemplateRenderer { + MCPTemplateRenderer(serverName: "tablepro", target: target(for: platform)) + } + + static let stdioEntry: JSONValue = .object([ + (key: "command", value: .string(bridge)), + (key: "args", value: .array([])), + ]) + + static func hasPlaceholder(_ text: String) -> Bool { + text.range(of: #"\{[A-Za-z|]+\}"#, options: .regularExpression) != nil + } + + static func parsed(_ text: String) throws -> NSDictionary { + let object = try JSONSerialization.jsonObject(with: Data(text.utf8), options: [.json5Allowed]) + return try #require(object as? NSDictionary) + } + + static func temporaryDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("TableProMCPClientsTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPConfigFileWriterTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPConfigFileWriterTests.swift new file mode 100644 index 0000000000..0958780b95 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPConfigFileWriterTests.swift @@ -0,0 +1,203 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct MCPConfigFileWriterTests { + private let fileManager = FileManager.default + private let writer = MCPConfigFileWriter() + + private func mode(_ url: URL) throws -> Int { + try #require((try fileManager.attributesOfItem(atPath: url.path)[.posixPermissions] as? NSNumber)?.intValue) + } + + private func contents(_ url: URL) throws -> String { + try String(contentsOf: url, encoding: .utf8) + } + + @Test("A new file is created at 0600 with its folders, and the transform sees nil") + func createsNewFile() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("a/b/mcp.json") + var seen: [String?] = [] + let result = try writer.update(url) { text in + seen.append(text) + return "{}\n" + } + #expect(seen == [nil]) + #expect(result == MCPConfigWriteResult(backupURL: nil, created: true)) + #expect(try contents(url) == "{}\n") + #expect(try mode(url) == 0o600) + } + + @Test("Replacing keeps the file's mode and leaves the old bytes in a sibling backup") + func replacesWithBackup() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + try "old".write(to: url, atomically: false, encoding: .utf8) + try fileManager.setAttributes([.posixPermissions: 0o640], ofItemAtPath: url.path) + + var seen: [String?] = [] + let result = try writer.update(url) { text in + seen.append(text) + return "new" + } + #expect(seen == ["old"]) + #expect(!result.created) + let backup = try #require(result.backupURL) + #expect(backup.lastPathComponent == "mcp.json.tablepro-backup") + #expect(try contents(url) == "new") + #expect(try contents(backup) == "old") + #expect(try mode(url) == 0o640) + #expect(try fileManager.contentsOfDirectory(atPath: directory.path).sorted() == ["mcp.json", "mcp.json.tablepro-backup"]) + + _ = try writer.update(url) { _ in "newer" } + #expect(try contents(backup) == "new") + } + + @Test("An unchanged result writes nothing") + func unchangedSkipsWrite() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + try "same".write(to: url, atomically: false, encoding: .utf8) + let result = try writer.update(url) { $0 ?? "" } + #expect(result == MCPConfigWriteResult(backupURL: nil, created: false)) + #expect(try fileManager.contentsOfDirectory(atPath: directory.path) == ["mcp.json"]) + } + + @Test("Writing through a symlink replaces its target and keeps the link") + func keepsSymlink() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let dotfiles = directory.appendingPathComponent("dotfiles", isDirectory: true) + try fileManager.createDirectory(at: dotfiles, withIntermediateDirectories: true) + let real = dotfiles.appendingPathComponent("settings.json") + try "old".write(to: real, atomically: false, encoding: .utf8) + try fileManager.setAttributes([.posixPermissions: 0o600], ofItemAtPath: real.path) + let link = directory.appendingPathComponent("settings.json") + try fileManager.createSymbolicLink(atPath: link.path, withDestinationPath: "dotfiles/settings.json") + + let result = try writer.update(link) { _ in "new" } + #expect(try fileManager.destinationOfSymbolicLink(atPath: link.path) == "dotfiles/settings.json") + #expect(try contents(real) == "new") + #expect(try contents(link) == "new") + #expect(try mode(real) == 0o600) + let backup = try #require(result.backupURL) + #expect(backup.deletingLastPathComponent().lastPathComponent == "dotfiles") + #expect(try contents(dotfiles.appendingPathComponent("settings.json.tablepro-backup")) == "old") + } + + @Test("A change made while the edit was computed is read again and the edit redone once") + func redoesOnce() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + try "first".write(to: url, atomically: false, encoding: .utf8) + + var seen: [String?] = [] + _ = try writer.update(url) { text in + seen.append(text) + if seen.count == 1 { + try "concurrent".write(to: url, atomically: false, encoding: .utf8) + } + return (text ?? "") + "+edit" + } + #expect(seen == ["first", "concurrent"]) + #expect(try contents(url) == "concurrent+edit") + } + + @Test("A file that keeps changing is refused") + func changingFileRefused() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + try "0".write(to: url, atomically: false, encoding: .utf8) + var calls = 0 + #expect(throws: MCPConfigWriteError.changedDuringWrite) { + try writer.update(url) { _ in + calls += 1 + try String(calls).write(to: url, atomically: false, encoding: .utf8) + return "edit" + } + } + #expect(try contents(url) == "2") + } + + @Test("A file that appears while a new one is prepared is redone against it") + func appearsDuringCreate() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + var seen: [String?] = [] + let result = try writer.update(url) { text in + seen.append(text) + if text == nil { + try "theirs".write(to: url, atomically: false, encoding: .utf8) + } + return (text ?? "") + "+ours" + } + #expect(seen == [nil, "theirs"]) + #expect(!result.created) + #expect(try contents(url) == "theirs+ours") + } + + @Test("A transform error leaves the file alone") + func transformError() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + try "keep".write(to: url, atomically: false, encoding: .utf8) + #expect(throws: MCPConfigEditError.duplicateKey(["mcpServers"])) { + try writer.update(url) { _ in throw MCPConfigEditError.duplicateKey(["mcpServers"]) } + } + #expect(try contents(url) == "keep") + #expect(try fileManager.contentsOfDirectory(atPath: directory.path) == ["mcp.json"]) + } + + @Test("Bytes that are not UTF-8 are refused") + func notUTF8() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("mcp.json") + try Data([0xFF, 0xFE, 0x00]).write(to: url) + #expect(throws: MCPConfigWriteError.io("notUTF8")) { + try writer.update(url) { _ in "x" } + } + } + + @Test("Upserting through the writer keeps a JSONC file's comments") + func endToEnd() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + defer { try? fileManager.removeItem(at: directory) } + let url = directory.appendingPathComponent("settings.json") + try "// Zed\n{\n \"theme\": \"One Dark\"\n}\n".write(to: url, atomically: false, encoding: .utf8) + _ = try writer.update(url) { text in + try JSONCMemberEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["context_servers"], in: text ?? "") + } + let written = try contents(url) + #expect(written.hasPrefix("// Zed\n{\n \"theme\": \"One Dark\",\n \"context_servers\": {")) + #expect(try JSONCMemberEditor.member("tablepro", at: ["context_servers"], in: written) == MCPClientsFixtures.stdioEntry) + } + + @Test("A write the folder refuses carries the system's reason, not only its code") + func refusedWriteNamesReason() throws { + let directory = try MCPClientsFixtures.temporaryDirectory() + let url = directory.appendingPathComponent("mcp.json") + try "old".write(to: url, atomically: false, encoding: .utf8) + try fileManager.setAttributes([.posixPermissions: 0o500], ofItemAtPath: directory.path) + defer { + try? fileManager.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) + try? fileManager.removeItem(at: directory) + } + do { + _ = try writer.update(url) { _ in "new" } + Issue.record("The write went through a read-only folder") + } catch MCPConfigWriteError.io(let detail) { + #expect(detail.contains("(NSCocoaErrorDomain ")) + #expect(!detail.hasPrefix("NSCocoaErrorDomain")) + } + #expect(try contents(url) == "old") + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPEntryUpdateTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPEntryUpdateTests.swift new file mode 100644 index 0000000000..58d394bd32 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPEntryUpdateTests.swift @@ -0,0 +1,150 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct MCPEntryUpdateTests { + private struct Rendered { + let entry: JSONValue + let template: JSONValue + } + + private func rendered(_ id: String, platform: MCPClientPlatform = .macos) throws -> Rendered { + let spec = try #require(try MCPClientsFixtures.client(id).spec(for: platform)) + let template = try #require(spec.entry) + return Rendered(entry: try MCPClientsFixtures.renderer(for: platform).entry(spec), template: template) + } + + private func updatedJSON(_ id: String, _ text: String, platform: MCPClientPlatform = .macos) throws -> JSONValue? { + let update = try rendered(id, platform: platform) + let result = try JSONCMemberEditor.updating( + "tablepro", entry: update.entry, template: update.template, at: ["mcpServers"], in: text + ) + return try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: result) + } + + private static let old = "/Volumes/Gone/TablePro.app/Contents/MacOS/tablepro-mcp" + + @Test("Update keeps Copilot CLI's tool list and the turned-off flag of Cline and Kiro") + func keepsRestrictions() throws { + let copilot = try #require(try updatedJSON("copilot-cli", """ + {"mcpServers": {"tablepro": {"type": "local", "command": "\(Self.old)", "args": [], "tools": ["list_connections"]}}} + """)) + #expect(copilot["tools"] == .array([.string("list_connections")])) + #expect(copilot["command"] == .string(MCPClientsFixtures.bridge)) + + for id in ["cline", "kiro"] { + let entry = try #require(try updatedJSON(id, """ + {"mcpServers": {"tablepro": {"command": "\(Self.old)", "args": [], "disabled": true}}} + """)) + #expect(entry["disabled"] == .bool(true)) + #expect(entry["command"] == .string(MCPClientsFixtures.bridge)) + } + } + + @Test("A member the catalog sets that the entry lacks is added, and type and args are always rewritten") + func fillsMissingMembers() throws { + let copilot = try #require(try updatedJSON("copilot-cli", """ + {"mcpServers": {"tablepro": {"type": "stdio", "command": "\(Self.old)", "args": ["--old"]}}} + """)) + #expect(copilot == .object([ + (key: "type", value: .string("local")), + (key: "command", value: .string(MCPClientsFixtures.bridge)), + (key: "args", value: .array([])), + (key: "tools", value: .array([.string("*")])), + ])) + } + + @Test("A placeholder at any depth is rewritten, and the user's own headers and keys stay") + func rewritesNestedConnectionMembers() throws { + let entry = try #require(try updatedJSON("cline", """ + {"mcpServers": {"tablepro": {"type": "sse", "url": "http://127.0.0.1:1/mcp", "timeout": 5, "disabled": true, + "headers": {"Authorization": "Bearer tp_old", "X-Trace": "1"}}}} + """, platform: .linux)) + #expect(entry["type"] == .string("streamableHttp")) + #expect(entry["url"] == .string(MCPClientsFixtures.httpURL)) + #expect(entry["headers"] == .object([ + (key: "Authorization", value: .string(MCPClientsFixtures.authorization)), + (key: "X-Trace", value: .string("1")), + ])) + #expect(entry["timeout"] == .number(5)) + #expect(entry["disabled"] == .bool(true)) + } + + @Test("An entry that reaches TablePro the other way is replaced whole, keeping only the user's choices") + func transportChangeReplacesTheEntry() throws { + let toHTTP = try updatedJSON("cursor", """ + {"mcpServers": {"tablepro": {"command": "/x", "args": [], "env": {"A": "1"}}}} + """, platform: .linux) + #expect(toHTTP == (try rendered("cursor", platform: .linux).entry)) + + let toStdio = try updatedJSON("cline", """ + {"mcpServers": {"tablepro": {"url": "http://127.0.0.1:23508/mcp", "headers": {"Authorization": "Bearer tp_x"}}}} + """) + #expect(toStdio == (try rendered("cline").entry)) + + let copilot = try updatedJSON("copilot-cli", """ + {"mcpServers": {"tablepro": {"type": "local", "command": "/x", "args": [], "env": {"A": "1"}, "tools": ["list_connections"]}}} + """, platform: .linux) + #expect(copilot == .object([ + (key: "type", value: .string("http")), + (key: "url", value: .string(MCPClientsFixtures.httpURL)), + (key: "headers", value: .object([(key: "Authorization", value: .string(MCPClientsFixtures.authorization))])), + (key: "tools", value: .array([.string("list_connections")])), + ])) + } + + @Test("A missing entry is added as rendered, and an unchanged one is left byte for byte") + func addsAndLeavesAlone() throws { + let update = try rendered("copilot-cli") + let empty = #"{"mcpServers": {}}"# + let added = try JSONCMemberEditor.updating("tablepro", entry: update.entry, template: update.template, at: ["mcpServers"], in: empty) + #expect(try JSONCMemberEditor.member("tablepro", at: ["mcpServers"], in: added) == update.entry) + #expect(try JSONCMemberEditor.updating("tablepro", entry: update.entry, template: update.template, at: ["mcpServers"], in: added) + == added) + + let current = """ + {"mcpServers": {"tablepro": {"type": "local", "command": "\(MCPClientsFixtures.bridge)", "args": [], "tools": ["a"], + "timeout": 1.0}}} + """ + #expect(try JSONCMemberEditor.updating("tablepro", entry: update.entry, template: update.template, at: ["mcpServers"], in: current) + == current) + } + + @Test("Codex's TOML table keeps a key the catalog only fills in, and its own headers") + func tomlKeepsRestrictions() throws { + let template: JSONValue = .object([ + (key: "command", value: .string("{command}")), + (key: "args", value: .array([])), + (key: "enabled", value: .bool(true)), + ]) + let entry = try MCPClientsFixtures.renderer(for: .macos).entry(template: template) + let text = "[mcp_servers.tablepro]\ncommand = \"\(Self.old)\"\nargs = []\nenabled = false\n" + let updated = try TOMLTableEditor.updating("tablepro", entry: entry, template: template, at: ["mcp_servers"], in: text) + #expect(updated == "[mcp_servers.tablepro]\ncommand = \"\(MCPClientsFixtures.bridge)\"\nargs = []\nenabled = false\n") + + let http = try rendered("codex", platform: .linux) + let table = """ + [mcp_servers.tablepro] + url = "http://127.0.0.1:1/mcp" + http_headers = { Authorization = "Bearer tp_old", X-Trace = "1" } + startup_timeout_sec = 30 + + """ + let merged = try TOMLTableEditor.updating("tablepro", entry: http.entry, template: http.template, at: ["mcp_servers"], in: table) + #expect(merged == """ + [mcp_servers.tablepro] + url = "\(MCPClientsFixtures.httpURL)" + http_headers = { Authorization = "\(MCPClientsFixtures.authorization)", X-Trace = "1" } + startup_timeout_sec = 30 + + """) + } + + @Test("A TOML table that reaches TablePro the other way is replaced with its subtables") + func tomlTransportChangeReplacesTheTable() throws { + let http = try rendered("codex", platform: .linux) + let text = "[mcp_servers.tablepro]\ncommand = \"/x\"\n\n[mcp_servers.tablepro.env]\nA = \"1\"\n" + let replaced = try TOMLTableEditor.updating("tablepro", entry: http.entry, template: http.template, at: ["mcp_servers"], in: text) + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: replaced) == http.entry) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPPathExpanderTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPPathExpanderTests.swift new file mode 100644 index 0000000000..9c77ba4295 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPPathExpanderTests.swift @@ -0,0 +1,49 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct MCPPathExpanderTests { + private let home = "/Users/me" + + @Test("A leading tilde is the home folder") + func tilde() { + #expect(MCPPathExpander.expand("~/.cursor/mcp.json", home: home, environment: [:]) == "/Users/me/.cursor/mcp.json") + #expect(MCPPathExpander.expand("~", home: home, environment: [:]) == "/Users/me") + #expect(MCPPathExpander.expand("~/x", home: "/Users/me/", environment: [:]) == "/Users/me/x") + #expect(MCPPathExpander.expand("/a/~/b", home: home, environment: [:]) == "/a/~/b") + } + + @Test("A set variable wins over its default") + func variableSet() { + let path = "${XDG_CONFIG_HOME:-~/.config}/zed/settings.json" + #expect(MCPPathExpander.expand(path, home: home, environment: ["XDG_CONFIG_HOME": "/xdg"]) == "/xdg/zed/settings.json") + } + + @Test("An unset or empty variable takes its default, tilde included") + func variableDefault() { + let path = "${XDG_CONFIG_HOME:-~/.config}/zed/settings.json" + #expect(MCPPathExpander.expand(path, home: home, environment: [:]) == "/Users/me/.config/zed/settings.json") + #expect(MCPPathExpander.expand(path, home: home, environment: ["XDG_CONFIG_HOME": ""]) == "/Users/me/.config/zed/settings.json") + #expect(MCPPathExpander.expand("${CLAUDE_CONFIG_DIR:-~}/.claude.json", home: home, environment: [:]) == "/Users/me/.claude.json") + #expect(MCPPathExpander.expand("${CLAUDE_CONFIG_DIR:-~}/.claude.json", home: home, environment: ["CLAUDE_CONFIG_DIR": "/cfg"]) + == "/cfg/.claude.json") + } + + @Test("A variable without a default expands to nothing when unset") + func variableWithoutDefault() { + #expect(MCPPathExpander.expand("/a/${NOPE}/b", home: home, environment: [:]) == "/a//b") + #expect(MCPPathExpander.expand("/a/${HOME}/b", home: home, environment: ["HOME": "/h"]) == "/a//h/b") + } + + @Test("Every catalog path expands to an absolute path") + func catalogPaths() throws { + let catalog = try MCPClientsFixtures.catalog() + for set in catalog.configs.values { + for location in [set.macos, set.linux].compactMap({ $0 }) { + let expanded = MCPPathExpander.expand(location.path, home: home, environment: [:]) + #expect(expanded.hasPrefix("/"), "\(location.path)") + #expect(!expanded.contains("$") && !expanded.contains("~"), "\(location.path)") + } + } + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPTemplateRendererTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPTemplateRendererTests.swift new file mode 100644 index 0000000000..60fc852863 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/MCPTemplateRendererTests.swift @@ -0,0 +1,179 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct MCPTemplateRendererTests { + private let stdio = MCPTemplateRenderer(serverName: "tablepro", target: .stdio(command: MCPClientsFixtures.betaBridge)) + + @Test("Cursor's config is base64 of the compact entry, percent-encoded") + func cursorDeeplink() throws { + let spec = try #require(try MCPClientsFixtures.catalog().client(id: "cursor")?.spec(for: .macos)) + guard case let .deeplink(_, template)? = spec.install.first else { + Issue.record("Cursor installs through a deeplink first") + return + } + let entry = try stdio.entry(spec) + #expect(entry.compactText == #"{"type":"stdio","command":"/Applications/TablePro Beta.app/Contents/MacOS/tablepro-mcp","args":[]}"#) + let url = try stdio.url(template, entry: entry) + #expect(url.absoluteString == "cursor://anysphere.cursor-deeplink/mcp/install?name=tablepro&config=" + + "eyJ0eXBlIjoic3RkaW8iLCJjb21tYW5kIjoiL0FwcGxpY2F0aW9ucy9UYWJsZVBybyBCZXRhLmFwcC9Db250ZW50cy9NYWNPUy90YWJsZXByby1tY3AiLCJhcmdzIjpbXX0%3D") + + let config = try #require(URLComponents(url: url, resolvingAgainstBaseURL: false)?.queryItems?.first { $0.name == "config" }?.value) + let decoded = try #require(Data(base64Encoded: config)) + #expect(String(bytes: decoded, encoding: .utf8) == entry.compactText) + } + + @Test("VS Code gets the percent-encoded entry with name first") + func vscodeDeeplink() throws { + let spec = try #require(try MCPClientsFixtures.catalog().client(id: "vscode")?.spec(for: .macos)) + guard case let .deeplink(scheme, template)? = spec.install.first else { + Issue.record("VS Code installs through a deeplink first") + return + } + let renderer = MCPClientsFixtures.renderer(for: .macos) + let url = try renderer.url(template, entry: try renderer.entry(spec)) + #expect(scheme == "vscode") + #expect(url.absoluteString == "vscode:mcp/install?" + + "%7B%22name%22%3A%22tablepro%22%2C%22type%22%3A%22stdio%22%2C%22command%22%3A%22%2FApplications%2FTablePro.app" + + "%2FContents%2FMacOS%2Ftablepro-mcp%22%2C%22args%22%3A%5B%5D%7D") + } + + @Test("entryWithName replaces a name the entry already has") + func entryWithNameReplacesName() throws { + let entry: JSONValue = .object([(key: "type", value: .string("stdio")), (key: "name", value: .string("old"))]) + let url = try stdio.url("x:{entryWithName|json|uri}", entry: entry) + #expect(url.absoluteString == "x:%7B%22name%22%3A%22tablepro%22%2C%22type%22%3A%22stdio%22%7D") + } + + @Test("Goose's deeplink percent-encodes the bridge path") + func gooseDeeplink() throws { + let spec = try #require(try MCPClientsFixtures.catalog().client(id: "goose")?.spec(for: .macos)) + guard case let .deeplink(_, template)? = spec.install.first else { + Issue.record("Goose installs through a deeplink") + return + } + let url = try stdio.url(template, entry: try stdio.entry(spec)) + #expect(url.absoluteString == "goose://extension?cmd=%2FApplications%2FTablePro%20Beta.app%2FContents%2FMacOS%2Ftablepro-mcp" + + "&id=tablepro&name=TablePro&description=TablePro%20databases") + } + + @Test("CLI arguments substitute every placeholder in place") + func cliArguments() throws { + let catalog = try MCPClientsFixtures.catalog() + let client = try #require(catalog.client(id: "claude-code")) + for (platform, expected) in [ + (MCPClientPlatform.macos, ["mcp", "add", "--scope", "user", "tablepro", "--", MCPClientsFixtures.bridge]), + (.linux, ["mcp", "add", "--scope", "user", "--transport", "http", "tablepro", MCPClientsFixtures.httpURL, + "--header", "Authorization: Bearer tp_test"]), + ] { + let spec = try #require(client.spec(for: platform)) + guard case .cli(let cli)? = spec.install.first else { + Issue.record("claude-code installs through its CLI on \(platform)") + continue + } + let renderer = MCPClientsFixtures.renderer(for: platform) + #expect(try renderer.arguments(cli.add, entry: try renderer.entry(spec)) == expected) + } + } + + @Test("Command lines quote only the words that need it") + func commandLineQuoting() { + let line = stdio.commandLine(binary: "claude", arguments: ["mcp", "add", "--scope", "user", "tablepro", "--", MCPClientsFixtures.betaBridge]) + #expect(line == "claude mcp add --scope user tablepro -- '/Applications/TablePro Beta.app/Contents/MacOS/tablepro-mcp'") + #expect(stdio.commandLine(binary: "x", arguments: ["it's", "", "a=b,c:d@e%f+g", "Authorization: Bearer t", "~/x", "$HOME"]) + == #"x 'it'\''s' '' a=b,c:d@e%f+g 'Authorization: Bearer t' '~/x' '$HOME'"#) + } + + @Test("Entries keep numbers as integers and booleans as booleans") + func scalarRendering() throws { + let spec = try #require(try MCPClientsFixtures.catalog().client(id: "goose")?.spec(for: .macos)) + let entry = try stdio.entry(spec) + #expect(entry.compactText.contains(#""enabled":true"#)) + #expect(entry.compactText.contains(#""timeout":300"#)) + #expect(entry["cmd"] == .string(MCPClientsFixtures.betaBridge)) + } + + @Test("A placeholder the target cannot fill throws") + func unavailablePlaceholder() { + let http = MCPTemplateRenderer(serverName: "tablepro", target: .http(url: "u", authorization: "a")) + #expect(throws: MCPTemplateError.unavailablePlaceholder("command")) { + try http.entry(template: .object([(key: "command", value: .string("{command}"))])) + } + #expect(throws: MCPTemplateError.unavailablePlaceholder("url")) { + try stdio.arguments(["{url}"], entry: .null) + } + #expect(throws: MCPTemplateError.unknownPlaceholder("bridge")) { + try stdio.arguments(["{bridge}"], entry: .null) + } + #expect(throws: MCPTemplateError.unknownFilter("hex")) { + try stdio.arguments(["{name|hex}"], entry: .null) + } + #expect(throws: MCPTemplateError.unknownFilter("base64")) { + try stdio.arguments(["{entry|base64}"], entry: .null) + } + } + + @Test("A spec without an entry throws") + func missingEntry() throws { + let spec = try #require(try MCPClientsFixtures.catalog().client(id: "chatgpt-classic")?.spec(for: .macos)) + #expect(throws: MCPTemplateError.missingEntry) { + try stdio.entry(spec) + } + } + + @Test("The JSON snippet nests the entry under the key path, two-space indented") + func jsonSnippet() { + let location = MCPConfigLocation(path: "~/.cursor/mcp.json", format: .json, keyPath: ["mcpServers"]) + #expect(stdio.snippet(for: location, entry: MCPClientsFixtures.stdioEntry) == """ + { + "mcpServers": { + "tablepro": { + "command": "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp", + "args": [] + } + } + } + """) + #expect(stdio.snippet(for: nil, entry: .object([(key: "command", value: .string("/b"))])) == """ + { + "tablepro": { + "command": "/b" + } + } + """) + } + + @Test("The TOML snippet is the server's table") + func tomlSnippet() { + let location = MCPConfigLocation(path: "~/.codex/config.toml", format: .toml, keyPath: ["mcp_servers"]) + let entry: JSONValue = .object([ + (key: "url", value: .string("http://127.0.0.1:23508/mcp")), + (key: "http_headers", value: .object([(key: "Authorization", value: .string("Bearer t"))])), + ]) + #expect(stdio.snippet(for: location, entry: entry) == """ + [mcp_servers.tablepro] + url = "http://127.0.0.1:23508/mcp" + http_headers = { Authorization = "Bearer t" } + """) + } + + @Test("The YAML snippet nests block mappings and quotes strings") + func yamlSnippet() { + let location = MCPConfigLocation(path: "~/.config/goose/config.yaml", format: .yaml, keyPath: ["extensions"], isWritable: false) + let entry: JSONValue = .object([ + (key: "cmd", value: .string("/b c")), + (key: "args", value: .array([])), + (key: "tools", value: .array([.string("*")])), + (key: "timeout", value: .number(300)), + ]) + #expect(stdio.snippet(for: location, entry: entry) == """ + extensions: + tablepro: + cmd: "/b c" + args: [] + tools: + - "*" + timeout: 300 + """) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/TOMLTableEditorTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/TOMLTableEditorTests.swift new file mode 100644 index 0000000000..76c8194594 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPClientsTests/TOMLTableEditorTests.swift @@ -0,0 +1,310 @@ +import Foundation +@testable import TableProMCPClients +import Testing + +struct TOMLTableEditorTests { + private let entry: JSONValue = .object([(key: "command", value: .string("/b"))]) + + private let codexConfig = """ + # Codex settings + model = "gpt-5" + approval_policy = "on-request" + + [mcp_servers.node_repl] + command = "/Applications/ChatGPT.app/Contents/Resources/cua_node/bin/node_repl" # keep this + args = [] + + [profiles.fast] + model = "gpt-5-mini" + + """ + + @Test("A missing server is appended after a blank line, and removing it restores the file") + func appendAndRemove() throws { + let result = try TOMLTableEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: codexConfig) + #expect(result == codexConfig + """ + + [mcp_servers.tablepro] + command = "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp" + args = [] + + """) + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: result) == MCPClientsFixtures.stdioEntry) + #expect(try TOMLTableEditor.removing("tablepro", at: ["mcp_servers"], in: result) == codexConfig) + } + + @Test("An existing table and its subtables are replaced in place; other bytes stay") + func replaceInPlace() throws { + let text = """ + model = "gpt-5" + + [mcp_servers.tablepro] + command = "/old" # old + args = ["--x"] + enabled = false + + [mcp_servers.tablepro.env] + TOKEN = "x" + + [mcp_servers.other] + command = "o" # inline comment survives + + """ + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) == .object([ + (key: "command", value: .string("/old")), + (key: "args", value: .array([.string("--x")])), + (key: "enabled", value: .bool(false)), + (key: "env", value: .object([(key: "TOKEN", value: .string("x"))])), + ])) + #expect(try TOMLTableEditor.upserting("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: text) == """ + model = "gpt-5" + + [mcp_servers.tablepro] + command = "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp" + args = [] + + [mcp_servers.other] + command = "o" # inline comment survives + + """) + #expect(try TOMLTableEditor.removing("tablepro", at: ["mcp_servers"], in: text) == """ + model = "gpt-5" + + [mcp_servers.other] + command = "o" # inline comment survives + + """) + } + + @Test("Inline tables, dotted keys inside the table and numbers are read") + func readsHTTPServer() throws { + let text = """ + [mcp_servers.tablepro] + url = "http://127.0.0.1:23508/mcp" + http_headers = { Authorization = "Bearer t" } + startup_timeout_sec = 1_000 + mask = 0xff + ratio = 0.5 + env.A = '1' + created = 1979-05-27 07:32:00Z + """ + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) == .object([ + (key: "url", value: .string("http://127.0.0.1:23508/mcp")), + (key: "http_headers", value: .object([(key: "Authorization", value: .string("Bearer t"))])), + (key: "startup_timeout_sec", value: .number(1_000)), + (key: "mask", value: .number(255)), + (key: "ratio", value: .number(0.5)), + (key: "env", value: .object([(key: "A", value: .string("1"))])), + (key: "created", value: .string("1979-05-27 07:32:00Z")), + ])) + } + + @Test("Nested values render as inline tables and arrays") + func rendersHTTPEntry() throws { + let value: JSONValue = .object([ + (key: "url", value: .string("http://127.0.0.1:23508/mcp")), + (key: "http_headers", value: .object([(key: "Authorization", value: .string("Bearer t"))])), + (key: "env", value: .object([])), + (key: "args", value: .array([.string("a b"), .number(2), .bool(true)])), + ]) + let result = try TOMLTableEditor.upserting("tablepro", value: value, at: ["mcp_servers"], in: "") + #expect(result == """ + [mcp_servers.tablepro] + url = "http://127.0.0.1:23508/mcp" + http_headers = { Authorization = "Bearer t" } + env = {} + args = ["a b", 2, true] + + """) + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: result) == value) + } + + @Test("Escapes round trip and odd keys are quoted") + func escapes() throws { + let value: JSONValue = .object([ + (key: "command", value: .string("/Apps/Tab\"le\\Pro é\n\u{1}\u{7F}")), + (key: "odd key", value: .string("x")), + ]) + let result = try TOMLTableEditor.upserting("my server", value: value, at: ["mcp_servers"], in: "") + #expect(result.hasPrefix(#"[mcp_servers."my server"]"#)) + #expect(result.contains(#""odd key" = "x""#)) + #expect(try TOMLTableEditor.member("my server", at: ["mcp_servers"], in: result) == value) + } + + @Test("Headers and arrays inside strings are not taken for tables") + func lookalikesInStrings() throws { + let text = """ + notes = \"\"\" + [mcp_servers.tablepro] + not a header + \"\"\" + args = [ + "[x]", # comment + 'y', + ] + literal = ''' + [mcp_servers.tablepro] + ''' + + """ + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) == nil) + let result = try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: text) + #expect(result == text + "\n[mcp_servers.tablepro]\ncommand = \"/b\"\n") + } + + @Test("Servers defined with dotted keys elsewhere do not stop a table from being added") + func otherDottedServers() throws { + let text = "mcp_servers.other.command = \"o\"\n" + let result = try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: text) + #expect(result == text + "\n[mcp_servers.tablepro]\ncommand = \"/b\"\n") + } + + @Test("A server defined any other way is refused", arguments: [ + "[mcp_servers]\ntablepro.command = \"x\"\n", + "[mcp_servers]\ntablepro = { command = \"x\" }\n", + "[mcp_servers]\n\"tablepro\" = { command = \"x\" }\n", + "mcp_servers.tablepro.command = \"x\"\n", + "mcp_servers = { tablepro = { command = \"x\" } }\n", + "[[mcp_servers.tablepro]]\ncommand = \"x\"\n", + "[[mcp_servers]]\nname = \"x\"\n", + ]) + func unsupportedShapes(text: String) { + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) + } + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: text) + } + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.removing("tablepro", at: ["mcp_servers"], in: text) + } + } + + @Test("An inline mcp_servers without this server reads as missing but cannot take a table") + func inlineContainer() throws { + let text = "mcp_servers = { other = { command = \"o\" } }\n" + #expect(try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) == nil) + #expect(try TOMLTableEditor.removing("tablepro", at: ["mcp_servers"], in: text) == text) + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: text) + } + } + + @Test("The same table twice is refused") + func duplicateTable() { + let text = "[mcp_servers.tablepro]\ncommand = \"a\"\n[mcp_servers.tablepro]\ncommand = \"b\"\n" + #expect(throws: MCPConfigEditError.duplicateKey(["mcp_servers", "tablepro"])) { + try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) + } + } + + @Test("CRLF files and files without a final line break get matching line breaks") + func lineBreaks() throws { + #expect(try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: "model = \"x\"\r\n") + == "model = \"x\"\r\n\r\n[mcp_servers.tablepro]\r\ncommand = \"/b\"\r\n") + #expect(try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: "model = \"x\"") + == "model = \"x\"\n\n[mcp_servers.tablepro]\ncommand = \"/b\"\n") + #expect(try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: "[mcp_servers.tablepro]\ncommand = \"/old\"") + == "[mcp_servers.tablepro]\ncommand = \"/b\"") + #expect(try TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: "") == "[mcp_servers.tablepro]\ncommand = \"/b\"\n") + #expect(try TOMLTableEditor.removing("tablepro", at: ["mcp_servers"], in: "") == "") + } + + @Test("A null value cannot be written") + func nullRefused() { + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.upserting("tablepro", value: .object([(key: "x", value: .null)]), at: ["mcp_servers"], in: "") + } + } + + @Test("Broken syntax is refused", arguments: [ + "model = \n", "[mcp_servers\n", "a = \"unterminated\n", "a = 1 b = 2\n", "a = [1, 2\n", "a = bare\n", + ]) + func invalidSyntax(text: String) { + #expect(throws: MCPConfigEditError.invalidSyntax) { + try TOMLTableEditor.member("tablepro", at: ["mcp_servers"], in: text) + } + } + + @Test("Merging rewrites only the keys the entry names and keeps every other key and subtable") + func mergeKeepsUserKeys() throws { + let text = """ + model = "gpt-5" + + [mcp_servers.tablepro] + command = "/old" # old + args = ["--x"] + enabled_tools = ["list_connections"] + startup_timeout_sec = 30 + + [mcp_servers.tablepro.env] + TOKEN = "x" + + [mcp_servers.other] + command = "o" + + """ + let result = try TOMLTableEditor.merging("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: text) + #expect(result == """ + model = "gpt-5" + + [mcp_servers.tablepro] + command = "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp" + args = [] + enabled_tools = ["list_connections"] + startup_timeout_sec = 30 + + [mcp_servers.tablepro.env] + TOKEN = "x" + + [mcp_servers.other] + command = "o" + + """) + #expect(try TOMLTableEditor.merging("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: result) == result) + } + + @Test("Merging adds a missing key after the table's last key, keeping CRLF and a missing final line break") + func mergeAddsKeys() throws { + let text = "[mcp_servers.tablepro]\r\ncommand = \"/old\"\r\nenabled = false\r\n\r\n[other]\r\nx = 1\r\n" + #expect(try TOMLTableEditor.merging("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: text) + == "[mcp_servers.tablepro]\r\ncommand = \"/Applications/TablePro.app/Contents/MacOS/tablepro-mcp\"\r\nenabled = false\r\n" + + "args = []\r\n\r\n[other]\r\nx = 1\r\n") + #expect(try TOMLTableEditor.merging("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: "[mcp_servers.tablepro]\nx = 1") + == "[mcp_servers.tablepro]\nx = 1\ncommand = \"/Applications/TablePro.app/Contents/MacOS/tablepro-mcp\"\nargs = []\n") + } + + @Test("A table the entry names is merged into, and rewritten inline only when its contents change") + func mergeNestedTables() throws { + let text = "[mcp_servers.tablepro]\ncommand = \"/b\"\n\n[mcp_servers.tablepro.env]\nTOKEN = \"x\"\n\n[profiles.fast]\nmodel = \"m\"\n" + let keepsEnv: JSONValue = .object([(key: "command", value: .string("/b")), (key: "env", value: .object([]))]) + #expect(try TOMLTableEditor.merging("tablepro", value: keepsEnv, at: ["mcp_servers"], in: text) == text) + + let addsToEnv: JSONValue = .object([(key: "env", value: .object([(key: "NEW", value: .string("y"))]))]) + let merged = try TOMLTableEditor.merging("tablepro", value: addsToEnv, at: ["mcp_servers"], in: text) + #expect(merged == "[mcp_servers.tablepro]\ncommand = \"/b\"\nenv = { TOKEN = \"x\", NEW = \"y\" }\n\n[profiles.fast]\nmodel = \"m\"\n") + + let dotted = "[mcp_servers.tablepro]\nenv.A = \"1\"\ncommand = \"/b\"\nenv.B = \"2\"\n" + #expect(try TOMLTableEditor.merging("tablepro", value: addsToEnv, at: ["mcp_servers"], in: dotted) + == "[mcp_servers.tablepro]\nenv = { A = \"1\", B = \"2\", NEW = \"y\" }\ncommand = \"/b\"\n") + } + + @Test("Merging into a server that has only subtables adds its table at the end") + func mergeWithoutMainTable() throws { + let text = "[mcp_servers.tablepro.env]\nTOKEN = \"x\"\n" + let result = try TOMLTableEditor.merging("tablepro", value: MCPClientsFixtures.stdioEntry, at: ["mcp_servers"], in: text) + #expect(result == text + "\n[mcp_servers.tablepro]\ncommand = \"/Applications/TablePro.app/Contents/MacOS/tablepro-mcp\"\nargs = []\n") + } + + @Test("Merging a missing server is the same as adding it, and other shapes are still refused") + func mergeFallsBack() throws { + #expect(try TOMLTableEditor.merging("tablepro", value: entry, at: ["mcp_servers"], in: codexConfig) + == TOMLTableEditor.upserting("tablepro", value: entry, at: ["mcp_servers"], in: codexConfig)) + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.merging("tablepro", value: entry, at: ["mcp_servers"], in: "[mcp_servers]\ntablepro = { command = \"x\" }\n") + } + #expect(throws: MCPConfigEditError.unsupportedShape) { + try TOMLTableEditor.merging("tablepro", value: entry, at: ["mcp_servers"], in: "mcp_servers = { other = {} }\n") + } + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Auth/MCPPrincipalToolAllowlistTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Auth/MCPPrincipalToolAllowlistTests.swift new file mode 100644 index 0000000000..d83aae0b91 --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Auth/MCPPrincipalToolAllowlistTests.swift @@ -0,0 +1,56 @@ +import Foundation +import TableProMCP +import Testing + +struct MCPPrincipalToolAllowlistTests { + private let tokenId = UUID() + + private func principal(toolAllowlist: Set? = nil) -> MCPPrincipal { + MCPPrincipal( + tokenFingerprint: "fp", + tokenId: tokenId, + scopes: [.toolsRead], + metadata: MCPPrincipalMetadata(label: "test", issuedAt: Date(timeIntervalSince1970: 0), expiresAt: nil), + toolAllowlist: toolAllowlist + ) + } + + @Test("A principal without an allowlist allows every tool its scopes reach") + func noAllowlistAllowsEveryTool() { + let open = principal() + + #expect(open.toolAllowlist == nil) + #expect(open.allowsTool("list_tables")) + #expect(open.allowsTool("execute_query")) + } + + @Test("An allowlist allows only the tools it names") + func allowlistIsMembership() { + let narrow = principal(toolAllowlist: ["list_tables", "describe_table"]) + + #expect(narrow.allowsTool("list_tables")) + #expect(narrow.allowsTool("describe_table")) + #expect(!narrow.allowsTool("execute_query")) + #expect(!narrow.allowsTool("")) + } + + @Test("An empty allowlist allows no tool") + func emptyAllowlistAllowsNothing() { + #expect(!principal(toolAllowlist: []).allowsTool("list_tables")) + } + + @Test("Two principals differing only in their allowlist are not equal") + func equalityIncludesTheAllowlist() { + let tables = principal(toolAllowlist: ["list_tables"]) + let sameTables = principal(toolAllowlist: ["list_tables"]) + + #expect(principal() != tables) + #expect(tables != principal(toolAllowlist: ["describe_table"])) + #expect(tables == sameTables) + } + + @Test("The in-app assistant carries no allowlist") + func inAppAssistantIsUnrestricted() { + #expect(MCPPrincipal.inAppAssistant.toolAllowlist == nil) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Bridge/BridgeProxyStampTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Bridge/BridgeProxyStampTests.swift new file mode 100644 index 0000000000..28f7f2fddc --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Bridge/BridgeProxyStampTests.swift @@ -0,0 +1,97 @@ +import Darwin +import Foundation +@testable import TableProMCP +import TableProMCPTestSupport +import TableProTestSupport +import Testing + +struct BridgeProxyStampTests { + private static let discovery = BridgeDiscovery( + supportedVersions: MCPProtocolVersion.supportedRawValues, + capabilities: .object([:]), + serverInfo: nil, + instructions: nil, + instanceId: "instance-1" + ) + + private static func app() async throws -> MockHttpServer { + let server = MockHttpServer() + await server.setHandler { request in + guard case .request(let rpc)? = try? JsonRpcCodec.decode(request.body) else { + return MockHttpResponse(status: 202, headers: [], body: Data()) + } + let body = try? JsonRpcCodec.encode(.successResponse(JsonRpcSuccessResponse(id: rpc.id, result: .object([:])))) + return MockHttpResponse(status: 200, headers: [("Content-Type", "application/json")], body: body ?? Data()) + } + try await server.start() + return server + } + + private static func forwarded(_ lines: [String]) async throws -> [Data] { + let server = try await app() + let credentials = MCPUpstreamCredentials(socketPath: server.socketPath, serverPID: getpid(), bearerToken: "stamp-token") + let upstream = MCPStreamableHttpClientTransport( + configuration: MCPStreamableHttpClientConfiguration(requestTimeout: .seconds(5)), + credentialsProvider: MCPCachedUpstreamCredentialsProvider(initial: credentials) { credentials }, + errorLogger: nil + ) + let hostToBridge = Pipe() + let bridgeToHost = Pipe() + let proxy = BridgeProxy( + upstream: upstream, + discovery: discovery, + logger: RecordingBridgeLogger(), + stdin: hostToBridge.fileHandleForReading, + stdout: bridgeToHost.fileHandleForWriting + ) + let run = Task { await proxy.run() } + for line in lines { + try hostToBridge.fileHandleForWriting.write(contentsOf: Data((line + "\n").utf8)) + } + var waitFailure: (any Error)? + do { + try await waitUntil { await server.requests.count == lines.count } + } catch { + waitFailure = error + } + try? hostToBridge.fileHandleForWriting.close() + run.cancel() + await run.value + try? bridgeToHost.fileHandleForWriting.close() + let bodies = await server.requests.map(\.body) + await server.stop() + if let waitFailure { throw waitFailure } + return bodies + } + + private static func method(of body: Data) throws -> String? { + try JSONDecoder().decode(JsonValue.self, from: body)["method"]?.stringValue + } + + private static func instance(in body: Data) throws -> String? { + try JSONDecoder().decode(JsonValue.self, from: body)["params"]?["_meta"]?[MCPMetaKeys.bridgeInstance]?.stringValue + } + + @Test("A modern request and a cancellation sent without metadata carry one bridge instance", .timeLimit(.minutes(1))) + func cancellationMeetsItsModernRequest() async throws { + let request = #""" + {"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"list_connections","_meta":\# + {"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{},\# + "io.modelcontextprotocol/clientInfo":{"name":"claude-code","version":"2.0.0"}}}} + """# + let cancel = #"{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":3}}"# + + let bodies = try await Self.forwarded([request, cancel]) + + // The two posts race, so each body is told apart by its method. + let call = try #require(try bodies.first { try Self.method(of: $0) == "tools/call" }) + let cancelled = try #require(try bodies.first { try Self.method(of: $0) == "notifications/cancelled" }) + let requestInstance = try #require(try Self.instance(in: call)) + #expect(try Self.instance(in: cancelled) == requestInstance) + let stamped = request.replacingOccurrences( + of: #""_meta":{"#, + with: #""_meta":{"\#(MCPMetaKeys.bridgeInstance)":"\#(requestInstance)","# + ) + #expect(String(bytes: call, encoding: .utf8) == stamped, "A modern request gains only the bridge instance") + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIPairingTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIPairingTests.swift index 4751d89d9c..879a11987f 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIPairingTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/CommandLine/CLIPairingTests.swift @@ -41,6 +41,10 @@ struct CLIPairingTests { requestTarget: "\(path)?error=access_denied&error_description=user_denied&state=s1", state: "s1" ) == .denied) + #expect(CLIPairingCallback.parse( + requestTarget: "\(path)?error=server_error&error_description=token_not_issued&state=s1", + state: "s1" + ) == .failed) #expect(CLIPairingCallback.parse(requestTarget: "\(path)?code=abc&state=other", state: "s1") == .ignored) #expect(CLIPairingCallback.parse(requestTarget: "/favicon.ico", state: "s1") == .ignored) #expect(CLIPairingCallback.parse(requestTarget: "\(path)?state=s1", state: "s1") == .ignored) diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Lifecycle/MCPStartRefusalTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Lifecycle/MCPStartRefusalTests.swift new file mode 100644 index 0000000000..009442cffe --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Lifecycle/MCPStartRefusalTests.swift @@ -0,0 +1,73 @@ +import Foundation +@testable import TableProMCP +import Testing + +struct MCPStartRefusalTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-refusal-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) + } + + @Test("A refusal written before refusals carried a reason reads as turned off") + func refusalWithoutAReasonIsTurnedOff() throws { + let refusal = try MCPHandshakeCoding.decoder().decode( + MCPStartRefusal.self, + from: Data(#"{"pid":1,"refusedAt":1000.5}"#.utf8) + ) + + #expect(refusal.reason == .turnedOff) + } + + @Test("A reason this build does not know still says the server is not coming up") + func unknownReasonIsAFailedStart() throws { + let refusal = try MCPHandshakeCoding.decoder().decode( + MCPStartRefusal.self, + from: Data(#"{"pid":1,"refusedAt":1000.5,"reason":"somethingNewer"}"#.utf8) + ) + + #expect(refusal.reason == .failedToStart) + } + + @Test("A failed start keeps its reason through the file") + func failedStartRoundTrips() throws { + let written = MCPStartRefusal(pid: 7, refusedAt: Date(timeIntervalSince1970: 1_000.25), reason: .failedToStart) + + let read = try MCPHandshakeCoding.decoder().decode( + MCPStartRefusal.self, + from: MCPHandshakeCoding.encoder().encode(written) + ) + + #expect(read == written) + } + + @Test("The bridge names a failed start instead of waiting out the timeout") + func bridgeReportsAFailedStart() async throws { + let directory = directory + let acquirer = MCPHandshakeAcquirer( + logger: SilentLogger(), + supportDirectory: directory, + pollTimeout: .seconds(30), + launchHostApp: { _ in + let refusal = MCPStartRefusal(pid: 1, refusedAt: Date(), reason: .failedToStart) + try MCPHandshakeCoding.encoder().encode(refusal).write(to: MCPHandshakeLocation.refusalFile(in: directory)) + } + ) + + let started = ContinuousClock().now + await #expect(throws: MCPHandshakeAcquisitionError.serverFailedToStart) { + _ = try await acquirer.acquire() + } + #expect(ContinuousClock().now - started < .seconds(5)) + let message = MCPHandshakeAcquisitionError.serverFailedToStart.localizedDescription + #expect(message.contains("Settings > MCP")) + #expect(!message.contains("not running")) + } +} + +private struct SilentLogger: MCPBridgeLogger { + func log(_ level: MCPBridgeLogLevel, _ message: String) {} +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Protocol/MCPInflightRegistryTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Protocol/MCPInflightRegistryTests.swift index 53f45b3391..d217c57f36 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Protocol/MCPInflightRegistryTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Protocol/MCPInflightRegistryTests.swift @@ -251,4 +251,126 @@ struct MCPInflightRegistryTests { #expect(key.requestId == .string("abc")) #expect(key == MCPInflightKey(clientFingerprint: "alice", requestId: .string("abc"))) } + + @Test("Two stdio clients on the bridge credential keep one request id apart") + func stdioClientsSharingTheBridgeCredentialAreTwoEntries() async { + let registry = MCPInflightRegistry() + let cursorToken = MCPCancellationToken() + let zedToken = MCPCancellationToken() + let cursorKey = MCPInflightKey(client: Self.stdioClient("cursor-vscode"), requestId: .number(1)) + let zedKey = MCPInflightKey(client: Self.stdioClient("Zed"), requestId: .number(1)) + #expect(cursorKey != zedKey) + + #expect(await registry.register(key: cursorKey, token: cursorToken, tokenId: nil, method: "tools/call", startedAt: .now)) + #expect(await registry.register(key: zedKey, token: zedToken, tokenId: nil, method: "tools/call", startedAt: .now)) + #expect(await registry.count() == 2) + + #expect(await registry.cancel(key: zedKey, reason: .clientRequested(nil))) + #expect(await zedToken.isCancelled) + #expect(!(await cursorToken.isCancelled)) + + await registry.remove(key: zedKey, token: zedToken) + #expect(await registry.contains(key: cursorKey)) + } + + @Test("A token one client holds keys its requests by the token alone") + func ownTokenKeysByTheTokenAlone() { + let identity = MCPClientIdentity( + clientName: "Raycast", + clientVersion: "1.0", + tokenId: alice.tokenId, + tokenName: "Raycast", + principalFingerprint: alice.tokenFingerprint, + address: .loopback, + transport: .http + ) + #expect(MCPInflightKey(client: identity, requestId: .number(7)) == MCPInflightKey(principal: alice, requestId: .number(7))) + } + + @Test("A stdio cancellation sent with the renewed bridge credential finds the request the old one started") + func stdioCancellationCrossesACredentialRotation() async { + let registry = MCPInflightRegistry() + let token = MCPCancellationToken() + let started = MCPInflightKey(client: Self.stdioClient("claude-code", fingerprint: "a"), requestId: .number(5)) + let cancelled = MCPInflightKey(client: Self.stdioClient("claude-code", fingerprint: "b"), requestId: .number(5)) + + await registry.register(key: started, token: token, tokenId: nil, method: "tools/call", startedAt: .now) + + #expect(await registry.cancel(key: cancelled, reason: .clientRequested(nil))) + #expect(await token.isCancelled) + } + + @Test("Two sessions of one stdio client keep one request id apart, and ending the client ends every session") + func sessionsOfOneStdioClientAreSeparateEntries() async { + let registry = MCPInflightRegistry() + let client = Self.stdioClient("claude-code") + let tokens = [MCPCancellationToken(), MCPCancellationToken(), MCPCancellationToken()] + let keys = ["one", "two", "three"].map { MCPInflightKey(client: client, bridgeInstance: $0, requestId: .number(1)) } + + for (key, token) in zip(keys, tokens) { + #expect(await registry.register(key: key, token: token, tokenId: nil, clientId: client.id, method: "tools/call", startedAt: .now)) + } + #expect(await registry.count() == 3) + + #expect(await registry.cancel(key: keys[2], reason: .clientRequested(nil))) + #expect(!(await tokens[0].isCancelled)) + #expect(!(await tokens[1].isCancelled)) + + #expect(await registry.cancelAll(matchingClientId: client.id, reason: .clientDisconnected) == 2) + #expect(await tokens[0].isCancelled) + #expect(await tokens[1].isCancelled) + } + + @Test("A stdio cancellation without clientInfo stops the request that had it, by bridge instance and request id") + func stdioCancellationWithoutClientInfo() async { + let registry = MCPInflightRegistry() + let token = MCPCancellationToken() + let named = Self.stdioClient("claude-code") + let unnamed = MCPClientIdentity( + clientName: MCPClientIdentity.unknownClientName, + clientVersion: nil, + tokenId: UUID(), + tokenName: "__stdio_bridge__", + principalFingerprint: "bridge", + address: .localSocket, + transport: .stdioBridge + ) + let started = MCPInflightKey(client: named, bridgeInstance: "one", requestId: .number(3)) + await registry.register(key: started, token: token, tokenId: nil, clientId: named.id, method: "tools/call", startedAt: .now) + + #expect(!(await registry.cancel(key: MCPInflightKey(client: unnamed, bridgeInstance: "two", requestId: .number(3)), reason: .clientRequested(nil)))) + #expect(await registry.cancel(key: MCPInflightKey(client: unnamed, bridgeInstance: "one", requestId: .number(3)), reason: .clientRequested(nil))) + #expect(await token.isCancelled) + #expect(MCPInflightKey(bridgeInstance: "one", requestId: .number(3)) == started) + } + + @Test("A client on a token of its own cannot pick a bridge instance") + func ownTokenIgnoresTheBridgeInstance() { + let identity = MCPClientIdentity( + clientName: "Raycast", + clientVersion: "1.0", + tokenId: alice.tokenId, + tokenName: "Raycast", + principalFingerprint: alice.tokenFingerprint, + address: .loopback, + transport: .http + ) + #expect( + MCPInflightKey(client: identity, bridgeInstance: "forged", requestId: .number(7)) + == MCPInflightKey(principal: alice, requestId: .number(7)) + ) + } + + private static func stdioClient(_ name: String, fingerprint: String = "bridge") -> MCPClientIdentity { + MCPClientIdentity( + clientName: name, + clientInfoName: name, + clientVersion: "1.0", + tokenId: UUID(), + tokenName: "__stdio_bridge__", + principalFingerprint: fingerprint, + address: .localSocket, + transport: .stdioBridge + ) + } } diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/BridgeStdoutTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/BridgeStdoutTests.swift index 1bea723d88..77342cdf2e 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/BridgeStdoutTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/BridgeStdoutTests.swift @@ -2,8 +2,8 @@ import Darwin import Foundation @testable import TableProMCP import TableProMCPTestSupport +import TableProProcess import TableProTestSupport -import TableProUtilities import Testing struct BridgeStdoutTests { diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/MCPStderrBridgeLoggerTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/MCPStderrBridgeLoggerTests.swift index f760887de4..9d5f2bd676 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/MCPStderrBridgeLoggerTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Transport/MCPStderrBridgeLoggerTests.swift @@ -1,8 +1,8 @@ import Darwin import Foundation @testable import TableProMCP +import TableProProcess import TableProTestSupport -import TableProUtilities import Testing struct MCPStderrBridgeLoggerTests { diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift index c40818145b..eb1c3c1459 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift @@ -11,9 +11,6 @@ import OSLog public enum OracleCellFormatting { public static let maxHexBytes = 4_096 - /// A region-named zone is sent as an id into Oracle's time zone file, which the client does not have. - public static let timeZoneRegionPlaceholder = unsupportedPlaceholder(typeName: "time zone region") - // MARK: - Datetimes /// A zone's hour and minute are each sent above a bias, so a zone west of UTC sends bytes below it. @@ -32,16 +29,25 @@ public enum OracleCellFormatting { return fields.text(fractionDigits: fractionDigits) } - /// `TIMESTAMP(p) WITH TIME ZONE` as the wall clock at the value's own offset, followed by that offset - /// with no space. The date and time bytes are UTC. - public static func timestampWithTimeZoneText(wire bytes: [UInt8], fractionDigits: Int) -> String? { + // The date and time bytes are UTC, whether the zone is an offset or a region. + public static func timestampWithTimeZoneText( + wire bytes: [UInt8], + fractionDigits: Int, + regions: OracleTimeZoneRegions = .none + ) -> String? { guard var fields = OracleWireDateTime(bytes) else { return nil } + if let id = OracleTimeZoneRegions.regionID(wire: bytes) { + guard let region = regions.region(forID: id), let zone = region.zone else { + return fields.text(fractionDigits: fractionDigits) + offsetText(minutes: 0) + } + let instant = Date(timeIntervalSince1970: TimeInterval(fields.secondsTreatingFieldsAsUTC)) + fields.shift(bySeconds: zone.secondsFromGMT(for: instant)) + // A space before the name, as Oracle's TZR spells it. + return fields.text(fractionDigits: fractionDigits) + " " + region.name + } guard bytes.count >= 13, bytes[11] != 0, bytes[12] != 0 else { return fields.text(fractionDigits: fractionDigits) + "+00:00" } - if bytes[11] & 0x80 != 0 { - return timeZoneRegionPlaceholder - } let offsetMinutes = (Int(bytes[11]) - zoneHourBias) * 60 + Int(bytes[12]) - zoneMinuteBias fields.shift(bySeconds: offsetMinutes * 60) return fields.text(fractionDigits: fractionDigits) + offsetText(minutes: offsetMinutes) @@ -324,11 +330,12 @@ struct OracleResultColumn: Sendable, Equatable { enum OracleCellDecoding { private static let logger = Logger(subsystem: "com.TablePro", category: "OracleCellDecoding") - /// `unsupported` hears the type name of a cell rendered as a placeholder, so the caller can log it once. + // `unsupported` hears the type name of a cell rendered as a placeholder, so the caller can log it once. static func decode( _ cell: OracleCell, column: OracleResultColumn, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions = .none, unsupported: (String) -> Void ) -> OracleRawCell { guard let bytes = cell.bytes else { return .null } @@ -339,7 +346,7 @@ enum OracleCellDecoding { do { guard let text = try text(of: cell, wire: Array(bytes.readableBytesView), column: column, zones: zones, - unsupported: unsupported) + regions: regions, unsupported: unsupported) else { return .null } return .string(text) } catch { @@ -355,6 +362,7 @@ enum OracleCellDecoding { wire: [UInt8], column: OracleResultColumn, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, unsupported: (String) -> Void ) throws -> String? { switch cell.dataType { @@ -380,7 +388,9 @@ enum OracleCellDecoding { return OracleCellFormatting.timestampText(wire: wire, fractionDigits: column.fractionDigits) case .timestampTZ: - return OracleCellFormatting.timestampWithTimeZoneText(wire: wire, fractionDigits: column.fractionDigits) + return OracleCellFormatting.timestampWithTimeZoneText( + wire: wire, fractionDigits: column.fractionDigits, regions: regions + ) case .timestampLTZ: return OracleCellFormatting.localTimestampText(wire: wire, fractionDigits: column.fractionDigits, zones: zones) diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift index 19c7fd4a89..fc53cd4a15 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift @@ -109,9 +109,10 @@ public final class OracleCoreConnection: @unchecked Sendable { var close = OracleCloseRecord() var serverRelease: OracleServerRelease? var loginWarning: String? - /// Read before the first statement of each session, and again before the statement after one that can - /// change them, so no result ever waits on them mid-stream. + // Read before a session's first statement and again after one that can change them, never mid-stream. var sessionZones: OracleSessionTimeZones? + // The server's time zone file fixes these, so only a new session clears them. + var timeZoneRegions: OracleTimeZoneRegions? var isReservedForDebugging = false } @@ -218,6 +219,7 @@ public final class OracleCoreConnection: @unchecked Sendable { current.serverRelease = release current.loginWarning = connection.loginWarning?.message current.sessionZones = nil + current.timeZoneRegions = nil current.close.clearOnConnect() return true } @@ -846,8 +848,13 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) let result = try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) + let regions = await timeZoneRegions(on: connection) return try await collectRows( - statement, options: OracleRowLimits.collecting(admitted.options), zones: zones, on: connection + statement, + options: OracleRowLimits.collecting(admitted.options), + zones: zones, + regions: regions, + on: connection ) } await recordSuccess(of: role, on: admitted.session) @@ -936,7 +943,7 @@ public final class OracleCoreConnection: @unchecked Sendable { } } - /// The app's own session statements, none of which reads a `TIMESTAMP WITH LOCAL TIME ZONE`. + // The app's own session statements, none of which reads a zoned timestamp. private func collectRows( _ query: String, options: StatementOptions, @@ -946,6 +953,7 @@ public final class OracleCoreConnection: @unchecked Sendable { OracleStatement(unsafeSQL: query), options: options, zones: .driverDefault, + regions: OracleTimeZoneRegions.none, on: connection ) } @@ -954,12 +962,13 @@ public final class OracleCoreConnection: @unchecked Sendable { _ statement: OracleStatement, options: StatementOptions, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, on connection: OracleNIO.OracleConnection ) async throws -> OracleRawResult { let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) - var decoder = CellDecoder(columns: columns, zones: zones) + var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) var allRows: [[OracleRawCell]] = [] var truncated = false @@ -993,7 +1002,18 @@ public final class OracleCoreConnection: @unchecked Sendable { public func streamQuery( _ query: String, + batching: OracleRowBatcher, continuation: AsyncThrowingStream.Continuation + ) async throws { + try await streamQuery(query, batching: batching, onElement: { continuation.yield($0) }) + continuation.finish() + } + + // The next row is read only once `onElement` returns, so a caller that waits in it holds the statement open. + public func streamQuery( + _ query: String, + batching: OracleRowBatcher, + onElement: @escaping @Sendable (OracleStreamElement) async -> Void ) async throws { let role = OracleTransactionRole(of: query) await queryGate.acquire() @@ -1004,12 +1024,20 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) - try await streamRows(query, options: admitted.options, zones: zones, on: connection, continuation: continuation) + let regions = await timeZoneRegions(on: connection) + try await streamRows( + query, + options: admitted.options, + zones: zones, + regions: regions, + batching: batching, + on: connection, + onElement: onElement + ) } await recordSuccess(of: role, on: admitted.session) forgetSessionZonesIfChanged(by: query) await queryGate.release() - continuation.finish() } catch { let mapped = mapExecutionError(error) forgetSessionZonesIfChanged(by: query) @@ -1023,24 +1051,36 @@ public final class OracleCoreConnection: @unchecked Sendable { _ query: String, options: StatementOptions, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, + batching: OracleRowBatcher, on connection: OracleNIO.OracleConnection, - continuation: AsyncThrowingStream.Continuation + onElement: @escaping @Sendable (OracleStreamElement) async -> Void ) async throws { let statement = OracleStatement(unsafeSQL: query) let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) - var decoder = CellDecoder(columns: columns, zones: zones) - continuation.yield(.header(columns: columns.map(\.descriptor))) + var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) + await onElement(.header(columns: columns.map(\.descriptor))) - for try await row in stream { - try Task.checkCancellation() - continuation.yield(.rows([decoder.decode(row)])) + var batcher = batching + do { + for try await row in stream { + try Task.checkCancellation() + if let batch = batcher.append(decoder.decode(row)) { + await onElement(.rows(batch)) + } + } + } catch { + // The rows read before a failure still reach the reader ahead of the error, as they did one by one. + if let batch = batcher.remainder() { await onElement(.rows(batch)) } + throw error } + if let batch = batcher.remainder() { await onElement(.rows(batch)) } await warnOnce(about: decoder.unsupportedTypeNames) guard columns.isEmpty else { return } - continuation.yield(.affectedRows(try await stream.affectedRows)) + await onElement(.affectedRows(try await stream.affectedRows)) } // MARK: - Session Time Zones @@ -1085,17 +1125,68 @@ public final class OracleCoreConnection: @unchecked Sendable { state.withLock { $0.sessionZones = nil } } + // MARK: - Time Zone Regions + + // A failed read keeps an empty map for the session, so a region value reads as its UTC wall clock and the + // read is not retried before every statement. + private func timeZoneRegions(on connection: OracleNIO.OracleConnection) async -> OracleTimeZoneRegions { + if let regions = state.withLock({ $0.timeZoneRegions }) { + return regions + } + do { + let regions = try await readTimeZoneRegions(on: connection) + state.withLock { $0.timeZoneRegions = regions } + return regions + } catch is CancellationError { + return OracleTimeZoneRegions.none + } catch { + osLogger.error( + "Reading the Oracle time zone regions failed: \(String(describing: type(of: error)), privacy: .public)" + ) + state.withLock { $0.timeZoneRegions = OracleTimeZoneRegions.none } + return OracleTimeZoneRegions.none + } + } + + // Read straight from the stream: the cell decode would turn the region column back into the UTC fallback. + private func readTimeZoneRegions(on connection: OracleNIO.OracleConnection) async throws -> OracleTimeZoneRegions { + let stream = try await connection.execute( + OracleStatement(unsafeSQL: OracleTimeZoneRegions.query), + options: OracleRowLimits.collecting(Self.sessionSetupOptions), + logger: nioLogger + ) + var rows: [(name: String, wire: [UInt8])] = [] + for try await row in stream { + let cells = Array(row) + guard cells.count == 2, let name = try? cells[0].decode(String.self), let wire = cells[1].bytes else { + continue + } + rows.append((name: name, wire: Array(wire.readableBytesView))) + } + let regions = OracleTimeZoneRegions(rows: rows) + let repeated = regions.duplicateIDs + let unknown = regions.unknownNames + if repeated > 0 || unknown > 0 { + osLogger.notice( + "Oracle time zone regions: \(regions.count, privacy: .public), repeated ids \(repeated, privacy: .public), unknown names \(unknown, privacy: .public)" + ) + } + return regions + } + // MARK: - Cell Decoding - /// Decodes the cells of one result, collecting the names of the types it had to render as placeholders. + // Collects the names of the types it had to render as placeholders, so each is logged once. private struct CellDecoder { let columns: [OracleResultColumn] let zones: OracleSessionTimeZones + let regions: OracleTimeZoneRegions private(set) var unsupportedTypeNames: Set = [] - init(columns: [OracleResultColumn], zones: OracleSessionTimeZones) { + init(columns: [OracleResultColumn], zones: OracleSessionTimeZones, regions: OracleTimeZoneRegions) { self.columns = columns self.zones = zones + self.regions = regions } mutating func decode(_ cells: some Sequence) -> [OracleRawCell] { @@ -1105,7 +1196,7 @@ public final class OracleCoreConnection: @unchecked Sendable { let column = columns.indices.contains(index) ? columns[index] : OracleResultColumn(name: cell.columnName, dataType: cell.dataType, scale: 9) - values.append(OracleCellDecoding.decode(cell, column: column, zones: zones) { name in + values.append(OracleCellDecoding.decode(cell, column: column, zones: zones, regions: regions) { name in unsupportedTypeNames.insert(name) }) } diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRowBatcher.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRowBatcher.swift new file mode 100644 index 0000000000..ebd111018a --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRowBatcher.swift @@ -0,0 +1,43 @@ +import Foundation + +// Groups decoded rows into stream elements, so a long read pays its stream hops once per batch rather than once per row. +public struct OracleRowBatcher: Sendable { + public static let batchRows = 5_000 + + private let rowsPerBatch: Int + // The rows a capped reader takes before it stops. The batch that completes them goes out at once, so the reader + // never waits on rows past its cap. + private let rowBudget: Int? + private var delivered = 0 + private var pending: [[OracleRawCell]] = [] + + public init(rowsPerBatch: Int, rowBudget: Int? = nil) { + self.rowsPerBatch = max(1, rowsPerBatch) + self.rowBudget = rowBudget.map { max(1, $0) } + } + + public mutating func append(_ row: [OracleRawCell]) -> [[OracleRawCell]]? { + if pending.isEmpty, delivered > 0 { pending.reserveCapacity(nextBatchCapacity) } + pending.append(row) + // The first row goes out alone, since a reader times it as the moment the server began to answer. + let isFirstRow = delivered == 0 + guard isFirstRow || pending.count >= rowsPerBatch || rowBudget == delivered + pending.count else { return nil } + return take() + } + + public mutating func remainder() -> [[OracleRawCell]]? { + pending.isEmpty ? nil : take() + } + + private var nextBatchCapacity: Int { + guard let rowBudget, rowBudget > delivered else { return rowsPerBatch } + return min(rowsPerBatch, rowBudget - delivered) + } + + private mutating func take() -> [[OracleRawCell]] { + let batch = pending + delivered += batch.count + pending = [] + return batch + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift index 66d1fcceb5..e7732e76a0 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift @@ -32,4 +32,14 @@ public struct OracleServerRelease: Sendable, Equatable { public var hasVectorInfo: Bool { major > 23 || (major == 23 && update >= 4) } + + // 12.1 added `ALL_CONSTRAINTS.SEARCH_CONDITION_VC`. Before it a check's text is only in the LONG `SEARCH_CONDITION`. + public var hasSearchConditionText: Bool { + major >= 12 + } + + // 23ai brought SQL's BOOLEAN type. Before it `TRUE` and `FALSE` are names, and a statement using one fails ORA-00904. + public var hasBooleanType: Bool { + major >= 23 + } } diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTableConstraint.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTableConstraint.swift new file mode 100644 index 0000000000..47ddaa87db --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTableConstraint.swift @@ -0,0 +1,192 @@ +import Foundation + +// A PRIMARY KEY, UNIQUE or CHECK constraint, as `CREATE TABLE` writes it after the columns. A dump needs the keys as +// constraints, not as unique indexes: a foreign key added later fails with ORA-02270 without one on the columns it +// references. +public struct OracleTableConstraint: Sendable, Equatable { + public enum Kind: Sendable, Equatable { + case primaryKey + case unique + case check + + init?(code: String?) { + switch code { + case "P"?: self = .primaryKey + case "U"?: self = .unique + case "C"?: self = .check + default: return nil + } + } + + var rank: Int { + switch self { + case .primaryKey: return 0 + case .unique: return 1 + case .check: return 2 + } + } + } + + public let name: String + public let kind: Kind + // In key order for a key. For a check, the columns its condition names. + public let columns: [String] + // Nil for a key. + public var condition: String? + // `SEARCH_CONDITION_VC` is a VARCHAR2(4000) that cuts a longer condition, so one of 4,000 bytes may be cut short. + public let conditionMayBeCut: Bool + public let isDeferrable: Bool + public let isInitiallyDeferred: Bool + public let isEnabled: Bool + // An enabled NOVALIDATE constraint was never checked against the rows it already had, so they may break it. + public let isValidated: Bool + // A `SYS_C` name is numbered across the source database, so on another one it may already be taken. + public let hasGeneratedName: Bool + + public init( + name: String, + kind: Kind, + columns: [String], + condition: String? = nil, + conditionMayBeCut: Bool = false, + isDeferrable: Bool = false, + isInitiallyDeferred: Bool = false, + isEnabled: Bool = true, + isValidated: Bool = true, + hasGeneratedName: Bool = false + ) { + self.name = name + self.kind = kind + self.columns = columns + self.condition = condition + self.conditionMayBeCut = conditionMayBeCut + self.isDeferrable = isDeferrable + self.isInitiallyDeferred = isInitiallyDeferred + self.isEnabled = isEnabled + self.isValidated = isValidated + self.hasGeneratedName = hasGeneratedName + } + + // Written disabled so the rows that break it still restore, then enabled after them without a check. + public var enablesAfterRows: Bool { + isEnabled && !isValidated + } + + // A check with a generated name has no name to enable it by, so it is added after the rows instead of written in + // the CREATE TABLE. + public var isAddedAfterRows: Bool { + enablesAfterRows && kind == .check && hasGeneratedName + } + + // A key with a generated name is enabled by what it is, which needs no name. Nil for a check added after the rows. + public func enableAfterRowsStatement(qualifiedTable: String, quote: (String) -> String) -> String? { + guard enablesAfterRows else { return nil } + guard hasGeneratedName else { + return "ALTER TABLE \(qualifiedTable) ENABLE NOVALIDATE CONSTRAINT \(quote(name))" + } + switch kind { + case .primaryKey: + return "ALTER TABLE \(qualifiedTable) ENABLE NOVALIDATE PRIMARY KEY" + case .unique: + return "ALTER TABLE \(qualifiedTable) ENABLE NOVALIDATE UNIQUE (\(columns.map(quote).joined(separator: ", ")))" + case .check: + return nil + } + } + + // Oracle keeps a NOT NULL as a check reading `"COL" IS NOT NULL`, which the column's own definition already + // writes when the column is NOT NULL. + public func restatesNotNull(of notNullColumns: Set) -> Bool { + guard kind == .check, isEnabled, isValidated, columns.count == 1, let column = columns.first, + notNullColumns.contains(column), let condition else { return false } + return condition.trimmingCharacters(in: .whitespacesAndNewlines) + == "\(OracleSchemaQueries.quoteIdentifier(column)) IS NOT NULL" + } + + // A generated name is left for the server to choose again. + public func clause(quote: (String) -> String) -> String { + var parts: [String] = [] + if !hasGeneratedName { + parts.append("CONSTRAINT \(quote(name))") + } + let columnList = columns.map(quote).joined(separator: ", ") + switch kind { + case .primaryKey: + parts.append("PRIMARY KEY (\(columnList))") + case .unique: + parts.append("UNIQUE (\(columnList))") + case .check: + let text = (condition ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + parts.append("CHECK (\(text))") + } + if isDeferrable { + parts.append(isInitiallyDeferred ? "DEFERRABLE INITIALLY DEFERRED" : "DEFERRABLE") + } + if !isEnabled || !isValidated { + parts.append("DISABLE") + } + return parts.joined(separator: " ") + } +} + +extension OracleSchemaQueries { + // One row per constraint column. Before 12.1 a check's text is only in a LONG oracle-nio cannot decode, so only + // the keys are read there. + public static func tableConstraints(schema: String, table: String, release: OracleServerRelease) -> String { + let readsChecks = release.hasSearchConditionText + let condition = readsChecks ? "c.SEARCH_CONDITION_VC" : "NULL" + let conditionBytes = readsChecks ? "LENGTHB(c.SEARCH_CONDITION_VC)" : "NULL" + let types = readsChecks ? "'P', 'U', 'C'" : "'P', 'U'" + return """ + SELECT c.CONSTRAINT_NAME, c.CONSTRAINT_TYPE, c.DEFERRABLE, c.DEFERRED, c.STATUS, c.GENERATED, + \(condition) AS SEARCH_CONDITION_VC, \(conditionBytes) AS CONDITION_BYTES, cc.COLUMN_NAME, + c.VALIDATED + FROM \(OracleDictionary.allConstraints) c + LEFT JOIN \(OracleDictionary.allConsColumns) cc ON cc.OWNER = c.OWNER + AND cc.CONSTRAINT_NAME = c.CONSTRAINT_NAME + WHERE c.OWNER = '\(escapeLiteral(schema))' + AND c.TABLE_NAME = '\(escapeLiteral(table))' + AND c.CONSTRAINT_TYPE IN (\(types)) + ORDER BY c.CONSTRAINT_NAME, cc.POSITION, cc.COLUMN_NAME + """ + } + + // Keys first, then checks, each by name, so a dump reads the same every time. + public static func parseTableConstraints(_ rows: [[OracleRawCell]]) -> [OracleTableConstraint] { + var order: [String] = [] + var firstRows: [String: (row: [OracleRawCell], kind: OracleTableConstraint.Kind)] = [:] + var columns: [String: [String]] = [:] + for row in rows { + guard let name = row[safe: 0]?.stringValue, + let kind = OracleTableConstraint.Kind(code: row[safe: 1]?.stringValue) else { continue } + if firstRows[name] == nil { + order.append(name) + firstRows[name] = (row, kind) + } + if let column = row[safe: 8]?.stringValue { + columns[name, default: []].append(column) + } + } + let constraints = order.compactMap { name -> OracleTableConstraint? in + guard let entry = firstRows[name] else { return nil } + let row = entry.row + let kind = entry.kind + let conditionBytes = row[safe: 7]?.stringValue.flatMap(OracleSchemaQueries.integer) + return OracleTableConstraint( + name: name, + kind: kind, + columns: columns[name] ?? [], + condition: kind == .check ? row[safe: 6]?.stringValue : nil, + conditionMayBeCut: kind == .check && (conditionBytes ?? 0) >= maxSearchConditionBytes, + isDeferrable: row[safe: 2]?.stringValue == "DEFERRABLE", + isInitiallyDeferred: row[safe: 3]?.stringValue == "DEFERRED", + isEnabled: row[safe: 4]?.stringValue != "DISABLED", + isValidated: row[safe: 9]?.stringValue != "NOT VALIDATED", + hasGeneratedName: row[safe: 5]?.stringValue == "GENERATED NAME" + ) + } + return constraints.sorted { ($0.kind.rank, $0.name) < ($1.kind.rank, $1.name) } + } + + static let maxSearchConditionBytes = 4_000 +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift new file mode 100644 index 0000000000..34ec98233f --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift @@ -0,0 +1,63 @@ +import Foundation + +// A TIMESTAMP WITH TIME ZONE stored with a region name carries an id into the server's time zone file, which a +// thin client does not have, so the names are read from the server once per connection. +public struct OracleTimeZoneRegions: Sendable, Equatable { + public struct Region: Sendable, Equatable { + public let name: String + public let zone: TimeZone? + } + + public static let none = OracleTimeZoneRegions(regions: [:], duplicateIDs: 0) + + // AT TIME ZONE converts an instant, so no name can fail on a wall clock its zone skips. + public static let query = "SELECT TZNAME, TIMESTAMP '2000-01-01 00:00:00 UTC' AT TIME ZONE TZNAME FROM " + + "(SELECT DISTINCT TZNAME FROM \(OracleDictionary.performanceView("V$TIMEZONE_NAMES"))) ORDER BY TZNAME" + + private static let regionFlag: UInt8 = 0x80 + + let regions: [Int: Region] + let duplicateIDs: Int + + private init(regions: [Int: Region], duplicateIDs: Int) { + self.regions = regions + self.duplicateIDs = duplicateIDs + } + + public init(rows: [(name: String, wire: [UInt8])]) { + var regions: [Int: Region] = [:] + var duplicateIDs = 0 + for row in rows { + guard let id = Self.regionID(wire: row.wire) else { continue } + guard regions[id] == nil else { + duplicateIDs += 1 + continue + } + regions[id] = Region(name: row.name, zone: TimeZone(identifier: row.name)) + } + self.init(regions: regions, duplicateIDs: duplicateIDs) + } + + public var isEmpty: Bool { + regions.isEmpty + } + + public var count: Int { + regions.count + } + + var unknownNames: Int { + regions.values.filter { $0.zone == nil }.count + } + + func region(forID id: Int) -> Region? { + regions[id] + } + + // An offset's hour byte is at most 20 + 14, so the flag alone marks a region; the low two bits of byte 12 are not + // part of the id. + public static func regionID(wire bytes: [UInt8]) -> Int? { + guard bytes.count >= 13, bytes[11] & regionFlag != 0 else { return nil } + return (Int(bytes[11] & 0x7F) << 6) | (Int(bytes[12] & 0xFC) >> 2) + } +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift index dbc3ed24d6..5e5f2c8306 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift @@ -15,12 +15,15 @@ struct OracleCellDecodingTests { _ bytes: [UInt8]?, as type: OracleDataType, declared: OracleDataType? = nil, - scale: Int = 0 + scale: Int = 0, + regions: OracleTimeZoneRegions = .none ) -> (OracleRawCell, Set) { let cell = OracleCell(bytes: bytes.map { ByteBuffer(bytes: $0) }, dataType: type, columnName: "C", columnIndex: 0) let column = OracleResultColumn(name: "C", dataType: declared ?? type, scale: scale) var unsupported: Set = [] - let value = OracleCellDecoding.decode(cell, column: column, zones: Self.utcPlusSeven) { unsupported.insert($0) } + let value = OracleCellDecoding.decode(cell, column: column, zones: Self.utcPlusSeven, regions: regions) { + unsupported.insert($0) + } return (value, unsupported) } @@ -73,6 +76,14 @@ struct OracleCellDecodingTests { #expect(decode(Self.hex("787c0102020506000003e8"), as: .timestampLTZ, scale: 6).0 == .string("2024-01-02 08:04:05.000001+07:00")) } + @Test func testRegionZoneDecodesThroughTheRegionMap() { + let paris = Self.hex("787c01020305061dcd650085f8") + let (named, unsupported) = decode(paris, as: .timestampTZ, scale: 6, regions: OracleTimeZoneRegionFixtures.paris) + #expect(named == .string("2024-01-02 03:04:05.500000 Europe/Paris")) + #expect(unsupported.isEmpty) + #expect(decode(paris, as: .timestampTZ, scale: 6).0 == .string("2024-01-02 02:04:05.500000+00:00")) + } + @Test func testUniversalRowIDReadsAsOracleSpellsIt() { #expect(decode(Array("*BAYAAesCwQL+".utf8), as: .uRowID).0 == .string("*BAYAAesCwQL+")) } diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift index 96687599e5..a766acd508 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift @@ -54,10 +54,58 @@ struct OracleCellFormattingTests { #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: utc, fractionDigits: 0) == "2024-01-01 00:30:00+05:00") } - /// A zone stored by region name is an id into Oracle's time zone file. It used to read ``. - @Test func testRegionNamedZoneIsAPlaceholder() { - let paris: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x03, 0x05, 0x06, 0x1d, 0xcd, 0x65, 0x00, 0x85, 0xf8] - #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: paris, fractionDigits: 6) == "") + private static let parisInJanuary: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x03, 0x05, 0x06, 0x1d, 0xcd, 0x65, 0x00, 0x85, 0xf8] + + private static func regionText(_ wire: [UInt8], fractionDigits: Int = 6, regions: OracleTimeZoneRegions) -> String? { + OracleCellFormatting.timestampWithTimeZoneText(wire: wire, fractionDigits: fractionDigits, regions: regions) + } + + // A zone stored by region name used to read ``, and export wrote that text. + @Test func testRegionZoneIsTheWallClockAndOraclesName() { + #expect(Self.regionText(Self.parisInJanuary, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.500000 Europe/Paris") + } + + @Test func testRegionZoneFollowsDaylightSaving() { + let july: [UInt8] = [0x78, 0x7c, 0x07, 0x01, 0x0d, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x85, 0xf8] + #expect(Self.regionText(july, fractionDigits: 0, regions: OracleTimeZoneRegionFixtures.paris) == "2024-07-01 14:00:00 Europe/Paris") + } + + @Test func testRegionZoneCanCrossAYearBoundary() { + let lastHalfHourOf2024: [UInt8] = [0x78, 0x7c, 0x0c, 0x1f, 0x18, 0x1f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x85, 0xf8] + let paris = Self.regionText(lastHalfHourOf2024, fractionDigits: 0, regions: OracleTimeZoneRegionFixtures.paris) + #expect(paris == "2025-01-01 00:30:00 Europe/Paris") + + // Id 65 is made up; only the map decides which name it stands for. + let losAngeles = OracleTimeZoneRegions(rows: [OracleTimeZoneRegionFixtures.row("America/Los_Angeles", zone: (0x81, 0x04))]) + let threeAMOn2025: [UInt8] = [0x78, 0x7d, 0x01, 0x01, 0x04, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x81, 0x04] + #expect(Self.regionText(threeAMOn2025, fractionDigits: 0, regions: losAngeles) == "2024-12-31 19:00:00 America/Los_Angeles") + } + + @Test func testRegionZoneIgnoresTheLowBitsOfByte12() { + var lowBitsSet = Self.parisInJanuary + lowBitsSet[12] = 0xf9 + #expect(Self.regionText(lowBitsSet, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.500000 Europe/Paris") + } + + // Without a name the text keeps the instant, as the UTC wall clock in the offset spelling the grid reads. + @Test func testRegionMissingFromTheMapReadsAsUTC() { + var otherID = Self.parisInJanuary + otherID[12] = 0xfc + #expect(Self.regionText(otherID, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 02:04:05.500000+00:00") + #expect(Self.regionText(Self.parisInJanuary, regions: .none) == "2024-01-02 02:04:05.500000+00:00") + #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: Self.parisInJanuary, fractionDigits: 6) == "2024-01-02 02:04:05.500000+00:00") + } + + @Test func testRegionFoundationDoesNotKnowReadsAsUTC() { + let unknown = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Not/AZone", zone: OracleTimeZoneRegionFixtures.parisZone), + ]) + #expect(Self.regionText(Self.parisInJanuary, regions: unknown) == "2024-01-02 02:04:05.500000+00:00") + } + + @Test func testRegionsLeaveAnOffsetValueAlone() { + let minusFiveThirty: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x09, 0x23, 0x06, 0x07, 0x5b, 0xca, 0x00, 0x0f, 0x1e] + #expect(Self.regionText(minusFiveThirty, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.123456-05:30") } /// The bytes are the value in the database's zone; the text is the session's wall clock and the session's @@ -113,6 +161,7 @@ struct OracleCellFormattingTests { "2024-01-02 03:04:05.123456-05:30", "2024-01-02 08:04:05.000001+07:00", "2024-07-01 14:00:00+02:00", + "2024-01-02 02:04:05.500000+00:00", ] for spelling in spellings { let range = NSRange(spelling.startIndex..., in: spelling) diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift index 3ae756dc8a..1b9ab70987 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift @@ -40,6 +40,8 @@ struct OracleDictionaryQualificationTests { ("columns 23ai", OracleSchemaQueries.columns(schema: "HR", table: "T", release: OracleServerRelease(major: 23))), ("indexes", OracleSchemaQueries.indexes(schema: "HR", table: "T")), ("foreignKeys", OracleSchemaQueries.foreignKeys(schema: "HR", table: "T")), + ("tableConstraints 11g", OracleSchemaQueries.tableConstraints(schema: "HR", table: "T", release: OracleServerRelease(major: 11))), + ("tableConstraints 23ai", OracleSchemaQueries.tableConstraints(schema: "HR", table: "T", release: OracleServerRelease(major: 23))), ("allColumns 11g", OracleSchemaQueries.allColumns(schema: "HR", release: OracleServerRelease(major: 11))), ("allColumns 23ai", OracleSchemaQueries.allColumns(schema: "HR", release: OracleServerRelease(major: 23))), ("allForeignKeys", OracleSchemaQueries.allForeignKeys(schema: "HR")), diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRowBatcherTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRowBatcherTests.swift new file mode 100644 index 0000000000..55b3a32e20 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRowBatcherTests.swift @@ -0,0 +1,93 @@ +import Foundation +import TableProOracleCore +import Testing + +struct OracleRowBatcherTests { + private static func row(_ index: Int) -> [OracleRawCell] { + [.string(String(index))] + } + + // The batch sizes the stream carries for `count` rows, in order, and the rows they carry. + private static func drive(_ batcher: OracleRowBatcher, rows count: Int) -> (sizes: [Int], rows: [[OracleRawCell]]) { + var batcher = batcher + var sizes: [Int] = [] + var rows: [[OracleRawCell]] = [] + for index in 0.. [OracleRawCell] { + [ + .string(name), .string(type), .string(deferrable), .string(deferred), .string(status), .string(generated), + condition.map(OracleRawCell.string) ?? .null, + conditionBytes.map { OracleRawCell.string(String($0)) } ?? .null, + column.map(OracleRawCell.string) ?? .null, + .string(validated) + ] + } + + // Rows that break a NOVALIDATE constraint are in the source, so it goes in disabled and is enabled after them. + @Test func testANovalidateConstraintIsWrittenDisabledAndEnabledAfterTheRows() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("PK_LOG", "P", column: "ID", validated: "NOT VALIDATED"), + row("SYS_C0042", "U", column: "CODE", generated: "GENERATED NAME", validated: "NOT VALIDATED"), + row("UQ_REF", "U", column: "REF") + ]) + let quote = OracleSchemaQueries.quoteIdentifier + + #expect(constraints.map { $0.clause(quote: quote) } == [ + "CONSTRAINT \"PK_LOG\" PRIMARY KEY (\"ID\") DISABLE", + "UNIQUE (\"CODE\") DISABLE", + "CONSTRAINT \"UQ_REF\" UNIQUE (\"REF\")" + ]) + #expect(constraints.compactMap { $0.enableAfterRowsStatement(qualifiedTable: "\"S\".\"LOG\"", quote: quote) } == [ + "ALTER TABLE \"S\".\"LOG\" ENABLE NOVALIDATE CONSTRAINT \"PK_LOG\"", + "ALTER TABLE \"S\".\"LOG\" ENABLE NOVALIDATE UNIQUE (\"CODE\")" + ]) + } + + @Test func testOnlyAValidatedNotNullCheckCountsAsRestated() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("NN_ID", "C", column: "ID", condition: "\"ID\" IS NOT NULL", conditionBytes: 16), + row("NN_ID2", "C", column: "ID", condition: "\"ID\" IS NOT NULL", conditionBytes: 16, validated: "NOT VALIDATED") + ]) + #expect(constraints.map { $0.restatesNotNull(of: ["ID"]) } == [true, false]) + } + + // A generated name was not written, so a key is enabled by what it is and a check is added after the rows. + @Test func testAGeneratedNameNovalidateConstraintNeedsNoName() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("SYS_C0001", "P", column: "ID", generated: "GENERATED NAME", validated: "NOT VALIDATED"), + row("SYS_C0002", "C", column: "QTY", generated: "GENERATED NAME", condition: "qty > 0", + conditionBytes: 7, validated: "NOT VALIDATED") + ]) + let quote = OracleSchemaQueries.quoteIdentifier + + #expect(constraints.map { $0.isAddedAfterRows } == [false, true]) + #expect(constraints.compactMap { $0.enableAfterRowsStatement(qualifiedTable: "T", quote: quote) } + == ["ALTER TABLE T ENABLE NOVALIDATE PRIMARY KEY"]) + } + + @Test func testRowsGroupIntoConstraintsKeysFirst() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("CK_QTY", "C", column: "QTY", condition: "qty > 0", conditionBytes: 7), + row("PK_ORDERS", "P", column: "TENANT"), + row("PK_ORDERS", "P", column: "ID"), + row("UQ_CODE", "U", column: "CODE") + ]) + + let names = constraints.map(\.name) + #expect(names == ["PK_ORDERS", "UQ_CODE", "CK_QTY"]) + #expect(constraints.first?.columns == ["TENANT", "ID"]) + #expect(constraints.last?.condition == "qty > 0") + } + + @Test func testAKeyIsWrittenAsAConstraintOnItsColumns() throws { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("PK_ORDERS", "P", column: "TENANT"), + row("PK_ORDERS", "P", column: "ID"), + row("UQ_CODE", "U", column: "CODE") + ]) + + #expect(constraints.map { $0.clause(quote: OracleSchemaQueries.quoteIdentifier) } == [ + "CONSTRAINT \"PK_ORDERS\" PRIMARY KEY (\"TENANT\", \"ID\")", + "CONSTRAINT \"UQ_CODE\" UNIQUE (\"CODE\")" + ]) + } + + @Test func testAGeneratedNameIsLeftForTheServer() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([ + row("SYS_C008123", "P", column: "ID", generated: "GENERATED NAME") + ]).first) + + #expect(constraint.clause(quote: OracleSchemaQueries.quoteIdentifier) == "PRIMARY KEY (\"ID\")") + } + + @Test func testDeferralAndADisabledStateAreKept() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("UQ_A", "U", column: "A", deferrable: "DEFERRABLE", deferred: "DEFERRED"), + row("UQ_B", "U", column: "B", deferrable: "DEFERRABLE"), + row("UQ_C", "U", column: "C", status: "DISABLED") + ]) + + #expect(constraints.map { $0.clause(quote: OracleSchemaQueries.quoteIdentifier) } == [ + "CONSTRAINT \"UQ_A\" UNIQUE (\"A\") DEFERRABLE INITIALLY DEFERRED", + "CONSTRAINT \"UQ_B\" UNIQUE (\"B\") DEFERRABLE", + "CONSTRAINT \"UQ_C\" UNIQUE (\"C\") DISABLE" + ]) + } + + @Test func testACheckKeepsItsConditionAsTheCatalogStoresIt() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([ + row("CK_STATUS", "C", column: "STATUS", condition: "status IN ('A', 'B')\n", conditionBytes: 21) + ]).first) + + #expect(constraint.clause(quote: OracleSchemaQueries.quoteIdentifier) == "CONSTRAINT \"CK_STATUS\" CHECK (status IN ('A', 'B'))") + #expect(!constraint.conditionMayBeCut) + } + + // `SEARCH_CONDITION_VC` holds 4,000 bytes, so a condition that fills it may have been cut. + @Test func testAConditionThatFillsTheColumnMayBeCut() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([ + row("CK_LONG", "C", column: "A", condition: String(repeating: "a", count: 4_000), conditionBytes: 4_000) + ]).first) + + #expect(constraint.conditionMayBeCut) + } + + @Test func testTheNotNullCheckOfANotNullColumnIsTheColumnsOwn() throws { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("SYS_C0081", "C", column: "ID", generated: "GENERATED NAME", condition: "\"ID\" IS NOT NULL"), + row("NN_NAME", "C", column: "Na\"me", condition: "\"Na\"\"me\" IS NOT NULL"), + row("CK_NOTE", "C", column: "NOTE", condition: "\"NOTE\" IS NOT NULL"), + row("CK_QTY", "C", column: "QTY", condition: "QTY > 0") + ]) + + let restated = constraints.filter { $0.restatesNotNull(of: ["ID", "Na\"me", "QTY"]) }.map(\.name) + #expect(restated == ["NN_NAME", "SYS_C0081"]) + } + + @Test func testAKeyNeverRestatesANotNull() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([row("PK_T", "P", column: "ID")]).first) + + #expect(!constraint.restatesNotNull(of: ["ID"])) + } + + @Test func testOtherConstraintTypesAreLeftOut() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("FK_T", "R", column: "PARENT_ID"), + row("V_T", "V", column: nil) + ]) + + #expect(constraints.isEmpty) + } + + @Test func testTheQueryReadsChecksFrom12On() { + let sql = OracleSchemaQueries.tableConstraints(schema: "HR", table: "ORDERS", release: OracleServerRelease(major: 19)) + + #expect(sql.contains("c.SEARCH_CONDITION_VC AS SEARCH_CONDITION_VC")) + #expect(sql.contains("LENGTHB(c.SEARCH_CONDITION_VC) AS CONDITION_BYTES")) + #expect(sql.contains("c.CONSTRAINT_TYPE IN ('P', 'U', 'C')")) + } + + // 11g keeps a check's text only in the LONG `SEARCH_CONDITION`, which oracle-nio cannot decode. + @Test func testTheQueryReadsOnlyKeysOn11g() { + let sql = OracleSchemaQueries.tableConstraints(schema: "HR", table: "ORDERS", release: OracleServerRelease(major: 11)) + + #expect(!sql.contains("SEARCH_CONDITION_VC)")) + #expect(sql.contains("NULL AS SEARCH_CONDITION_VC")) + #expect(sql.contains("c.CONSTRAINT_TYPE IN ('P', 'U')")) + } + + @Test func testTheQueryEscapesTheOwnerAndTable() { + let sql = OracleSchemaQueries.tableConstraints( + schema: "O'BRIEN", table: "IT'S", release: OracleServerRelease(major: 23) + ) + + #expect(sql.contains("c.OWNER = 'O''BRIEN'")) + #expect(sql.contains("c.TABLE_NAME = 'IT''S'")) + } + + @Test func testBooleanLiteralsArriveWith23ai() { + #expect(!OracleServerRelease(major: 21).hasBooleanType) + #expect(OracleServerRelease(major: 23).hasBooleanType) + } +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift new file mode 100644 index 0000000000..da3f079302 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift @@ -0,0 +1,77 @@ +import Foundation +@testable import TableProOracleCore +import Testing + +enum OracleTimeZoneRegionFixtures { + // Bytes 11 and 12 of a Europe/Paris value measured on 23ai: region id 382. + static let parisZone: (UInt8, UInt8) = (0x85, 0xf8) + + // A row as the map query returns it: 2000-01-01 00:00:00 UTC at the named zone. + static func row(_ name: String, zone: (UInt8, UInt8)) -> (name: String, wire: [UInt8]) { + (name, [0x78, 0x64, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, zone.0, zone.1]) + } + + static let paris = OracleTimeZoneRegions(rows: [row("Europe/Paris", zone: parisZone)]) +} + +struct OracleTimeZoneRegionsTests { + private static func wire(zone: (UInt8, UInt8)) -> [UInt8] { + OracleTimeZoneRegionFixtures.row("", zone: zone).wire + } + + @Test func testRegionIDSpansBytes11And12() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: OracleTimeZoneRegionFixtures.parisZone)) == 382) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0xff, 0xfc))) == 8_191) + } + + @Test func testRegionIDIgnoresTheLowBitsOfByte12() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0xf9))) == 382) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0xfb))) == 382) + } + + // The flag alone marks a region, so an id whose low six bits are zero is not read as a zero offset. + @Test func testRegionIDWithAZeroByte12IsStillARegion() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0x00))) == 320) + } + + @Test func testOffsetZonesHaveNoRegionID() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x14, 0x3c))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x0f, 0x1e))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x00, 0x00))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Array(Self.wire(zone: (0x85, 0xf8)).prefix(11))) == nil) + } + + @Test func testMapDropsOffsetRowsAndKeepsTheFirstOfARepeatedID() throws { + let regions = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Europe/Paris", zone: OracleTimeZoneRegionFixtures.parisZone), + OracleTimeZoneRegionFixtures.row("Europe/Monaco", zone: OracleTimeZoneRegionFixtures.parisZone), + OracleTimeZoneRegionFixtures.row("UTC", zone: (0x14, 0x3c)), + ]) + #expect(regions.count == 1) + #expect(regions.duplicateIDs == 1) + let paris = try #require(regions.region(forID: 382)) + #expect(paris.name == "Europe/Paris") + #expect(paris.zone?.identifier == "Europe/Paris") + } + + @Test func testMapKeepsANameFoundationDoesNotKnowWithoutAZone() throws { + let regions = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Not/AZone", zone: OracleTimeZoneRegionFixtures.parisZone), + ]) + let region = try #require(regions.region(forID: 382)) + #expect(region.name == "Not/AZone") + #expect(region.zone == nil) + #expect(regions.unknownNames == 1) + } + + @Test func testNoneHoldsNoRegions() { + #expect(OracleTimeZoneRegions.none.isEmpty) + #expect(OracleTimeZoneRegions(rows: []) == OracleTimeZoneRegions.none) + } + + // A schema can hold its own V$TIMEZONE_NAMES, which an unqualified name resolves to first. + @Test func testQueryReadsTheQualifiedPerformanceView() { + #expect(OracleTimeZoneRegions.query.contains("SELECT DISTINCT TZNAME FROM SYS.V_$TIMEZONE_NAMES)")) + #expect(OracleTimeZoneRegions.query.hasSuffix("ORDER BY TZNAME")) + } +} diff --git a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift index 4d776c917f..20e586fa24 100644 --- a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift +++ b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift @@ -451,26 +451,30 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { + PluginRowStream.make { continuation, abort in + let task = Task { do { - let result = try await perform { [self] () -> PluginQueryResult in + try Task.checkCancellation() + let result = try await self.perform { [self] () -> PluginQueryResult in if let bql = Self.extractBQLQuery(from: query) { return try executeBQL(query: bql) } return try executeSQLite(query: query, parameters: []) } - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: result.rows.count - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) continuation.yield(.rows(result.rows)) continuation.finish() } catch { continuation.finish(throwing: error) } } + abort.onAbort { task.cancel() } } } diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift index 10e7bb644d..64979b93ea 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver+Execution.swift @@ -83,7 +83,7 @@ extension BigQueryPluginDriver { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.runningStatements.run { @@ -93,9 +93,7 @@ extension BigQueryPluginDriver { continuation.finish(throwing: BigQueryError.wrap(error)) } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } @@ -260,7 +258,7 @@ extension BigQueryPluginDriver { let firstRows = BigQueryTypeMapper.flattenRows(from: firstPage, schema: schema) if !firstRows.isEmpty { - continuation.yield(.rows(firstRows)) + _ = await PluginRowStream.deliver(.rows(firstRows), to: continuation) } var pageToken = firstPage.pageToken @@ -273,7 +271,7 @@ extension BigQueryPluginDriver { ) let nextRows = BigQueryTypeMapper.flattenRows(from: nextPage, schema: schema) if !nextRows.isEmpty { - continuation.yield(.rows(nextRows)) + _ = await PluginRowStream.deliver(.rows(nextRows), to: continuation) } pageToken = nextPage.pageToken } diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift index 5bbd4314cd..a67fb55d13 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift @@ -71,6 +71,7 @@ internal final class BigQueryPluginDriver: PluginDatabaseDriver, @unchecked Send .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, ] } @@ -414,6 +415,8 @@ internal final class BigQueryPluginDriver: PluginDatabaseDriver, @unchecked Send ) } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let datasetId = dataset(for: schema) guard !datasetId.isEmpty else { return [:] } diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift index 0ee163ee61..c75a935967 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift @@ -6,8 +6,6 @@ import TableProPluginKit internal struct BigQueryStatementGenerator { typealias Statement = (statement: String, parameters: [PluginCellValue]) - static let defaultSentinel = "__DEFAULT__" - private static let logger = Logger(subsystem: "com.TablePro", category: "BigQueryStatementGenerator") private static let placeholder = "?" @@ -81,7 +79,7 @@ internal struct BigQueryStatementGenerator { var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, column) in columns.enumerated() { - guard let value = values[column], !Self.isDefaultSentinel(value) else { continue } + guard let value = values[column], !change.isColumnDefault(index) else { continue } names.append(GoogleSQLLiteral.quotedIdentifier(column)) if let sql = Self.expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(sql) @@ -124,7 +122,7 @@ internal struct BigQueryStatementGenerator { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cellChange -> String in let column = GoogleSQLLiteral.quotedIdentifier(cellChange.columnName) - guard !Self.isDefaultSentinel(cellChange.newValue) else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cellChange.columnIndex) else { return "\(column) = DEFAULT" } if let sql = Self.expression( cellChange.newValue, isMarked: expressionColumns.contains(cellChange.columnIndex) @@ -185,10 +183,6 @@ internal struct BigQueryStatementGenerator { ) } - private static func isDefaultSentinel(_ value: PluginCellValue) -> Bool { - value.asText == defaultSentinel - } - // The host checked a marked cell's text against the engine's vocabulary before handing it over. private static func expression(_ value: PluginCellValue, isMarked: Bool) -> String? { guard isMarked, case .text(let sql) = value else { return nil } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift index 6f92b8c0a4..9c08165f7f 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -3,8 +3,6 @@ import TableProBigtableCore import TableProPluginKit extension BigtablePluginDriver { - private static let defaultMarker = "__DEFAULT__" - func generateRowWrites( table: String, schema: String?, @@ -14,18 +12,37 @@ extension BigtablePluginDriver { insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { try changes.compactMap { change in do { - return try request(for: change, table: table, columns: columns, insertedRowData: insertedRowData) - .map { PluginRowWrite(statement: $0.text, rowIndices: [change.rowIndex]) } + return try request( + for: change, table: table, columns: columns, insertedRowData: insertedRowData, + locator: context.rowLocators[change.rowIndex] + ).map { PluginRowWrite(statement: $0.text, rowIndices: [change.rowIndex]) } } catch { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: error.localizedDescription) } } } - /// Restoring a deleted row writes it back under its own key, refused if that key is taken again. func generateIdentityPreservingInsert( table: String, schema: String?, @@ -34,8 +51,32 @@ extension BigtablePluginDriver { rows: [[PluginCellValue]], absentCells: [Int: Set] ) -> [(statement: String, parameters: [PluginCellValue])]? { - let statements = rows.enumerated().compactMap { index, row in - try? BigtableRowWrites.insert(table: table, columns: columns, values: cells(row, absent: absentCells[index] ?? [])).text + generateIdentityPreservingInsert( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, rows: rows, + absentCells: absentCells, context: PluginRowWriteContext() + ) + } + + // Restoring a deleted row writes it back under its own key, refused if that key is taken again. + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + absentCells: [Int: Set], + context: PluginRowWriteContext + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let keyIndex = columns.firstIndex(of: BigtableRowSet.keyColumn) + let statements = rows.enumerated().compactMap { index, row -> String? in + var values = cells(row, absent: absentCells[index] ?? []) + if let keyIndex, keyIndex < values.count { + guard let key = try? BigtableRowWrites.originalKey(locator: context.rowLocators[index], keyCell: values[keyIndex]) else { + return nil + } + values[keyIndex] = .bytes(key) + } + return try? BigtableRowWrites.insert(table: table, columns: columns, values: values).text } return statements.count == rows.count ? statements.map { ($0, []) } : nil } @@ -44,41 +85,47 @@ extension BigtablePluginDriver { for change: PluginRowChange, table: String, columns: [String], - insertedRowData: [Int: [PluginCellValue]] + insertedRowData: [Int: [PluginCellValue]], + locator: String? ) throws -> BigtableRequest? { let absent = change.absentColumns ?? [] switch change.type { case .insert: return try BigtableRowWrites.insert( - table: table, columns: columns, values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent) + table: table, + columns: columns, + values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent.union(change.defaultColumns ?? [])) ) case .update: let changes = try change.cellChanges.map { cell in - guard cell.newValue.asText != Self.defaultMarker else { + guard !change.isColumnDefault(cell.columnIndex) else { throw BigtableError.unsupported(String(localized: "Bigtable has no default values. Delete the cell, or enter a value.")) } return (column: cell.columnName, value: absent.contains(cell.columnIndex) ? BigtableRowSet.Cell.absent : Self.cell(cell.newValue)) } - return try BigtableRowWrites.update(table: table, key: try originalKey(change, columns: columns), changes: changes) + return try BigtableRowWrites.update( + table: table, key: try originalKey(change, columns: columns, locator: locator), changes: changes + ) case .delete: - return BigtableRowWrites.delete(table: table, key: try originalKey(change, columns: columns)) + return BigtableRowWrites.delete(table: table, key: try originalKey(change, columns: columns, locator: locator)) } } - private func originalKey(_ change: PluginRowChange, columns: [String]) throws -> Data { - guard let index = columns.firstIndex(of: BigtableRowSet.keyColumn), - let row = change.originalRow, index < row.count, - let key = try BigtableRowWrites.key(Self.cell(row[index])) else { + private func originalKey(_ change: PluginRowChange, columns: [String], locator: String?) throws -> Data { + var keyCell: BigtableRowSet.Cell? + if let index = columns.firstIndex(of: BigtableRowSet.keyColumn), let row = change.originalRow, index < row.count { + keyCell = Self.cell(row[index]) + } + guard let key = try BigtableRowWrites.originalKey(locator: locator, keyCell: keyCell) else { throw BigtableError.configuration(String(localized: "The row has no _key, so the row it came from is unknown. Reload the table.")) } return key } - /// A new row holds the host's Default marker where it has no value yet. Bigtable generates - /// nothing, so that cell is left out, and a `_key` left that way is refused. + // Bigtable generates nothing, so a cell left to the default is left out, and a `_key` left that way is refused. private func cells(_ row: [PluginCellValue], absent: Set) -> [BigtableRowSet.Cell] { row.enumerated().map { index, value in - absent.contains(index) || value.asText == Self.defaultMarker ? .absent : Self.cell(value) + absent.contains(index) ? .absent : Self.cell(value) } } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift index 1b3c890c81..71d84b3060 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -149,16 +149,18 @@ extension BigtablePluginDriver { // MARK: - Export func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if case .browse(let spec)? = try? BigtableStatement.parse(query), spec.limit == 0 { try await self.export(spec, continuation: continuation) } else { let result = try await self.execute(query: query) - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } } continuation.finish() @@ -166,7 +168,7 @@ extension BigtablePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -188,7 +190,7 @@ extension BigtablePluginDriver { ))) } if !rows.isEmpty { - continuation.yield(.rows(page.rows(alignedTo: columns).map { $0.map(Self.cell) })) + _ = await PluginRowStream.deliver(.rows(page.rows(alignedTo: columns).map { $0.map(Self.cell) }), to: continuation) } after = rows.count == Self.exportPageSize ? rows.last?.key : nil } while after != nil diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift index 6a6b3de9c0..c67626f445 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift @@ -22,7 +22,7 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send } var serverVersion: String? { "Google Cloud Bigtable" } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift index 2fdbd570a2..cd7942fb4b 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift @@ -79,7 +79,8 @@ public struct CSVExportOptions: Equatable, Codable, Sendable { public var quoteHandling: CSVQuoteHandling = .asNeeded public var lineBreak: CSVLineBreak = .lf public var decimalFormat: CSVDecimalFormat = .period - public var sanitizeFormulas: Bool = true + // Off: RFC 4180 carries a field as written, and OWASP scopes the quote prefix to files meant for a spreadsheet. + public var sanitizeFormulas: Bool = false /// What the file's bytes are written in. UTF-8 keeps every existing export byte for byte. public var encoding: PluginTextEncoding = .utf8 diff --git a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift index a35bd63f80..f38791eced 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift @@ -23,7 +23,11 @@ struct CSVExportOptionsView: View { Toggle("Sanitize formula-like values", isOn: $plugin.settings.sanitizeFormulas) .toggleStyle(.checkbox) - .help("Prevent CSV formula injection by prefixing values starting with =, +, -, @ with a single quote") + .help(""" + For a file opened in a spreadsheet. Values that would run as formulas, such as those starting \ + with =, +, - or @, get a leading single quote, which stays in the value for anything that \ + imports the file. + """) } Divider() diff --git a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift index 6f11d384b2..f4f2bf17b3 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift @@ -59,6 +59,7 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let lineBreak = options.lineBreak.value let encoding = options.encoding var report = CSVEncodingReport() + var neutralisedCount = 0 if options.writesByteOrderMark, !encoding.byteOrderMark.isEmpty { try fileHandle.write(contentsOf: Data(encoding.byteOrderMark)) @@ -80,7 +81,7 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } var isFirstBatch = true - var columns: [String] = [] + var columnNumbers: [PluginColumnNumbers] = [] let stream = dataSource.streamRows(for: table) for try await element in stream { @@ -88,17 +89,27 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi switch element { case .header(let header): - columns = header.columns + columnNumbers = header.columnTypeNames.map { PluginColumnNumbers(typeName: $0) } if isFirstBatch && options.includeFieldNames { - let headerLine = columns - .map { escapeCSVField($0, options: options) } + if options.sanitizeFormulas { + neutralisedCount += header.columns.filter(PluginRowWriters.isFormulaLike).count + } + let headerLine = header.columns + .map { escapeCSVField($0, options: options, sanitizes: options.sanitizeFormulas) } .joined(separator: options.delimiter.actualValue) try write(headerLine + lineBreak, to: fileHandle, encoding: encoding, report: &report) } isFirstBatch = false case .rows(let rows): for row in rows { - try writeCSVRow(row, options: options, to: fileHandle, report: &report) + try writeCSVRow( + row, + columnNumbers: columnNumbers, + options: options, + to: fileHandle, + report: &report, + neutralisedCount: &neutralisedCount + ) progress.incrementRow() } } @@ -114,7 +125,22 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) committed = true progress.finalizeTable() - return ExportFormatResult(warnings: report.warnings(for: encoding)) + return ExportFormatResult(warnings: report.warnings(for: encoding) + Self.neutralisedWarnings(neutralisedCount)) + } + + // The quote is part of the value for anything that imports the file, so the export says how many it changed. + static func neutralisedWarnings(_ count: Int) -> [String] { + guard count > 0 else { return [] } + return [String( + format: String( + localized: """ + Values a spreadsheet would read as formulas, written with a leading single quote: %lld. \ + Turn off Sanitize formula-like values to export them unchanged. + """, + bundle: .main + ), + Int64(count) + )] } // MARK: - Private @@ -143,26 +169,31 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try fileHandle.write(contentsOf: encoded.data) } + // A text column holding "1.10" is a version or a code; a column with no usable type keeps the decimal comma. private func writeCSVRow( _ row: [PluginCellValue], + columnNumbers: [PluginColumnNumbers], options: CSVExportOptions, to fileHandle: FileHandle, - report: inout CSVEncodingReport + report: inout CSVEncodingReport, + neutralisedCount: inout Int ) throws { - let delimiter = options.delimiter.actualValue - let lineBreak = options.lineBreak.value + var fields: [String] = [] + fields.reserveCapacity(row.count) - let rowLine = row.map { cell -> String in + for (index, cell) in row.enumerated() { let val: String switch cell { case .null: - return options.convertNullToEmpty ? "" : "NULL" + fields.append(options.convertNullToEmpty ? "" : "NULL") + continue case .text(let s): val = s case .bytes(let d): val = "0x" + d.map { String(format: "%02X", $0) }.joined() } + let takesDecimalComma = index >= columnNumbers.count || columnNumbers[index] != .text var processed = val // `\r\n` is one Character, so only a scalar test sees a Windows line break. let hadLineBreaks = val.unicodeScalars.contains { $0 == "\n" || $0 == "\r" } @@ -174,40 +205,54 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi .replacingOccurrences(of: "\n", with: " ", options: .literal) } - if options.decimalFormat == .comma { + var isCommaDecimal = false + if options.decimalFormat == .comma, takesDecimalComma { let range = NSRange(processed.startIndex..., in: processed) if Self.decimalFormatRegex.firstMatch(in: processed, range: range) != nil { processed = processed.replacingOccurrences(of: ".", with: ",") + isCommaDecimal = true } } - return escapeCSVField(processed, options: options, originalHadLineBreaks: hadLineBreaks) - }.joined(separator: delimiter) + // `-1,5` is still a number, and the quote would turn it into text. + let sanitizes = options.sanitizeFormulas && !isCommaDecimal + if sanitizes, PluginRowWriters.isFormulaLike(processed) { + neutralisedCount += 1 + } + fields.append(escapeCSVField( + processed, options: options, sanitizes: sanitizes, originalHadLineBreaks: hadLineBreaks + )) + } - try write(rowLine + lineBreak, to: fileHandle, encoding: options.encoding, report: &report) + try write( + fields.joined(separator: options.delimiter.actualValue) + options.lineBreak.value, + to: fileHandle, + encoding: options.encoding, + report: &report + ) } - /// Escaping and quoting live in `PluginRowWriters`, so this format, the other export formats - /// and the MCP tool spell a value the same way. Only the option mapping is this plugin's own. - /// - /// `originalHadLineBreaks` says the value's breaks were already replaced with spaces upstream, - /// and it still forces quoting: the source text spanned lines, and a reader that splits on the - /// delimiter has no way to know the space it now sees was one. - private func escapeCSVField(_ field: String, options: CSVExportOptions, originalHadLineBreaks: Bool = false) -> String { - let escaped = PluginRowWriters.csvField(field, options: writeOptions(options)) + // `originalHadLineBreaks` still forces quoting after the breaks became spaces: the source text spanned lines. + private func escapeCSVField( + _ field: String, + options: CSVExportOptions, + sanitizes: Bool, + originalHadLineBreaks: Bool = false + ) -> String { + let escaped = PluginRowWriters.csvField(field, options: writeOptions(options, sanitizes: sanitizes)) guard originalHadLineBreaks, options.quoteHandling == .asNeeded, escaped.unicodeScalars.first != "\"" else { return escaped } return "\"\(escaped.replacingOccurrences(of: "\"", with: "\"\"", options: .literal))\"" } - private func writeOptions(_ options: CSVExportOptions) -> PluginCsvWriteOptions { + private func writeOptions(_ options: CSVExportOptions, sanitizes: Bool) -> PluginCsvWriteOptions { PluginCsvWriteOptions( delimiter: options.delimiter.actualValue, quoteHandling: quoteHandling(options.quoteHandling), lineEnding: options.lineBreak.value, nullAsEmpty: true, - sanitizesFormulas: options.sanitizeFormulas, + sanitizesFormulas: sanitizes, flattensLineBreaks: false ) } diff --git a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift index e81c968cb2..a319f836b9 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -103,10 +103,27 @@ enum CSVImportParsing { } } - static func isBlank(_ fields: [String]) -> Bool { + // RFC 4180 makes an empty line a record of one empty field, which is how CSV export writes a NULL in a one-column + // file. A record of several empty fields, `,,`, is an all-NULL row. + static func isBlank(_ fields: [String], columnCount: Int) -> Bool { + columnCount > 1 && fields.count == 1 && fields[0].isEmpty + } + + // A record with nothing in it adds no sample to detection. + static func holdsNoValue(_ fields: [String]) -> Bool { fields.allSatisfy { $0.isEmpty } } + // A spreadsheet saves the empty rows under its data as lines of delimiters, the same text as all-NULL rows at + // the very end of an export. Only that trailing run is left out; empty records between rows are rows. + static func dataEnd(recordCount: Int, fields: (Int) -> [String]) -> Int { + var end = recordCount + while end > 0, holdsNoValue(fields(end - 1)) { + end -= 1 + } + return end + } + static func detectFields( in data: Data, options: CSVImportOptions, @@ -141,7 +158,7 @@ enum CSVImportParsing { for range in dataRanges { if sampled >= limit { break } let fields = parser.parseRow(buffer, range: range) - if isBlank(fields) { continue } + if holdsNoValue(fields) { continue } for column in 0.. RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } @@ -58,10 +58,12 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi ) async throws -> PluginImportResult { let startTime = Date() let url = source.fileURL() + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings let text: CSVImportText do { - text = try CSVImportText.contents(of: url, encoding: settings.encoding, isCancelled: { progress.isCancelled }) + text = try CSVImportText.contents(of: url, encoding: options.encoding, isCancelled: { progress.isCancelled }) } catch is TabularCancellation { throw PluginImportCancellationError() } catch { @@ -80,9 +82,9 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } let data = text.data - let dialect = CSVImportParsing.resolveDialect(in: data, options: settings) + let dialect = CSVImportParsing.resolveDialect(in: data, options: options) let parser = CSVStreamingParser(dialect: dialect) - let hasHeader = settings.hasHeaderRow + let hasHeader = options.hasHeaderRow let (dataRanges, columnNames) = indexRowsAndNames(in: data, parser: parser, hasHeader: hasHeader) guard !columnNames.isEmpty else { @@ -93,7 +95,7 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi let lineOffset = hasHeader ? 2 : 1 var cursor = 0 - let configuration = runConfiguration + let configuration = Self.runConfiguration(options) var pendingFailure: PluginImportError? var malformedCount = 0 var malformedErrors: [PluginImportResult.ImportStatementError] = [] @@ -111,7 +113,8 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi ranges: dataRanges[cursor..] if hasHeader { let header = parser.parseRow(buffer, range: ranges[0]) - let names = CSVImportParsing.columnNames(header: header, columnCount: header.count) - return (Array(ranges.dropFirst()), names) + names = CSVImportParsing.columnNames(header: header, columnCount: header.count) + records = Array(ranges.dropFirst()) + } else { + let firstCount = parser.parseRow(buffer, range: ranges[0]).count + names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount) + records = ranges } - let firstCount = parser.parseRow(buffer, range: ranges[0]).count - let names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount) - return (ranges, names) + let end = CSVImportParsing.dataEnd(recordCount: records.count) { parser.parseRow(buffer, range: records[$0]) } + return (Array(records[..>, startIndex: Int, lineOffset: Int, - columnNames: [String] + columnNames: [String], + options: CSVImportOptions ) -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) { data.withUnsafeBytes { raw -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) in guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return ([], []) } @@ -182,15 +191,15 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi out.reserveCapacity(ranges.count) for (offset, range) in ranges.enumerated() { let fields = parser.parseRow(buffer, range: range) - if CSVImportParsing.isBlank(fields) { continue } + if CSVImportParsing.isBlank(fields, columnCount: columnNames.count) { continue } let line = startIndex + offset + lineOffset if let error = CSVImportParsing.shapeError( - fields: fields, columnCount: columnNames.count, options: settings + fields: fields, columnCount: columnNames.count, options: options ) { malformed.append((line, error)) continue } - out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: settings))) + out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: options))) } return (out, malformed) } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift b/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift index cd8c0b396a..73d1b395c2 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift @@ -141,7 +141,7 @@ enum CassandraValueParser { case .blob: return .bytes(data) case .text: - guard let text = String(data: data, encoding: .utf8) else { + guard let text = CassandraWireText.text(data) else { throw CassandraValueRefusal.invalid("0x" + data.map { String(format: "%02x", $0) }.joined(), typeName: type.name) } return .string(text) diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift b/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift index 310d047813..cadd7debd4 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift @@ -19,6 +19,9 @@ struct CassandraBrowseStatement: Equatable, Sendable { let limit: Int let values: [String] let refusal: String? + // The table the CQL reads, for the TTL() selectors the run adds. A header written before them has neither. + var keyspace: String? + var table: String? } static let opening = "/* TablePro browse " @@ -34,15 +37,23 @@ struct CassandraBrowseStatement: Equatable, Sendable { return Self.opening + header + Self.closing + cql } - /// The same browse read one row past a query tab's cap, so a result the cap trims is still reported as trimmed. + // The same browse read one row past a query tab's cap, so a result the cap trims is still reported as trimmed. func cappedAt(rowCap: Int) -> CassandraBrowseStatement { let limit = min(window.limit, rowCap + 1) return CassandraBrowseStatement( - window: Window(offset: window.offset, limit: limit, values: window.values, refusal: window.refusal), + window: Window( + offset: window.offset, limit: limit, values: window.values, refusal: window.refusal, + keyspace: window.keyspace, table: window.table + ), cql: cql ) } + // The run keys its resume points on the CQL it sends. + func sending(_ cql: String) -> CassandraBrowseStatement { + CassandraBrowseStatement(window: window, cql: cql) + } + static func parse(_ text: String) -> CassandraBrowseStatement? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard trimmed.hasPrefix(opening), @@ -111,7 +122,10 @@ enum CassandraBrowseRenderer { if sorted { throw CassandraBrowseRefusal.sorting } let condition = try whereCondition(filters: filters, matchAll: matchAll) return CassandraBrowseStatement( - window: .init(offset: max(offset, 0), limit: max(limit, 0), values: condition.values, refusal: nil), + window: .init( + offset: max(offset, 0), limit: max(limit, 0), values: condition.values, refusal: nil, + keyspace: keyspace?.isEmpty == false ? keyspace : nil, table: table + ), cql: appending(condition, to: source) ) } catch { diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift b/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift index bbc86f6312..e3b4a7d825 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift @@ -347,13 +347,15 @@ actor CassandraConnectionActor { return extractResult(from: result, startTime: startTime) } - /// Reads the rows a table browse shows, walking the paging state past the ones before its offset. A page - /// is counted by the rows it carried rather than by its size, because ScyllaDB answers a filtered page short. + // Reads the rows a table browse shows, walking the paging state past the ones before its offset. A page is + // counted by the rows it carried rather than by its size, because ScyllaDB answers a filtered page short. + // `fallback` runs instead when the server refuses to prepare `browse`. func executeBrowse( _ browse: CassandraBrowseStatement, + fallback: CassandraBrowseStatement? = nil, consistency: CassandraConsistency?, cancellation: CassandraCancellation - ) throws -> CassandraRawResult { + ) throws -> (result: CassandraRawResult, ranFallback: Bool) { if let refusal = browse.window.refusal { throw CassandraBrowseRefusal(pluginErrorMessage: refusal) } @@ -363,7 +365,18 @@ actor CassandraConnectionActor { let startTime = Date() try cancellation.check() - let prepared = try prepare(browse.cql, on: session, cancellation: cancellation) + var browse = browse + var ranFallback = false + let prepared: OpaquePointer + do { + prepared = try prepare(browse.cql, on: session, cancellation: cancellation) + } catch let error where !(error is CancellationError) { + guard let fallback else { throw error } + Self.logger.info("Browse refused, running it without TTL selectors: \(error.localizedDescription)") + browse = fallback + ranFallback = true + prepared = try prepare(browse.cql, on: session, cancellation: cancellation) + } defer { cass_prepared_free(prepared) } let pageSize = CassandraResumePoints.pageSize(forLimit: browse.window.limit) @@ -385,13 +398,14 @@ actor CassandraConnectionActor { read = try self.walk(walk, from: nil, prepared: prepared, session: session, cancellation: cancellation) } - return CassandraRawResult( + let result = CassandraRawResult( columns: read.header?.columns ?? [], columnTypeNames: read.header?.typeNames ?? [], rows: read.rows, rowsAffected: read.rows.count, executionTime: Date().timeIntervalSince(startTime) ) + return (result, ranFallback) } private func walk( @@ -669,16 +683,8 @@ actor CassandraConnectionActor { case CASS_VALUE_TYPE_ASCII, CASS_VALUE_TYPE_TEXT, CASS_VALUE_TYPE_VARCHAR: var output: UnsafePointer? var outputLength: Int = 0 - let rc = cass_value_get_string(value, &output, &outputLength) - if rc == CASS_OK, let output { - return String( - bytesNoCopy: UnsafeMutableRawPointer(mutating: output), - length: outputLength, - encoding: .utf8, - freeWhenDone: false - ) - } - return nil + guard cass_value_get_string(value, &output, &outputLength) == CASS_OK, let output else { return nil } + return wireText(output, length: outputLength) case CASS_VALUE_TYPE_INT: var intVal: Int32 = 0 @@ -817,21 +823,19 @@ actor CassandraConnectionActor { return CassandraVarint.decimalString(unscaled: Data(bytes: unscaled, count: length), scale: scale) default: - // Fallback: try reading as string var output: UnsafePointer? var outputLength: Int = 0 if cass_value_get_string(value, &output, &outputLength) == CASS_OK, let output { - return String( - bytesNoCopy: UnsafeMutableRawPointer(mutating: output), - length: outputLength, - encoding: .utf8, - freeWhenDone: false - ) + return wireText(output, length: outputLength) } return "" } } + private static func wireText(_ pointer: UnsafePointer, length: Int) -> String? { + CassandraWireText.text(UnsafeRawBufferPointer(start: pointer, count: length)) + } + private static func extractCollectionString( _ value: OpaquePointer, open: String, @@ -871,7 +875,7 @@ actor CassandraConnectionActor { var namePointer: UnsafePointer? var nameLength: Int = 0 let name = cass_iterator_get_user_type_field_name(iterator, &namePointer, &nameLength) == CASS_OK - ? namePointer.flatMap { String(bytes: UnsafeRawBufferPointer(start: $0, count: nameLength), encoding: .utf8) } + ? namePointer.flatMap { wireText($0, length: nameLength) } : nil let fieldValue = cass_iterator_get_user_type_field_value(iterator).flatMap(nestedText) ?? "null" fields.append("\(name ?? "?"): \(fieldValue)") @@ -955,10 +959,8 @@ actor CassandraConnectionActor { return "Unknown error" } - /// The whole loop is one non-suspending actor job holding the cooperative-pool thread, and - /// cass_future_wait blocks it, so nothing outside can interrupt a page. `abort` is the only - /// way the consumer's departure reaches this body, and it is read between pages and between - /// rows rather than mid-request. + // One non-suspending actor job, blocked in cass_future_wait and in `deliverBlocking`, so the consumer's departure + // reaches it only through `abort`, read between pages and between rows rather than mid-request. func streamQuery( _ cql: String, consistency: CassandraConsistency?, @@ -1036,6 +1038,7 @@ actor CassandraConnectionActor { let colCount = cass_result_column_count(result) let iterator = cass_iterator_from_result(result) + var page: [PluginRow] = [] if let iterator { var rowsThisPage = 0 @@ -1047,7 +1050,7 @@ actor CassandraConnectionActor { } let row = cass_iterator_get_row(iterator) guard let row else { continue } - continuation.yield(.rows([Self.decodeRow(row, columnCount: colCount)])) + page.append(Self.decodeRow(row, columnCount: colCount)) } cass_iterator_free(iterator) } @@ -1060,6 +1063,10 @@ actor CassandraConnectionActor { cass_result_free(result) + if !page.isEmpty, !aborted { + _ = PluginRowStream.deliverBlocking(.rows(page), to: continuation) + } + if !hasMore || aborted { break } } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index f59b0a903d..f11d9e7f63 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -135,6 +135,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen private let config: DriverConnectionConfig let connectionActor = CassandraConnectionActor() let activeBrowse = CassandraActiveBrowse() + let cellLifetimes = CassandraCellLifetimeStore() private let stateLock = NSLock() nonisolated(unsafe) private var _currentKeyspace: String? nonisolated(unsafe) private var _sessionConsistency: CassandraConsistency? @@ -172,6 +173,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen .alterTableDDL, .cellExpressions, .uuidCellText, + .columnDefaultMarks, ] } @@ -286,6 +288,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen func disconnect() { activeBrowse.cancel() + cellLifetimes.removeAll() Task.detached(priority: .utility) { [connectionActor] in await connectionActor.close() } @@ -324,7 +327,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen let rawResult = try await connectionActor.executeQuery( query, consistency: consistency, cancellation: cancellation ) - if let refusal = CassandraRowWriter.unappliedInsertRefusal( + if let refusal = CassandraRowWriter.unappliedWriteRefusal( statement: query, columns: rawResult.columns, rows: rawResult.rows ) { throw CassandraPluginError.queryFailed(refusal) @@ -371,7 +374,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen let rawResult = try await connectionActor.executePrepared( query, parameters: parameters, consistency: consistency ) - if let refusal = CassandraRowWriter.unappliedInsertRefusal( + if let refusal = CassandraRowWriter.unappliedWriteRefusal( statement: query, columns: rawResult.columns, rows: rawResult.rows ) { throw CassandraPluginError.queryFailed(refusal) @@ -480,7 +483,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen return streamBrowse(browse, consistency: consistency) } let cql = stripTrailingSemicolon(query) - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.connectionActor.streamQuery( @@ -491,7 +494,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen continuation.finish(throwing: error) } } - _ = streamTask + abort.onAbort { streamTask.cancel() } } } @@ -558,6 +561,8 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let ks = resolveKeyspace(schema) let query = """ diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift index 7e072991f0..c7de4f443b 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift @@ -4,8 +4,12 @@ // import Foundation +import os +import TableProCassandraCore import TableProPluginKit +private let logger = Logger(subsystem: "com.TablePro.CassandraDriver", category: "Browse") + extension CassandraPluginDriver { func buildBrowseQuery( table: String, @@ -125,7 +129,19 @@ extension CassandraPluginDriver { ) async throws -> PluginQueryResult { let cancellation = activeBrowse.begin(consumerLeft: consumerLeft) defer { activeBrowse.end(cancellation) } - let raw = try await connectionActor.executeBrowse(browse, consistency: consistency, cancellation: cancellation) + let capture = try await lifetimeCapture(for: browse, cancellation: cancellation) + // Taken before the request, so the time a kept TTL has run counts the whole request. + let readAt = ContinuousClock.now + let run = try await connectionActor.executeBrowse( + capture.map { browse.sending($0.plan.cql) } ?? browse, + fallback: capture == nil ? nil : browse, + consistency: consistency, + cancellation: cancellation + ) + var raw = run.result + if let capture, !run.ranFallback { + raw = keepLifetimes(of: raw, capture: capture, readAt: readAt) + } let isTruncated = rowCap.map { raw.rows.count > $0 } ?? false let rows = isTruncated ? Array(raw.rows.prefix(rowCap ?? raw.rows.count)) : raw.rows return PluginQueryResult( @@ -166,6 +182,134 @@ extension CassandraPluginDriver { } } + private struct LifetimeCapture: Sendable { + let plan: CassandraLifetimeRead.Plan + let table: CassandraTableIdentity + let layout: CassandraTableLayout + } + + // A browse from the first row reads the table's columns again, so a column added since is shown. A read larger + // than the store holds, such as a copy or an export, runs as written. + private func lifetimeCapture( + for browse: CassandraBrowseStatement, + cancellation: CassandraCancellation + ) async throws -> LifetimeCapture? { + let window = browse.window + guard window.refusal == nil, window.limit <= CassandraCellLifetimeStore.maximumRows, + let tableName = window.table, + let keyspace = window.keyspace.flatMap({ $0.isEmpty ? nil : $0 }) ?? currentSchema + else { return nil } + let table = CassandraTableIdentity(keyspace: keyspace, table: tableName) + guard let layout = try await tableLayout(table, refresh: window.offset == 0, cancellation: cancellation), + let plan = CassandraLifetimeRead.plan( + cql: browse.cql, + target: CassandraBrowseRenderer.qualifiedTable(keyspace: window.keyspace, table: tableName), + layout: layout + ) + else { return nil } + return LifetimeCapture(plan: plan, table: table, layout: layout) + } + + private func tableLayout( + _ table: CassandraTableIdentity, + refresh: Bool, + cancellation: CassandraCancellation + ) async throws -> CassandraTableLayout? { + if !refresh, let known = cellLifetimes.layout(of: table) { return known.layout } + let columns: CassandraRawResult + let options: CassandraRawResult + do { + columns = try await connectionActor.executeQuery( + CassandraTableLayout.columnsQuery(keyspace: table.keyspace, table: table.table), + consistency: nil, + cancellation: cancellation + ) + options = try await connectionActor.executeQuery( + CassandraTableLayout.defaultTimeToLiveQuery(keyspace: table.keyspace, table: table.table), + consistency: nil, + cancellation: cancellation + ) + } catch let error as CancellationError { + throw error + } catch { + logger.info("Browse runs without TTL selectors, the table's columns were not read: \(error.localizedDescription)") + return nil + } + let layout = Self.layout(columns: columns.rows, options: options.rows) + cellLifetimes.setLayout(layout, of: table) + return layout + } + + private static func layout(columns: [[PluginCellValue]], options: [[PluginCellValue]]) -> CassandraTableLayout? { + var parsed: [CassandraTableLayout.Column] = [] + for row in columns { + guard let column = CassandraTableLayout.column( + name: row[safe: 0]?.asText, + type: row[safe: 1]?.asText, + kind: row[safe: 2]?.asText, + position: row[safe: 3]?.asText + ) else { return nil } + parsed.append(column) + } + let defaultTimeToLive = options.first?[safe: 0]?.asText.flatMap { Int($0) } + return CassandraTableLayout(columns: parsed, defaultTimeToLive: defaultTimeToLive) + } + + // The host sees the columns it asked for. Every row read replaces what was kept for its key, or drops it. + private func keepLifetimes( + of raw: CassandraRawResult, + capture: LifetimeCapture, + readAt: ContinuousClock.Instant + ) -> CassandraRawResult { + let plan = capture.plan + let shownCount = plan.shownColumns.count + guard raw.columns.count == shownCount + plan.lifetimeColumns.count else { return raw } + var read: [(row: CassandraRowIdentity, lifetimes: CassandraRowLifetimes?)] = [] + read.reserveCapacity(raw.rows.count) + let rows = raw.rows.map { row -> [PluginCellValue] in + if let key = Self.key(of: row, plan: plan) { + read.append(( + CassandraRowIdentity(table: capture.table, key: key), + Self.lifetimes(of: row, capture: capture, readAt: readAt) + )) + } + return Array(row.prefix(shownCount)) + } + cellLifetimes.keep(read) + return CassandraRawResult( + columns: Array(raw.columns.prefix(shownCount)), + columnTypeNames: Array(raw.columnTypeNames.prefix(shownCount)), + rows: rows, + rowsAffected: raw.rowsAffected, + executionTime: raw.executionTime + ) + } + + private static func key(of row: [PluginCellValue], plan: CassandraLifetimeRead.Plan) -> [String: PluginCellValue]? { + var key: [String: PluginCellValue] = [:] + for (column, index) in plan.keyIndices { + guard index < row.count, !row[index].isNull else { return nil } + key[column] = row[index] + } + return key + } + + private static func lifetimes( + of row: [PluginCellValue], + capture: LifetimeCapture, + readAt: ContinuousClock.Instant + ) -> CassandraRowLifetimes? { + let plan = capture.plan + let lifetimes = plan.lifetimeValueIndices.enumerated().map { selector, valueIndex -> CassandraCellLifetime? in + let timeToLiveIndex = plan.shownColumns.count + selector + guard valueIndex < row.count, timeToLiveIndex < row.count else { return nil } + return CassandraCellLifetime(valueIsNull: row[valueIndex].isNull, timeToLive: row[timeToLiveIndex].asText) + } + return CassandraRowLifetimes( + columns: plan.lifetimeColumns, lifetimes: lifetimes, readAt: readAt, layout: capture.layout + ) + } + private static func matchesAll(_ logicMode: String) -> Bool { logicMode.lowercased() != "or" } @@ -203,3 +347,64 @@ final class CassandraActiveBrowse: @unchecked Sendable { running?.cancel() } } + +struct CassandraTableIdentity: Hashable, Sendable { + let keyspace: String + let table: String +} + +struct CassandraRowIdentity: Hashable, Sendable { + let table: CassandraTableIdentity + let key: [String: PluginCellValue] +} + +// What the browses on one connection read about cell lifetimes, so a later edit or restore writes each cell with +// the TTL it had left. Rows read longest ago drop out first. +final class CassandraCellLifetimeStore: @unchecked Sendable { + static let maximumRows = 20_000 + static let maximumTables = 64 + + // A table system_schema lists no layout for is remembered too, so its browse pages ask once. + struct KnownLayout: Sendable { + let layout: CassandraTableLayout? + } + + private let lock = NSLock() + private var rows = CassandraRecentEntries( + capacity: CassandraCellLifetimeStore.maximumRows + ) + private var layouts = CassandraRecentEntries( + capacity: CassandraCellLifetimeStore.maximumTables + ) + + func layout(of table: CassandraTableIdentity) -> KnownLayout? { + lock.withLock { layouts[table] } + } + + func setLayout(_ layout: CassandraTableLayout?, of table: CassandraTableIdentity) { + lock.withLock { layouts.set(KnownLayout(layout: layout), for: table) } + } + + func keep(_ read: [(row: CassandraRowIdentity, lifetimes: CassandraRowLifetimes?)]) { + lock.withLock { + for entry in read { + if let lifetimes = entry.lifetimes { + rows.set(lifetimes, for: entry.row) + } else { + rows.removeValue(for: entry.row) + } + } + } + } + + func lifetimes(of row: CassandraRowIdentity) -> CassandraRowLifetimes? { + lock.withLock { rows[row] } + } + + func removeAll() { + lock.withLock { + rows.removeAll() + layouts.removeAll() + } + } +} diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift index 75d0a49af6..4d509259a8 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProCassandraCore import TableProPluginKit extension CassandraPluginDriver { @@ -23,7 +24,8 @@ extension CassandraPluginDriver { columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, - insertedRowData: insertedRowData + insertedRowData: insertedRowData, + lifetimes: keptLifetimes(keyspace: schema, table: table) ) } @@ -35,6 +37,24 @@ extension CassandraPluginDriver { rows: [[PluginCellValue]], absentCells: [Int: Set] ) -> [(statement: String, parameters: [PluginCellValue])]? { - CassandraRowWriter.restoreInserts(keyspace: schema, table: table, columns: columns, rows: rows) + CassandraRowWriter.restoreInserts( + keyspace: schema, + table: table, + columns: columns, + primaryKeyColumns: primaryKeyColumns, + rows: rows, + lifetimes: keptLifetimes(keyspace: schema, table: table) + ) + } + + // An unqualified write lands in the session's keyspace, the one an unqualified browse read. + private func keptLifetimes(keyspace schema: String?, table: String) -> CassandraRowWriter.Lifetimes { + let keyspace = schema.flatMap { $0.isEmpty ? nil : $0 } ?? currentSchema + return { [cellLifetimes] key in + guard let keyspace else { return nil } + return cellLifetimes.lifetimes( + of: CassandraRowIdentity(table: CassandraTableIdentity(keyspace: keyspace, table: table), key: key) + ) + } } } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift index 0c7ce00b73..be924c7b54 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift @@ -4,16 +4,17 @@ // import Foundation +import TableProCassandraCore import TableProPluginKit -/// The CQL a grid save writes: one statement per row, each naming the row by its whole primary key. -/// -/// CQL has no `OR` and no parenthesised groups, so several rows cannot share one `DELETE`, and an `INSERT` is an -/// upsert, so a new row whose key is taken would silently replace the row already there. A new row is therefore -/// written `IF NOT EXISTS`, and anything CQL cannot do to a row is refused by name rather than sent. +// One statement per row, each naming the row by its whole primary key: CQL has no `OR` and no parenthesised groups. +// A new row is written `IF NOT EXISTS` and an edit `IF EXISTS` (CassandraConditionalWrite), and anything CQL cannot +// do to a row is refused by name rather than sent. enum CassandraRowWriter { - static let defaultSentinel = "__DEFAULT__" - static let unappliedMarker = "[applied]" + static let unappliedMarker = CassandraConditionalWrite.appliedColumn + + // What a browse kept for the row with these primary key values, by column name. + typealias Lifetimes = ([String: PluginCellValue]) -> CassandraRowLifetimes? static func rowWrites( keyspace: String?, @@ -21,10 +22,13 @@ enum CassandraRowWriter { columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], - insertedRowData: [Int: [PluginCellValue]] + insertedRowData: [Int: [PluginCellValue]], + lifetimes: @escaping Lifetimes = { _ in nil } ) throws -> [PluginRowWrite] { let target = CassandraBrowseRenderer.qualifiedTable(keyspace: keyspace, table: table) - let writer = Writer(target: target, columns: columns, primaryKeyColumns: primaryKeyColumns) + let writer = Writer( + target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, lifetimes: lifetimes + ) return try changes.compactMap { change in switch change.type { case .insert: @@ -37,38 +41,92 @@ enum CassandraRowWriter { } } - /// Writes deleted rows back as they were, key included, to undo a delete. A plain `INSERT` rather than - /// `IF NOT EXISTS`: restoring a row means making it what it was, whatever took its place since. + // Writes deleted rows back as they were, key included and each cell with the TTL it had left, to undo a delete. + // A plain `INSERT` rather than `IF NOT EXISTS`: restoring a row means making it what it was, whatever took its + // place since. static func restoreInserts( keyspace: String?, table: String, columns: [String], - rows: [[PluginCellValue]] + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + lifetimes: Lifetimes = { _ in nil } ) -> [(statement: String, parameters: [PluginCellValue])] { let target = CassandraBrowseRenderer.qualifiedTable(keyspace: keyspace, table: table) + let now = ContinuousClock.now return rows.map { row in let width = min(row.count, columns.count) - let names = columns.prefix(width).map(CassandraBrowseRenderer.quote).joined(separator: ", ") - let markers = Array(repeating: "?", count: width).joined(separator: ", ") - return ("INSERT INTO \(target) (\(names)) VALUES (\(markers))", Array(row.prefix(width))) + let names = Array(columns.prefix(width)) + let values = Array(row.prefix(width)) + let kept = rowKey(primaryKeyColumns, columns: names, row: values).flatMap(lifetimes) + let keys = kept.flatMap { kept -> CassandraLifetimeWrite.RowKeys? in + guard let whole = condition(naming: primaryKeyColumns, columns: names, row: values), + let partition = condition(naming: kept.layout.partitionKey, columns: names, row: values) + else { return nil } + return CassandraLifetimeWrite.RowKeys(row: whole, partition: partition) + } + let cells = zip(names, values).map { name, value in + CassandraLifetimeWrite.RestoredCell( + column: name, + value: value, + isKey: primaryKeyColumns.contains(name), + isNull: value.isNull, + isStatic: kept?.layout.staticColumns.contains(name) ?? false, + timeToLive: kept?.timeToLive(of: name, at: now) + ) + } + let write = CassandraLifetimeWrite.restore(target: target, cells: cells, keys: keys) + return (write.cql, write.parameters) + } + } + + static func rowKey( + _ keyColumns: [String], + columns: [String], + row: [PluginCellValue] + ) -> [String: PluginCellValue]? { + guard !keyColumns.isEmpty else { return nil } + var key: [String: PluginCellValue] = [:] + for keyColumn in keyColumns { + guard let index = columns.firstIndex(of: keyColumn), index < row.count, !row[index].isNull else { + return nil + } + key[keyColumn] = row[index] } + return key + } + + static func condition( + naming keyColumns: [String], + columns: [String], + row: [PluginCellValue] + ) -> CassandraLifetimeWrite.KeyCondition? { + guard let key = rowKey(keyColumns, columns: columns, row: row) else { return nil } + return CassandraLifetimeWrite.KeyCondition( + cql: keyColumns.map { "\(CassandraBrowseRenderer.quote($0)) = ?" }.joined(separator: " AND "), + values: keyColumns.compactMap { key[$0] } + ) } - /// An `INSERT … IF NOT EXISTS` that found its key taken answers with `[applied] = false` and the row already - /// there, which the server reports as a success. - static func unappliedInsertRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { - let normalized = statement.uppercased() - guard normalized.hasPrefix("INSERT"), normalized.hasSuffix("IF NOT EXISTS"), - columns.first == unappliedMarker, - rows.first?.first?.asText?.lowercased() == "false" - else { return nil } - return String(localized: "A row with this primary key already exists. Edit that row instead.") + static func unappliedWriteRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { + let refusal = CassandraConditionalWrite.refusal( + statement: statement, firstColumn: columns.first, firstValue: rows.first?.first?.asText + ) + switch refusal { + case .keyTaken: + return String(localized: "A row with this primary key already exists. Edit that row instead.") + case .rowMissing: + return String(localized: "No row has this primary key, so nothing was updated. Another client may have deleted it.") + case nil: + return nil + } } private struct Writer { let target: String let columns: [String] let primaryKeyColumns: [String] + let lifetimes: Lifetimes func insert(_ change: PluginRowChange, values: [PluginCellValue]?) throws -> PluginRowWrite { guard !primaryKeyColumns.isEmpty else { @@ -77,12 +135,16 @@ enum CassandraRowWriter { )) } let row = values ?? rowFromCellChanges(change) + // A new row built from its edits leaves every column it never set to the server. + let leftOut = (change.defaultColumns ?? []).union( + values == nil ? Set(columns.indices).subtracting(change.cellChanges.map(\.columnIndex)) : [] + ) let expressionColumns = change.expressionColumns ?? [] var names: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, value) in row.enumerated() where index < columns.count { - if value.asText == defaultSentinel { continue } + if leftOut.contains(index) { continue } names.append(CassandraBrowseRenderer.quote(columns[index])) if let call = expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(call) @@ -93,23 +155,25 @@ enum CassandraRowWriter { } for keyColumn in primaryKeyColumns { guard let index = columns.firstIndex(of: keyColumn), index < row.count, !row[index].isNull, - row[index].asText != defaultSentinel + !leftOut.contains(index) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( format: String(localized: "Enter a value for the primary key column %@."), keyColumn )) } } - let statement = "INSERT INTO \(target) (\(names.joined(separator: ", "))) " - + "VALUES (\(placeholders.joined(separator: ", "))) IF NOT EXISTS" + let statement = CassandraConditionalWrite.insertStatement(target: target, names: names, values: placeholders) return PluginRowWrite(statement: statement, parameters: parameters, rowIndices: [change.rowIndex]) } func update(_ change: PluginRowChange) throws -> PluginRowWrite? { guard !change.cellChanges.isEmpty else { return nil } let expressionColumns = change.expressionColumns ?? [] - var assignments: [String] = [] - var parameters: [PluginCellValue] = [] + let kept = change.originalRow + .flatMap { CassandraRowWriter.rowKey(primaryKeyColumns, columns: columns, row: $0) } + .flatMap(lifetimes) + let now = ContinuousClock.now + var assignments: [CassandraLifetimeWrite.Assignment] = [] for cell in change.cellChanges { if primaryKeyColumns.contains(cell.columnName) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( @@ -119,22 +183,36 @@ enum CassandraRowWriter { cell.columnName )) } - if cell.newValue.asText == defaultSentinel { + if change.isColumnDefault(cell.columnIndex) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( localized: "Cassandra columns have no default value." )) } - let column = CassandraBrowseRenderer.quote(cell.columnName) - if let call = expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) { - assignments.append("\(column) = \(call)") - } else { - assignments.append("\(column) = ?") - parameters.append(cell.newValue) + if kept?.hasExpired(cell.columnName, at: now) == true { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( + format: String( + localized: "The value in %@ expired after the grid read it. Refresh, then edit the row again." + ), + cell.columnName + )) } + let column = CassandraBrowseRenderer.quote(cell.columnName) + let call = expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) + assignments.append(CassandraLifetimeWrite.Assignment( + cql: "\(column) = \(call ?? "?")", + parameters: call == nil ? [cell.newValue] : [], + timeToLive: kept?.timeToLive(of: cell.columnName, at: now), + isStatic: kept?.layout.staticColumns.contains(cell.columnName) ?? false + )) } let key = try keyCondition(for: change) - let statement = "UPDATE \(target) SET \(assignments.joined(separator: ", ")) WHERE \(key.cql)" - return PluginRowWrite(statement: statement, parameters: parameters + key.values, rowIndices: [change.rowIndex]) + let partition = change.originalRow.flatMap { original in + kept.flatMap { CassandraRowWriter.condition(naming: $0.layout.partitionKey, columns: columns, row: original) } + } + let write = CassandraLifetimeWrite.update( + target: target, assignments: assignments, row: key, partition: partition + ) + return PluginRowWrite(statement: write.cql, parameters: write.parameters, rowIndices: [change.rowIndex]) } func delete(_ change: PluginRowChange) throws -> PluginRowWrite { @@ -144,28 +222,19 @@ enum CassandraRowWriter { ) } - private func keyCondition(for change: PluginRowChange) throws -> (cql: String, values: [PluginCellValue]) { - guard !primaryKeyColumns.isEmpty, let original = change.originalRow else { + private func keyCondition(for change: PluginRowChange) throws -> CassandraLifetimeWrite.KeyCondition { + guard let original = change.originalRow, + let key = CassandraRowWriter.condition(naming: primaryKeyColumns, columns: columns, row: original) + else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( localized: "The row's primary key is not loaded, so the row cannot be found." )) } - var conditions: [String] = [] - var values: [PluginCellValue] = [] - for keyColumn in primaryKeyColumns { - guard let index = columns.firstIndex(of: keyColumn), index < original.count, !original[index].isNull else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( - localized: "The row's primary key is not loaded, so the row cannot be found." - )) - } - conditions.append("\(CassandraBrowseRenderer.quote(keyColumn)) = ?") - values.append(original[index]) - } - return (conditions.joined(separator: " AND "), values) + return key } private func rowFromCellChanges(_ change: PluginRowChange) -> [PluginCellValue] { - var row = [PluginCellValue](repeating: .text(defaultSentinel), count: columns.count) + var row = [PluginCellValue](repeating: .null, count: columns.count) for cell in change.cellChanges where cell.columnIndex < row.count { row[cell.columnIndex] = cell.newValue } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift index 704497b82a..c2e1f90f3c 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseHTTPChunks.swift @@ -89,14 +89,14 @@ nonisolated internal final class ClickHouseHTTPChunks: NSObject, URLSessionDataD } guard !isFailure else { return } - continuation.yield(data) - let shouldSuspend = lock.withLock { () -> Bool in - guard !isTransferSuspended else { return false } + // Suspended before the reader sees the chunk: a reader done with a short chunk first would find nothing to + // resume, and the suspend after it would stop the transfer for good. + lock.withLock { + guard !isTransferSuspended else { return } isTransferSuspended = true - return true + dataTask.suspend() } - guard shouldSuspend else { return } - dataTask.suspend() + continuation.yield(data) } internal func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift index b0ea856105..69b6a98efd 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift @@ -223,6 +223,8 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked .uuidCellText, .repeatingPrimaryKey, .readOnlyStatements, + .enforcesReadOnlyMode, + .columnDefaultMarks, ] } @@ -339,15 +341,15 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked // MARK: - Query Execution func execute(query: String) async throws -> PluginQueryResult { - try await runStatement(query, parameters: [], readOnly: false) + try await runStatement(query, parameters: [], readOnly: nil) } func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { - try await runStatement(query, parameters: [], readOnly: context.readOnly && context.origin == .external) + try await runStatement(query, parameters: [], readOnly: ClickHouseReadOnlyReason(context)) } func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { - try await runStatement(query, parameters: parameters, readOnly: false) + try await runStatement(query, parameters: parameters, readOnly: nil) } func executeParameterized( @@ -355,19 +357,24 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await runStatement(query, parameters: parameters, readOnly: context.readOnly && context.origin == .external) + try await runStatement(query, parameters: parameters, readOnly: ClickHouseReadOnlyReason(context)) } + // A capped read-only read with no parameters is the bounded stream, which ends on the server one row past the cap. + // Parameters ride only on the buffered request, so a read that has them is read whole and cut here. func executeUserQuery( query: String, rowCap: Int?, parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - guard context.readOnly, context.origin == .external else { + guard let reason = ClickHouseReadOnlyReason(context) else { return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } - let raw = try await runStatement(query, parameters: parameters ?? [], readOnly: true) + if let cap = rowCap, cap > 0, parameters?.isEmpty ?? true { + return try await boundedRead(query: query, rowCap: cap, readOnly: reason, byteLimit: context.resultByteLimit) + } + let raw = try await runStatement(query, parameters: parameters ?? [], readOnly: reason) guard let cap = rowCap, cap > 0, raw.rows.count > cap else { return raw } return PluginQueryResult( columns: raw.columns, @@ -379,7 +386,11 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked ) } - private func runStatement(_ query: String, parameters: [PluginCellValue], readOnly: Bool) async throws -> PluginQueryResult { + private func runStatement( + _ query: String, + parameters: [PluginCellValue], + readOnly: ClickHouseReadOnlyReason? + ) async throws -> PluginQueryResult { let startTime = Date() let queryId = UUID().uuidString let result: CHQueryResult @@ -578,6 +589,30 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked /// both ask for names its columns, so neither pays a `LIMIT 0` probe to learn them. That probe /// appended to the statement, which a query already carrying a `LIMIT` rejects outright. func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await boundedRead(query: query, rowCap: rowCap, readOnly: nil, byteLimit: nil) + } + + // A context `readonly` does not cover goes back to the host as PluginKit's default sends it. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + guard let reason = ClickHouseReadOnlyReason(context) else { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await executeBoundedQuery(query: query, rowCap: rowCap) + } + return try await boundedRead(query: query, rowCap: rowCap, readOnly: reason, byteLimit: context.resultByteLimit) + } + + private func boundedRead( + query: String, + rowCap: Int, + readOnly: ClickHouseReadOnlyReason?, + byteLimit: Int? + ) async throws -> PluginQueryResult { let started = Date() let isOutputCut = OSAllocatedUnfairLock(initialState: false) let stream = PluginRowStream.make { continuation, abort in @@ -586,6 +621,8 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked try await self.performBoundedStreamRows( query: query, rowCap: rowCap, + readOnly: readOnly, + byteLimit: byteLimit, isOutputCut: isOutputCut, continuation: continuation ) @@ -610,6 +647,8 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked private func performBoundedStreamRows( query: String, rowCap: Int, + readOnly: ClickHouseReadOnlyReason?, + byteLimit: Int?, isOutputCut: OSAllocatedUnfairLock, continuation: AsyncThrowingStream.Continuation ) async throws { @@ -620,17 +659,23 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked database: database, queryId: queryId, executionTimeLimit: executionTimeLimit, - shape: .streamed(rowCap: rowCap) + shape: .streamed(rowCap: rowCap), + readOnly: readOnly != nil ) request.timeoutInterval = _queryTimeout.requestTimeoutInterval try await killingOnCancel(queryId: queryId) { - try await streamTabSeparatedRows( - request: request, - session: session, - batchSize: min(5_000, rowCap + 1), - boundedOutputCut: isOutputCut, - continuation: continuation - ) + do { + try await streamTabSeparatedRows( + request: request, + session: session, + batchSize: min(5_000, rowCap + 1), + boundedOutputCut: isOutputCut, + byteLimit: byteLimit, + continuation: continuation + ) + } catch { + throw Self.readOnlyRefusal(error, reason: readOnly) + } } } @@ -648,6 +693,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked session: URLSession, batchSize: Int, boundedOutputCut: OSAllocatedUnfairLock? = nil, + byteLimit: Int? = nil, continuation: AsyncThrowingStream.Continuation ) async throws { var decoder = ClickHouseTabSeparatedRowDecoder() @@ -656,6 +702,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked var headerSent = false var batch: [PluginRow] = [] batch.reserveCapacity(batchSize) + var bytes = 0 func sendHeader(_ header: ClickHouseTabSeparatedRowDecoder.Header) { guard !headerSent else { return } @@ -667,6 +714,20 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked ))) } + // Keeps the row that takes the decoded rows past `byteLimit`, and says it came; the rest is never read. + func take(_ rows: [PluginRow]) -> Bool { + guard let byteLimit else { + batch.append(contentsOf: rows) + return false + } + for row in rows { + batch.append(row) + bytes += Self.decodedByteCount(of: row) + if bytes > byteLimit { return true } + } + return false + } + let chunks = ClickHouseHTTPChunks(session: session, request: request) do { try await chunks.forEachChunk { chunk in @@ -693,11 +754,18 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked if let header = decoder.header { sendHeader(header) } - batch.append(contentsOf: rows) + if take(rows) { throw ByteLimitReached() } guard batch.count >= batchSize else { return } continuation.yield(.rows(batch)) batch.removeAll(keepingCapacity: true) } + } catch is ByteLimitReached { + if !batch.isEmpty { + continuation.yield(.rows(batch)) + } + boundedOutputCut?.withLock { $0 = true } + continuation.finish() + return } catch is OtherFormatBodyFull {} if let otherFormatBody { @@ -722,12 +790,14 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked exceptionTag: chunks.headerValue("X-ClickHouse-Exception-Tag"), header: decoder.header ) - batch.append(contentsOf: decoder.consume(rest)) - batch.append(contentsOf: decoder.finish()) + let stoppedAtByteLimit = take(decoder.consume(rest)) || take(decoder.finish()) sendHeader(decoder.header ?? ClickHouseTabSeparatedRowDecoder.Header(columns: [], columnTypeNames: [])) if !batch.isEmpty { continuation.yield(.rows(batch)) } + if stoppedAtByteLimit { + boundedOutputCut?.withLock { $0 = true } + } continuation.finish() } @@ -737,15 +807,17 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked private struct OtherFormatBodyFull: Error {} + private struct ByteLimitReached: Error {} + func streamRows(query: String) -> AsyncThrowingStream { - stream(query, readOnly: false) + stream(query, readOnly: nil) } func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - stream(query, readOnly: context.readOnly && context.origin == .external) + stream(query, readOnly: ClickHouseReadOnlyReason(context)) } - private func stream(_ query: String, readOnly: Bool) -> AsyncThrowingStream { + private func stream(_ query: String, readOnly: ClickHouseReadOnlyReason?) -> AsyncThrowingStream { AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in let streamTask = Task { do { @@ -762,7 +834,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked private func performStreamRows( query: String, - readOnly: Bool, + readOnly: ClickHouseReadOnlyReason?, continuation: AsyncThrowingStream.Continuation ) async throws { let queryId = UUID().uuidString @@ -773,7 +845,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked queryId: queryId, executionTimeLimit: executionTimeLimit, shape: .streamed(rowCap: nil), - readOnly: readOnly + readOnly: readOnly != nil ) request.timeoutInterval = _queryTimeout.requestTimeoutInterval try await killingOnCancel(queryId: queryId) { @@ -785,7 +857,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked continuation: continuation ) } catch { - throw readOnly ? Self.readOnlyRefusal(error) : error + throw Self.readOnlyRefusal(error, reason: readOnly) } } } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift index d95fde6e89..242463a761 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Http.swift @@ -18,7 +18,7 @@ nonisolated extension ClickHousePluginDriver { _ query: String, queryId: String? = nil, requestTimeout: TimeInterval? = nil, - readOnly: Bool = false + readOnly: ClickHouseReadOnlyReason? = nil ) async throws -> CHQueryResult { try await sendStatement( query, @@ -34,7 +34,7 @@ nonisolated extension ClickHousePluginDriver { _ query: String, params: [String: String?], queryId: String? = nil, - readOnly: Bool = false + readOnly: ClickHouseReadOnlyReason? = nil ) async throws -> CHQueryResult { try await sendStatement( query, @@ -51,7 +51,7 @@ nonisolated extension ClickHousePluginDriver { params: [String: String?]?, executionTimeLimit: Int?, requestTimeout: TimeInterval? = nil, - readOnly: Bool = false + readOnly: ClickHouseReadOnlyReason? = nil ) async throws -> CHQueryResult { let (session, database) = try lock.withLock { () throws -> (URLSession, String) in guard let session = self.session else { throw ClickHouseError.notConnected } @@ -68,26 +68,34 @@ nonisolated extension ClickHousePluginDriver { queryId: queryId, executionTimeLimit: executionTimeLimit, shape: .buffered(parameters: params), - readOnly: readOnly + readOnly: readOnly != nil ) request.timeoutInterval = requestTimeout ?? _queryTimeout.requestTimeoutInterval return try await killingOnCancel(queryId: queryId) { do { return try await perform(request: request, session: session) } catch { - throw readOnly ? Self.readOnlyRefusal(error) : error + throw Self.readOnlyRefusal(error, reason: readOnly) } } } // 164 is READONLY: the request's own `readonly` refused the statement. - static func readOnlyRefusal(_ error: Error) -> Error { - guard let failure = error as? ClickHouseError, failure.exceptionCode == 164 else { return error } - return ClickHouseError( - message: String(localized: "This request is read only, and the statement is not a read."), - exceptionCode: failure.exceptionCode, - detail: failure.message - ) + static func readOnlyRefusal(_ error: Error, reason: ClickHouseReadOnlyReason?) -> Error { + guard let reason, let failure = error as? ClickHouseError, failure.exceptionCode == 164 else { return error } + return ClickHouseError(message: reason.refusalMessage, exceptionCode: failure.exceptionCode, detail: failure.message) + } + + // Text as UTF-8, bytes as they are, null as nothing: never more than the host counts for the same row, so a read + // never stops before the host's own budget is full. + static func decodedByteCount(of row: PluginRow) -> Int { + row.reduce(0) { total, cell in + switch cell { + case .null: total + case .text(let text): total + text.utf8.count + case .bytes(let data): total + data.count + } + } } // Two callers can share this driver, so a cancelled statement stops its own query, never the one sent last. @@ -330,6 +338,32 @@ nonisolated extension ClickHousePluginDriver { } } +// Who asked for `readonly`, which is what a refusal names. ClickHouse has no transaction a request could find open. +nonisolated enum ClickHouseReadOnlyReason: Sendable, Equatable { + case externalClients + case safeMode + + // An app read marked `readOnly` alone, a catalog read or a dashboard tile, runs as it is. + init?(_ context: PluginStatementContext) { + if context.readOnly, context.origin == .external { + self = .externalClients + } else if context.enforcesReadOnly { + self = .safeMode + } else { + return nil + } + } + + var refusalMessage: String { + switch self { + case .externalClients: + String(localized: "This request is read only, and the statement is not a read.") + case .safeMode: + String(localized: "Safe Mode is set to Read-Only for this connection, and the database refused the statement.") + } + } +} + /// How the server hands a result back. nonisolated enum ClickHouseResponseShape: Equatable { /// The server holds the whole result until the query ends, so a failure is an HTTP status and diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift index c9b04b9c9b..91ee397d3d 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Schema.swift @@ -118,6 +118,8 @@ nonisolated extension ClickHousePluginDriver { """ } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { // Pre-fetch PK columns for all tables. Falls back to sorting_key when // primary_key is empty (MergeTree without explicit PRIMARY KEY clause). diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift index 94d7513dfc..961f6f6185 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift @@ -17,8 +17,6 @@ nonisolated enum ClickHouseRowWrites { let delete: (_ table: String, _ predicate: String) -> String } - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func writes( table: String, schema: String?, @@ -52,7 +50,8 @@ nonisolated enum ClickHouseRowWrites { let values = insertedRowData[change.rowIndex] else { continue } writes.append(insert( into: target, columns: columns, values: values, expressions: change.expressionColumns ?? [], - serverOwned: context.serverOwnedColumns, rowIndex: change.rowIndex + defaults: change.defaultColumns ?? [], serverOwned: context.serverOwnedColumns, + rowIndex: change.rowIndex )) case .update: guard !change.cellChanges.isEmpty else { continue } @@ -78,13 +77,14 @@ nonisolated enum ClickHouseRowWrites { columns: [String], values: [PluginCellValue], expressions: Set, + defaults: Set = [], serverOwned: Set = [], rowIndex: Int ) -> PluginRowWrite { var names: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] - for (index, value) in values.enumerated() where index < columns.count && value != defaultMarker { + for (index, value) in values.enumerated() where index < columns.count && !defaults.contains(index) { names.append(clickHouseQuotedIdentifier(columns[index])) if expressions.contains(index), case .text(let sql) = value { placeholders.append(sql) diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift index 6c16be41f6..d1ef570299 100644 --- a/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift @@ -248,6 +248,7 @@ struct ClickHouseDMLStatementTests { func insertStaysPlain() throws { var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) change.expressionColumns = [1] + change.defaultColumns = [2] let write = try #require(try writes( [change], columns: ["id", "at", "note"], inserted: [0: ["7", "now()", "__DEFAULT__"]] )?.first) @@ -257,9 +258,21 @@ struct ClickHouseDMLStatementTests { #expect(write.lookup == nil) } + @Test("The text __DEFAULT__ in a column with no default mark is written as text") + func unmarkedDefaultTextIsAValue() throws { + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let write = try #require(try writes( + [change], columns: ["id", "note"], inserted: [0: ["7", "__DEFAULT__"]] + )?.first) + + #expect(write.statement == "INSERT INTO `users` (`id`, `note`) VALUES (?, ?)") + #expect(write.parameters == ["7", "__DEFAULT__"]) + } + @Test("A new row that sets nothing takes every default, named through a column the server does not fill") func emptyInsertTakesDefaults() throws { - let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + change.defaultColumns = [0, 1] var context = PluginRowWriteContext() context.serverOwnedColumns = ["id"] let write = try #require(try waitingDriver.generateRowWrites( diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseReadOnlyRequestTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseReadOnlyRequestTests.swift index 9396f7bd61..59f34fd5c3 100644 --- a/apple/Plugins/ClickHouseDriverTests/ClickHouseReadOnlyRequestTests.swift +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseReadOnlyRequestTests.swift @@ -145,8 +145,140 @@ struct ClickHouseReadOnlyRequestTests { } } - @Test("The driver says it runs a read-only statement in ClickHouse's own read-only mode") + @Test("The driver says it runs a read-only statement in ClickHouse's own read-only mode, from outside and at Read-Only") func declaresReadOnlyStatements() { - #expect(ClickHouseStubServer().connectedDriver().capabilities.contains(.readOnlyStatements)) + let capabilities = ClickHouseStubServer().connectedDriver().capabilities + #expect(capabilities.contains(.readOnlyStatements)) + #expect(capabilities.contains(.enforcesReadOnlyMode)) + } + + private static let safeMode = PluginStatementContext(enforcesReadOnly: true) + + @Test("An external read names external clients, Read-Only from any origin names Safe Mode, an app read alone neither") + func reasonFollowsTheContext() { + var appRead = PluginStatementContext() + appRead.readOnly = true + var externalWrite = PluginStatementContext() + externalWrite.origin = .external + externalWrite.enforcesReadOnly = true + var externalSafeModeRead = Self.externalRead + externalSafeModeRead.enforcesReadOnly = true + + #expect(ClickHouseReadOnlyReason(Self.externalRead) == .externalClients) + #expect(ClickHouseReadOnlyReason(externalSafeModeRead) == .externalClients) + #expect(ClickHouseReadOnlyReason(Self.safeMode) == .safeMode) + #expect(ClickHouseReadOnlyReason(externalWrite) == .safeMode) + #expect(ClickHouseReadOnlyReason(appRead) == nil) + #expect(ClickHouseReadOnlyReason(PluginStatementContext()) == nil) + } + + @Test("At Read-Only every entry point carries readonly=1, and an app read marked readOnly alone carries none") + func safeModeEntryPointsCarryIt() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + var appRead = PluginStatementContext() + appRead.readOnly = true + + _ = try await driver.execute(query: "SELECT 1", context: Self.safeMode) + _ = try await driver.executeUserQuery(query: "SELECT 2", rowCap: 10, parameters: nil, context: Self.safeMode) + _ = try await driver.executeUserQuery(query: "SELECT ?", rowCap: nil, parameters: [.text("3")], context: Self.safeMode) + _ = try await driver.executeParameterized(query: "SELECT ?", parameters: [.text("4")], context: Self.safeMode) + for try await _ in driver.streamRows(query: "SELECT 5", context: Self.safeMode) {} + _ = try await driver.executeBoundedQuery(query: "SELECT 6", rowCap: 10, context: Self.safeMode) + _ = try await driver.executeBoundedQuery(query: "SELECT 7", rowCap: 10, context: Self.externalRead) + + _ = try await driver.execute(query: "SELECT 8", context: appRead) + _ = try await driver.executeUserQuery(query: "SELECT 9", rowCap: 10, parameters: nil, context: appRead) + + let statements = server.requests.filter { !$0.body.hasPrefix("KILL QUERY") } + let guarded = statements.filter { $0.item("readonly") == "1" }.map(\.body) + let unguarded = statements.filter { $0.item("readonly") == nil }.map(\.body) + #expect(guarded == ["SELECT 1", "SELECT 2", "SELECT {p1:String}", "SELECT {p1:String}", "SELECT 5", "SELECT 6", "SELECT 7"]) + #expect(unguarded == ["SELECT 8", "SELECT 9"]) + } + + @Test("A bounded read the read-only mode does not cover goes back to the host") + func boundedReadGoesBackWhenNotCovered() async throws { + let driver = ClickHouseStubServer().connectedDriver() + var appRead = PluginStatementContext() + appRead.readOnly = true + var catalogRead = appRead + catalogRead.readsCatalog = true + + #expect(try await driver.executeBoundedQuery(query: "SELECT 1", rowCap: 10, context: appRead) == nil) + #expect(try await driver.executeBoundedQuery(query: "SELECT 1", rowCap: 10, context: catalogRead) == nil) + let plain = try await driver.executeBoundedQuery(query: "SELECT 1", rowCap: 10, context: PluginStatementContext()) + #expect(plain?.rows.count == 1) + } + + @Test("At Read-Only a statement the server refuses names Safe Mode, and the server's text stays") + func safeModeRefusalNamesSafeMode() async throws { + let serverText = "Code: 164. DB::Exception: default: Cannot execute query in readonly mode. (READONLY) (version 26.8.36.3 (official build))" + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 500, body: serverText, headers: ["X-ClickHouse-Exception-Code": "164"]) + } + let driver = server.connectedDriver() + + do { + _ = try await driver.execute(query: "SELECT purge_orders()", context: Self.safeMode) + Issue.record("expected a refusal") + } catch let error as ClickHouseError { + #expect(error.message == ClickHouseReadOnlyReason.safeMode.refusalMessage) + #expect(error.message != ClickHouseReadOnlyReason.externalClients.refusalMessage) + #expect(error.exceptionCode == 164) + #expect(error.pluginErrorDetail == serverText) + } + + do { + _ = try await driver.executeBoundedQuery(query: "SELECT purge_orders()", rowCap: 10, context: Self.safeMode) + Issue.record("expected a refusal") + } catch let error as ClickHouseError { + #expect(error.message == ClickHouseReadOnlyReason.safeMode.refusalMessage) + #expect(error.exceptionCode == 164) + } + } + + @Test("An external capped read is the bounded stream: the server stops one row past the cap, and readonly stays") + func externalCappedReadEndsOnTheServer() async throws { + let server = ClickHouseStubServer() + let driver = server.connectedDriver() + driver.serverProfile = Self.server258 + + _ = try await driver.executeUserQuery(query: "SELECT 2", rowCap: 10, parameters: nil, context: Self.externalRead) + _ = try await driver.executeUserQuery(query: "SELECT ?", rowCap: 10, parameters: [.text("3")], context: Self.externalRead) + + let capped = server.requests.first { $0.body == "SELECT 2" } + #expect(capped?.item("max_result_rows") == "11") + #expect(capped?.item("result_overflow_mode") == "break") + #expect(capped?.item("readonly") == "1") + let bound = server.requests.first { $0.body == "SELECT {p1:String}" } + #expect(bound?.item("max_result_rows") == nil) + #expect(bound?.item("readonly") == "1") + } + + @Test("A read-only read stops at the first row past the byte limit and keeps that row") + func readOnlyReadStopsAtTheByteLimit() async throws { + let server = ClickHouseStubServer { _ in + ClickHouseStubReply(statusCode: 200, body: "s\nString\naaaaaa\nbbbbbb\ncccccc\n") + } + let driver = server.connectedDriver() + var limited = Self.externalRead + limited.resultByteLimit = 10 + var safeModeLimited = Self.safeMode + safeModeLimited.resultByteLimit = 10 + + let external = try await driver.executeUserQuery(query: "SELECT s", rowCap: 100, parameters: nil, context: limited) + #expect(external.rows.compactMap { $0.first?.asText } == ["aaaaaa", "bbbbbb"]) + #expect(external.isTruncated) + + let bounded = try await driver.executeBoundedQuery(query: "SELECT s", rowCap: 100, context: safeModeLimited) + #expect(bounded?.rows.count == 2) + #expect(bounded?.isTruncated == true) + + var roomy = Self.externalRead + roomy.resultByteLimit = 1_000 + let whole = try await driver.executeUserQuery(query: "SELECT s", rowCap: 100, parameters: nil, context: roomy) + #expect(whole.rows.count == 3) + #expect(!whole.isTruncated) } } diff --git a/apple/Plugins/CloudflareD1DriverPlugin/CloudflareD1PluginDriver.swift b/apple/Plugins/CloudflareD1DriverPlugin/CloudflareD1PluginDriver.swift index 3ef3e11102..cb3e9b4506 100644 --- a/apple/Plugins/CloudflareD1DriverPlugin/CloudflareD1PluginDriver.swift +++ b/apple/Plugins/CloudflareD1DriverPlugin/CloudflareD1PluginDriver.swift @@ -361,6 +361,8 @@ nonisolated final class CloudflareD1PluginDriver: PluginDatabaseDriver, @uncheck } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let query = """ SELECT m.name AS tbl, p.cid, p.name, p.type, p."notnull", p.dflt_value, p.pk diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json index 89595dcb46..2c58ed400f 100644 --- a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json @@ -146,7 +146,8 @@ { "type": "TIMESTAMP WITH TIME ZONE", "values": [ - {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"} + {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"}, + {"sql": "TIMESTAMP '2026-01-01 00:00:00 Europe/Paris'", "cell": "2026-01-01 00:00:00.000000 Europe/Paris"} ], "edit": {"text": "2000-01-01 00:00:00.000000+00:00", "cell": "2000-01-01 00:00:00.000000+00:00"}, "known": { diff --git a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift index cc80a87298..46a2450be2 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift @@ -160,7 +160,7 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var capabilities: PluginCapabilities { [ .parameterizedQueries, .transactions, .alterTableDDL, .multiSchema, .schemaCompare, .dataCompare, - .rowWrites, .cellExpressions, + .rowWrites, .cellExpressions, .columnDefaultMarks, ] } @@ -323,10 +323,9 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let task = Task { do { let result = try await self.execute(query: query, context: context) - continuation.yield(.header(PluginStreamHeader( - columns: result.columns, - columnTypeNames: result.columnTypeNames - ))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } diff --git a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift index cb50bfa039..f0c495b41e 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift @@ -19,7 +19,10 @@ extension DamengPluginDriver { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex), let row = insertedRowData[change.rowIndex], - let statement = insertStatement(table: qualifiedTable, columns: columns, values: row, expressions: expressions) + let statement = insertStatement( + table: qualifiedTable, columns: columns, values: row, expressions: expressions, + defaults: change.defaultColumns ?? [] + ) else { continue } writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) case .update: @@ -28,16 +31,18 @@ extension DamengPluginDriver { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cell -> String in let column = quoteIdentifier(cell.columnName) - if cell.newValue == Self.defaultMarker { return "\(column) = DEFAULT" } + if change.isColumnDefault(cell.columnIndex) { return "\(column) = DEFAULT" } if expressions.contains(cell.columnIndex), case .text(let sql) = cell.newValue { return "\(column) = \(sql)" } parameters.append(cell.newValue) return "\(column) = ?" } + // Measured on DM8: the count takes in a row already holding the value and leaves out rows a trigger writes. writes.append(PluginRowWrite( statement: "UPDATE \(qualifiedTable) SET \(assignments.joined(separator: ", ")) WHERE \(match.predicate) AND ROWNUM = 1", parameters: parameters + match.parameters, rowIndices: [change.rowIndex], - lookup: match.lookup + lookup: match.lookup, + expectedRowCount: 1 )) case .delete: guard deletedRowIndices.contains(change.rowIndex) else { continue } @@ -46,7 +51,8 @@ extension DamengPluginDriver { statement: "DELETE FROM \(qualifiedTable) WHERE \(match.predicate) AND ROWNUM = 1", parameters: match.parameters, rowIndices: [change.rowIndex], - lookup: match.lookup + lookup: match.lookup, + expectedRowCount: 1 )) } } @@ -61,11 +67,11 @@ extension DamengPluginDriver { rows: [[PluginCellValue]] ) -> [(statement: String, parameters: [PluginCellValue])]? { let qualifiedTable = qualifiedName(schema: schema, object: table) - return rows.compactMap { insertStatement(table: qualifiedTable, columns: columns, values: $0, expressions: []) } + return rows.compactMap { + insertStatement(table: qualifiedTable, columns: columns, values: $0, expressions: [], defaults: []) + } } - private static let defaultMarker = PluginCellValue.text("__DEFAULT__") - func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { guard !definition.columns.isEmpty else { return nil } let table = qualifiedName(schema: nil, object: definition.tableName) @@ -200,14 +206,15 @@ extension DamengPluginDriver { table: String, columns: [String], values: [PluginCellValue], - expressions: Set + expressions: Set, + defaults: Set ) -> (statement: String, parameters: [PluginCellValue])? { var insertColumns: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, value) in values.enumerated() where index < columns.count { insertColumns.append(quoteIdentifier(columns[index])) - if value == Self.defaultMarker { + if defaults.contains(index) { placeholders.append("DEFAULT") } else if expressions.contains(index), case .text(let sql) = value { placeholders.append(sql) diff --git a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Schema.swift b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Schema.swift index 8b49aa22e7..50aae42be1 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Schema.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Schema.swift @@ -103,6 +103,8 @@ extension DamengPluginDriver { } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let owner = effectiveSchema(schema) let result = try await executeParameterized( diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift index 321389ab22..5f9519a15e 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift @@ -214,29 +214,39 @@ actor DuckDBConnectionActor { } noteActivity(query) - var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandon) + var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandonWhenIdle) defer { duckdb_destroy_result(&opened.result) } try streamResultRows(opened, continuation: continuation) } // The state check, the BEGIN, the statement and the ROLLBACK share one actor turn, so nothing lands in between. // The statement text is not tracked: the ROLLBACK leaves the session as idle as it was found. - func executeReadOnly(_ query: String, parameters: [PluginCellValue], rowCap: Int? = nil) throws -> DuckDBRawResult { + // A stop at `byteLimit` ends only the read's own read-only transaction. + func executeReadOnly( + _ query: String, + parameters: [PluginCellValue], + rowCap: Int? = nil, + byteLimit: Int? = nil, + reason: DuckDBReadOnlyStatement.Reason + ) throws -> DuckDBRawResult { let conn = try readOnlyConnection() let startTime = Date() - return try DuckDBReadOnlyStatement.run(on: conn, session: sessionTransactionState()) { + return try DuckDBReadOnlyStatement.run(on: conn, session: sessionTransactionState(), reason: reason) { var opened = try openResult(query: query, parameters: parameters, connection: conn, unread: .abandon) defer { duckdb_destroy_result(&opened.result) } - return try extractResult(from: opened, limit: rowCap ?? PluginRowLimits.emergencyMax, startTime: startTime) + return try extractResult( + from: opened, limit: rowCap ?? PluginRowLimits.emergencyMax, byteLimit: byteLimit, startTime: startTime + ) } } func streamReadOnly( _ query: String, + reason: DuckDBReadOnlyStatement.Reason, continuation: AsyncThrowingStream.Continuation ) throws { let conn = try readOnlyConnection() - try DuckDBReadOnlyStatement.run(on: conn, session: sessionTransactionState()) { + try DuckDBReadOnlyStatement.run(on: conn, session: sessionTransactionState(), reason: reason) { var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandon) defer { duckdb_destroy_result(&opened.result) } try streamResultRows(opened, continuation: continuation) @@ -655,20 +665,39 @@ actor DuckDBConnectionActor { return ColumnSchema(names: names, logicalTypes: logicalTypes) } - private func extractResult(from opened: OpenedResult, limit: Int, startTime: Date) throws -> DuckDBRawResult { + // With `byteLimit` the read stops at the row that takes the decoded rows past it and keeps that row. A chunk is + // decoded whole before it is measured. + private func extractResult( + from opened: OpenedResult, + limit: Int, + byteLimit: Int? = nil, + startTime: Date + ) throws -> DuckDBRawResult { var rows: [[PluginCellValue]] = [] if let rowCount = opened.rowCount { rows.reserveCapacity(Int(min(rowCount, UInt64(max(0, limit))))) } var budget = DuckDBFetchBudget(limit: limit) + var bytes = 0 + var stoppedAtByteLimit = false let reachedEnd = try readRows(of: opened) { batch in let kept = budget.take(batch.count) - rows.append(contentsOf: kept == batch.count ? batch[...] : batch.prefix(kept)) - return !budget.isExceeded + var slice = kept == batch.count ? batch[...] : batch.prefix(kept) + if let byteLimit { + for (offset, row) in slice.enumerated() { + bytes += Self.decodedByteCount(of: row) + guard bytes > byteLimit else { continue } + slice = slice.prefix(offset + 1) + stoppedAtByteLimit = true + break + } + } + rows.append(contentsOf: slice) + return !budget.isExceeded && !stoppedAtByteLimit } - if !reachedEnd, opened.isStreaming, !opened.abandonsUnreadRows { - try Self.drain(opened.result) + if !reachedEnd { + try finishUnreadRows(of: opened) } return DuckDBRawResult( @@ -677,10 +706,22 @@ actor DuckDBConnectionActor { rows: rows, rowsAffected: Int(opened.rowsChanged), executionTime: Date().timeIntervalSince(startTime), - isTruncated: budget.isExceeded + isTruncated: budget.isExceeded || stoppedAtByteLimit ) } + // Text as UTF-8, bytes as they are, null as nothing: never more than the host counts for the same row, so a read + // never stops before the host's own budget is full. + static func decodedByteCount(of row: [PluginCellValue]) -> Int { + row.reduce(0) { total, cell in + switch cell { + case .null: total + case .text(let text): total + text.utf8.count + case .bytes(let data): total + data.count + } + } + } + private func streamResultRows( _ opened: OpenedResult, continuation: AsyncThrowingStream.Continuation @@ -692,10 +733,19 @@ actor DuckDBConnectionActor { ))) var cancelled = false - try readRows(of: opened) { batch in - cancelled = Task.isCancelled - || !DuckDBBoundedStream.deliver(PluginStreamElement.rows(batch), to: continuation) - return !cancelled + let reachedEnd: Bool + do { + reachedEnd = try readRows(of: opened) { batch in + cancelled = Task.isCancelled + || !DuckDBBoundedStream.deliver(PluginStreamElement.rows(batch), to: continuation) + return !cancelled + } + } catch is CancellationError { + try finishUnreadRows(of: opened) + throw CancellationError() + } + if !reachedEnd { + try finishUnreadRows(of: opened) } if cancelled { continuation.finish(throwing: CancellationError()) @@ -743,6 +793,13 @@ actor DuckDBConnectionActor { } } + // A consumer that stops reading inside a transaction must not abort it: a SELECT calling nextval left unfinished + // fails the transaction (scripts/probes/check-duckdb-streaming-results.sh). + private func finishUnreadRows(of opened: OpenedResult) throws { + guard opened.isStreaming, !opened.abandonsUnreadRows else { return } + try Self.drain(opened.result) + } + // Completes a statement whose rows are no longer wanted, so that what it changed is kept. private static func drain(_ result: duckdb_result) throws { var source = result diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift index 19da1b5586..eab2cca22e 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift @@ -277,6 +277,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { .schemaCompare, .dataCompare, .readOnlyStatements, + .enforcesReadOnlyMode, ] } @@ -695,8 +696,10 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { - guard context.readOnly, context.origin == .external else { return try await execute(query: query) } - return Self.queryResult(try await connectionActor.executeReadOnly(query, parameters: [])) + guard let reason = DuckDBReadOnlyStatement.reason(for: context) else { return try await execute(query: query) } + return Self.queryResult(try await connectionActor.executeReadOnly( + query, parameters: [], byteLimit: context.resultByteLimit, reason: reason + )) } func executeParameterized( @@ -711,8 +714,12 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - guard context.readOnly, context.origin == .external else { return try await executeParameterized(query: query, parameters: parameters) } - return Self.queryResult(try await connectionActor.executeReadOnly(query, parameters: parameters)) + guard let reason = DuckDBReadOnlyStatement.reason(for: context) else { + return try await executeParameterized(query: query, parameters: parameters) + } + return Self.queryResult(try await connectionActor.executeReadOnly( + query, parameters: parameters, byteLimit: context.resultByteLimit, reason: reason + )) } func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { @@ -720,6 +727,24 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return Self.queryResult(try await connectionActor.executeBounded(query, parameters: [], rowCap: rowCap)) } + // A context the read-only mode does not cover goes back to the host as PluginKit's default sends it. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + guard let reason = DuckDBReadOnlyStatement.reason(for: context) else { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await executeBoundedQuery(query: query, rowCap: rowCap) + } + guard rowCap > 0 else { return nil } + return Self.queryResult(try await connectionActor.executeReadOnly( + query, parameters: [], rowCap: rowCap, byteLimit: context.resultByteLimit, reason: reason + )) + } + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { guard let cap = rowCap, cap > 0 else { return Self.queryResult(try await connectionActor.executePrepared(query, parameters: parameters ?? [])) @@ -733,11 +758,13 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - guard context.readOnly, context.origin == .external else { + guard let reason = DuckDBReadOnlyStatement.reason(for: context) else { return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } - return Self.queryResult(try await connectionActor.executeReadOnly(query, parameters: parameters ?? [], rowCap: cap)) + return Self.queryResult(try await connectionActor.executeReadOnly( + query, parameters: parameters ?? [], rowCap: cap, byteLimit: context.resultByteLimit, reason: reason + )) } private static func queryResult(_ rawResult: DuckDBRawResult) -> PluginQueryResult { @@ -772,12 +799,12 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - guard context.readOnly, context.origin == .external else { return streamRows(query: query) } + guard let reason = DuckDBReadOnlyStatement.reason(for: context) else { return streamRows(query: query) } let actor = connectionActor return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(DuckDBBoundedStream.bufferedBatches)) { continuation in Task { do { - try await actor.streamReadOnly(query, continuation: continuation) + try await actor.streamReadOnly(query, reason: reason, continuation: continuation) } catch { continuation.finish(throwing: error) } @@ -859,6 +886,8 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let schemaName = resolveSchema(schema) let catalog = try requireCatalog() diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBReadOnlyStatement.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBReadOnlyStatement.swift index 802331d537..e0e502323e 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBReadOnlyStatement.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBReadOnlyStatement.swift @@ -9,29 +9,51 @@ import TableProPluginKit // Depends on nothing else in the plugin, so TableProMobileTests compiles it against a DuckDB connection of its own. nonisolated enum DuckDBReadOnlyStatement { + // Who asked for the read-only mode, which is what a refusal names. + nonisolated enum Reason: Sendable, Equatable { + case externalClients + case safeMode + } + nonisolated enum Refusal: Error, PluginDriverError, Equatable { case transactionOpen + case safeModeTransactionOpen case beginFailed(String) var pluginErrorMessage: String { switch self { case .transactionOpen: String(localized: "This connection has a transaction open. Commit or roll it back in TablePro, then try again.") + // Read-Only offers no Commit, so the way out is a rollback. + case .safeModeTransactionOpen: + String( + localized: "Safe Mode is set to Read-Only for this connection, and it has a transaction open. Roll it back, then try again." + ) case .beginFailed(let message): message } } } + // An app read marked `readOnly` alone, a catalog read or a dashboard tile, runs as it is. + static func reason(for context: PluginStatementContext) -> Reason? { + if context.readOnly, context.origin == .external { return .externalClients } + if context.enforcesReadOnly { return .safeMode } + return nil + } + // DuckDB refuses every database write inside a read-only transaction. A BEGIN inside the user's own transaction // fails and aborts it (measured on v2.0.0), so only a session known to be idle gets one, and the caller reads that // state in the same actor turn. static func run( on connection: duckdb_connection, session: PluginSessionTransactionState, + reason: Reason = .externalClients, _ statement: () throws -> T ) throws -> T { - guard session == .idle else { throw Refusal.transactionOpen } + guard session == .idle else { + throw reason == .safeMode ? Refusal.safeModeTransactionOpen : Refusal.transactionOpen + } if let failure = send("BEGIN TRANSACTION READ ONLY", on: connection) { throw Refusal.beginFailed(failure) } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift index ee013d3fe9..55f3946ad1 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProWireIO extension DynamoDBPluginDriver { func execute(query: String) async throws -> PluginQueryResult { @@ -340,14 +341,11 @@ extension DynamoDBPluginDriver { // MARK: - Streaming - /// Streams every item a read returns, for an export, a copy or a compare. - /// - /// A stream states its columns once, before its first row, and a DynamoDB item may carry an - /// attribute no earlier item had. The items are therefore written to a temporary file while the - /// union of their attributes is collected, and the rows are sent once the read has finished, so - /// an attribute first seen on the last page still gets its column. + // A stream states its columns before its first row, and an item may carry an attribute no earlier item had, so + // the items go to a temporary file while their attributes are collected and the rows follow once the read ends. + // The rows are read back from the file only as fast as the consumer takes them. func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.run { session in @@ -358,7 +356,7 @@ extension DynamoDBPluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -373,38 +371,37 @@ extension DynamoDBPluginDriver { source = try await spoolSource(for: statement, session: session) } catch DynamoDBError.invalidStatement(let message) where message == Self.notAReadMarker { let result = try await runStatement(text, parameters: [], rowCap: PluginRowLimits.emergencyMax) - continuation.yield(.header(PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } return } - let spoolURL = FileManager.default.temporaryDirectory - .appendingPathComponent("TablePro-DynamoDB-\(UUID().uuidString).jsonl") - guard FileManager.default.createFile(atPath: spoolURL.path, contents: nil) else { + let spool: SpoolFile + do { + spool = try SpoolFile(prefix: "TablePro-DynamoDB") + } catch { throw DynamoDBError.transport(String(localized: "Could not create a temporary file for the export")) } - defer { try? FileManager.default.removeItem(at: spoolURL) } let sortsLocally = !source.clientOrder.isEmpty let readOffset = sortsLocally ? 0 : source.window.offset let readLimit = sortsLocally ? Int.max : max(source.window.limit ?? Int.max, 0) - var collector = SpoolCollector(handle: try FileHandle(forWritingTo: spoolURL)) + var collector = SpoolCollector(spool: spool) try await source.read(readOffset, readLimit) { item in try collector.add(item) return true } - try collector.handle.close() let columns = collector.columns(preferred: source.preferredColumns, schema: source.schema) continuation.yield(.header(PluginStreamHeader(columns: columns, columnTypeNames: collector.typeNames(for: columns, schema: source.schema)))) if sortsLocally { var items: [DynamoDBItem] = [] - var lines = try SpoolLines(url: spoolURL) - defer { lines.close() } - while let line = try lines.next() { + while let record = try spool.next() { try Task.checkCancellation() - items.append(try DynamoDBItem(wireItem: DynamoDBJSON.parse(line))) + items.append(try DynamoDBItem(wireItem: DynamoDBJSON.parse(Data(record)))) } let sorted = DynamoDBItemTable.sorted(items, by: source.clientOrder) let end = source.window.limit.map { limit in @@ -413,26 +410,25 @@ extension DynamoDBPluginDriver { } ?? sorted.count let slice = source.window.offset < end ? Array(sorted[source.window.offset.. Data? { - while true { - if let newline = buffer[start...].firstIndex(of: UInt8(ascii: "\n")) { - let line = buffer[start..() private var typeCounts: [String: [DynamoDBAttributeType: Int]] = [:] - init(handle: FileHandle) { - self.handle = handle + init(spool: SpoolFile) { + self.spool = spool } mutating func add(_ item: DynamoDBItem) throws { - try handle.write(contentsOf: Data((item.wireJSON.serialized() + "\n").utf8)) + try spool.append(Array(item.wireJSON.serialized().utf8)) for (name, value) in item { attributes.insert(name) guard value != .null else { continue } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift index b7a62af00f..435ad7aaca 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Schema.swift @@ -73,6 +73,8 @@ extension DynamoDBPluginDriver { ) } + var fetchesAllColumnsInOneRequest: Bool { true } + /// Columns for autocomplete, from the tables already described, with no request at all. The /// app asks for every table on connect, and answering by sampling each one read up to 300 /// tables in the background. diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift index 41e0c101de..e02600add0 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift @@ -34,7 +34,7 @@ final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return client.endpoint.isLocal ? "DynamoDB Local" : "DynamoDB \(client.endpoint.signingRegion)" } - var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery] } + var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift index 85756fc6c9..07585c2818 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift @@ -10,8 +10,6 @@ import TableProSQLQuoting /// the grid loaded, so it neither recreates an item someone deleted nor overwrites one someone /// changed. struct DynamoDBWriteStatements { - static let defaultSentinel = "__DEFAULT__" - let table: String let columns: [String] let keyColumns: [String] @@ -55,13 +53,12 @@ struct DynamoDBWriteStatements { return writes.map { (statement: $0.statement, parameters: $0.parameters) } } - func insert(row: [PluginCellValue]) -> (statement: String, parameters: [PluginCellValue]) { + func insert(row: [PluginCellValue], defaults: Set = []) -> (statement: String, parameters: [PluginCellValue]) { var names: [String] = [] var parameters: [PluginCellValue] = [] - for (index, column) in columns.enumerated() where index < row.count { + for (index, column) in columns.enumerated() where index < row.count && !defaults.contains(index) { let value = row[index] if case .null = value { continue } - if case .text(let text) = value, text == Self.defaultSentinel { continue } names.append(column) parameters.append(value) } @@ -73,14 +70,15 @@ struct DynamoDBWriteStatements { _ change: PluginRowChange, rowData: [PluginCellValue]? ) -> (statement: String, parameters: [PluginCellValue]) { + let defaults = change.defaultColumns ?? [] if let rowData { - return insert(row: rowData) + return insert(row: rowData, defaults: defaults) } var row = [PluginCellValue](repeating: .null, count: columns.count) for cell in change.cellChanges where cell.columnIndex < row.count { row[cell.columnIndex] = cell.newValue } - return insert(row: row) + return insert(row: row, defaults: defaults) } private func update(_ change: PluginRowChange) throws -> (statement: String, parameters: [PluginCellValue])? { @@ -92,7 +90,7 @@ struct DynamoDBWriteStatements { var guardValues: [PluginCellValue] = [] for cell in change.cellChanges { - if case .text(let text) = cell.newValue, text == Self.defaultSentinel { + if change.isColumnDefault(cell.columnIndex) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( format: String(localized: "DynamoDB has no default values, so %@ cannot be set to DEFAULT."), cell.columnName @@ -239,7 +237,7 @@ struct DynamoDBParameterBinder { } return .binary(data) case .text(let text): - guard !text.isEmpty, text != DynamoDBWriteStatements.defaultSentinel else { + guard !text.isEmpty else { throw DynamoDBError.invalidValue( attribute: attribute, reason: String(localized: "A key attribute needs a value") ) diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift index ed8324ba2b..00fae138a7 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchConnection.swift @@ -42,6 +42,25 @@ internal struct ElasticsearchResponse { let rawText: String } +// A search can answer with a new point-in-time id, and the next request has to name the newest one. +internal final class ElasticsearchPointInTime: @unchecked Sendable { + private let lock = NSLock() + private var current: String + + init(id: String) { + current = id + } + + var id: String { + lock.withLock { current } + } + + func adopt(from response: ElasticsearchResponse) { + guard let next = (response.json as? [String: Any])?["pit_id"] as? String, !next.isEmpty else { return } + lock.withLock { current = next } + } +} + internal struct ElasticsearchIndexInfo { let name: String let docsCount: Int? @@ -264,6 +283,27 @@ internal final class ElasticsearchConnection: NSObject, @unchecked Sendable { return id } + // A shard whose context is gone would otherwise answer 200 without its hits, and the page would lose its rows. + func searchPointInTime(body: [String: Any]) async throws -> ElasticsearchResponse { + let response = try await request( + method: "POST", + path: "/_search?allow_partial_search_results=false", + body: try serialize(body), + isRead: true + ) + guard response.statusCode == 200 else { throw mapError(response, fallback: "Search failed") } + return response + } + + // A search for no hits that names the point-in-time renews its keep-alive on every shard and reads nothing. + func renewPointInTime(id: String, keepAlive: String) async throws -> ElasticsearchResponse { + try await searchPointInTime(body: [ + "size": 0, + "track_total_hits": false, + "pit": ["id": id, "keep_alive": keepAlive] + ]) + } + func closePointInTime(id: String) async { let body = try? serialize(["id": id]) _ = try? await request(method: "DELETE", path: "/_pit", body: body, isRead: true) diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift index 63a0b38d19..c786699062 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift @@ -56,7 +56,7 @@ extension ElasticsearchPluginDriver { guard let index = ElasticsearchOperations.decodeExport(query) else { return defaultStreamRows(query: query) } - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.streamIndex(index, continuation: continuation) @@ -65,13 +65,15 @@ extension ElasticsearchPluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } /// A point-in-time pins the index for the whole export, so a document written while it runs /// cannot shift the sort and be exported twice or skipped. `search_after` walks it, which is the /// only way past the 10,000 `max_result_window` ceiling. + /// + /// A reader slower than the cluster would otherwise outlive the point-in-time's keep-alive. private func streamIndex( _ index: String, continuation: AsyncThrowingStream.Continuation @@ -84,8 +86,8 @@ extension ElasticsearchPluginDriver { index: index, from: 0, size: Self.deepPageBatchSize, sorts: [], filters: [], logicMode: "AND" ) - let pit = try await conn.openPointInTime(index: index, keepAlive: Self.pitKeepAlive) - defer { Task { await conn.closePointInTime(id: pit) } } + let pit = ElasticsearchPointInTime(id: try await conn.openPointInTime(index: index, keepAlive: Self.pitKeepAlive)) + defer { Task { await conn.closePointInTime(id: pit.id) } } var columns: [String]? var searchAfter: [Any]? @@ -95,8 +97,10 @@ extension ElasticsearchPluginDriver { for: parsed, fields: fields, size: Self.deepPageBatchSize, tiebreaker: true, searchAfter: searchAfter, supportsCaseInsensitive: supportsCaseInsensitiveSearch ) - body["pit"] = ["id": pit, "keep_alive": Self.pitKeepAlive] - let response = try await conn.search(index: nil, body: body) + body["pit"] = ["id": pit.id, "keep_alive": Self.pitKeepAlive] + body["track_total_hits"] = false + let response = try await conn.searchPointInTime(body: body) + pit.adopt(from: response) let hits = extractHits(response) guard !hits.isEmpty else { break } @@ -106,16 +110,18 @@ extension ElasticsearchPluginDriver { if columns == nil { let resolved = ElasticsearchMappingFlattener.columns(forHits: hits, mappingColumns: mappingColumns) columns = resolved - continuation.yield(.header(PluginStreamHeader( + let header = PluginStreamElement.header(PluginStreamHeader( columns: resolved, columnTypeNames: resolved.map { exportTypeName($0, fields: fields) }, estimatedRowCount: nil - ))) + )) + guard try await deliver(header, to: continuation, renewing: pit, on: conn) else { return } } if let resolved = columns { - continuation.yield(.rows(ElasticsearchMappingFlattener.rows( + let rows = PluginStreamElement.rows(ElasticsearchMappingFlattener.rows( forHits: hits, columns: resolved, nestedParents: nestedParents, length: .whole - ))) + )) + guard try await deliver(rows, to: continuation, renewing: pit, on: conn) else { return } } searchAfter = hits.last?["sort"] as? [Any] @@ -124,11 +130,23 @@ extension ElasticsearchPluginDriver { if columns == nil { let resolved = ElasticsearchMappingFlattener.columns(forHits: [], mappingColumns: mappingColumns) - continuation.yield(.header(PluginStreamHeader( + let header = PluginStreamElement.header(PluginStreamHeader( columns: resolved, columnTypeNames: resolved.map { exportTypeName($0, fields: fields) }, estimatedRowCount: nil - ))) + )) + _ = try await deliver(header, to: continuation, renewing: pit, on: conn) + } + } + + private func deliver( + _ element: PluginStreamElement, + to continuation: AsyncThrowingStream.Continuation, + renewing pit: ElasticsearchPointInTime, + on conn: ElasticsearchConnection + ) async throws -> Bool { + try await PluginRowStream.deliver(element, to: continuation, keepingAliveEvery: Self.pitRenewalInterval) { + pit.adopt(from: try await conn.renewPointInTime(id: pit.id, keepAlive: Self.pitKeepAlive)) } } @@ -193,6 +211,7 @@ extension ElasticsearchPluginDriver { """) let hits = try await fetchHits(index: parsed.index, parsed: parsed, fields: fields, conn: conn) + rememberArrayText(in: hits, index: parsed.index, mappingColumns: mappingColumns) return renderHits(hits, mappingColumns: mappingColumns, fields: fields, length: length, startTime: startTime) } @@ -223,13 +242,13 @@ extension ElasticsearchPluginDriver { fields: [String: ElasticsearchFieldInfo], conn: ElasticsearchConnection ) async throws -> [[String: Any]] { - let pit = try await conn.openPointInTime(index: index, keepAlive: Self.pitKeepAlive) + let pit = ElasticsearchPointInTime(id: try await conn.openPointInTime(index: index, keepAlive: Self.pitKeepAlive)) do { let hits = try await collectDeepHits(parsed: parsed, fields: fields, pit: pit, conn: conn) - await conn.closePointInTime(id: pit) + await conn.closePointInTime(id: pit.id) return hits } catch { - await conn.closePointInTime(id: pit) + await conn.closePointInTime(id: pit.id) throw error } } @@ -237,7 +256,7 @@ extension ElasticsearchPluginDriver { private func collectDeepHits( parsed: ElasticsearchParsedSearch, fields: [String: ElasticsearchFieldInfo], - pit: String, + pit: ElasticsearchPointInTime, conn: ElasticsearchConnection ) async throws -> [[String: Any]] { var searchAfter: [Any]? @@ -250,8 +269,10 @@ extension ElasticsearchPluginDriver { for: parsed, fields: fields, size: Self.deepPageBatchSize, tiebreaker: true, searchAfter: searchAfter, supportsCaseInsensitive: supportsCaseInsensitiveSearch ) - body["pit"] = ["id": pit, "keep_alive": Self.pitKeepAlive] + body["pit"] = ["id": pit.id, "keep_alive": Self.pitKeepAlive] + body["track_total_hits"] = false let response = try await conn.search(index: nil, body: body) + pit.adopt(from: response) let hits = extractHits(response) guard !hits.isEmpty else { break } @@ -396,7 +417,7 @@ extension ElasticsearchPluginDriver { return fields[column]?.type ?? "" } } - return PluginQueryResult( + var result = PluginQueryResult( columns: columns, columnTypeNames: typeNames, rows: rows, @@ -404,6 +425,8 @@ extension ElasticsearchPluginDriver { executionTime: Date().timeIntervalSince(startTime), columnMeta: ElasticsearchTypeClassification.columnMeta(columns: columns, typeNames: typeNames) ) + result.rowLocators = hits.map { ElasticsearchDocumentLocator(hit: $0)?.locator } + return result } private func renderObjects( diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift index 2d51700557..e7f6755c6f 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift @@ -7,6 +7,7 @@ import Foundation import os +import TableProDocumentPath import TableProPluginKit internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked Sendable { @@ -14,11 +15,14 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked private let lock = NSLock() private var _connection: ElasticsearchConnection? private var _mappingCache: [String: [ElasticsearchColumn]] = [:] + private var _arrayTextColumns: [String: Set] = [:] static let logger = Logger(subsystem: "com.TablePro", category: "ElasticsearchPluginDriver") static let maxResultWindow = 10_000 static let deepPageBatchSize = 1_000 static let pitKeepAlive = "1m" + // Half the keep-alive, so a renewal that waits on a slow node still lands before the point-in-time expires. + static let pitRenewalInterval: TimeInterval = 30 var connection: ElasticsearchConnection? { lock.withLock { _connection } } @@ -35,7 +39,7 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } init(config: DriverConnectionConfig) { self.config = config @@ -58,6 +62,7 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked _connection?.disconnect() _connection = nil _mappingCache.removeAll() + _arrayTextColumns.removeAll() } } @@ -248,12 +253,37 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext + ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes.filter { $0.type != .insert }) { + throw refusal + } + var insertedRowData = insertedRowData + let changes = changes.map { Self.leavingDefaultsOut($0, insertedRowData: &insertedRowData) } let typeNames = columnTypeNames(for: columns, index: table) let generator = ElasticsearchStatementGenerator( index: table, columns: columns, columnTypeNames: typeNames, - columnPathKeys: columnPathKeys(index: table) + columnPathKeys: columnPathKeys(index: table), + rowLocators: context.rowLocators, + arrayTextColumns: lock.withLock { _arrayTextColumns[table] } ?? [] ) return try generator.generateRowWrites( from: changes, @@ -263,6 +293,29 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked ) } + // A new document leaves out a field left to its default, and one whose `_id` is left to it gets an id from the server. + private static func leavingDefaultsOut( + _ change: PluginRowChange, + insertedRowData: inout [Int: [PluginCellValue]] + ) -> PluginRowChange { + guard change.type == .insert, let defaults = change.defaultColumns, !defaults.isEmpty else { return change } + if var row = insertedRowData[change.rowIndex] { + for index in defaults where row.indices.contains(index) { row[index] = .null } + insertedRowData[change.rowIndex] = row + } + var cleared = PluginRowChange( + rowIndex: change.rowIndex, + type: change.type, + cellChanges: change.cellChanges.map { cell in + defaults.contains(cell.columnIndex) ? (cell.columnIndex, cell.columnName, cell.oldValue, .null) : cell + }, + originalRow: change.originalRow + ) + cleared.absentColumns = change.absentColumns + cleared.expressionColumns = change.expressionColumns + return cleared + } + // MARK: - Caching func cachedMappingColumns(_ index: String) async throws -> [ElasticsearchColumn] { @@ -280,7 +333,28 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked /// nothing. The whole cache goes rather than one index: a console request can name a list, a /// wildcard or an alias, and refilling it costs one `GET //_mapping` per index reopened. func invalidateMappingCache() { - lock.withLock { _mappingCache.removeAll() } + lock.withLock { + _mappingCache.removeAll() + _arrayTextColumns.removeAll() + } + } + + // The grid shows a string and an array that read the same alike, so a new row's text in such a field stays text. + func rememberArrayText(in hits: [[String: Any]], index: String, mappingColumns: [ElasticsearchColumn]) { + let stringColumns = mappingColumns.filter { ElasticsearchStatementGenerator.stringTypes.contains($0.type) } + guard !stringColumns.isEmpty else { return } + var found: Set = [] + for hit in hits { + guard let source = hit["_source"] as? [String: Any] else { continue } + for column in stringColumns where !found.contains(column.name) { + if let text = DocumentPath.value(in: source, atPath: column.name) as? String, + ElasticsearchStatementGenerator.parsedArray(text) != nil { + found.insert(column.name) + } + } + } + guard !found.isEmpty else { return } + lock.withLock { _arrayTextColumns[index, default: []].formUnion(found) } } private func columnPathKeys(index: String) -> [String: [String]] { diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift index 0bfc05bcd3..548e7ed1e6 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift @@ -17,12 +17,54 @@ struct ElasticsearchWriteRequest: Equatable { // The fields a partial update sets, each as its mapping keys. The stored document decides how // each one is spelled, so the request is aligned to it when it runs. let leafPaths: [[String]] + // Leaves of `leafPaths` sent as text that reads as a JSON array. The grid shows the string `["a"]` and the array + // `["a"]` alike, so the stored document decides which one the edit is. + let arrayCandidates: [[String]] - init(method: String, path: String, body: String?, leafPaths: [[String]] = []) { + init(method: String, path: String, body: String?, leafPaths: [[String]] = [], arrayCandidates: [[String]] = []) { self.method = method self.path = path self.body = body self.leafPaths = leafPaths + self.arrayCandidates = arrayCandidates + } +} + +// Where a hit lives: its concrete index (a search can go through an alias or a wildcard) and, for a document indexed +// with custom routing, the routing that finds its shard again. With custom routing an `_id` is unique only per shard. +struct ElasticsearchDocumentLocator: Equatable { + let index: String + let id: String + let routing: String? + + private static let indexKey = "_index" + private static let idKey = "_id" + private static let routingKey = "_routing" + + init(index: String, id: String, routing: String?) { + self.index = index + self.id = id + self.routing = routing + } + + init?(hit: [String: Any]) { + guard let index = hit[Self.indexKey] as? String, let id = hit[Self.idKey] as? String else { return nil } + self.init(index: index, id: id, routing: hit[Self.routingKey] as? String) + } + + init?(locator: String) { + guard let object = try? JSONSerialization.jsonObject(with: Data(locator.utf8)) as? [String: Any] else { + return nil + } + self.init(hit: object) + } + + // Sorted keys, so the same hit always gives the same text. + var locator: String? { + var object: [String: Any] = [Self.indexKey: index, Self.idKey: id] + if let routing { object[Self.routingKey] = routing } + guard let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) else { return nil } + return String(data: data, encoding: .utf8) } } @@ -32,10 +74,16 @@ struct ElasticsearchStatementGenerator { // Plain `PUT _doc/` replaces a document that already has the id. private static let createOnlyQuery = "&op_type=create" private static let updateEndpoint = "_update" + private static let documentEndpoint = "_doc" + private static let routingParameter = "routing=" let index: String let columns: [String] let columnTypeNames: [String] + // By row index, the `ElasticsearchDocumentLocator` text each hit was read with. + let rowLocators: [Int: String] + // String fields some hit held a string reading as a JSON array in. + let arrayTextColumns: Set /// A `nested` column carries the whole array of objects, and its dotted leaves are views of /// those same bytes. Writing both makes Elasticsearch expand the dotted key into the object the @@ -45,11 +93,20 @@ struct ElasticsearchStatementGenerator { private let nestedParentByLeaf: [String: String] private let columnPathKeys: [String: [String]] - init(index: String, columns: [String], columnTypeNames: [String], columnPathKeys: [String: [String]] = [:]) { + init( + index: String, + columns: [String], + columnTypeNames: [String], + columnPathKeys: [String: [String]] = [:], + rowLocators: [Int: String] = [:], + arrayTextColumns: Set = [] + ) { self.index = index self.columns = columns self.columnTypeNames = columnTypeNames self.columnPathKeys = columnPathKeys + self.rowLocators = rowLocators + self.arrayTextColumns = arrayTextColumns self.nestedParentByLeaf = Self.nestedParents(columns: columns, typeNames: columnTypeNames) } @@ -202,12 +259,11 @@ struct ElasticsearchStatementGenerator { private func updateRequest(for change: PluginRowChange) throws -> ElasticsearchWriteRequest? { guard !change.cellChanges.isEmpty else { return nil } - guard let id = documentId(from: change) else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) - } + let target = try document(for: change) var doc: [String: Any] = [:] var leafPaths: [[String]] = [] + var arrayCandidates: [[String]] = [] for cellChange in change.cellChanges { let column = cellChange.columnName if metaColumns.contains(column) { @@ -222,8 +278,15 @@ struct ElasticsearchStatementGenerator { Self.isShortened(cellChange.newValue) || Self.isShortened(cellChange.oldValue) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.shortenedValueReason(column)) } - let value: Any = cellChange.newValue.asText.map { jsonValue($0, for: column) } ?? NSNull() let keys = pathKeys(for: column) + let text = cellChange.newValue.asText + let value: Any + if let text, holdsTextOrArray(column), Self.parsedArray(text) != nil { + value = text + arrayCandidates.append(keys) + } else { + value = text.map { jsonValue($0, for: column) } ?? NSNull() + } guard DocumentPathPlacement.set(value, atKeys: keys, in: &doc) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.notJSONReason) } @@ -235,9 +298,10 @@ struct ElasticsearchStatementGenerator { } return .init( method: "POST", - path: "/\(encodedIndex)/\(Self.updateEndpoint)/\(encodePathComponent(id))\(Self.refreshQuery)", + path: documentPath(Self.updateEndpoint, for: target), body: body, - leafPaths: leafPaths + leafPaths: leafPaths, + arrayCandidates: arrayCandidates ) } @@ -247,11 +311,16 @@ struct ElasticsearchStatementGenerator { request.path.hasSuffix(createOnlyQuery) } + // A routed document is found only on its routing's shard, so the read carries the update's routing. static func storedDocumentPath(for request: ElasticsearchWriteRequest) -> String? { guard !request.leafPaths.isEmpty else { return nil } - let parts = (request.path.split(separator: "?", maxSplits: 1).first ?? "").split(separator: "/") + let pathAndQuery = request.path.split(separator: "?", maxSplits: 1) + let parts = (pathAndQuery.first ?? "").split(separator: "/") guard parts.count == 3, parts[1] == updateEndpoint else { return nil } - return "/\(parts[0])/_doc/\(parts[2])" + let routing = pathAndQuery.dropFirst().first? + .split(separator: "&") + .first(where: { $0.hasPrefix(routingParameter) }) + return "/\(parts[0])/\(documentEndpoint)/\(parts[2])" + (routing.map { "?\($0)" } ?? "") } static func aligned( @@ -267,7 +336,11 @@ struct ElasticsearchStatementGenerator { var placed: [String: Any] = [:] for keys in request.leafPaths { - guard let value = DocumentPathPlacement.value(in: doc, atKeys: keys) else { return request } + guard var value = DocumentPathPlacement.value(in: doc, atKeys: keys) else { return request } + if request.arrayCandidates.contains(keys), let text = value as? String, let array = parsedArray(text), + storedHoldsArray(atKeys: keys, in: stored) { + value = array + } try DocumentPathPlacement.place(value, atKeys: keys, in: &placed, following: stored) } guard let alignedBody = serialize(["doc": placed]) else { return request } @@ -279,13 +352,33 @@ struct ElasticsearchStatementGenerator { return ElasticsearchWriteRequest(method: request.method, path: path, body: alignedBody) } + // Every spelling the stored document has for the field holds an array. + private static func storedHoldsArray(atKeys keys: [String], in stored: [String: Any]) -> Bool { + let atoms = keys.flatMap { $0.split(separator: ".", omittingEmptySubsequences: false).map(String.init) } + let found = storedValues(of: atoms[...], in: stored) + return !found.isEmpty && found.allSatisfy { $0 is [Any] } + } + + // The same walk as `DocumentPathPlacement.place`: a level can spell the next part of the path as one dotted key. + private static func storedValues(of atoms: ArraySlice, in level: [String: Any]) -> [Any] { + var found: [Any] = [] + for end in atoms.indices.map({ $0 + 1 }) { + let key = atoms[atoms.startIndex.. ElasticsearchWriteRequest { - guard let id = documentId(from: change) else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) - } - return .init(method: "DELETE", path: docPath(id: id), body: nil) + let target = try document(for: change) + return .init(method: "DELETE", path: documentPath(Self.documentEndpoint, for: target), body: nil) } // MARK: - Refusals @@ -317,6 +410,20 @@ struct ElasticsearchStatementGenerator { // MARK: - Helpers + // A row read from a table tab carries its hit's locator. Without one, the tab's index and the `_id` cell. + private func document(for change: PluginRowChange) throws -> ElasticsearchDocumentLocator { + if let locator = rowLocators[change.rowIndex] { + guard let document = ElasticsearchDocumentLocator(locator: locator) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) + } + return document + } + guard let id = documentId(from: change) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) + } + return ElasticsearchDocumentLocator(index: index, id: id, routing: nil) + } + private func documentId(from change: PluginRowChange) -> String? { guard let originalRow = change.originalRow, let idIndex = columns.firstIndex(of: ElasticsearchMappingFlattener.idColumn), @@ -332,7 +439,13 @@ struct ElasticsearchStatementGenerator { } private func docPath(id: String) -> String { - "/\(encodedIndex)/_doc/\(encodePathComponent(id))\(Self.refreshQuery)" + documentPath(Self.documentEndpoint, for: ElasticsearchDocumentLocator(index: index, id: id, routing: nil)) + } + + private func documentPath(_ endpoint: String, for document: ElasticsearchDocumentLocator) -> String { + let path = "/\(encodePathComponent(document.index))/\(endpoint)/\(encodePathComponent(document.id))\(Self.refreshQuery)" + guard let routing = document.routing else { return path } + return path + "&" + Self.routingParameter + Self.encodeQueryValue(routing) } private var encodedIndex: String { @@ -341,17 +454,49 @@ struct ElasticsearchStatementGenerator { private static let pathComponentAllowed: CharacterSet = .urlPathAllowed.subtracting(CharacterSet(charactersIn: "/")) + // RFC 3986 unreserved: `&`, `=`, `+`, `#` and `%` in a routing value would end or change the parameter. + private static let queryValueAllowed = CharacterSet( + charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + ) private static let structuredTypes: Set = ["object", "nested", "flattened", "join"] + static let stringTypes: Set = [ + "text", "keyword", "wildcard", "constant_keyword", "match_only_text", "search_as_you_type", "annotated_text" + ] private func encodePathComponent(_ value: String) -> String { value.addingPercentEncoding(withAllowedCharacters: Self.pathComponentAllowed) ?? value } - private func jsonValue(_ text: String, for column: String) -> Any { - let typeName = columns.firstIndex(of: column).flatMap { index in + private static func encodeQueryValue(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: queryValueAllowed) ?? value + } + + private func mappedType(of column: String) -> String { + columns.firstIndex(of: column).flatMap { index in index < columnTypeNames.count ? columnTypeNames[index] : nil } ?? "" + } + + // A string field, or one whose type is unknown, can hold a string that reads as a JSON array. + private func holdsTextOrArray(_ column: String) -> Bool { + let typeName = mappedType(of: column) + return typeName.isEmpty || Self.stringTypes.contains(typeName) + } + + // JSON allows white space before the bracket; anything else is not an array and is never copied and parsed. + static func parsedArray(_ text: String) -> [Any]? { + guard text.unicodeScalars.first(where: { !" \t\n\r".unicodeScalars.contains($0) }) == "[" else { return nil } + return try? JSONSerialization.jsonObject(with: Data(text.utf8)) as? [Any] + } + + private func jsonValue(_ text: String, for column: String) -> Any { + let typeName = mappedType(of: column) + if Self.stringTypes.contains(typeName) { + // A new row has no stored value to compare, so an array is read as one unless the field held such text. + if !arrayTextColumns.contains(column), let array = Self.parsedArray(text) { return array } + return text + } if ElasticsearchQueryBuilder.numericTypes.contains(typeName) { if let intVal = Int(text) { return intVal } if let doubleVal = Double(text) { return doubleVal } @@ -385,7 +530,11 @@ struct ElasticsearchStatementGenerator { guard !request.leafPaths.isEmpty, let leafData = try? JSONSerialization.data(withJSONObject: request.leafPaths) else { return statement } - return "\(statement):\(leafData.base64EncodedString())" + let withLeaves = "\(statement):\(leafData.base64EncodedString())" + guard !request.arrayCandidates.isEmpty, + let candidateData = try? JSONSerialization.data(withJSONObject: request.arrayCandidates) + else { return withLeaves } + return "\(withLeaves):\(candidateData.base64EncodedString())" } static func decode(_ statement: String) -> ElasticsearchWriteRequest? { @@ -396,17 +545,19 @@ struct ElasticsearchStatementGenerator { let path = decodeBase64(parts[1]) else { return nil } let body = decodeBase64(parts[2]) - let leafPaths = parts.count > 3 - ? Data(base64Encoded: parts[3]).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [[String]] } - : nil return ElasticsearchWriteRequest( method: method, path: path, body: (body?.isEmpty ?? true) ? nil : body, - leafPaths: leafPaths ?? [] + leafPaths: parts.count > 3 ? decodePaths(parts[3]) : [], + arrayCandidates: parts.count > 4 ? decodePaths(parts[4]) : [] ) } + private static func decodePaths(_ part: String) -> [[String]] { + Data(base64Encoded: part).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [[String]] } ?? [] + } + static func isTaggedStatement(_ statement: String) -> Bool { statement.hasPrefix(writeTag) } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift b/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift index 3e021e148c..0ab463f54e 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift @@ -63,8 +63,11 @@ extension EtcdOperation { case .put, .create, .leaseGrant, .leaseKeepAlive, .authEnable, .userAdd, .roleAdd, .userGrantRole, .userRevokeRole: return .write - // A move takes the key away from its name, as a del does. - case .move, .del, .compaction, .leaseRevoke, .authDisable, .userDelete, .roleDelete: + // One key by name is a row change, as a keyed DELETE is. A prefix can take every key. + case .del(_, let prefix): + return prefix ? .destructive : .write + // A move takes the key away from its name. + case .move, .compaction, .leaseRevoke, .authDisable, .userDelete, .roleDelete: return .destructive case .unknown: return nil @@ -309,9 +312,14 @@ struct EtcdCommandParser { guard let key = positional.first?.bytes else { throw EtcdParseError.missingArgument("del requires a key") } + // etcdctl deletes a range for a range end, `--from-key` or `--prefix=true`, which `.del` cannot carry. + guard positional.count == 1, !flags.has("from-key"), flags.value(for: "from-key") == nil, + flags.value(for: "prefix") == nil + else { + throw EtcdParseError.invalidArgument("del takes one key, or a prefix with --prefix") + } - let prefix = flags.has("prefix") - return .del(key: key, prefix: prefix) + return .del(key: key, prefix: flags.has("prefix")) } private static func parseWatch(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift b/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift index 892dea24a8..aa33f6d91a 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift @@ -88,6 +88,7 @@ internal struct EtcdRangeRequest: Encodable { var sortTarget: String? var keysOnly: Bool? var countOnly: Bool? + var revision: Int64? private enum CodingKeys: String, CodingKey { case key @@ -97,10 +98,12 @@ internal struct EtcdRangeRequest: Encodable { case sortTarget = "sort_target" case keysOnly = "keys_only" case countOnly = "count_only" + case revision } } internal struct EtcdRangeResponse: Decodable { + let header: EtcdResponseHeader? let kvs: [EtcdKeyValue]? let count: String? let more: Bool? diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift b/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift index fdb2d44a09..d1125f1722 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdKeyspace.swift @@ -40,11 +40,14 @@ internal struct EtcdKeyspace: Equatable, Sendable { return name } - func tables(forKeys keys: [Data]) -> [Table] { - var keyCounts: [String: Int] = [:] + func count(_ keys: [Data], into keyCounts: inout [String: Int]) { for key in keys { keyCounts[tableName(forKey: key), default: 0] += 1 } + } + + func tables(fromKeyCounts counts: [String: Int]) -> [Table] { + var keyCounts = counts let rootKeyCount = keyCounts.removeValue(forKey: Self.rootTableName) let prefixTables = keyCounts .sorted { $0.key < $1.key } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index 60e0518d0d..bcf8aef47a 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -17,6 +17,13 @@ private extension Array where Element == String { var asCells: [PluginCellValue] { map(PluginCellValue.text) } } +private extension Error { + var isEtcdRevisionCompacted: Bool { + guard let fault = (self as? EtcdError)?.serverFault else { return false } + return EtcdCompaction.isCompacted(grpcCode: fault.grpcCode, message: fault.message) + } +} + final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private let config: DriverConnectionConfig private var _httpClient: EtcdHttpClient? @@ -30,7 +37,9 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "EtcdPluginDriver") private static let maxKeys = PluginRowLimits.emergencyMax - + // Values make a response as large as the data, so a batch of them is kept far smaller than a batch of keys. + private static let valueBatchSize = 1_000 + private static let keyBatchSize = 10_000 private static let columns = EtcdColumn.allCases.map(\.rawValue) private static let columnTypeNames = ["String", "String", "Int64", "Int64", "Int64", "String"] @@ -42,7 +51,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { false } var capabilities: PluginCapabilities { - [.cancelQuery, .statementPolicy] + [.cancelQuery, .statementPolicy, .columnDefaultMarks] } // etcd has no transactions. @@ -176,7 +185,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.performStreamRows(query: query, context: context, continuation: continuation) @@ -185,9 +194,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } @@ -233,31 +240,57 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation: AsyncThrowingStream.Continuation ) async throws { let matcher = try filter.matcher() - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: Self.columns, columnTypeNames: Self.columnTypeNames, estimatedRowCount: nil - ))) + ) let range = Self.prefixRange(prefix) - var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) - req.sortOrder = sortAscending ? "ASCEND" : "DESCEND" - req.sortTarget = "KEY" - - let response = try await client.rangeRequest(req) - var rows: [PluginRow] = [] - - for kv in response.kvs ?? [] { - try Task.checkCancellation() - let row = Self.filterRow(kv) - guard matcher.matches(row) else { continue } - rows.append(row.cells) + guard sortAscending else { + // etcd reads only in ascending key order and sorts a descending range whole, so the range is one response. + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) + req.sortOrder = "DESCEND" + req.sortTarget = "KEY" + let response = try await client.rangeRequest(req) + let rows = (response.kvs ?? []).map(Self.filterRow).filter(matcher.matches).map(\.cells) + // `more` says the range held keys past the limit. + if response.more == true { header.rowLimit = rows.count } + continuation.yield(.header(header)) + if !rows.isEmpty { + continuation.yield(.rows(rows)) + } + continuation.finish() + return } - if !rows.isEmpty { - continuation.yield(.rows(rows)) + continuation.yield(.header(header)) + // The scan hands each batch over synchronously, so a batch goes out just before the next is read, where the + // read can wait for the consumer. + var unsent: [PluginRow] = [] + do { + try await scanKeyValues( + in: range, + client: client, + beforeNextBatch: { + guard !unsent.isEmpty else { return } + _ = await PluginRowStream.deliver(.rows(unsent), to: continuation) + unsent = [] + }, + body: { kvs in + unsent = kvs.map(Self.filterRow).filter(matcher.matches).map(\.cells) + return true + } + ) + } catch let error where error.isEtcdRevisionCompacted { + // Rows already went out, so the read cannot start over at a newer revision without repeating them. + throw EtcdError.serverError(String( + localized: "etcd compacted the revision this export started from, so it stopped part way. Run it again." + )) + } + if !unsent.isEmpty { + _ = await PluginRowStream.deliver(.rows(unsent), to: continuation) } - continuation.finish() } @@ -278,17 +311,24 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { throw EtcdError.notConnected } - let range = Self.prefixRange(keyspace.root) - - let response = try await client.rangeRequest(EtcdRangeRequest( - key: range.key, - rangeEnd: range.rangeEnd, - limit: Int64(Self.maxKeys), - keysOnly: true - )) - - let keys = (response.kvs ?? []).map(\.key) - return keyspace.tables(forKeys: keys).map { + let keyCounts: [String: Int] = try await restartingOnCompaction { + var keyCounts: [String: Int] = [:] + try await EtcdRangeScan.scan( + Self.prefixRange(keyspace.root), + batchSize: Self.keyBatchSize, + maxItems: Self.maxKeys, + key: { $0 }, + fetch: { range, limit, revision in + try await Self.keyBatch(in: range, limit: limit, revision: revision, client: client) + }, + body: { (keys: [Data]) in + keyspace.count(keys, into: &keyCounts) + return true + } + ) + return keyCounts + } + return keyspace.tables(fromKeyCounts: keyCounts).map { PluginTableInfo(name: $0.name, type: "PREFIX", rowCount: $0.keyCount) } } @@ -419,6 +459,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let generator = EtcdStatementGenerator( prefix: keyspace.prefix(forTable: table), columns: columns @@ -841,15 +882,17 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } if let parsed = EtcdQueryBuilder.parseRangeQuery(query) { - return try await fetchKeysPage( - prefix: parsed.prefix, - offset: parsed.offset, - limit: parsed.limit, - sortAscending: parsed.sortAscending, - filter: parsed.filter, - client: client, - startTime: startTime - ) + return try await restartingOnCompaction { + try await fetchKeysPage( + prefix: parsed.prefix, + offset: parsed.offset, + limit: parsed.limit, + sortAscending: parsed.sortAscending, + filter: parsed.filter, + client: client, + startTime: startTime + ) + } } if let parsed = EtcdQueryBuilder.parseCountQuery(query) { @@ -879,20 +922,129 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) async throws -> PluginQueryResult { let matcher = try filter.matcher() let range = Self.prefixRange(prefix) - let fetchLimit = filter.isUnfiltered ? Int64(min(offset + limit, Self.maxKeys)) : Int64(Self.maxKeys) + let pageLimit = min(limit, max(0, Self.maxKeys - offset)) + guard pageLimit > 0 else { return emptyResult(startTime: startTime) } + + guard filter.isUnfiltered else { + // Past the row ceiling an ascending scan stops at the smallest keys, while a descending page reads from + // the largest, so that page still comes from one sorted request. + if !sortAscending, try await countKeys(prefix: prefix, client: client) > Self.maxKeys { + return try await sortedDescendingPage( + of: range, offset: offset, limit: limit, matcher: matcher, client: client, startTime: startTime + ) + } + var collector = EtcdPageCollector(offset: offset, limit: limit, ascending: sortAscending) + try await scanKeyValues(in: range, client: client) { kvs in + for kv in kvs { + let row = Self.filterRow(kv) + guard matcher.matches(row) else { continue } + guard collector.add(row) else { return false } + } + return true + } + return result(of: collector.page, startTime: startTime) + } + + let kvs = sortAscending + ? try await ascendingPage(of: range, offset: offset, limit: pageLimit, client: client) + : try await descendingPage(of: range, offset: offset, limit: pageLimit, client: client) + return result(of: kvs.map(Self.filterRow), startTime: startTime) + } - var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: fetchLimit) - req.sortOrder = sortAscending ? "ASCEND" : "DESCEND" + private func sortedDescendingPage( + of range: EtcdKeyRange, + offset: Int, + limit: Int, + matcher: EtcdRowMatcher, + client: EtcdHttpClient, + startTime: Date + ) async throws -> PluginQueryResult { + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(Self.maxKeys)) + req.sortOrder = "DESCEND" req.sortTarget = "KEY" + let rows = (try await client.rangeRequest(req).kvs ?? []).map(Self.filterRow).filter(matcher.matches) + guard offset < rows.count else { return emptyResult(startTime: startTime) } + return result(of: Array(rows[offset ..< min(offset + limit, rows.count)]), startTime: startTime) + } - let response = try await client.rangeRequest(req) - let rows = (response.kvs ?? []).map(Self.filterRow).filter(matcher.matches) + private func restartingOnCompaction(_ read: () async throws -> Value) async throws -> Value { + try await EtcdCompaction.restartingOnce(when: { $0.isEtcdRevisionCompacted }, read) + } - guard offset < rows.count else { - return emptyResult(startTime: startTime) + // Keys alone carry the read up to the page, then one request reads the page's values from where it starts. + private func ascendingPage( + of range: EtcdKeyRange, offset: Int, limit: Int, client: EtcdHttpClient + ) async throws -> [EtcdKeyValue] { + let start = try await EtcdRangeScan.pageStart(of: range, skipping: offset, batchSize: Self.keyBatchSize) { + try await Self.keyBatch(in: $0, limit: $1, revision: $2, client: client) } - let pageEnd = min(offset + limit, rows.count) - return result(of: Array(rows[offset ..< pageEnd]), startTime: startTime) + guard let start else { return [] } + var req = EtcdRangeRequest(key: start.range.key, rangeEnd: start.range.rangeEnd, limit: Int64(limit)) + req.revision = start.revision + return try await client.rangeRequest(req).kvs ?? [] + } + + // A descending read sorts the whole range on the server however it is split, so the skip stays one request, + // of keys alone. + private func descendingPage( + of range: EtcdKeyRange, offset: Int, limit: Int, client: EtcdHttpClient + ) async throws -> [EtcdKeyValue] { + var pageRange = range + var revision: Int64? + if offset > 0 { + var skip = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: Int64(offset), keysOnly: true) + skip.sortOrder = "DESCEND" + skip.sortTarget = "KEY" + let skipped = try await client.rangeRequest(skip) + guard skipped.more == true, let last = skipped.kvs?.last else { return [] } + pageRange = EtcdRangeScan.range(before: last.key, in: range) + revision = Self.revision(of: skipped) + } + var req = EtcdRangeRequest(key: pageRange.key, rangeEnd: pageRange.rangeEnd, limit: Int64(limit)) + req.sortOrder = "DESCEND" + req.sortTarget = "KEY" + req.revision = revision + return try await client.rangeRequest(req).kvs ?? [] + } + + // Ascending key order, a bounded batch per response, all at one revision. + private func scanKeyValues( + in range: EtcdKeyRange, + client: EtcdHttpClient, + beforeNextBatch: () async -> Void = {}, + body: ([EtcdKeyValue]) throws -> Bool + ) async throws { + try await EtcdRangeScan.scan( + range, + batchSize: Self.valueBatchSize, + maxItems: Self.maxKeys, + key: { $0.key }, + fetch: { range, limit, revision in + await beforeNextBatch() + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: limit) + req.revision = revision + let response = try await client.rangeRequest(req) + return EtcdRangeBatch( + items: response.kvs ?? [], more: response.more ?? false, revision: Self.revision(of: response) + ) + }, + body: body + ) + } + + private static func keyBatch( + in range: EtcdKeyRange, limit: Int64, revision: Int64?, client: EtcdHttpClient + ) async throws -> EtcdRangeBatch { + var req = EtcdRangeRequest(key: range.key, rangeEnd: range.rangeEnd, limit: limit, keysOnly: true) + req.revision = revision + let response = try await client.rangeRequest(req) + return EtcdRangeBatch( + items: (response.kvs ?? []).map(\.key), more: response.more ?? false, revision: Self.revision(of: response) + ) + } + + private static func revision(of response: EtcdRangeResponse) -> Int64? { + response.header?.revision.flatMap { Int64($0) } } private func countKeys(prefix: String, client: EtcdHttpClient) async throws -> Int { diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift b/apple/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift index a9de882c51..937f902eca 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebaseCredentialFactory.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseClient import TableProFirebaseCore import TableProGoogleCloud import TableProPluginKit diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePlugin.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePlugin.swift index 42264bf53b..3693ee2e3e 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePlugin.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePlugin.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseClient import TableProFirebaseCore import TableProPluginKit diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift index 3e89356275..5c1681ca9c 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseClient import TableProFirebaseCore import TableProPluginKit @@ -35,6 +36,7 @@ extension FirebasePluginDriver { insertedRowIndices: Set, context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let types = writeColumnTypes(table, host: context.columnTypeNames) return try changes.compactMap { change in do { diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift index f60e627413..e5715c3b6a 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Execution.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseClient import TableProFirebaseCore import TableProPluginKit @@ -234,7 +235,7 @@ extension FirebasePluginDriver { // MARK: - Export func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if case .browse(let spec)? = try? FirebaseStatement.parse(query), spec.limit == 0 { @@ -250,7 +251,7 @@ extension FirebasePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -264,7 +265,7 @@ extension FirebasePluginDriver { continuation.yield(.header(PluginStreamHeader(columns: header ?? [], columnTypeNames: types, estimatedRowCount: nil))) } let fixed = header ?? [] - continuation.yield(.rows(rows.map { row in Self.align(row, columns: columns, to: fixed) })) + _ = await PluginRowStream.deliver(.rows(rows.map { row in Self.align(row, columns: columns, to: fixed) }), to: continuation) } if header == nil { continuation.yield(.header(PluginStreamHeader(columns: [Self.extraColumn], columnTypeNames: ["map"], estimatedRowCount: nil))) @@ -282,7 +283,7 @@ extension FirebasePluginDriver { private func forEachExportPage( _ spec: FirebaseBrowseSpec, - page: ([String], [String], [[PluginCellValue]]) throws -> Void + page: ([String], [String], [[PluginCellValue]]) async throws -> Void ) async throws { switch variant { case .firestore: @@ -309,13 +310,13 @@ extension FirebasePluginDriver { operation: String(format: String(localized: "export %@"), spec.table)) guard !documents.isEmpty else { return } let rows = FirestoreRowSet(documents: documents, target: target) - try page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(Self.cell) }) + try await page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(Self.cell) }) cursor = documents.count == Self.exportPageSize ? documents.last?.path : nil } while cursor != nil case .realtimeDatabase: guard spec.filters.isEmpty else { let result = try await browse(spec) - try page(result.columns, result.columnTypeNames, result.rows) + try await page(result.columns, result.columnTypeNames, result.rows) return } let client = try requireRealtimeDatabase() @@ -330,7 +331,7 @@ extension FirebasePluginDriver { let children = try await client.children(node, plan: plan) guard !children.isEmpty else { return } let rows = RealtimeDatabaseRowSet(children: children, nodePath: node) - try page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(PluginCellValue.fromOptional) }) + try await page(rows.columns, rows.columnTypes, rows.rows.map { $0.map(PluginCellValue.fromOptional) }) lastKey = children.count == Self.exportPageSize ? children.last?.key : nil } while lastKey != nil } diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Metadata.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Metadata.swift index 2cd43b56d8..66d1058169 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Metadata.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Metadata.swift @@ -1,4 +1,5 @@ import Foundation +import TableProFirebaseClient import TableProFirebaseCore import TableProPluginKit diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift index abad5c8f93..af0a2a17a4 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift @@ -1,5 +1,6 @@ import Foundation import os +import TableProFirebaseClient import TableProFirebaseCore import TableProGoogleCloud import TableProLogRedaction @@ -34,7 +35,7 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send variant == .firestore ? "Cloud Firestore" : "Firebase Realtime Database" } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift b/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift new file mode 100644 index 0000000000..db59df3638 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift @@ -0,0 +1,174 @@ +// +// CSVExportFormulaAndDecimalTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private final class TypedCSVSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + private let header: PluginStreamHeader + private let rows: [PluginRow] + + init(columns: [String], columnTypeNames: [String], rows: [PluginRow]) { + header = PluginStreamHeader(columns: columns, columnTypeNames: columnTypeNames) + self.rows = rows + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let header = header + let rows = rows + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +struct CSVExportFormulaAndDecimalTests { + private func export( + _ options: CSVExportOptions, + columns: [String], + types: [String], + rows: [PluginRow] + ) async throws -> (text: String, warnings: [String]) { + let output = try await CSVExportHarness.shared.bytes( + options: options, + tables: [PluginExportTable(name: "users", databaseName: "", tableType: "table", schema: nil, kind: .table)], + dataSource: TypedCSVSource(columns: columns, columnTypeNames: types, rows: rows) + ) + return (String(decoding: output.data, as: UTF8.self), output.result.warnings) + } + + // A database export is read back by importers, which keep the quote as part of the value. + @Test("The stock export writes a value that starts like a formula as it is") + func stockExportKeepsValues() async throws { + let output = try await export( + CSVExportOptions(), + columns: ["handle"], + types: ["text"], + rows: [["@handle"], ["+84 90 123 4567"], ["-pending"]] + ) + #expect(output.text == "handle\n@handle\n+84 90 123 4567\n-pending\n") + #expect(output.warnings.isEmpty) + } + + @Test("Sanitizing says how many values it gave a leading quote") + func sanitizingCountsWhatItChanged() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let output = try await export( + options, + columns: ["handle"], + types: ["text"], + rows: [["@handle"], ["plain"], ["=1+1"], ["-5"], ["-pending"]] + ) + #expect(output.text == "handle\n\"'@handle\"\nplain\n\"'=1+1\"\n-5\n\"'-pending\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(3)) + } + + @Test("Nothing changed means no warning about changes") + func noCountWithoutChanges() { + #expect(CSVExportPlugin.neutralisedWarnings(0).isEmpty) + } + + // A text column holding "1.10" is a version or a code, not a number. + @Test("A decimal comma leaves text columns alone and keeps columns with no usable type") + func decimalCommaSkipsTextColumns() async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export( + options, + columns: ["version", "amount", "untyped"], + types: ["varchar(16)", "numeric(10,2)", ""], + rows: [["1.10", "1.10", "1.10"]] + ) + #expect(output.text == "version;amount;untyped\n1.10;1,10;1,10\n") + } + + @Test("A decimal comma reaches every engine's numeric type", arguments: [ + "Nullable(Float64)", "Nullable(Decimal(10, 2))", "money", "smallmoney", "BINARY_DOUBLE", "BINARY_FLOAT", + "BIGNUMERIC", "double precision", "NUMBER(10,2)" + ]) + func decimalCommaReachesNumericTypes(typeName: String) async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export(options, columns: ["n"], types: [typeName], rows: [["1.5"]]) + #expect(output.text == "n\n1,5\n") + } + + @Test("A decimal comma never reaches a text type", arguments: [ + "varchar(16)", "String", "LowCardinality(String)", "nvarchar(10)", "TEXT", "jsonb" + ]) + func decimalCommaSkipsTextTypes(typeName: String) async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export(options, columns: ["n"], types: [typeName], rows: [["1.5"]]) + #expect(output.text == "n\n1.5\n") + } + + @Test("A header name sanitizing changes is counted with the values") + func headerNamesAreCounted() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let output = try await export(options, columns: ["=total"], types: ["text"], rows: [["=1"]]) + #expect(output.text == "\"'=total\"\n\"'=1\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(2)) + } + + // `'-1,5` is text to a spreadsheet, so a negative number written with a decimal comma lost its value. + @Test("A number written with a decimal comma is never given a quote") + func commaDecimalsAreNotSanitized() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export( + options, + columns: ["amount", "note"], + types: ["numeric", "text"], + rows: [["-1.5", "-1.5x"]] + ) + #expect(output.text == "amount;note\n-1,5;\"'-1.5x\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(1)) + } +} + +struct PluginRowWritersFormulaLeadTests { + // OWASP lists a leading line feed and the full-width operators, which a Japanese locale reads as formulas. + @Test("A leading line feed or a full-width operator counts as a formula lead") + func owaspLeadsAreNeutralised() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("\n=1+1", options: options).hasPrefix("\"'")) + for lead in ["\u{FF1D}", "\u{FF0B}", "\u{FF0D}", "\u{FF20}"] { + #expect(PluginRowWriters.csvField("\(lead)SUM(A1)", options: options) == "\"'\(lead)SUM(A1)\"") + } + } + + @Test("Only a value that would be prefixed is formula-like") + func formulaLikeMatchesTheWriter() { + #expect(PluginRowWriters.isFormulaLike("=1+1")) + #expect(PluginRowWriters.isFormulaLike("\t=1+1")) + #expect(!PluginRowWriters.isFormulaLike("-5")) + #expect(!PluginRowWriters.isFormulaLike("plain")) + #expect(!PluginRowWriters.isFormulaLike("")) + } +} diff --git a/apple/Plugins/FormatPluginTests/CSVImportPluginTests.swift b/apple/Plugins/FormatPluginTests/CSVImportPluginTests.swift index 4cc2cf9be0..964cdeb071 100644 --- a/apple/Plugins/FormatPluginTests/CSVImportPluginTests.swift +++ b/apple/Plugins/FormatPluginTests/CSVImportPluginTests.swift @@ -152,11 +152,21 @@ struct CSVImportPluginTests { #expect(CSVImportParsing.importFieldType(for: .date) == .text) } - @Test("Blank rows are detected") + @Test("Only an empty line in a file of several columns is blank; empty fields make a row") func testIsBlank() { - #expect(CSVImportParsing.isBlank([""])) - #expect(CSVImportParsing.isBlank(["", ""])) - #expect(!CSVImportParsing.isBlank(["", "x"])) + #expect(CSVImportParsing.isBlank([""], columnCount: 3)) + #expect(!CSVImportParsing.isBlank([""], columnCount: 1)) + #expect(!CSVImportParsing.isBlank(["", ""], columnCount: 2)) + #expect(!CSVImportParsing.isBlank(["", "x"], columnCount: 2)) + } + + @Test("The data ends at the last record that holds a value") + func testDataEnd() { + let records: [[String]] = [["1", "a"], ["", ""], ["2", ""], ["", ""], [""]] + let end = CSVImportParsing.dataEnd(recordCount: records.count) { records[$0] } + let allEmpty = CSVImportParsing.dataEnd(recordCount: 2) { _ in ["", ""] } + #expect(end == 3) + #expect(allEmpty == 0) } // MARK: - Field detection diff --git a/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift b/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift index 1c0b39b90b..657d10a150 100644 --- a/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift +++ b/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift @@ -107,6 +107,34 @@ struct CSVImportRowShapeTests { #expect(sink.rows.map { $0["id"] } == [.text("1"), .text("3")]) } + @Test("A record of empty fields, how export writes an all-NULL row, imports as one; an empty line does not") + func emptyFieldsAreARow() async throws { + let (outcome, sink) = try await runImport( + "id,name,city\n1,Ada,Paris\n,,\n\n3,Lin,Hanoi\n", errorHandling: .stopAndRollback) + let result = try outcome.get() + #expect(result.executedStatements == 3) + #expect(sink.rows.map { $0["id"] } == [.text("1"), .null, .text("3")]) + #expect(sink.rows[1] == ["id": .null, "name": .null, "city": .null]) + } + + @Test("The empty rows a spreadsheet saves under its data, and a trailing empty line, are not rows") + func trailingEmptyRecordsAreNotRows() async throws { + let (outcome, sink) = try await runImport("id,name\n1,Ada\n,\n,\n", errorHandling: .stopAndRollback) + #expect(try outcome.get().executedStatements == 1) + #expect(sink.rows.map { $0["id"] } == [.text("1")]) + + let (oneColumn, oneColumnSink) = try await runImport("email\na@x.io\nb@x.io\n\n", errorHandling: .stopAndRollback) + _ = try oneColumn.get() + #expect(oneColumnSink.rows.map { $0["email"] } == [.text("a@x.io"), .text("b@x.io")]) + } + + @Test("In a one-column file an empty line is a NULL, as export writes one") + func emptyLineInOneColumnIsARow() async throws { + let (outcome, sink) = try await runImport("name\nAda\n\nLin\n", errorHandling: .stopAndRollback) + _ = try outcome.get() + #expect(sink.rows.map { $0["name"] } == [.text("Ada"), .null, .text("Lin")]) + } + @Test("A row that lacks trailing fields imports with them NULL") func shortRowIsPadded() async throws { let (outcome, sink) = try await runImport( diff --git a/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift b/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift index 29503ea388..fef8f82850 100644 --- a/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift +++ b/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift @@ -85,6 +85,14 @@ struct ExportImportRoundTripTests { try RoundTrip.expectSame(target, RoundTripFixture.table, FormatLimits.csv.carried(RoundTripFixture.rows)) } + @Test("A CSV file sanitized for a spreadsheet imports back with the quote it gave formula-like values") + func csvSanitizedImportsBackWithItsQuotes() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let target = try await RoundTrip.csv(options) + try RoundTrip.expectSame(target, RoundTripFixture.table, FormatLimits.csvSanitized.carried(RoundTripFixture.rows)) + } + @Test("A CSV file written with a byte order mark imports back") func csvWithByteOrderMarkImportsBack() async throws { var options = CSVExportOptions() @@ -166,9 +174,14 @@ struct ExportImportRoundTripTests { extension FormatLimits { // CSV has no NULL and no binary: both NULL and '' are written as an empty field and read as - // NULL, and a binary value is written as hex text. A value a spreadsheet would run as a - // formula gains a leading quote. + // NULL, and a binary value is written as hex text. static let csv = FormatLimits( + emptyTextIsNull: true, + binary: { .text(FormatLimits.hex($0)) } + ) + + // Sanitizing for a spreadsheet gives a value that would run as a formula a leading quote, which the import keeps. + static let csvSanitized = FormatLimits( emptyTextIsNull: true, binary: { .text(FormatLimits.hex($0)) }, text: { FormatLimits.csvNeutralised[$0] ?? $0 } @@ -183,11 +196,12 @@ extension FormatLimits { // JSON has no binary type: a binary value is written as base64 text. static let json = FormatLimits(binary: { .text($0.base64EncodedString()) }) - // An empty cell is not written, so '' reads back as NULL; a binary value is written as hex text, - // and XML 1.0 cannot hold a control character other than tab and the two line breaks. + // An empty cell is not written, so '' reads back as NULL; a binary value, zero bytes too, is + // written as hex text, and XML 1.0 cannot hold a control character other than tab and the two + // line breaks. static let xlsx = FormatLimits( emptyTextIsNull: true, - binary: { $0.isEmpty ? .null : .text(FormatLimits.hex($0)) }, + binary: { .text(FormatLimits.hex($0)) }, text: { text in String(String.UnicodeScalarView(text.unicodeScalars.filter { scalar in scalar.value >= 0x20 || scalar == "\t" || scalar == "\n" || scalar == "\r" diff --git a/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift new file mode 100644 index 0000000000..f995008f40 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift @@ -0,0 +1,185 @@ +// +// FormatSettingsChangedMidRunTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +// One plugin instance serves every window, so a dialog open in another window changes `settings` while a run reads. +// Each source here changes them between its first and second batch, the way that dialog would. +private final class TwoBatchSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "SQLite" + private let header: PluginStreamHeader + private let batches: [[PluginRow]] + private let betweenBatches: @Sendable () -> Void + + init(columns: [String], batches: [[PluginRow]], betweenBatches: @escaping @Sendable () -> Void) { + header = PluginStreamHeader(columns: columns, columnTypeNames: columns.map { _ in "TEXT" }) + self.batches = batches + self.betweenBatches = betweenBatches + } + + // Pulled, so the change lands only after the plugin has written the first batch. + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let header = header + let batches = batches + let betweenBatches = betweenBatches + let pulls = Counter() + return AsyncThrowingStream(unfolding: { + let pull = pulls.next() + if pull == 0 { return .header(header) } + guard pull <= batches.count else { return nil } + if pull == 2 { betweenBatches() } + return .rows(batches[pull - 1]) + }) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +private final class Counter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + func next() -> Int { + lock.lock() + defer { lock.unlock() } + defer { value += 1 } + return value + } +} + +private final class ForeignKeyRecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "SQLite" + var onExecute: () -> Void = {} + private(set) var disabled = 0 + private(set) var enabled = 0 + + func execute(statement: String) async throws { onExecute() } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + func disableForeignKeyChecks() async throws { disabled += 1 } + func enableForeignKeyChecks() async throws { enabled += 1 } +} + +private final class OneStatementSource: PluginImportSource, @unchecked Sendable { + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { continuation in + continuation.yield((statement: "INSERT INTO t VALUES (1);", lineNumber: 1)) + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + + func fileSizeBytes() -> Int64 { 0 } +} + +@Suite(.serialized) +struct FormatSettingsChangedMidRunTests { + private let table = PluginExportTable(name: "t", databaseName: "", tableType: "table", schema: nil, kind: .table) + private let rowWithNull: PluginRow = [.text("1"), .null] + + private func export( + _ plugin: any ExportFormatPlugin, + from source: TwoBatchSource, + fileExtension: String + ) async throws -> Data { + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("\(UUID().uuidString).\(fileExtension)") + defer { try? FileManager.default.removeItem(at: destination) } + _ = try await plugin.export( + tables: [table], + dataSource: source, + destination: destination, + progress: PluginExportProgress(progress: Progress(totalUnitCount: 1)) + ) + return try Data(contentsOf: destination) + } + + @Test("JSON keeps writing null keys after another window turns Include NULL values off") + func jsonKeepsItsNullChoice() async throws { + let plugin = JSONExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = JSONExportOptions() + options.layout = .newlineDelimited + options.includeNullValues = true + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.includeNullValues = false + } + let text = String(decoding: try await export(plugin, from: source, fileExtension: "ndjson"), as: UTF8.self) + #expect(text == "{\"id\": \"1\", \"note\": null}\n{\"id\": \"1\", \"note\": null}\n") + } + + @Test("Markdown keeps its NULL placeholder after another window changes it") + func markdownKeepsItsPlaceholder() async throws { + let plugin = MarkdownExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = MarkdownExportOptions() + options.alignsColumns = false + options.includesTableNames = false + options.nullPlaceholder = "NULL" + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.nullPlaceholder = "(none)" + } + let text = String(decoding: try await export(plugin, from: source, fileExtension: "md"), as: UTF8.self) + #expect(!text.contains("(none)")) + #expect(text.components(separatedBy: "| 1 | NULL |").count == 3) + } + + @Test("XLSX keeps writing NULL as an empty cell after another window turns the option off") + func xlsxKeepsItsNullChoice() async throws { + let plugin = XLSXExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = XLSXExportOptions() + options.convertNullToEmpty = true + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.convertNullToEmpty = false + } + let written = try await export(plugin, from: source, fileExtension: "xlsx") + let text = try #require(String(bytes: written, encoding: .isoLatin1)) + #expect(!text.contains("NULL")) + } + + @Test("A SQL import turns foreign key checks back on after another window unticks the option") + func sqlImportRestoresTheChecksItTurnedOff() async throws { + let plugin = SQLImportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = SQLImportOptions() + options.disableForeignKeyChecks = true + plugin.settings = options + + let sink = ForeignKeyRecordingSink() + sink.onExecute = { plugin.settings.disableForeignKeyChecks = false } + _ = try await plugin.performImport( + source: OneStatementSource(), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + #expect(sink.disabled == 1) + #expect(sink.enabled == 1) + } +} diff --git a/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift new file mode 100644 index 0000000000..7a5792e4a7 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift @@ -0,0 +1,446 @@ +// +// JSONDocumentStreamingTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +private func writeJSON(_ data: Data) throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("json-document-\(UUID().uuidString).json") + try data.write(to: url) + return url +} + +private func writeJSON(_ text: String) throws -> URL { + try writeJSON(Data(text.utf8)) +} + +private func drain(_ elements: any JSONElementSequence) throws -> [Any] { + var elements = elements + defer { elements.close() } + var values: [Any] = [] + while let element = try elements.next() { + values.append(element) + } + return values +} + +struct JSONDocumentStreamingTests { + private static let trickyRows = [ + #"{"id":1,"name":"Ada \"the\" {first} [one]","path":"C:\\temp\\","tags":["a","]","}"],"meta":{"k":[1,{"x":null}]}}"#, + #"{"id":9007199254740993,"score":-3.9192320754595876e-07,"ok":true,"none":null,"city":"Hà Nội 東京 \u00e9"}"#, + #"{"note":"line\nbreak \u2028 sep","empty":"","nested":[[[]]],"obj":{}}"#, + #"{}"#, + ] + + private func streamedRows(_ url: URL, targetTable: String? = nil, chunkSize: Int) throws -> [Any] { + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, _ in + try reader.skipValue() + } + guard case .array(let offset, let count, _) = rows else { + reader.close() + Issue.record("Expected the rows of an array, got \(rows)") + return [] + } + let values = try drain(JSONStreamedElements(reader: reader, arrayAt: offset)) + #expect(values.count == count) + return values + } + + private func importedElements(_ json: String, targetTable: String? = nil) throws -> [Any] { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + return try drain(JSONImportParsing.rowElements(at: url, targetTable: targetTable, checkCancellation: {}).elements) + } + + private func parsedElements(_ json: String, targetTable: String? = nil) throws -> [Any] { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + return try JSONImportParsing.parseElements(at: url, targetTable: targetTable) + } + + // The rows each element becomes, nil for one that is not an object. + private func rows(_ elements: [Any]) -> [[String: PluginCellValue]?] { + elements.map { try? JSONImportParsing.row(from: $0) } + } + + private func importedRows(_ data: Data, chunkSize: Int = JSONDocumentReader.defaultChunkSize) throws -> [[String: PluginCellValue]] { + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize, checkCancellation: {}) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: nil) { reader, _ in try reader.skipValue() } + switch layout { + case .array(let offset, _, _): + return try drain(JSONStreamedElements(reader: reader, arrayAt: offset)).map { try JSONImportParsing.row(from: $0) } + case .singleObject: + reader.close() + return try drain(JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}).elements) + .map { try JSONImportParsing.row(from: $0) } + } + } + + private static let encodings: [String.Encoding] = [.utf16BigEndian, .utf16LittleEndian, .utf32BigEndian, .utf32LittleEndian] + + // Foundation writes no byte order mark for an encoding that names its byte order. + private static func encoded(_ text: String, _ encoding: String.Encoding, withMark: Bool) -> Data { + let mark: [UInt8] = switch encoding { + case .utf16BigEndian: [0xFE, 0xFF] + case .utf16LittleEndian: [0xFF, 0xFE] + case .utf32BigEndian: [0x00, 0x00, 0xFE, 0xFF] + default: [0xFF, 0xFE, 0x00, 0x00] + } + return (withMark ? Data(mark) : Data()) + (text.data(using: encoding) ?? Data()) + } + + @Test("Elements read one at a time equal a whole-file parse, whatever the chunk size", arguments: [1, 3, 64, 1 << 20]) + func streamedElementsMatchAWholeFileParse(chunkSize: Int) throws { + let array = "[\n " + Self.trickyRows.joined(separator: ",\n ") + "\n]" + let wrapped = #"{"public.other":[{"x":1}], "public.people" : "# + array + "}" + for (json, target) in [(array, nil), (wrapped, "people")] as [(String, String?)] { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + let expected = try JSONImportParsing.parseElements(at: url, targetTable: target) + let streamed = try streamedRows(url, targetTable: target, chunkSize: chunkSize) + #expect((streamed as NSArray).isEqual(to: expected)) + } + } + + @Test("The reader holds one chunk and one element, not the file") + func readerHoldsOneChunkAndOneElement() throws { + let rows = (1...5_000).map { #"{"id":\#($0),"name":"value \#($0)","pad":"\#(String(repeating: "x", count: 50))"}"# } + let json = #"{"public.t":["# + rows.joined(separator: ",") + "]}" + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + let chunkSize = 256 + let longestElement = rows.map(\.utf8.count).max() ?? 0 + + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: "t") { reader, _ in try reader.skipValue() } + var mostHeld = reader.heldByteCount + guard case .array(let offset, let count, _) = layout else { + Issue.record("Expected the rows of an array, got \(layout)") + return + } + var elements = try JSONStreamedElements(reader: reader, arrayAt: offset) + defer { elements.close() } + var read = 0 + while try elements.next() != nil { + read += 1 + mostHeld = max(mostHeld, elements.heldByteCount) + } + + #expect(count == 5_000) + #expect(read == 5_000) + #expect(mostHeld <= chunkSize + longestElement) + #expect(json.utf8.count > 100 * (chunkSize + longestElement)) + } + + @Test("Every shape a whole-file parse accepts gives the same rows", arguments: [ + (#"[{"a":1},{"a":2}]"#, nil), + (#" [ ] "#, nil), + (#"[{"a":1}, 5, [1], "s", {"a":2}]"#, nil), + (#"{"users":[{"id":1}]}"#, nil), + (#"{"users":[{"id":1}],"orders":[{"id":1},{"id":2}]}"#, "orders"), + (#"{"public.users":[{"id":1}]}"#, "users"), + (#"{"id":1,"tags":["a"]}"#, nil), + (#"{}"#, nil), + (#"{"t":[]}"#, nil), + (#"{"t":[{"a":1}],"t":[{"a":2},{"a":3}]}"#, nil), + (#"{"t":5,"t":[{"a":1}]}"#, nil), + (#"{"x":[{"a":1}],"y":[1]}"#, nil), + (#"[{"a":1},]"#, nil), + (#"{"t":[{"a":1} , ],}"#, nil), + ] as [(String, String?)]) + // A lone object streams as a built row, so it is compared as rows; an array's elements are compared as values. + func shapesMatchAWholeFileParse(json: String, targetTable: String?) throws { + let imported = try importedElements(json, targetTable: targetTable) + let parsed = try parsedElements(json, targetTable: targetTable) + if imported.contains(where: { $0 is [String: PluginCellValue] }) { + #expect(rows(imported) == rows(parsed)) + } else { + #expect((imported as NSArray).isEqual(to: parsed)) + } + } + + // A whole-file parse hands the tables over as a dictionary, whose order changes from launch to launch. + @Test("Two tables that both match the target table resolve to the first in the file", arguments: [ + #"{"a.users":[{"n":1}],"b.users":[{"n":2}]}"#, + #"{"Users":[{"n":1}],"users":[{"n":2}]}"#, + ]) + func ambiguousTableResolvesToTheFirst(json: String) throws { + let elements = try importedElements(json, targetTable: "USERS") + #expect((elements as NSArray).isEqual(to: [["n": 1]])) + } + + @Test("A file whose structure breaks fails before its first row", arguments: [ + #"[{"a":1},{"a":2},{"a":"#, + #"{"t":[{"a":1}]"#, + #"[{"a":1}] x"#, + #"[{"a":1} {"a":2}]"#, + #"[{"a":1},,{"a":2}]"#, + #"[,]"#, + #"[{"a":1]"#, + #"{"a" 1}"#, + "42", + "", + ]) + func brokenStructureFailsUpFront(json: String) throws { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("A UTF-16 file with a byte order mark streams its rows and fields") + func utf16FileStillReads() throws { + let url = try writeJSON(try #require(#"{"t":[{"name":"Ada"},{"name":"Grace"}]}"#.data(using: .utf16))) + defer { try? FileManager.default.removeItem(at: url) } + let document = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + #expect(document.count == 2) + #expect((try drain(document.elements) as NSArray).isEqual(to: [["name": "Ada"], ["name": "Grace"]])) + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["name"]) + } + + // `JSONSerialization` reads UTF-32LE with a byte order mark as UTF-16LE and fails, so the oracle + // is the parse of the same text without its mark, which it reads in every encoding. + @Test("UTF-16 and UTF-32 text streams to the rows a whole-file parse gives, with or without a mark", arguments: [1, 3, 7, 1 << 20]) + func decodedTextMatchesAWholeFileParse(chunkSize: Int) throws { + let json = "[\n " + Self.trickyRows.joined(separator: ",\n ") + #",{"emoji":"😀 𝄞 a"}]"# + for encoding in Self.encodings { + let url = try writeJSON(Self.encoded(json, encoding, withMark: false)) + defer { try? FileManager.default.removeItem(at: url) } + let expected = rows(try JSONImportParsing.parseElements(at: url, targetTable: nil)) + #expect(expected.count == Self.trickyRows.count + 1) + for withMark in [true, false] { + let data = Self.encoded(json, encoding, withMark: withMark) + #expect(try importedRows(data, chunkSize: chunkSize).map(Optional.some) == expected) + } + } + } + + @Test("A decoded file holds one chunk and one element, not the file") + func decodedReaderHoldsOneChunkAndOneElement() throws { + let rows = (1...2_000).map { #"{"id":\#($0),"name":"Hà Nội \#($0)"}"# } + let data = Self.encoded("[" + rows.joined(separator: ",") + "]", .utf16LittleEndian, withMark: true) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + let chunkSize = 128 + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: nil) { reader, _ in try reader.skipValue() } + guard case .array(let offset, let count, _) = layout else { + Issue.record("Expected the rows of an array, got \(layout)") + return + } + var elements = try JSONStreamedElements(reader: reader, arrayAt: offset) + defer { elements.close() } + var mostHeld = 0 + while try elements.next() != nil { + mostHeld = max(mostHeld, elements.heldByteCount) + } + let longestElement = rows.map(\.utf8.count).max() ?? 0 + #expect(count == 2_000) + #expect(mostHeld <= chunkSize * 2 + longestElement) + #expect(data.count > 100 * (chunkSize * 2 + longestElement)) + } + + @Test("An error in decoded text points at the byte of the file", arguments: [String.Encoding.utf16LittleEndian, .utf32BigEndian]) + func decodedErrorNamesTheFileByte(encoding: String.Encoding) throws { + let text = #"[{"a":"é😀"} x]"# + let data = Self.encoded(text, encoding, withMark: true) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + let mark = encoding == .utf16LittleEndian ? 2 : 4 + let unitsBefore = Array(#"[{"a":"é😀"} "#.utf16).count + let expected = encoding == .utf16LittleEndian ? mark + unitsBefore * 2 : mark + (#"[{"a":"é😀"} "#.unicodeScalars.count) * 4 + #expect(throws: JSONDocumentError.unexpectedCharacter(byteOffset: expected)) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("Text that is not valid UTF-16 fails instead of importing", arguments: [ + Data([0xFF, 0xFE, 0x5B, 0x00, 0x00, 0xD8, 0x5D, 0x00]), + Data([0xFF, 0xFE, 0x5B, 0x00, 0x5D]), + ]) + func invalidDecodedTextFails(data: Data) throws { + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("A lone object is one row, read member by member, keeping the first of two members with one name") + func singleObjectStreamsItsMembers() throws { + let tags = "[" + (1...3_000).map { #""tag \#($0)""# }.joined(separator: ",") + "]" + let json = #"{"id":7,"tags":"# + tags + #","id":8,"meta":{"k":null},"ok":true}"# + let data = Data(json.utf8) + let streamed = try importedRows(data, chunkSize: 64) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + #expect(streamed.map(Optional.some) == rows(try JSONImportParsing.parseElements(at: url, targetTable: nil))) + #expect(streamed.first?["id"] == .text("7")) + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["id", "meta", "ok", "tags"]) + } + + @Test("A file that starts with a UTF-8 byte order mark streams") + func byteOrderMarkIsSkipped() throws { + let url = try writeJSON(Data([0xEF, 0xBB, 0xBF]) + Data(#"[{"a":1}]"#.utf8)) + defer { try? FileManager.default.removeItem(at: url) } + #expect((try streamedRows(url, chunkSize: 2) as NSArray).isEqual(to: [["a": 1]])) + } + + @Test("Detection fails on a value that does not parse, in any table, as a whole-file parse did") + func detectionValidatesEveryTable() throws { + let url = try writeJSON(#"{"users":[{"id":1}],"orders":[{"id":tru}]}"#) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.detectFields(at: url, targetTable: "users") + } + } + + @Test("Detection reports the fields of the first of two tables with one name") + func detectionUsesTheFirstOfTwoTablesWithOneName() throws { + let url = try writeJSON(#"{"t":[{"a":1}],"t":[{"b":2}]}"#) + defer { try? FileManager.default.removeItem(at: url) } + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["a"]) + } +} + +private final class RecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? + + private(set) var rows: [[String: PluginCellValue]] = [] + + init(targetTable: String?) { + self.targetTable = targetTable + } + + func execute(statement: String) async throws {} + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { self.rows.append(contentsOf: rows) } + func deleteAllRowsFromTargetTable() async throws {} + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + func disableForeignKeyChecks() async throws {} + func enableForeignKeyChecks() async throws {} +} + +private final class DocumentSource: PluginImportSource, @unchecked Sendable { + private let url: URL + + init(url: URL) { self.url = url } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { $0.finish() } + } + + func fileURL() -> URL { url } + func fileSizeBytes() -> Int64 { 0 } +} + +// Serialized because `JSONImportPlugin.settings` persists through plugin storage. +@Suite(.serialized) +struct JSONDocumentImportTests { + private func runImport( + _ json: String, + errorHandling: ImportErrorHandling, + sink: RecordingSink, + progress: PluginImportProgress = PluginImportProgress(progress: Progress()) + ) async throws -> Result { + let url = try writeJSON(json) + defer { try? FileManager.default.removeItem(at: url) } + + // `settings` persists, so the developer's own preference is put back. + let plugin = JSONImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings.errorHandling = errorHandling + plugin.settings.wrapInTransaction = true + plugin.settings.deleteExistingRows = false + + do { + return .success(try await plugin.performImport(source: DocumentSource(url: url), sink: sink, progress: progress)) + } catch { + return .failure(error) + } + } + + @Test("An array element that is not an object is reported as a skipped row, not left out unseen") + func nonObjectElementIsReported() async throws { + let sink = RecordingSink(targetTable: "people") + let outcome = try await runImport( + #"[{"name":"Ada"}, 5, ["x"], {"name":"Grace"}]"#, errorHandling: .skipAndContinue, sink: sink + ) + guard case .success(let result) = outcome else { + Issue.record("Skip and Continue must not abort the import: \(outcome)") + return + } + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 2) + #expect(result.errors.map { $0.line } == [2, 3]) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace")]) + } + + @Test("A stop mode fails on an array element that is not an object") + func nonObjectElementStopsAStopMode() async throws { + let outcome = try await runImport( + #"[{"name":"Ada"}, [1, "Grace"]]"#, errorHandling: .stopAndRollback, sink: RecordingSink(targetTable: "people") + ) + guard case .failure = outcome else { + Issue.record("Stop and Rollback must not import a file with a row it cannot read") + return + } + } + + // A whole-file parse refused such a file outright, so no error mode passes over the element. + @Test( + "An element that does not parse fails the import in every error mode", + arguments: [ImportErrorHandling.skipAndContinue, .stopAndRollback, .stopAndCommit] + ) + func unreadableElementFailsTheImport(errorHandling: ImportErrorHandling) async throws { + let sink = RecordingSink(targetTable: "people") + let outcome = try await runImport( + #"[{"name":"Ada"}, {"name":tru}, {"name":"Grace"}]"#, errorHandling: errorHandling, sink: sink + ) + guard case .failure = outcome else { + Issue.record("An element that does not parse must fail the import: \(outcome)") + return + } + #expect(sink.rows.isEmpty) + } + + @Test("A truncated export imports nothing, even in Skip and Continue") + func truncatedFileImportsNothing() async throws { + let sink = RecordingSink(targetTable: "people") + let outcome = try await runImport( + #"{"public.people":[{"name":"Ada"},{"name":"Grace"},{"name":"Ka"#, errorHandling: .skipAndContinue, sink: sink + ) + guard case .failure = outcome else { + Issue.record("A truncated file must fail: \(outcome)") + return + } + #expect(sink.rows.isEmpty) + } + + @Test("A table-keyed export imports the target table's rows in file order, with an exact progress total") + func tableKeyedExportImportsTheTargetTable() async throws { + let sink = RecordingSink(targetTable: "people") + let progress = PluginImportProgress(progress: Progress()) + let json = #"{"public.other":[{"name":"x"}],"public.people":[{"name":"Ada"},{"name":"Grace"},{"name":"Hedy"}]}"# + let outcome = try await runImport(json, errorHandling: .stopAndRollback, sink: sink, progress: progress) + guard case .success(let result) = outcome else { + Issue.record("A valid file must import: \(outcome)") + return + } + #expect(result.executedStatements == 3) + #expect(progress.estimatedTotalStatements == 3) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace"), .text("Hedy")]) + } +} diff --git a/apple/Plugins/FormatPluginTests/SQLImportCopyRowsTests.swift b/apple/Plugins/FormatPluginTests/SQLImportCopyRowsTests.swift new file mode 100644 index 0000000000..37e845c375 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLImportCopyRowsTests.swift @@ -0,0 +1,172 @@ +// +// SQLImportCopyRowsTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private struct CopyRefusal: Error {} + +private final class CopySink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + private let lock = NSLock() + private var log: [String] = [] + private var received = Data() + var refusesCopy = false + var onPiece: @Sendable () -> Void = {} + + var calls: [String] { lock.withLock { log } } + var copied: Data { lock.withLock { received } } + + func execute(statement: String) async throws { + lock.withLock { log.append(statement) } + } + + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + lock.withLock { log.append("\(statement) @\(line)") } + if refusesCopy { throw CopyRefusal() } + while let piece = try rows.nextPiece() { + lock.withLock { received.append(piece) } + onPiece() + } + } + + func deleteAllRowsFromTargetTable() async throws {} + func beginTransaction() async throws { lock.withLock { log.append("BEGIN") } } + func commitTransaction() async throws { lock.withLock { log.append("COMMIT") } } + func rollbackTransaction() async throws { lock.withLock { log.append("ROLLBACK") } } +} + +private final class ElementSource: PluginImportSource, @unchecked Sendable { + private let elements: [PluginImportElement] + + init(_ elements: [PluginImportElement]) { + self.elements = elements + } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { $0.finish() } + } + + func elements() async throws -> AsyncThrowingStream { + let elements = self.elements + return AsyncThrowingStream { continuation in + elements.forEach { continuation.yield($0) } + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + func fileSizeBytes() -> Int64 { 0 } +} + +private final class PieceList: @unchecked Sendable { + private let lock = NSLock() + private var pieces: [Data] + + init(_ pieces: [String]) { + self.pieces = pieces.map { Data($0.utf8) } + } + + var data: PluginCopyData { + PluginCopyData { [self] in + lock.withLock { pieces.isEmpty ? nil : pieces.removeFirst() } + } + } +} + +@Suite("SQL import COPY rows", .serialized) +struct SQLImportCopyRowsTests { + private func runImport( + _ elements: [PluginImportElement], + sink: CopySink, + errorHandling: ImportErrorHandling = .stopAndRollback, + progress: PluginImportProgress = PluginImportProgress(progress: Progress()) + ) async -> Result { + let plugin = SQLImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings.errorHandling = errorHandling + plugin.settings.wrapInTransaction = true + plugin.settings.disableForeignKeyChecks = false + do { + return .success(try await plugin.performImport(source: ElementSource(elements), sink: sink, progress: progress)) + } catch { + return .failure(error) + } + } + + @Test("A COPY element goes to the sink with its rows, inside the import's transaction, counted as one statement") + func copyRunsInsideTheTransaction() async throws { + let sink = CopySink() + let result = try await runImport([ + .statement("CREATE TABLE t (a int)", lineNumber: 1), + .copyFromStandardInput(statement: "COPY t (a) FROM stdin", lineNumber: 2, rows: PieceList(["1\n2\n", "3\n"]).data), + .statement("SELECT 1", lineNumber: 6), + ], sink: sink).get() + + #expect(sink.calls == ["BEGIN", "CREATE TABLE t (a int)", "COPY t (a) FROM stdin @2", "SELECT 1", "COMMIT"]) + #expect(sink.copied == Data("1\n2\n3\n".utf8)) + #expect(result.executedStatements == 3) + } + + @Test("A COPY the sink refuses fails the import at its line and rolls back") + func refusedCopyStopsTheImport() async throws { + let sink = CopySink() + sink.refusesCopy = true + let outcome = await runImport([ + .copyFromStandardInput(statement: "COPY t FROM stdin", lineNumber: 7, rows: PieceList(["1\n"]).data), + ], sink: sink) + + guard case .failure(let error) = outcome, + case .statementFailed(let statement, let line, _) = error as? PluginImportError + else { + Issue.record("Expected statementFailed, got \(outcome)") + return + } + #expect(statement == "COPY t FROM stdin") + #expect(line == 7) + #expect(sink.calls.last == "ROLLBACK") + } + + @Test("A Stop while rows are sent ends the COPY as a cancelled import") + func stopDuringRows() async throws { + let sink = CopySink() + let progress = PluginImportProgress(progress: Progress()) + sink.onPiece = { progress.cancel() } + let outcome = await runImport([ + .copyFromStandardInput(statement: "COPY t FROM stdin", lineNumber: 1, rows: PieceList(["1\n", "2\n", "3\n"]).data), + ], sink: sink, progress: progress) + + guard case .failure(let error) = outcome else { + Issue.record("Expected a cancellation, got \(outcome)") + return + } + #expect(error is PluginImportCancellationError) + #expect(sink.copied == Data("1\n".utf8)) + } + + @Test("A source with statements alone still runs through the default elements") + func statementOnlySourceRuns() async throws { + final class LegacySource: PluginImportSource, @unchecked Sendable { + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { continuation in + continuation.yield((statement: "SELECT 1", lineNumber: 3)) + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + func fileSizeBytes() -> Int64 { 0 } + } + + var lines: [Int] = [] + for try await element in try await LegacySource().elements() { + lines.append(element.lineNumber) + #expect(element.statementText == "SELECT 1") + } + #expect(lines == [3]) + } +} diff --git a/apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift b/apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift new file mode 100644 index 0000000000..cc68d53a73 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift @@ -0,0 +1,43 @@ +// +// XLSXSheetRowsTests.swift +// FormatPluginTests +// + +import Foundation +import Testing + +struct XLSXSheetRowsTests { + private typealias Chunk = XLSXSheetRows.Chunk + + @Test("One batch past several sheets fills each to the limit, header included") + func oneBatchFillsSeveralSheets() { + var sheets = XLSXSheetRows(limit: 5, headerRowCount: 1) + + #expect(sheets.chunks(of: 12) == [ + Chunk(rows: 0..<4, startsNewSheet: false), + Chunk(rows: 4..<8, startsNewSheet: true), + Chunk(rows: 8..<12, startsNewSheet: true), + ]) + } + + @Test("A batch that fills a sheet exactly starts the next sheet only when another row comes") + func exactFillStartsNoEmptySheet() { + var sheets = XLSXSheetRows(limit: 5, headerRowCount: 1) + + #expect(sheets.chunks(of: 4) == [Chunk(rows: 0..<4, startsNewSheet: false)]) + #expect(sheets.chunks(of: 0).isEmpty) + #expect(sheets.chunks(of: 1) == [Chunk(rows: 0..<1, startsNewSheet: true)]) + } + + @Test("Batches add up across calls, and without a header a sheet holds the whole limit") + func batchesAddUpWithoutHeader() { + var sheets = XLSXSheetRows(limit: 5, headerRowCount: 0) + + #expect(sheets.chunks(of: 3) == [Chunk(rows: 0..<3, startsNewSheet: false)]) + #expect(sheets.chunks(of: 9) == [ + Chunk(rows: 0..<2, startsNewSheet: false), + Chunk(rows: 2..<7, startsNewSheet: true), + Chunk(rows: 7..<9, startsNewSheet: true), + ]) + } +} diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift index 3d9980f0bb..367fe36762 100644 --- a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift @@ -14,15 +14,15 @@ struct XLSXWriterCellTypeTests { databaseTypeId: String = "" ) async throws -> String { let writer = XLSXWriter(databaseTypeId: databaseTypeId) - writer.beginSheet( + try writer.beginSheet( name: "t", columns: columnTypeNames.indices.map { "c\($0)" }, columnTypeNames: columnTypeNames, includeHeader: false, convertNullToEmpty: true ) - writer.addRows([row], convertNullToEmpty: true) - writer.finishSheet() + try writer.addRows([row], convertNullToEmpty: true) + try writer.finishSheet() let destination = FileManager.default.temporaryDirectory .appendingPathComponent("\(UUID().uuidString).xlsx") defer { try? FileManager.default.removeItem(at: destination) } diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift index d6445fbe1e..5d66558ff8 100644 --- a/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXWriterSheetNameTests.swift @@ -9,9 +9,9 @@ import Testing struct XLSXWriterSheetNameTests { private static let fortyCharacterName = "customer_subscription_billing_events_log" - private func addSheet(_ name: String, to writer: XLSXWriter) { - writer.beginSheet(name: name, columns: ["id"], includeHeader: false, convertNullToEmpty: true) - writer.finishSheet() + private func addSheet(_ name: String, to writer: XLSXWriter) throws { + try writer.beginSheet(name: name, columns: ["id"], includeHeader: false, convertNullToEmpty: true) + try writer.finishSheet() } private func workbookSheetNames(of writer: XLSXWriter) async throws -> [String] { @@ -35,16 +35,16 @@ struct XLSXWriterSheetNameTests { @Test("Two tables with the same name get two sheet names") func sameNameTwiceIsNumbered() async throws { let writer = XLSXWriter() - addSheet("users", to: writer) - addSheet("users", to: writer) + try addSheet("users", to: writer) + try addSheet("users", to: writer) #expect(try await workbookSheetNames(of: writer) == ["users", "users (2)"]) } @Test("Names that differ only in case count as the same name, as Excel compares them") func caseOnlyDifferenceIsNumbered() async throws { let writer = XLSXWriter() - addSheet("users", to: writer) - addSheet("Users", to: writer) + try addSheet("users", to: writer) + try addSheet("Users", to: writer) let names = try await workbookSheetNames(of: writer) #expect(names == ["users", "Users (2)"]) #expect(areDistinctIgnoringCase(names)) @@ -53,17 +53,17 @@ struct XLSXWriterSheetNameTests { @Test("A number already taken by another table is skipped") func takenNumberIsSkipped() async throws { let writer = XLSXWriter() - addSheet("users", to: writer) - addSheet("users (2)", to: writer) - addSheet("users", to: writer) + try addSheet("users", to: writer) + try addSheet("users (2)", to: writer) + try addSheet("users", to: writer) #expect(try await workbookSheetNames(of: writer) == ["users", "users (2)", "users (3)"]) } @Test("Two long names that share their first 31 characters get two sheet names within the limit") func longNamesSharingAPrefixStayDistinct() async throws { let writer = XLSXWriter() - addSheet(Self.fortyCharacterName, to: writer) - addSheet(Self.fortyCharacterName + "_archive", to: writer) + try addSheet(Self.fortyCharacterName, to: writer) + try addSheet(Self.fortyCharacterName + "_archive", to: writer) let names = try await workbookSheetNames(of: writer) #expect(names.count == 2) #expect(names.allSatisfy { $0.count <= 31 }) @@ -73,14 +73,14 @@ struct XLSXWriterSheetNameTests { @Test("A continuation sheet of a long table keeps its number within the 31 character limit") func continuationOfLongNameKeepsItsSuffix() async throws { let writer = XLSXWriter() - writer.beginSheet(name: Self.fortyCharacterName, columns: ["id"], includeHeader: false, convertNullToEmpty: true) - writer.continueSheet( + try writer.beginSheet(name: Self.fortyCharacterName, columns: ["id"], includeHeader: false, convertNullToEmpty: true) + try writer.continueSheet( baseName: Self.fortyCharacterName, columns: ["id"], includeHeader: false, convertNullToEmpty: true ) - writer.finishSheet() + try writer.finishSheet() let names = try await workbookSheetNames(of: writer) #expect(names.count == 2) #expect(names.allSatisfy { $0.count <= 31 }) diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift new file mode 100644 index 0000000000..ee35e25170 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift @@ -0,0 +1,151 @@ +// +// XLSXWriterStreamingTests.swift +// TableProTests +// + +import CryptoKit +import Foundation +import TableProPluginKit +import Testing + +struct XLSXWriterStreamingTests { + private static let columns = ["id", "name", "score", "blob", "note"] + private static let columnTypeNames = ["int4", "text", "float8", "bytea", "varchar(8)"] + + // SHA-256 of the workbook below; every flush threshold has to produce it. + private static let workbookDigest = "0e3615d4591e87affd46978b977d3122f9cf93f459873835a08080ae9048f333" + + private static let closingTags = "".utf8.count + + private func temporaryURL(_ name: String) -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("xlsx-streaming-\(UUID().uuidString)-\(name)") + } + + private func fileSize(_ url: URL) throws -> Int { + let attributes = try FileManager.default.attributesOfItem(atPath: url.path(percentEncoded: false)) + return try #require((attributes[.size] as? NSNumber)?.intValue) + } + + private func buildWorkbook(_ writer: XLSXWriter) throws { + try writer.beginSheet( + name: "users", columns: Self.columns, columnTypeNames: Self.columnTypeNames, + includeHeader: true, convertNullToEmpty: false + ) + try writer.addRows([ + [.text("1"), .text("Ada & "), .text("3.5"), .bytes(Data([0xDE, 0xAD])), .null], + [.text("2"), .text("Grace \"Hopper\"\r\n'x'"), .text("1e-7"), .bytes(Data()), .text("007")], + [.text("1234567890123456789"), .text(""), .text("NaN"), .null, .text("\u{01}bell")], + ], convertNullToEmpty: false) + try writer.continueSheet( + baseName: "users", columns: Self.columns, columnTypeNames: Self.columnTypeNames, + includeHeader: true, convertNullToEmpty: false + ) + try writer.addRows([[.text("3"), .text("Hà Nội 東京"), .text("-0.50"), .null, .text("x")]], convertNullToEmpty: false) + try writer.finishSheet() + try writer.beginSheet(name: "Users", columns: ["a", "b"], includeHeader: false, convertNullToEmpty: true) + try writer.addRows((1...3_000).map { [.text(String($0)), .text("v\($0)")] }, convertNullToEmpty: true) + try writer.addRows([[.null, .text("1E5")]], convertNullToEmpty: true) + try writer.finishSheet() + try writer.beginSheet(name: "a/b:c", columns: [], includeHeader: false, convertNullToEmpty: true) + try writer.finishSheet() + } + + @Test( + "The workbook keeps every byte it had when sheets were built in memory", + arguments: [0, 64, XLSXWriter.defaultFlushThreshold] + ) + func workbookBytesAreUnchanged(flushThreshold: Int) async throws { + let destination = temporaryURL("workbook.xlsx") + defer { try? FileManager.default.removeItem(at: destination) } + try Data("an older export".utf8).write(to: destination) + + let writer = XLSXWriter(flushThreshold: flushThreshold) + try buildWorkbook(writer) + try await writer.write(to: destination) + + let digest = SHA256.hash(data: try Data(contentsOf: destination)).map { String(format: "%02x", $0) }.joined() + #expect(digest == Self.workbookDigest) + } + + @Test("A sheet holds at most one flush of XML in memory while its rows arrive") + func sheetXMLReachesItsFileAsRowsArrive() throws { + let parts = temporaryURL("parts") + let threshold = 4_096 + let writer = XLSXWriter(partsDirectory: parts, flushThreshold: threshold) + try writer.beginSheet(name: "t", columns: ["a", "b"], includeHeader: true, convertNullToEmpty: true) + try writer.addRows((1...10_000).map { [.text(String($0)), .text("value \($0)")] }, convertNullToEmpty: true) + + let sheet = parts.appendingPathComponent("sheet1.xml") + let writtenBeforeFinishing = try fileSize(sheet) + try writer.finishSheet() + let sheetSize = try fileSize(sheet) + + #expect(sheetSize > 100 * threshold) + #expect(sheetSize - writtenBeforeFinishing < threshold + Self.closingTags) + } + + @Test("A workbook past the ZIP size limit fails while its rows are still arriving") + func oversizedWorkbookFailsEarly() throws { + let writer = XLSXWriter(flushThreshold: 0, maximumArchiveSize: 20_000) + try writer.beginSheet(name: "t", columns: ["a"], includeHeader: false, convertNullToEmpty: true) + var acceptedBatches = 0 + var failure: (any Error)? + do { + for _ in 0..<1_000 { + try writer.addRows((1...100).map { [.text("row \($0)")] }, convertNullToEmpty: true) + acceptedBatches += 1 + } + } catch { + failure = error + } + #expect(failure != nil) + #expect(acceptedBatches < 10) + } + + @Test("A writer that goes away, as a cancelled export's does, removes its sheet files") + func abandonedWriterRemovesItsSheetFiles() throws { + let parts = temporaryURL("parts") + do { + let writer = XLSXWriter(partsDirectory: parts, flushThreshold: 0) + try writer.beginSheet(name: "t", columns: ["a"], includeHeader: true, convertNullToEmpty: true) + try writer.addRows([[.text("1")]], convertNullToEmpty: true) + withExtendedLifetime(writer) { + #expect(FileManager.default.fileExists(atPath: parts.appendingPathComponent("sheet1.xml").path(percentEncoded: false))) + } + } + #expect(!FileManager.default.fileExists(atPath: parts.path(percentEncoded: false))) + } + + @Test("A new export removes the sheet folders an export that crashed left behind, and only those") + func abandonedSheetFoldersAreSwept() throws { + let parent = temporaryURL("temporary") + try FileManager.default.createDirectory(at: parent, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: parent) } + let folder = { (name: String) in parent.appendingPathComponent(name, isDirectory: true) } + let exists = { (url: URL) in FileManager.default.fileExists(atPath: url.path(percentEncoded: false)) } + + let crashed = folder(XLSXWriter.partsDirectoryPrefix + "crashed") + let recent = folder(XLSXWriter.partsDirectoryPrefix + "recent") + let unrelated = folder("unrelated") + for url in [crashed, recent, unrelated] { + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + } + try Data("".utf8).write(to: crashed.appendingPathComponent("sheet1.xml")) + let running = folder(XLSXWriter.partsDirectoryPrefix + "running") + let writer = XLSXWriter(partsDirectory: running) + try writer.beginSheet(name: "t", columns: ["a"], includeHeader: true, convertNullToEmpty: true) + + let twoDaysAgo = Date(timeIntervalSinceNow: -2 * 86_400) + for url in [crashed, unrelated, running] { + try FileManager.default.setAttributes([.modificationDate: twoDaysAgo], ofItemAtPath: url.path(percentEncoded: false)) + } + XLSXWriter.removeAbandonedPartsDirectories(in: parent, modifiedBefore: Date(timeIntervalSinceNow: -3_600)) + + withExtendedLifetime(writer) { + #expect(!exists(crashed)) + #expect(exists(running)) + #expect(exists(recent)) + #expect(exists(unrelated)) + } + } +} diff --git a/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift b/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift index 962b4b563c..c1a4784456 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift @@ -121,6 +121,42 @@ struct HanaExplainRequest: Encodable, Sendable { let timeoutSeconds: Int } +struct HanaCursorRequest: Encodable, Sendable { + let sql: String + let timeoutSeconds: Int + let batchRows: Int +} + +struct HanaCursorHeader: Decodable, Equatable, Sendable { + let columns: [String] + let columnTypeNames: [String] + let columnClassifications: [String?] + let rowsAffected: Int64 + let hasResultSet: Bool +} + +struct HanaCursorBatch: Decodable, Equatable, Sendable { + let rows: [[HanaBridgeCell]] + let truncatedLobCount: Int + let done: Bool + let sessionLost: Bool +} + +struct HanaCursorOpening: Decodable, Sendable { + let header: HanaCursorHeader + let batch: HanaCursorBatch + + // The helper sends the header and the first batch as one flat object. + init(from decoder: any Decoder) throws { + header = try HanaCursorHeader(from: decoder) + batch = try HanaCursorBatch(from: decoder) + } +} + +struct HanaCursorClosing: Decodable, Sendable { + let sessionLost: Bool +} + struct HanaConnectResult: Decodable, Equatable, Sendable { let serverVersion: String let currentSchema: String diff --git a/apple/Plugins/HanaDriverPlugin/HanaConnection.swift b/apple/Plugins/HanaDriverPlugin/HanaConnection.swift index f129f13637..e8cfcba775 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaConnection.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaConnection.swift @@ -21,6 +21,7 @@ final class HanaConnection: HanaSession, @unchecked Sendable { var hasLostConnection = false } + private static let cursorBatchRows = 1_000 private static let logger = Logger(subsystem: "com.TablePro", category: "HanaConnection") private let bridge: any HanaNativeBridge @@ -105,6 +106,28 @@ final class HanaConnection: HanaSession, @unchecked Sendable { } } + func stream( + sql: String, + cancellation: HanaOperationSlot, + onHeader: @escaping @Sendable (HanaCursorHeader) -> Void, + onBatch: @escaping @Sendable (HanaCursorBatch) -> Void + ) async throws { + let request = HanaCursorRequest(sql: sql, timeoutSeconds: queryTimeoutSeconds, batchRows: Self.cursorBatchRows) + let requestJSON = try JSONEncoder().encode(request) + // Every fetch stays in this one queue item. The cursor holds the helper's operation slot, so a read queued + // between two fetches would wait on that slot while the next fetch waited behind it on this queue. + try await perform(.connected, cancellation: cancellation) { bridge, ticket in + try self.drainCursor( + bridge, + ticket: ticket, + request: requestJSON, + cancellation: cancellation, + onHeader: onHeader, + onBatch: onBatch + ) + } + } + func cancel(_ cancellation: HanaOperationSlot) { guard let ticket = cancellation.cancel() else { return } bridge.cancel(ticket) @@ -195,6 +218,65 @@ final class HanaConnection: HanaSession, @unchecked Sendable { } } + private func drainCursor( + _ bridge: any HanaNativeBridge, + ticket: HanaOperationTicket, + request: Data, + cancellation: HanaOperationSlot, + onHeader: (HanaCursorHeader) -> Void, + onBatch: (HanaCursorBatch) -> Void + ) throws { + let opened = try bridge.openCursor(ticket, request: request) + // A failure reply means the helper already closed the cursor. Any other exit leaves it open until closed here. + var holdsCursor = true + defer { + if holdsCursor { + closeCursor(bridge, ticket: ticket) + } + } + let opening = try Self.decode(HanaCursorOpening.self, from: opened) + onHeader(opening.header) + var batch = opening.batch + while true { + if batch.sessionLost { + Self.logger.error("SAP HANA session was lost while its cursor was open") + markLost(ticket.session) + } + if !batch.rows.isEmpty { + onBatch(batch) + } + if batch.done { + holdsCursor = false + return + } + guard !cancellation.isCancelled else { + throw HanaBridgeFailure(kind: .cancelled) + } + let fetched: Data + do { + fetched = try bridge.fetch(ticket) + } catch { + holdsCursor = false + throw error + } + batch = try Self.decode(HanaCursorBatch.self, from: fetched) + } + } + + private func closeCursor(_ bridge: any HanaNativeBridge, ticket: HanaOperationTicket) { + do { + let closing = try JSONDecoder().decode(HanaCursorClosing.self, from: bridge.closeCursor(ticket)) + if closing.sessionLost { + markLost(ticket.session) + } + } catch { + if (error as? HanaBridgeFailure)?.kind == .connectionLost { + markLost(ticket.session) + } + Self.logger.error("SAP HANA cursor did not close: \(error.localizedDescription, privacy: .private)") + } + } + private func perform( _ target: Target, cancellation slot: HanaOperationSlot, diff --git a/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift b/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift index 93afd8b101..337323aff8 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift @@ -152,6 +152,18 @@ final class HanaHelperBridge: HanaNativeBridge, @unchecked Sendable { try call(.explain, ticket: ticket) { HanaHelperMessage.statement($0, request: request) } } + func openCursor(_ ticket: HanaOperationTicket, request: Data) throws -> Data { + try call(.openCursor, ticket: ticket) { HanaHelperMessage.statement($0, request: request) } + } + + func fetch(_ ticket: HanaOperationTicket) throws -> Data { + try call(.fetch, ticket: ticket) { HanaHelperMessage.operation($0) } + } + + func closeCursor(_ ticket: HanaOperationTicket) throws -> Data { + try call(.closeCursor, ticket: ticket) { HanaHelperMessage.operation($0) } + } + func ping(_ ticket: HanaOperationTicket) throws { _ = try call(.ping, ticket: ticket) { HanaHelperMessage.operation($0) } } diff --git a/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift b/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift index 74f17e800d..764e5d2605 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift @@ -8,6 +8,9 @@ enum HanaHelperOpcode: UInt8, Sendable { case ping = 5 case cancel = 6 case close = 7 + case openCursor = 8 + case fetch = 9 + case closeCursor = 10 } enum HanaHelperFrameError: Error, Equatable, Sendable { @@ -135,7 +138,7 @@ struct HanaHelperGreeting: Equatable, Sendable { } enum HanaHelperHandshake { - static let protocolVersion = 1 + static let protocolVersion = 2 static let frameID: UInt64 = 0 static let maximumBodyLength = 4_096 static let acceptedForcedSeverGraceSeconds = 1...3_600 diff --git a/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift b/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift index 7e72d072b1..ec789006e5 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift @@ -5,6 +5,9 @@ protocol HanaNativeBridge: Sendable { func connect(_ ticket: HanaOperationTicket) throws -> Data func execute(_ ticket: HanaOperationTicket, request: Data) throws -> Data func explain(_ ticket: HanaOperationTicket, request: Data) throws -> Data + func openCursor(_ ticket: HanaOperationTicket, request: Data) throws -> Data + func fetch(_ ticket: HanaOperationTicket) throws -> Data + func closeCursor(_ ticket: HanaOperationTicket) throws -> Data func ping(_ ticket: HanaOperationTicket) throws func cancel(_ ticket: HanaOperationTicket) func close(session: UInt64) diff --git a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift index 6bc959cc7c..7ea6ea662c 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift @@ -100,6 +100,22 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try await run(query, parameters: parameters, rowCap: nil) } + // Collected here rather than through boundedQueryFromStream, which would drop the column hints and the + // large object warning that execute returns. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + let report = HanaStreamReport() + let collected = try await PluginBoundedStream.collect( + stream(query, report: report, refusesCutValues: false), + rowCap: rowCap, + startedAt: Date() + ) + return report.completing(collected) + } + + func streamRows(query: String) -> AsyncThrowingStream { + stream(query, report: HanaStreamReport(), refusesCutValues: true) + } + func switchSchema(to schema: String) async throws { guard !schema.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw HanaError(kind: .configuration, message: String(localized: "The SAP HANA schema name cannot be empty.")) @@ -159,6 +175,88 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + // A copy, export or compare reads the stream, and would write a value cut at 64 MiB as if it were whole. + private func stream( + _ sql: String, + report: HanaStreamReport, + refusesCutValues: Bool + ) -> AsyncThrowingStream { + PluginRowStream.makeBounded { continuation, abort in + let driver = self + let producer = Task { + do { + try await driver.produce( + sql, into: continuation, report: report, refusesCutValues: refusesCutValues + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + report.logTruncatedLargeObjects() + } + // The cancel takes the Stop path: the helper ends a fetch in flight, or the cursor loop closes the cursor. + abort.onAbort { producer.cancel() } + } + } + + private func produce( + _ sql: String, + into continuation: AsyncThrowingStream.Continuation, + report: HanaStreamReport, + refusesCutValues: Bool + ) async throws { + if HanaExplainStatement.explainedStatement(in: sql) != nil { + let result = try await run(sql, parameters: nil, rowCap: nil) + _ = await PluginRowStream.deliver(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames + )), to: continuation) + if !result.rows.isEmpty { + _ = await PluginRowStream.deliver(.rows(result.rows), to: continuation) + } + return + } + let query = beginQuery() + defer { endQuery(query) } + do { + try await session.stream( + sql: sql, + cancellation: query.cancellation, + onHeader: { header in + report.record(header) + _ = PluginRowStream.deliverBlocking(.header(PluginStreamHeader( + columns: header.columns, + columnTypeNames: header.columnTypeNames + )), to: continuation, stops: { query.cancellation.isCancelled }) + }, + onBatch: { batch in + report.record(truncatedLargeObjects: batch.truncatedLobCount) + // The cursor loop sees the slot cancelled after this batch and closes the cursor. + if refusesCutValues, batch.truncatedLobCount > 0 { + _ = query.cancellation.cancel() + return + } + // Runs on the session's own queue, which waits here while the reader is behind. + let delivered = PluginRowStream.deliverBlocking( + .rows(HanaResultMapping.pluginRows(batch.rows)), + to: continuation, + stops: { query.cancellation.isCancelled } + ) + if !delivered { _ = query.cancellation.cancel() } + } + ) + } catch let failure as HanaBridgeFailure { + if refusesCutValues, report.hasCutLargeObjects { + throw HanaResultMapping.cutLargeObjectError + } + let requested = Task.isCancelled || cancellationGate.isCancelled(query.generation) + throw HanaFailureMapping.error(for: failure, cancellationRequested: requested) + } + if refusesCutValues, report.hasCutLargeObjects { + throw HanaResultMapping.cutLargeObjectError + } + } + private func beginQuery() -> RunningQuery { let cancellation = HanaOperationSlot() return stateLock.withLock { diff --git a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift index 2da12083d7..8d1f4f83f9 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift @@ -1,4 +1,5 @@ import Foundation +import os import TableProPluginKit enum HanaResultMapping { @@ -6,15 +7,23 @@ enum HanaResultMapping { PluginQueryResult( columns: envelope.columns, columnTypeNames: envelope.columnTypeNames, - rows: envelope.rows.map { row in row.map(\.pluginValue) }, + rows: pluginRows(envelope.rows), rowsAffected: Int(clamping: envelope.rowsAffected), timing: PluginQueryTiming(total: envelope.executionTime), isTruncated: envelope.isTruncated, statusMessage: statusMessage(truncatedLobCount: envelope.truncatedLobCount), - columnMeta: columnMeta(for: envelope) + columnMeta: columnMeta( + columns: envelope.columns, + columnTypeNames: envelope.columnTypeNames, + classifications: envelope.columnClassifications + ) ) } + static func pluginRows(_ rows: [[HanaBridgeCell]]) -> [PluginRow] { + rows.map { row in row.map(\.pluginValue) } + } + static func statusMessage(truncatedLobCount: Int) -> String? { guard truncatedLobCount > 0 else { return nil } return String( @@ -23,17 +32,29 @@ enum HanaResultMapping { ) } - static func columnMeta(for envelope: HanaResultEnvelope) -> [PluginColumnInfo]? { - let hints = envelope.columnClassifications + static var cutLargeObjectError: HanaError { + HanaError( + kind: .invalidResult, + message: String( + localized: "A large object value is longer than the 64 MiB TablePro reads from SAP HANA, so it cannot be copied or exported whole." + ) + ) + } + + static func columnMeta( + columns: [String], + columnTypeNames: [String], + classifications hints: [String?] + ) -> [PluginColumnInfo]? { guard hints.contains(where: { $0 != nil }), - hints.count == envelope.columns.count, - envelope.columnTypeNames.count == envelope.columns.count else { + hints.count == columns.count, + columnTypeNames.count == columns.count else { return nil } - return envelope.columns.indices.map { index in + return columns.indices.map { index in PluginColumnInfo( - name: envelope.columns[index], - dataType: envelope.columnTypeNames[index], + name: columns[index], + dataType: columnTypeNames[index], isNullable: true, isPrimaryKey: false, defaultValue: nil, @@ -55,3 +76,52 @@ enum HanaResultMapping { } } } + +final class HanaStreamReport: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "HanaPluginDriver") + + private let lock = NSLock() + private var header: HanaCursorHeader? + private var truncatedLargeObjects = 0 + + func record(_ header: HanaCursorHeader) { + lock.withLock { self.header = header } + } + + func record(truncatedLargeObjects count: Int) { + guard count > 0 else { return } + lock.withLock { truncatedLargeObjects += count } + } + + var hasCutLargeObjects: Bool { + lock.withLock { truncatedLargeObjects > 0 } + } + + func logTruncatedLargeObjects() { + let count = lock.withLock { truncatedLargeObjects } + guard count > 0 else { return } + Self.logger.warning("SAP HANA cut \(count) large object values in a streamed result at 64 MiB") + } + + func completing(_ collected: PluginQueryResult) -> PluginQueryResult { + let (header, count) = lock.withLock { (self.header, self.truncatedLargeObjects) } + var result = PluginQueryResult( + columns: collected.columns, + columnTypeNames: collected.columnTypeNames, + rows: collected.rows, + rowsAffected: collected.rowsAffected, + timing: collected.timing, + isTruncated: collected.isTruncated, + statusMessage: HanaResultMapping.statusMessage(truncatedLobCount: count), + columnMeta: header.flatMap { + HanaResultMapping.columnMeta( + columns: $0.columns, + columnTypeNames: $0.columnTypeNames, + classifications: $0.columnClassifications + ) + } + ) + result.columnOrigins = collected.columnOrigins + return result + } +} diff --git a/apple/Plugins/HanaDriverPlugin/HanaSession.swift b/apple/Plugins/HanaDriverPlugin/HanaSession.swift index 3efa43e337..7405b5be75 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaSession.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaSession.swift @@ -13,6 +13,12 @@ protocol HanaSession: AnyObject, Sendable { cancellation: HanaOperationSlot ) async throws -> HanaResultEnvelope func explain(sql: String, cancellation: HanaOperationSlot) async throws -> HanaResultEnvelope + func stream( + sql: String, + cancellation: HanaOperationSlot, + onHeader: @escaping @Sendable (HanaCursorHeader) -> Void, + onBatch: @escaping @Sendable (HanaCursorBatch) -> Void + ) async throws func cancel(_ cancellation: HanaOperationSlot) func applyQueryTimeout(seconds: Int) } diff --git a/apple/Plugins/HanaDriverPlugin/README.md b/apple/Plugins/HanaDriverPlugin/README.md index 0a7efd020c..7834573d17 100644 --- a/apple/Plugins/HanaDriverPlugin/README.md +++ b/apple/Plugins/HanaDriverPlugin/README.md @@ -11,7 +11,8 @@ What the plugin does: - User name and password login, with TLS in every TablePro mode. Preferred and Required encrypt without checking the certificate, Verify CA checks the chain only, and Verify Identity checks the chain and the host name, against the system roots unless a CA file is set. A client certificate and key can be added for mutual TLS. The TLS Server Name field overrides the name checked by Verify Identity when the certificate names another host. - Schema browsing, columns with full types, primary keys, identity and generated columns, indexes, foreign keys, row counts, table DDL and view definitions. - SQL execution with bound parameters, so grid edits, inserts and deletes work. +- Exports and capped editor reads stream. The helper keeps the statement's result open as a cursor and sends it in batches of up to 1,000 rows or 4 MiB, so the row count has no limit. The query timeout covers running the statement and stops once its result is ready. A stream that ends early, or Stop after the result is ready, ends the fetch at its next row and closes the result without `ALTER SYSTEM CANCEL SESSION`. - Stop and the query timeout. Both send `ALTER SYSTEM CANCEL SESSION` for the connection's own session, which keeps the session open. The bridge closes the session instead, and TablePro reconnects, when the server refuses the cancel or does not answer it within 10 seconds, or when the statement is still running `hana.ForcedSeverGrace` (30 seconds) later. After Stop, and not after a timeout, the plugin also stops the helper when the operation is still pending 10 seconds past that grace, which the helper announces as `forcedSeverGraceSeconds` in its first frame. - `EXPLAIN PLAN FOR` runs as one step that saves the plan, reads it and deletes it again. Reading plans needs the `OPTIMIZER ADMIN` privilege on recent HANA versions. -Not supported: structure editing, transactions, LDAP, JWT and SSO logins. Large object values longer than 64 MiB are cut short, and the result says so. A reply frame holds at most 2 GiB - 1 bytes, so a larger result comes back as an internal error, "the result is larger than 2 GiB". +Not supported: structure editing, transactions, LDAP, JWT and SSO logins. Large object values longer than 64 MiB are cut short. A query result says so; an export only logs it, since a stream has no field for the warning. A reply frame holds at most 2 GiB - 1 bytes, so a statement that is not streamed and returns more comes back as an internal error, "the result is larger than 2 GiB". diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift index 1f73cebab5..673794c1e0 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift @@ -26,6 +26,9 @@ internal final class InfluxDBConnection: @unchecked Sendable { private let lock = NSLock() private var session: URLSession? + // A streamed read holds one response for as long as its consumer takes, and a suspended task still counts toward + // the resource limit, so streams run on a session that keeps the system default. + private var streamSession: URLSession? private var inFlight: [ObjectIdentifier: PluginURLSessionTaskBox] = [:] private let queryTimeout: HttpQueryTimeoutBox @@ -44,6 +47,9 @@ internal final class InfluxDBConnection: @unchecked Sendable { configuration.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout let delegate = skipTLSVerify ? InfluxDBTrustingDelegate() : PluginRedirectRefusingDelegate() session = URLSession(configuration: configuration, delegate: delegate, delegateQueue: nil) + // The session made its own copy of the configuration above. + configuration.timeoutIntervalForResource = URLSessionConfiguration.default.timeoutIntervalForResource + streamSession = URLSession(configuration: configuration, delegate: delegate, delegateQueue: nil) } func close() { @@ -51,6 +57,8 @@ internal final class InfluxDBConnection: @unchecked Sendable { lock.withLock { session?.invalidateAndCancel() session = nil + streamSession?.invalidateAndCancel() + streamSession = nil } } @@ -63,17 +71,7 @@ internal final class InfluxDBConnection: @unchecked Sendable { } func send(_ request: InfluxRequest, timeout: TimeInterval? = nil) async throws -> InfluxDBResponse { - guard let url = request.url(base: baseURL) else { - throw InfluxError.invalidURL(baseURL.absoluteString + request.path) - } - var urlRequest = URLRequest(url: url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = timeout ?? queryTimeout.requestTimeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - + let urlRequest = try makeURLRequest(request, timeout: timeout) let taskBox = PluginURLSessionTaskBox() let key = ObjectIdentifier(taskBox) lock.withLock { inFlight[key] = taskBox } @@ -115,6 +113,48 @@ internal final class InfluxDBConnection: @unchecked Sendable { task.resume() } } + + // The body chunk by chunk, the next one read only once `body` returns. A status outside 2xx throws what `send`'s + // caller would, and `cancelAll` stops it like any other request. + func stream(_ request: InfluxRequest, body: (Data) async throws -> Void) async throws { + let urlRequest = try makeURLRequest(request, timeout: nil) + let taskBox = PluginURLSessionTaskBox() + let key = ObjectIdentifier(taskBox) + lock.withLock { inFlight[key] = taskBox } + defer { lock.withLock { _ = inFlight.removeValue(forKey: key) } } + // Made under the lock `close()` invalidates with, as in `perform`. + guard let task = lock.withLock({ streamSession?.dataTask(with: urlRequest) }) else { + throw InfluxError.notConnected + } + let chunks = InfluxDBHTTPChunks(task: task) + taskBox.set(task) + task.resume() + try await withTaskCancellationHandler { + do { + try await chunks.forEachChunk(body) + } catch let error as URLError where error.code == .cancelled { + throw CancellationError() + } + // A cancelled reader ends the chunk sequence without an error. + try Task.checkCancellation() + } onCancel: { + taskBox.cancel() + } + } + + private func makeURLRequest(_ request: InfluxRequest, timeout: TimeInterval?) throws -> URLRequest { + guard let url = request.url(base: baseURL) else { + throw InfluxError.invalidURL(baseURL.absoluteString + request.path) + } + var urlRequest = URLRequest(url: url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = timeout ?? queryTimeout.requestTimeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + return urlRequest + } } private final class InfluxDBTrustingDelegate: PluginRedirectRefusingDelegate, @unchecked Sendable { diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift new file mode 100644 index 0000000000..b492c1455f --- /dev/null +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift @@ -0,0 +1,109 @@ +import Foundation +import TableProInfluxDBCore + +// The body of a streamed read as the socket delivers it. It is the task's delegate, not the session's, so the +// session's own delegate still answers the server trust challenge. The transfer is suspended as each chunk is handed +// over and resumed once the decoder is done with it, so undecoded body never piles up. +final class InfluxDBHTTPChunks: PluginRedirectRefusingDelegate, URLSessionDataDelegate, @unchecked Sendable { + // An error body is a message for the user; InfluxDB's run to a few hundred bytes. + private static let errorBodyByteCap = 65_536 + + private let stream: AsyncThrowingStream + private let continuation: AsyncThrowingStream.Continuation + private let lock = NSLock() + private var task: URLSessionDataTask? + private var isTransferSuspended = false + private var refusal: Error? + private var failedResponse: HTTPURLResponse? + private var failureBody = Data() + + // Takes over the task's delegate; the caller resumes it. + init(task: URLSessionDataTask) { + (stream, continuation) = AsyncThrowingStream.makeStream() + self.task = task + super.init() + task.delegate = self + continuation.onTermination = { _ in task.cancel() } + } + + // The task is torn down on every way out. A reader that throws leaves it suspended mid-body with nothing left to + // resume it, which would hold the connection open until the driver disconnects. + func forEachChunk(_ body: (Data) async throws -> Void) async throws { + defer { endTransfer() } + for try await chunk in stream { + try await body(chunk) + resumeTransfer() + } + } + + func urlSession( + _ session: URLSession, + dataTask: URLSessionDataTask, + didReceive response: URLResponse, + completionHandler: @escaping (URLSession.ResponseDisposition) -> Void + ) { + guard let http = response as? HTTPURLResponse else { + lock.withLock { refusal = InfluxError.malformedResponse("Not an HTTP response") } + completionHandler(.cancel) + return + } + if !(200 ..< 300).contains(http.statusCode) { + lock.withLock { failedResponse = http } + } + completionHandler(.allow) + } + + func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { + let isFailure = lock.withLock { () -> Bool in + guard failedResponse != nil else { return false } + if failureBody.count < Self.errorBodyByteCap { + failureBody.append(data.prefix(Self.errorBodyByteCap - failureBody.count)) + } + return true + } + guard !isFailure else { return } + + // Suspended under the lock and before the reader sees the chunk, so the flag and the task never disagree. + lock.withLock { + guard !isTransferSuspended else { return } + isTransferSuspended = true + dataTask.suspend() + } + continuation.yield(data) + } + + // The same error `send` gives for the same response, so a refused statement reads alike on both paths. + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { + let failure = lock.withLock { () -> Error? in + if let refusal { return refusal } + if let error { return error } + guard let response = failedResponse else { return nil } + return InfluxDBResponse(statusCode: response.statusCode, headers: response.allHeaderFields, data: failureBody) + .error + } + guard let failure else { + continuation.finish() + return + } + continuation.finish(throwing: failure) + } + + private func resumeTransfer() { + lock.withLock { + guard isTransferSuspended else { return } + isTransferSuspended = false + task?.resume() + } + } + + private func endTransfer() { + let outstanding = lock.withLock { () -> URLSessionDataTask? in + let current = task + task = nil + isTransferSuspended = false + return current + } + outstanding?.cancel() + continuation.finish() + } +} diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift new file mode 100644 index 0000000000..4524dde441 --- /dev/null +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift @@ -0,0 +1,106 @@ +import Foundation +import TableProInfluxDBCore +import TableProPluginKit + +// Only the grid's own pages and exports stream: their columns are the measurement's, known before the first row. Flux +// and editor statements keep the collected read, whose columns are the union of everything that comes back. +extension InfluxDBPluginDriver { + func streamRows(query: String) -> AsyncThrowingStream { + guard let remembered = streamedGrid(query) else { return collectedStream(query) } + return PluginRowStream.makeBounded { continuation, abort in + let producer = Task { + do { + try await self.streamGrid(remembered, abort: abort, continuation: continuation) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { producer.cancel() } + } + } + + // A stream carries no column metadata, so the classification `execute` gives each column is added back. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + guard streamedGrid(query) != nil else { return nil } + let result = try await boundedQueryFromStream(query: query, rowCap: rowCap) + return PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: result.rows, + rowsAffected: 0, + timing: result.timing, + isTruncated: result.isTruncated, + columnMeta: Self.columnMeta(columns: result.columns, typeNames: result.columnTypeNames) + ) + } + + private func streamedGrid(_ query: String) -> RememberedGridQuery? { + guard let remembered = rememberedGrid(query), remembered.query.shape != .count else { return nil } + return remembered + } + + private func streamGrid( + _ remembered: RememberedGridQuery, + abort: PluginStreamAbort, + continuation: AsyncThrowingStream.Continuation + ) async throws { + guard !abort.isAborted else { throw CancellationError() } + let (_, server) = try requireConnection() + let (statement, schema) = try await gridStatementFromSchema( + remembered.query, + hostKinds: remembered.hostKinds, + generation: server.generation + ) + let (connection, current, request) = try await queryRequest(statement, streamed: true) + guard !abort.isAborted else { throw CancellationError() } + + var decoder = InfluxGridStream( + body: current.generation == .v3 ? .csv : .influxQLChunks, + grid: remembered.query, + schema: schema + ) + var headerSent = false + func send(_ batches: [InfluxResultTable]) async { + for batch in batches { + if !headerSent { + headerSent = true + continuation.yield(.header(PluginStreamHeader(columns: batch.columns, columnTypeNames: batch.typeNames))) + } + if !batch.rows.isEmpty { + _ = await PluginRowStream.deliver(.rows(Self.cells(batch.rows)), to: continuation) + } + } + } + try await connection.stream(request) { chunk in + guard !abort.isAborted else { throw CancellationError() } + let batches = try decoder.consume(chunk) + await send(batches) + } + let rest = try decoder.finish() + await send(rest) + } + + // PluginKit's default stream, which the override above replaces for every statement. + private func collectedStream(_ query: String) -> AsyncThrowingStream { + PluginRowStream.make { continuation, abort in + let producer = Task { + do { + try Task.checkCancellation() + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames + ))) + if !result.rows.isEmpty { + continuation.yield(.rows(result.rows)) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { producer.cancel() } + } + } +} diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift index 99a420548f..e2bc432c05 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift @@ -28,7 +28,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send var serverVersion: String? { lock.withLock { server?.version } } var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var currentDatabase: String? { lock.withLock { database.isEmpty ? nil : database } } func beginTransaction() async throws {} @@ -152,7 +152,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send let started = Date() let (_, server) = try requireConnection() let table: InfluxResultTable - if let remembered = lock.withLock({ gridQueries[query] }) { + if let remembered = rememberedGrid(query) { table = try await runGrid(remembered.query, hostKinds: remembered.hostKinds, generation: server.generation) } else { table = try await run(InfluxStatement.route(query, generation: server.generation)) @@ -160,22 +160,30 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send return PluginQueryResult( columns: table.columns, columnTypeNames: table.typeNames, - rows: table.rows.map { row in row.map { $0.map(PluginCellValue.text) ?? .null } }, + rows: Self.cells(table.rows), rowsAffected: 0, executionTime: Date().timeIntervalSince(started), - columnMeta: zip(table.columns, table.typeNames).map { name, typeName in - Self.columnInfo(name: name, typeName: typeName, role: nil) - } + columnMeta: Self.columnMeta(columns: table.columns, typeNames: table.typeNames) ) } - // A grid statement built before the measurement's columns were known is written again from them here. - // A filter on a column the cached schema lacks means the measurement gained one since; the schema is read again. private func runGrid( _ grid: InfluxGridQuery, hostKinds: [String: InfluxColumnKind], generation: InfluxGeneration ) async throws -> InfluxResultTable { + let (statement, schema) = try await gridStatementFromSchema(grid, hostKinds: hostKinds, generation: generation) + let table = try await run(statement).typed(by: schema) + return table.columns.isEmpty ? grid.emptyResult(schema: schema) : table + } + + // A grid statement built before the measurement's columns were known is written again from them here. + // A filter on a column the cached schema lacks means the measurement gained one since; the schema is read again. + func gridStatementFromSchema( + _ grid: InfluxGridQuery, + hostKinds: [String: InfluxColumnKind], + generation: InfluxGeneration + ) async throws -> (statement: InfluxStatement, schema: [InfluxColumn]) { var schema = try await columns(of: grid.measurement) let known = Set(schema.map(\.name)) if grid.filters.contains(where: { !known.contains($0.column) }) { @@ -186,20 +194,31 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send guard let text = grid.statement(language: language, kinds: kinds, marked: false) else { throw InfluxError.filterNotExpressible(language) } - var table = try await run(InfluxStatement(language: language, text: text)).typed(by: schema) - if table.columns.isEmpty { - let names = grid.projection.isEmpty ? schema.map(\.name) : grid.projection - let types = Dictionary(schema.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) - table = InfluxResultTable( - columns: names, - typeNames: names.map { types[$0] ?? InfluxTypeName.string }, - rows: [] - ) - } - return table + return (InfluxStatement(language: language, text: text), schema) + } + + func rememberedGrid(_ statement: String) -> RememberedGridQuery? { + lock.withLock { gridQueries[statement] } } func run(_ statement: InfluxStatement) async throws -> InfluxResultTable { + let (connection, server, request) = try await queryRequest(statement) + let response = try await connection.send(request) + guard response.isSuccess else { throw response.error } + switch (server.generation, statement.language) { + case (.v3, _): + return try InfluxResponseDecoder.decodeV3CSV(response.data) + case (_, .flux): + return try InfluxResponseDecoder.decodeFluxCSV(response.data) + default: + return try InfluxResponseDecoder.decodeV1JSON(response.data) + } + } + + func queryRequest( + _ statement: InfluxStatement, + streamed: Bool = false + ) async throws -> (connection: InfluxDBConnection, server: InfluxServerInfo, request: InfluxRequest) { let (connection, server) = try requireConnection() let (database, organization) = lock.withLock { (self.database, self.organization) } if server.generation == .v3, statement.language == .sql, database.isEmpty { @@ -221,18 +240,10 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send retentionPolicy: target.retentionPolicy, organization: organization, generation: server.generation, - credentials: connection.credentials + credentials: connection.credentials, + streamed: streamed ) - let response = try await connection.send(request) - guard response.isSuccess else { throw response.error } - switch (server.generation, statement.language) { - case (.v3, _): - return try InfluxResponseDecoder.decodeV3CSV(response.data) - case (_, .flux): - return try InfluxResponseDecoder.decodeFluxCSV(response.data) - default: - return try InfluxResponseDecoder.decodeV1JSON(response.data) - } + return (connection, server, request) } // Safe Mode read this script with TablePro's lexer. The server parses it first, and a script it reads as writing or @@ -360,10 +371,18 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send columnCheck: nil ) } + + static func columnMeta(columns: [String], typeNames: [String]) -> [PluginColumnInfo] { + zip(columns, typeNames).map { name, typeName in columnInfo(name: name, typeName: typeName, role: nil) } + } + + static func cells(_ rows: [[String?]]) -> [PluginRow] { + rows.map { row in row.map { $0.map(PluginCellValue.text) ?? .null } } + } } // The statements the driver built for its grid, so `execute` runs them knowing the measurement they read. -private struct RememberedGridQuery { +struct RememberedGridQuery { let query: InfluxGridQuery let hostKinds: [String: InfluxColumnKind] } diff --git a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift index cf23a29b92..60adecb627 100644 --- a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift +++ b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift @@ -53,10 +53,11 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } } - /// NDJSON is one row per line by definition, so the pretty-print setting cannot apply to it - /// and the wrapping object has no place to go. - let isNewlineDelimited = settings.layout == .newlineDelimited - let prettyPrint = settings.prettyPrint && !isNewlineDelimited + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings + // NDJSON is one row per line by definition, so pretty-printing and the wrapping object do not apply. + let isNewlineDelimited = options.layout == .newlineDelimited + let prettyPrint = options.prettyPrint && !isNewlineDelimited let indent = prettyPrint ? " " : "" let newline = prettyPrint ? "\n" : "" @@ -104,7 +105,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug for (colIndex, column) in columns.enumerated() { if colIndex < row.count { let value = row[colIndex] - if settings.includeNullValues || !value.isNull { + if options.includeNullValues || !value.isNull { if !isFirstField { rowString += ", " } @@ -117,7 +118,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let jsonValue = formatJSONValue( value, columnTypeName: colTypeName, - preserveAsString: settings.preserveAllAsStrings + preserveAsString: options.preserveAllAsStrings ) rowString += "\"\(escapedKey)\": \(jsonValue)" } diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift new file mode 100644 index 0000000000..4aa0e25e01 --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift @@ -0,0 +1,110 @@ +// +// JSONDocumentOutline.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +enum JSONDocumentRows: Equatable { + // The rows are the elements of the array that opens at `offset`. `slot` is the one `visit` was given for them. + case array(offset: Int, elementCount: Int, slot: Int) + case singleObject(offset: Int) +} + +// Finds which array of a `.json` file holds the rows, by the rules `JSONImportParsing.extractElements` +// applies to a parsed file, in one pass that builds nothing but member names. +enum JSONDocumentOutline { + private struct Member { + let name: String + let slot: Int + let offset: Int + let elementCount: Int + let holdsRows: Bool + } + + // `visit` gets the reader at each array element and at each member value that is not an array, + // and moves it past that value. `slot` is 0 for the elements of a top-level array, the member's + // position for the elements of a member's array, and nil for a value that cannot become a row. + static func rows( + in reader: inout JSONDocumentReader, + targetTable: String?, + visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void + ) throws -> JSONDocumentRows { + try reader.startDocument() + guard let first = try reader.nextSignificantByte() else { throw JSONDocumentError.endsEarly } + switch first { + case JSONDocumentReader.openBracket: + let offset = reader.offset + let array = try elements(of: &reader, slot: 0, visit: visit) + try reader.expectEnd() + return .array(offset: offset, elementCount: array.count, slot: 0) + case JSONDocumentReader.openBrace: + let offset = reader.offset + return try memberRows(in: &reader, at: offset, targetTable: targetTable, visit: visit) + case 0x22, 0x2D, 0x30...0x39, 0x66, 0x6E, 0x74: + throw JSONImportParsing.unexpectedShapeError + default: + throw JSONDocumentError.unexpectedCharacter(byteOffset: reader.fileOffset) + } + } + + private static func elements( + of reader: inout JSONDocumentReader, + slot: Int?, + visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void + ) throws -> (count: Int, allObjects: Bool) { + guard try reader.enterArray() else { return (0, true) } + var count = 0 + var allObjects = true + repeat { + if try reader.nextSignificantByte() != JSONDocumentReader.openBrace { + allObjects = false + } + try visit(&reader, slot) + count += 1 + } while try reader.continueArray() + return (count, allObjects) + } + + private static func memberRows( + in reader: inout JSONDocumentReader, + at objectOffset: Int, + targetTable: String?, + visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void + ) throws -> JSONDocumentRows { + var members: [Member] = [] + var names: Set = [] + if try reader.enterObject() { + repeat { + let name = try reader.memberName() + // A parsed object keeps the first of two members that share a name. + let slot = names.insert(name).inserted ? members.count : nil + let offset = reader.offset + var array = (count: 0, allObjects: false) + if try reader.nextSignificantByte() == JSONDocumentReader.openBracket { + array = try elements(of: &reader, slot: slot, visit: visit) + } else { + try visit(&reader, nil) + } + if let slot { + members.append(Member( + name: name, slot: slot, offset: offset, + elementCount: array.count, holdsRows: array.allObjects + )) + } + } while try reader.continueObject() + } + try reader.expectEnd() + + guard !members.isEmpty, members.allSatisfy({ $0.holdsRows }) else { return .singleObject(offset: objectOffset) } + let match = targetTable.flatMap { target in + members.first { $0.name.caseInsensitiveCompare(target) == .orderedSame } + ?? members.first { $0.name.split(separator: ".").last?.caseInsensitiveCompare(target) == .orderedSame } + } + guard let table = match ?? (members.count == 1 ? members[0] : nil) else { + throw JSONImportParsing.noMatchingTableError + } + return .array(offset: table.offset, elementCount: table.elementCount, slot: table.slot) + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift new file mode 100644 index 0000000000..8bace6ac1c --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift @@ -0,0 +1,367 @@ +// +// JSONDocumentReader.swift +// JSONImportPlugin +// + +import Foundation + +enum JSONDocumentError: LocalizedError, Equatable { + case endsEarly + case unexpectedCharacter(byteOffset: Int) + case invalidText + + var errorDescription: String? { + switch self { + case .endsEarly: + return String(localized: "The JSON file ends before its last value is complete.") + case .unexpectedCharacter(let byteOffset): + return String(format: String(localized: "The JSON file is not valid at byte %@."), (byteOffset + 1).formatted()) + case .invalidText: + return String(localized: "The JSON file has text that is not valid in its encoding.") + } + } +} + +// Walks a JSON file one chunk at a time. It finds where each value starts and ends without +// building it, so the file is held one chunk and one value at a time. `JSONSerialization` parses +// each value a caller asks for, so every value reads exactly as it would in a whole-file parse. +// UTF-16 and UTF-32 text is turned into UTF-8 as it is read, so `offset` counts UTF-8 bytes and +// `fileOffset` the file's own. +struct JSONDocumentReader { + static let defaultChunkSize = 1 << 20 + + private static let quote: UInt8 = 0x22 + private static let backslash: UInt8 = 0x5C + private static let comma: UInt8 = 0x2C + private static let colon: UInt8 = 0x3A + static let openBracket: UInt8 = 0x5B + private static let closeBracket: UInt8 = 0x5D + static let openBrace: UInt8 = 0x7B + private static let closeBrace: UInt8 = 0x7D + + private let handle: FileHandle + private let chunkSize: Int + private let checkCancellation: () throws -> Void + private var buffer: [UInt8] = [] + private var position = 0 + private var bufferStart = 0 + private var valueStart: Int? + private var reachedEnd = false + private var decoder: JSONTextDecoder? + // Where the text starts in the file, past any byte order mark, and how many of its code units + // the buffer has already dropped. Only a decoded file uses them. + private var textStart = 0 + private var droppedUnits = 0 + + init( + url: URL, + chunkSize: Int = Self.defaultChunkSize, + checkCancellation: @escaping () throws -> Void = { try Task.checkCancellation() } + ) throws { + handle = try FileHandle(forReadingFrom: url) + self.chunkSize = max(1, chunkSize) + self.checkCancellation = checkCancellation + } + + var offset: Int { bufferStart + position } + + var fileOffset: Int { fileOffset(atBufferIndex: position) } + + var heldByteCount: Int { buffer.count } + + func close() { + try? handle.close() + } + + // Steps over a byte order mark and, for UTF-16 or UTF-32 text, decodes what was read so far. + mutating func startDocument() throws { + while buffer.count - position < 4, try refill() {} + let detected = JSONTextEncoding.detect(buffer[position...]) + guard detected.encoding != .utf8 else { + position += detected.markLength + return + } + var decoder = JSONTextDecoder(encoding: detected.encoding) + textStart = bufferStart + position + detected.markLength + let raw = Data(buffer[(position + detected.markLength)...]) + buffer = try decoder.utf8(from: raw) + if reachedEnd { try decoder.finish() } + self.decoder = decoder + bufferStart = 0 + position = 0 + droppedUnits = 0 + } + + // `offset` is one `self.offset` gave. Decoded text has no file position for it, so the text is + // decoded again from its start up to there. + mutating func seek(to offset: Int) throws { + buffer.removeAll(keepingCapacity: true) + position = 0 + valueStart = nil + reachedEnd = false + guard decoder != nil else { + try handle.seek(toOffset: UInt64(offset)) + bufferStart = offset + return + } + try handle.seek(toOffset: UInt64(textStart)) + decoder?.reset() + bufferStart = 0 + droppedUnits = 0 + while bufferStart + buffer.count < offset { + position = buffer.count + guard try refill() else { throw JSONDocumentError.endsEarly } + } + position = offset - bufferStart + } + + // The next byte that is not JSON whitespace, left unread, or nil at the end of the file. + mutating func nextSignificantByte() throws -> UInt8? { + while true { + while position < buffer.count { + switch buffer[position] { + case 0x20, 0x09, 0x0A, 0x0D: + position += 1 + default: + return buffer[position] + } + } + guard try refill() else { return nil } + } + } + + // MARK: - Containers + + mutating func enterArray() throws -> Bool { + try enter(opening: Self.openBracket, closing: Self.closeBracket) + } + + mutating func enterObject() throws -> Bool { + try enter(opening: Self.openBrace, closing: Self.closeBrace) + } + + mutating func continueArray() throws -> Bool { + try continueAfterValue(closing: Self.closeBracket) + } + + mutating func continueObject() throws -> Bool { + try continueAfterValue(closing: Self.closeBrace) + } + + mutating func memberName() throws -> String { + guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard byte == Self.quote else { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } + let name = try readValue() + guard let decoded = try JSONSerialization.jsonObject(with: name, options: .fragmentsAllowed) as? String else { + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset(atBufferIndex: position - name.count)) + } + try consume(Self.colon) + return decoded + } + + mutating func expectEnd() throws { + guard try nextSignificantByte() == nil else { + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) + } + } + + // Consumes the opening bracket and tells whether an element or member follows it. + private mutating func enter(opening: UInt8, closing: UInt8) throws -> Bool { + try consume(opening) + guard let next = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard next == closing else { return true } + position += 1 + return false + } + + // Consumes the comma and returns true, or consumes the closing bracket and returns false. + // `JSONSerialization` takes one comma before a closing bracket, so a file it reads reads here too. + private mutating func continueAfterValue(closing: UInt8) throws -> Bool { + guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard byte == Self.comma || byte == closing else { + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) + } + position += 1 + if byte == closing { return false } + guard try nextSignificantByte() == closing else { return true } + position += 1 + return false + } + + private mutating func consume(_ expected: UInt8) throws { + guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } + guard byte == expected else { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } + position += 1 + } + + // MARK: - Values + + // The bytes of the next value. They stay in the buffer while it is read, so a value costs its own size and no more. + mutating func readValue() throws -> Data { + guard try nextSignificantByte() != nil else { throw JSONDocumentError.endsEarly } + valueStart = position + defer { valueStart = nil } + try passValue() + guard let start = valueStart else { return Data() } + return Data(buffer[start.. Bool in + while index < bytes.count { + let byte = bytes[index] + index += 1 + if escaped { + escaped = false + } else if byte == Self.backslash { + escaped = true + } else if byte == Self.quote { + return true + } + } + return false + } + position = index + if closed { return } + guard try refill() else { throw JSONDocumentError.endsEarly } + } + } + + private mutating func passContainer() throws { + var closers: [UInt8] = [] + var inString = false + var escaped = false + while true { + var index = position + var mismatched = false + let closed = buffer.withUnsafeBufferPointer { bytes -> Bool in + while index < bytes.count { + let byte = bytes[index] + index += 1 + if inString { + if escaped { + escaped = false + } else if byte == Self.backslash { + escaped = true + } else if byte == Self.quote { + inString = false + } + continue + } + switch byte { + case Self.quote: + inString = true + case Self.openBrace: + closers.append(Self.closeBrace) + case Self.openBracket: + closers.append(Self.closeBracket) + case Self.closeBrace, Self.closeBracket: + guard closers.last == byte else { + mismatched = true + return true + } + closers.removeLast() + if closers.isEmpty { return true } + default: + break + } + } + return false + } + position = index + if mismatched { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset(atBufferIndex: position - 1)) } + if closed { return } + guard try refill() else { throw JSONDocumentError.endsEarly } + } + } + + // A number, `true`, `false` or `null` runs to the next delimiter. `JSONSerialization` checks its spelling. + private mutating func passLiteral() throws { + while true { + while position < buffer.count { + switch buffer[position] { + case 0x20, 0x09, 0x0A, 0x0D, Self.comma, Self.colon, Self.quote, + Self.openBracket, Self.closeBracket, Self.openBrace, Self.closeBrace: + return + default: + position += 1 + } + } + guard try refill() else { return } + } + } + + // MARK: - Buffer + + // Drops what was read before the value in progress, then appends the next chunk. + private mutating func refill() throws -> Bool { + guard !reachedEnd else { return false } + try checkCancellation() + let keep = valueStart ?? position + if keep > 0 { + if decoder != nil { + droppedUnits += codeUnits(in: buffer[0.. Int { + guard let decoder else { return bufferStart + index } + let units = droppedUnits + codeUnits(in: buffer[0..) -> Int { + let countsPairs = decoder?.encoding.unitWidth == 2 + var units = 0 + for byte in bytes where byte & 0xC0 != 0x80 { + units += countsPairs && byte >= 0xF0 ? 2 : 1 + } + return units + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift new file mode 100644 index 0000000000..d11b26c8b3 --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift @@ -0,0 +1,116 @@ +// +// JSONElementBatches.swift +// JSONImportPlugin +// + +import Foundation +import TableProPluginKit + +// The values of a `.json` file that become rows. A value `JSONSerialization` cannot read throws in +// every error mode, as it failed the whole-file parse the import used to run. +protocol JSONElementSequence { + mutating func next() throws -> Any? + func close() +} + +// The elements of one array of the file, each read and parsed as it is reached. +struct JSONStreamedElements: JSONElementSequence { + private var reader: JSONDocumentReader + private var hasMore: Bool + + init(reader: JSONDocumentReader, arrayAt offset: Int) throws { + self.reader = reader + try self.reader.seek(to: offset) + hasMore = try self.reader.enterArray() + } + + var heldByteCount: Int { reader.heldByteCount } + + mutating func next() throws -> Any? { + guard hasMore else { return nil } + let value = try reader.readValue() + hasMore = try reader.continueArray() + let object: Any = try autoreleasepool { try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed) } + return object + } + + func close() { + reader.close() + } +} + +// A file whose top-level object is the one row, built member by member. +struct JSONSingleRow: JSONElementSequence { + private var row: [String: PluginCellValue]? + + init(_ row: [String: PluginCellValue]) { + self.row = row + } + + mutating func next() -> Any? { + defer { row = nil } + return row + } + + func close() {} +} + +// Hands a `.json` file's rows to `RowImportRunner` a batch at a time, numbered by their place in the +// array, and keeps the elements that are not objects, which Skip and Continue passes over. +struct JSONElementBatches { + private var elements: any JSONElementSequence + private let elementsPerBatch: Int + private let skipsUnreadableElements: Bool + private let maxRecordedErrors: Int + + private(set) var elementsRead = 0 + private(set) var unreadableElements: [PluginImportResult.ImportStatementError] = [] + + // Every unreadable element, counted past the end of the capped `unreadableElements` list. + private(set) var unreadableElementCount = 0 + + init( + elements: any JSONElementSequence, + elementsPerBatch: Int, + skipsUnreadableElements: Bool, + maxRecordedErrors: Int + ) { + self.elements = elements + self.elementsPerBatch = max(1, elementsPerBatch) + self.skipsUnreadableElements = skipsUnreadableElements + self.maxRecordedErrors = maxRecordedErrors + } + + mutating func next() throws -> [RowImportRunner.Entry]? { + var batch: [RowImportRunner.Entry] = [] + var elementsInBatch = 0 + while elementsInBatch < elementsPerBatch, let element = try elements.next() { + elementsInBatch += 1 + elementsRead += 1 + let number = elementsRead + do { + let row = try autoreleasepool { try JSONImportParsing.row(from: element) } + guard !row.isEmpty else { continue } + batch.append((number, row)) + } catch { + guard skipsUnreadableElements else { throw error } + recordUnreadableElement(number, error: error) + } + } + return elementsInBatch == 0 ? nil : batch + } + + func close() { + elements.close() + } + + private mutating func recordUnreadableElement(_ number: Int, error: any Error) { + unreadableElementCount += 1 + guard unreadableElements.count < maxRecordedErrors else { return } + unreadableElements.append(.init( + statement: "row \(number)", + line: number, + errorMessage: error.localizedDescription + )) + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift index 636afd94f9..f4c5acf4a8 100644 --- a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift +++ b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift @@ -15,10 +15,14 @@ struct JSONFieldSurvey { mutating func add(_ row: NSDictionary) { row.enumerateKeysAndObjects { key, value, _ in guard let name = key as? String else { return } - profiles[name, default: JSONFieldProfile()].add(value) + add(value, named: name) } } + mutating func add(_ value: Any, named name: String) { + profiles[name, default: JSONFieldProfile()].add(value) + } + var fields: [PluginImportField] { profiles.keys.sorted().map { name in let profile = profiles[name] ?? JSONFieldProfile() diff --git a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift index 9389cba392..e92faf3392 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -2,10 +2,6 @@ // JSONImportParsing.swift // JSONImportPlugin // -// Pure JSON parsing, row extraction, and field inference. Kept free of the -// plugin's loadable-bundle and SwiftUI surface so it can be compiled into the -// test target directly (a loadable .tableplugin cannot be linked by tests). -// import Foundation import TableProNumberFormatting @@ -33,19 +29,93 @@ enum JSONImportParsing { line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D } } - static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] { + static var unexpectedShapeError: PluginImportError { + .importFailed("Expected a JSON array of objects or a table-keyed object") + } + + static var noMatchingTableError: PluginImportError { + .importFailed("The file contains multiple tables and none matches the target table") + } + + // The values of a `.json` file that become rows, and how many there are. The file is read one + // element at a time after a pass that finds its rows and checks its structure, so a broken file + // fails before the first row is imported. + static func rowElements( + at url: URL, + targetTable: String?, + checkCancellation: @escaping () throws -> Void + ) throws -> (elements: any JSONElementSequence, count: Int) { + var reader = try JSONDocumentReader(url: url, checkCancellation: checkCancellation) + do { + let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, _ in + try reader.skipValue() + } + switch rows { + case .array(let offset, let count, _): + return (try JSONStreamedElements(reader: reader, arrayAt: offset), count) + case .singleObject(let offset): + defer { reader.close() } + var row: [String: PluginCellValue] = [:] + try members(ofObjectAt: offset, in: &reader) { name, value in + row[name] = cellValue(from: value) + } + return (JSONSingleRow(row), 1) + } + } catch { + reader.close() + throw error + } + } + + // Each member of the object at `offset`, parsed one at a time. A parsed object keeps the first + // of two members that share a name, so the later one is passed over. + static func members( + ofObjectAt offset: Int, + in reader: inout JSONDocumentReader, + _ body: (String, Any) throws -> Void + ) throws { + try reader.seek(to: offset) + guard try reader.enterObject() else { return } + var names: Set = [] + repeat { + let name = try reader.memberName() + let value = try reader.readValue() + guard names.insert(name).inserted else { continue } + try autoreleasepool { + try body(name, try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed)) + } + } while try reader.continueObject() + } + + static func row(from element: Any) throws -> [String: PluginCellValue] { + if let row = element as? [String: PluginCellValue] { + return row + } + guard let object = element as? NSDictionary else { + throw PluginImportError.importFailed(String(localized: "Each array element must be a JSON object")) + } + return convertRow(object) + } + + static func parseElements(at url: URL, targetTable: String?) throws -> [Any] { let data = try Data(contentsOf: url) let object = try JSONSerialization.jsonObject(with: data) - return try extractRows(from: object, targetTable: targetTable) + return try extractElements(from: object, targetTable: targetTable) } static func extractRows(from object: Any, targetTable: String?) throws -> [NSDictionary] { + try extractElements(from: object, targetTable: targetTable).compactMap { $0 as? NSDictionary } + } + + // A top-level array's elements come back whatever they are, so the import can report each one + // that is not an object instead of leaving it out unseen. + static func extractElements(from object: Any, targetTable: String?) throws -> [Any] { if let array = object as? [Any] { - return array.compactMap { $0 as? NSDictionary } + return array } guard let dict = object as? [String: Any] else { - throw PluginImportError.importFailed("Expected a JSON array of objects or a table-keyed object") + throw unexpectedShapeError } let tables = dict.compactMapValues { value -> [Any]? in @@ -59,12 +129,12 @@ enum JSONImportParsing { } if let targetTable, let match = matchTable(in: tables, to: targetTable) { - return match.compactMap { $0 as? NSDictionary } + return match } if tables.count == 1, let only = tables.values.first { - return only.compactMap { $0 as? NSDictionary } + return only } - throw PluginImportError.importFailed("The file contains multiple tables and none matches the target table") + throw noMatchingTableError } private static func matchTable(in tables: [String: [Any]], to target: String) -> [Any]? { @@ -112,11 +182,38 @@ enum JSONImportParsing { static func detectFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { try Task.checkCancellation() guard isLineDelimited(url) else { - return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) + return try detectFields(inDocumentAt: url, targetTable: targetTable) } return try detectFields(inLinesAt: url) } + // Every value is parsed once, one at a time, and each array keeps a survey of its own, so the + // target table's fields are known when the pass ends. A value that does not parse fails the + // detection, as it fails a whole-file parse. + static func detectFields(inDocumentAt url: URL, targetTable: String?) throws -> [PluginImportField] { + var reader = try JSONDocumentReader(url: url) + defer { reader.close() } + var surveys: [Int: JSONFieldSurvey] = [:] + let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, slot in + let value = try reader.readValue() + try autoreleasepool { + let object = try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed) + guard let slot, let row = object as? NSDictionary else { return } + surveys[slot, default: JSONFieldSurvey()].add(row) + } + } + switch rows { + case .array(_, _, let slot): + return surveys[slot]?.fields ?? [] + case .singleObject(let offset): + var survey = JSONFieldSurvey() + try members(ofObjectAt: offset, in: &reader) { name, value in + survey.add(value, named: name) + } + return survey.fields + } + } + static func detectFields(inLinesAt url: URL) throws -> [PluginImportField] { var survey = JSONFieldSurvey() var lines = try JSONLineReader(url: url) diff --git a/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift index a765985859..78a8533992 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -47,9 +47,9 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug } private static let batchSize = 500 - /// One budget shared with the rows the runner itself recorded, so the two lists together stay - /// inside the cap the import dialog documents. The skip *count* is deliberately uncapped: a - /// truncated list must not also under-report how much of the file was left out. + // One budget shared with the rows the runner itself recorded, so the two lists together stay + // inside the cap the import dialog documents. The skip count is uncapped: a truncated list must + // not also under-report how much of the file was left out. private static let maxRecordedErrors = 1_000 private let lineDelimitedFields = JSONFieldDetectionCache() @@ -83,22 +83,24 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug unreadableLines = batches.unreadableLines unreadableLineCount = batches.unreadableLineCount } else { - let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) - progress.setEstimatedTotal(rawRows.count) - var cursor = 0 + let document = try JSONImportParsing.rowElements( + at: url, targetTable: sink.targetTable, checkCancellation: progress.checkCancellation + ) + progress.setEstimatedTotal(document.count) + var batches = JSONElementBatches( + elements: document.elements, + elementsPerBatch: Self.batchSize, + skipsUnreadableElements: settings.errorHandling == .skipAndContinue, + maxRecordedErrors: Self.maxRecordedErrors + ) + defer { batches.close() } outcome = try await RowImportRunner.run( configuration: configuration, sink: sink, progress: progress ) { - guard cursor < rawRows.count else { return nil } - let end = min(cursor + Self.batchSize, rawRows.count) - let batch = (cursor.. RowImportRunner.Entry? in - let row = JSONImportParsing.convertRow(rawRows[index]) - guard !row.isEmpty else { return nil } - return (index + 1, row) - } - cursor = end - return batch + try batches.next() } + unreadableLines = batches.unreadableElements + unreadableLineCount = batches.unreadableElementCount } return PluginImportResult( diff --git a/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift new file mode 100644 index 0000000000..6d937f2826 --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift @@ -0,0 +1,109 @@ +// +// JSONTextDecoder.swift +// JSONImportPlugin +// + +import Foundation + +// The five encodings `JSONSerialization` reads, with or without a byte order mark. +enum JSONTextEncoding: Equatable { + case utf8 + case utf16BigEndian + case utf16LittleEndian + case utf32BigEndian + case utf32LittleEndian + + var unitWidth: Int { + switch self { + case .utf8: return 1 + case .utf16BigEndian, .utf16LittleEndian: return 2 + case .utf32BigEndian, .utf32LittleEndian: return 4 + } + } + + // RFC 4627, section 3: the first two characters are ASCII, so the zero bytes in the first four + // name the encoding when no byte order mark does. + static func detect(_ head: some Collection) -> (encoding: JSONTextEncoding, markLength: Int) { + let bytes = Array(head.prefix(4)) + if bytes.starts(with: [0xEF, 0xBB, 0xBF]) { return (.utf8, 3) } + if bytes.starts(with: [0x00, 0x00, 0xFE, 0xFF]) { return (.utf32BigEndian, 4) } + if bytes.starts(with: [0xFF, 0xFE, 0x00, 0x00]) { return (.utf32LittleEndian, 4) } + if bytes.starts(with: [0xFE, 0xFF]) { return (.utf16BigEndian, 2) } + if bytes.starts(with: [0xFF, 0xFE]) { return (.utf16LittleEndian, 2) } + guard bytes.count >= 2 else { return (.utf8, 0) } + if bytes.count >= 4 { + if bytes[0] == 0, bytes[1] == 0, bytes[2] == 0 { return (.utf32BigEndian, 0) } + if bytes[1] == 0, bytes[2] == 0, bytes[3] == 0 { return (.utf32LittleEndian, 0) } + } + if bytes[0] == 0 { return (.utf16BigEndian, 0) } + if bytes[1] == 0 { return (.utf16LittleEndian, 0) } + return (.utf8, 0) + } +} + +// Turns UTF-16 or UTF-32 text into UTF-8 a chunk at a time. A code unit, or a surrogate pair, that a +// chunk cuts in two waits for the next chunk. +struct JSONTextDecoder { + let encoding: JSONTextEncoding + private var pending: [UInt8] = [] + + init(encoding: JSONTextEncoding) { + self.encoding = encoding + } + + mutating func reset() { + pending.removeAll(keepingCapacity: true) + } + + mutating func utf8(from chunk: Data) throws -> [UInt8] { + guard encoding != .utf8 else { return Array(chunk) } + pending.append(contentsOf: chunk) + var usable = pending.count - pending.count % encoding.unitWidth + var output: [UInt8] = [] + output.reserveCapacity(usable) + let failed: Bool + switch encoding { + case .utf8: + return Array(chunk) + case .utf16BigEndian, .utf16LittleEndian: + var units: [UInt16] = [] + units.reserveCapacity(usable / 2) + var index = 0 + while index < usable { + let first = UInt16(pending[index]) + let second = UInt16(pending[index + 1]) + units.append(encoding == .utf16BigEndian ? first << 8 | second : second << 8 | first) + index += 2 + } + if let last = units.last, (0xD800...0xDBFF).contains(last) { + units.removeLast() + usable -= 2 + } + failed = transcode(units.makeIterator(), from: UTF16.self, to: UTF8.self, stoppingOnError: true) { + output.append($0) + } + case .utf32BigEndian, .utf32LittleEndian: + var units: [UInt32] = [] + units.reserveCapacity(usable / 4) + var index = 0 + while index < usable { + let bytes = pending[index..<(index + 4)].map(UInt32.init) + units.append(encoding == .utf32BigEndian + ? bytes[0] << 24 | bytes[1] << 16 | bytes[2] << 8 | bytes[3] + : bytes[3] << 24 | bytes[2] << 16 | bytes[1] << 8 | bytes[0]) + index += 4 + } + failed = transcode(units.makeIterator(), from: UTF32.self, to: UTF8.self, stoppingOnError: true) { + output.append($0) + } + } + if failed { throw JSONDocumentError.invalidText } + pending.removeSubrange(0.. KafkaBrowsePage { let metadata = try await cluster.metadata(topics: [query.topic]) @@ -58,10 +64,46 @@ enum KafkaBrowseEngine { // Each partition contributes at most `wanted` records, because the merge cannot know // in advance how the messages are distributed: one partition may hold the whole page. var collected: [KafkaRecord] = [] - var truncated = false + var truncated = query.skip + query.limit > maximumOverFetch + var walkedStarts = window.start for partition in selected { try Task.checkCancellation() guard let start = window.start[partition] else { continue } + // A tail scan must not read past the end it was anchored to, or page two would show + // messages produced after page one and the window would not be a window at all. + if window.readsBackward, let end = window.tail[partition] { + var walk = KafkaTailWalk( + wanted: wanted, + end: end, + firstStart: start, + floor: window.floor[partition] ?? start + ) + // The page's own range is read as before; the ranges past it share the budget. + var walkBytesLeft: Int? + while let range = walk.nextRange { + try Task.checkCancellation() + let result = try await KafkaFetchRequest.fetch( + topic: query.topic, + partition: partition, + startOffset: range.start, + maximumRecords: walk.needed, + endOffset: range.end, + keepingNewest: walk.needed, + byteBudget: walkBytesLeft, + cluster: cluster + ) + if result.stoppedAtByteBudget { + truncated = true + break + } + walkBytesLeft = walkBytesLeft.map { $0 - result.bytesRead } ?? tailWalkByteBudget + if result.truncated { truncated = true } + walk.absorb(result.records) + } + collected.append(contentsOf: walk.records) + walkedStarts[partition] = walk.start + continue + } let result = try await KafkaFetchRequest.fetch( topic: query.topic, partition: partition, @@ -69,13 +111,7 @@ enum KafkaBrowseEngine { maximumRecords: wanted, cluster: cluster ) - var records = result.records - // A tail scan must not read past the end it was anchored to, or page two would show - // messages produced after page one and the window would not be a window at all. - if let end = window.tail[partition] { - records = records.filter { $0.offset < end } - } - collected.append(contentsOf: records) + collected.append(contentsOf: result.records) if result.truncated { truncated = true } } @@ -87,7 +123,10 @@ enum KafkaBrowseEngine { readsBackward: window.readsBackward ) if ordered.count > query.skip + query.limit { truncated = true } - return KafkaBrowsePage(records: page, window: window, truncated: truncated) + let walked = KafkaScanWindow( + start: walkedStarts, tail: window.tail, readsBackward: window.readsBackward, floor: window.floor + ) + return KafkaBrowsePage(records: page, window: walked, truncated: truncated) } /// The partitions to read, or an error naming the ones the topic does not have. @@ -132,14 +171,16 @@ enum KafkaBrowseEngine { cluster: cluster ) var starts: [Int32: Int64] = [:] + var floors: [Int32: Int64] = [:] for (partition, end) in kept { // No fallback offset. A partition whose earliest offset did not come back has // an unknown floor, and anchoring it at zero would send the page to the start of // a log whose first surviving message may be far past it. guard let floor = earliest[partition] else { continue } starts[partition] = KafkaOffsetMath.stepBack(from: end, by: step, floor: floor) + floors[partition] = floor } - return KafkaScanWindow(start: starts, tail: kept, readsBackward: true) + return KafkaScanWindow(start: starts, tail: kept, readsBackward: true, floor: floors) case .oldest: let starts = try await KafkaOffsetsRequest.listOffsets( @@ -200,11 +241,13 @@ enum KafkaBrowseEngine { // says and what the partition count would otherwise hide. var starts: [Int32: Int64] = [:] var ends: [Int32: Int64] = [:] + var floors: [Int32: Int64] = [:] for bound in bounds { starts[bound.partition] = KafkaOffsetMath.stepBack(from: bound.latest, by: step, floor: bound.earliest) ends[bound.partition] = bound.latest + floors[bound.partition] = bound.earliest } - return KafkaScanWindow(start: starts, tail: ends, readsBackward: true) + return KafkaScanWindow(start: starts, tail: ends, readsBackward: true, floor: floors) } } } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift index 9365bfa506..0558392347 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift @@ -9,6 +9,13 @@ struct KafkaFetchResult: Sendable { /// return more. Distinguishing this from "the partition ended" is what keeps a page from /// silently stopping early. let truncated: Bool + // Where the next read starts, past control, aborted and compacted batches too. For one reply, nil when no batch + // came back whole; for a whole fetch, nil only when nothing can follow Int64.max. + let nextOffset: Int64? + // The replies passed `byteBudget` before the read reached its end, so it returns no records. + var stoppedAtByteBudget = false + // Record batch bytes the replies carried, aborted and control batches included. + var bytesRead = 0 } enum KafkaFetchRequest { @@ -20,39 +27,62 @@ enum KafkaFetchRequest { private static let defaultPartitionBudget: Int32 = 1 * 1_024 * 1_024 private static let maximumPartitionBudget: Int32 = 64 * 1_024 * 1_024 - // Kafka guarantees progress only for the first message in range (KIP-74), so the byte budget grows on an empty reply. + // Kafka guarantees progress only for the first message in range (KIP-74), so the byte budget grows on a reply + // that held no whole batch. `endOffset` stops the read there, so a tail window never reads past its anchor. + // With `keepingNewest` the read runs to `endOffset` and holds only that many of the newest records it passed. + // It gives up once the replies carried more than `byteBudget` bytes. static func fetch( topic: String, partition: Int32, startOffset: Int64, maximumRecords: Int, + endOffset: Int64? = nil, + keepingNewest: Int? = nil, + byteBudget: Int? = nil, cluster: KafkaCluster ) async throws -> KafkaFetchResult { var budget = defaultPartitionBudget var collected: [KafkaRecord] = [] var highWatermark: Int64 = startOffset - var offset = startOffset + var offset: Int64? = startOffset var sawTruncation = false var decompression: DecompressionBudget? + var bytesRead = 0 - while collected.count < maximumRecords { + while keepingNewest != nil || collected.count < maximumRecords, let from = offset { + if let endOffset, from >= endOffset { break } try Task.checkCancellation() let allowance = try KafkaCompression.fetchBudget(requesting: budget, after: decompression) let (page, spent) = try await fetchOnce( topic: topic, partition: partition, - startOffset: offset, + startOffset: from, budget: budget, decompression: allowance, cluster: cluster ) decompression = spent highWatermark = page.highWatermark + bytesRead += page.bytesRead + if let byteBudget, bytesRead > byteBudget { + return KafkaFetchResult( + records: [], highWatermark: page.highWatermark, truncated: true, nextOffset: nil, + stoppedAtByteBudget: true, bytesRead: bytesRead + ) + } if page.records.isEmpty { - // Nothing came back but the log says there is more: the next message does not + // Commit and abort markers, aborted runs and batches compaction emptied hold offsets but no + // records, so a reply of only those is progress, not a message too big for the budget. + if let next = page.nextOffset, next > from { + offset = next + budget = defaultPartitionBudget + if next >= page.highWatermark { break } + continue + } + // No whole batch came back but the log says there is more: the next message does not // fit the budget. Grow it and try once more before giving up on the partition. - let hasMore = offset < page.highWatermark + let hasMore = from < page.highWatermark if hasMore, budget < maximumPartitionBudget { budget = min(budget * 4, maximumPartitionBudget) continue @@ -61,18 +91,26 @@ enum KafkaFetchRequest { break } - collected.append(contentsOf: page.records) - guard let last = page.records.last, let next = KafkaOffsetMath.offset(after: last.offset) else { break } - offset = next + let kept = endOffset.map { end in page.records.filter { $0.offset < end } } ?? page.records + collected.append(contentsOf: kept) + if let keepingNewest, collected.count > keepingNewest { + collected.removeFirst(collected.count - keepingNewest) + } + let afterRecords = page.records.last.flatMap { KafkaOffsetMath.offset(after: $0.offset) } + offset = afterRecords.map { max($0, page.nextOffset ?? $0) } budget = defaultPartitionBudget - if offset >= page.highWatermark { break } + if let next = offset, next >= page.highWatermark { break } } - if collected.count > maximumRecords { + if keepingNewest == nil, collected.count > maximumRecords { collected = Array(collected.prefix(maximumRecords)) sawTruncation = true + offset = collected.last.flatMap { KafkaOffsetMath.offset(after: $0.offset) } } - return KafkaFetchResult(records: collected, highWatermark: highWatermark, truncated: sawTruncation) + return KafkaFetchResult( + records: collected, highWatermark: highWatermark, truncated: sawTruncation, nextOffset: offset, + bytesRead: bytesRead + ) } // Returns the decompression budget as the reply left it, so the next page continues from it. @@ -170,7 +208,7 @@ enum KafkaFetchRequest { if flexible { try partitionReader.taggedFields() } guard let blob, !blob.isEmpty else { - return KafkaFetchResult(records: [], highWatermark: highWatermark, truncated: false) + return KafkaFetchResult(records: [], highWatermark: highWatermark, truncated: false, nextOffset: nil) } let decoded = try KafkaRecordBatchDecoder.decode( blob: blob, @@ -185,7 +223,9 @@ enum KafkaFetchRequest { return KafkaFetchResult( records: trimmed, highWatermark: highWatermark, - truncated: decoded.truncated + truncated: decoded.truncated, + nextOffset: decoded.nextOffset, + bytesRead: blob.count ) } if flexible { try reader.taggedFields() } @@ -193,7 +233,7 @@ enum KafkaFetchRequest { } // One topic and one partition were requested, so there is one answer. let result = topics.flatMap { $0 }.first - ?? KafkaFetchResult(records: [], highWatermark: startOffset, truncated: false) + ?? KafkaFetchResult(records: [], highWatermark: startOffset, truncated: false, nextOffset: nil) return (result, spent) } } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift index 9f811a0625..4cf3de5658 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift @@ -99,11 +99,11 @@ final class KafkaLiveSubscription: PluginLiveSubscription, @unchecked Sendable { for record in page.records { continuation.yield(KafkaFollowCursor.message(from: record, topic: cursor.topic)) } - if let last = page.records.last { - guard let next = KafkaOffsetMath.offset(after: last.offset) else { return } - offset = next - } else if ContinuousClock.now - started < Self.idleWait / 2 { - // A trailing transaction marker answers at once and reads as no record, + guard let next = page.nextOffset else { return } + let moved = next > offset + offset = next + if !moved, ContinuousClock.now - started < Self.idleWait / 2 { + // A reply that answers at once with nothing to move past would spin, // where an idle partition makes the broker wait out the long poll. try await Task.sleep(for: Self.idleWait) } diff --git a/apple/Plugins/KingbaseESDriverPlugin/KingbaseESPluginDriver.swift b/apple/Plugins/KingbaseESDriverPlugin/KingbaseESPluginDriver.swift index 19e459e8d6..5d99829c88 100644 --- a/apple/Plugins/KingbaseESDriverPlugin/KingbaseESPluginDriver.swift +++ b/apple/Plugins/KingbaseESDriverPlugin/KingbaseESPluginDriver.swift @@ -14,6 +14,9 @@ final class KingbaseESPluginDriver: PostgreSQLPluginDriver, @unchecked Sendable ) } + // Not measured against a KingbaseES catalog yet, so no Extensions group. + override var serverObjectKinds: [PluginServerObjectKind] { [] } + override func probeAfterConnect() async throws { let result = try await catalogRead(KingbaseESCompatibilityMode.probeQuery) let detected = try KingbaseESCompatibilityMode(setting: result.rows.first?.first?.asText) diff --git a/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift b/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift index c60cbb0d99..a726914a4f 100644 --- a/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift +++ b/apple/Plugins/LibSQLDriverPlugin/LibSQLPluginDriver.swift @@ -283,47 +283,46 @@ final class LibSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in + let spill = PluginRowSpill(continuation: continuation) let streamTask = Task { do { - try await self.performStreamRows(query: query, continuation: continuation) + try await self.performStreamRows(query: query, into: spill) } catch { - continuation.finish(throwing: error) + spill.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in + abort.onAbort { + spill.abandon() streamTask.cancel() } } } - private func performStreamRows( - query: String, - continuation: AsyncThrowingStream.Continuation - ) async throws { + private func performStreamRows(query: String, into spill: PluginRowSpill) async throws { switch getBackend() { case .remote(let client): let result = try await client.execute(sql: query) let columns = result.cols.map(\.name) let columnTypeNames = result.cols.map { $0.decltype ?? "" } - continuation.yield(.header(PluginStreamHeader( + spill.yield(PluginStreamHeader( columns: columns, columnTypeNames: columnTypeNames, estimatedRowCount: nil - ))) + )) if !result.rows.isEmpty { let rows = result.rows.map { rawRow in rawRow.map(\.stringValue).map(PluginCellValue.fromOptional) } - continuation.yield(.rows(rows)) + spill.deliver(rows) } - continuation.finish() + spill.finish() case .local(let localBackend): - try await localBackend.streamQuery(query, continuation: continuation) + try await localBackend.streamQuery(query, into: spill) case nil: throw LibSQLError.notConnected } @@ -383,6 +382,8 @@ final class LibSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let query = """ SELECT m.name AS tbl, p.cid, p.name, p.type, p."notnull", p.dflt_value, p.pk diff --git a/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift b/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift index b0d9e348e9..fd456427d9 100644 --- a/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift +++ b/apple/Plugins/LibSQLDriverPlugin/SQLiteLocalBackend.swift @@ -149,10 +149,7 @@ actor SQLiteLocalBackend { ) } - func streamQuery( - _ query: String, - continuation: AsyncThrowingStream.Continuation - ) throws { + func streamQuery(_ query: String, into spill: PluginRowSpill) throws { guard let db else { throw LibSQLError.notConnected } @@ -166,11 +163,11 @@ actor SQLiteLocalBackend { let firstStep = runtime.stepFirst(statement) let columnCount = firstStep.count - continuation.yield(.header(PluginStreamHeader( + spill.yield(PluginStreamHeader( columns: firstStep.names, columnTypeNames: firstStep.typeNames, estimatedRowCount: nil - ))) + )) let batchSize = 5_000 var batch: [PluginRow] = [] @@ -180,31 +177,32 @@ actor SQLiteLocalBackend { while stepResult == SQLITE_ROW { if Task.isCancelled { if !batch.isEmpty { - continuation.yield(.rows(batch)) + spill.deliver(batch) } - continuation.finish(throwing: CancellationError()) + spill.finish(throwing: CancellationError()) return } batch.append(rowValues(of: statement, count: columnCount)) if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // Synchronous on the actor's thread: an await here would let another call reach the handle mid-statement. + guard spill.deliver(batch) else { return } batch.removeAll(keepingCapacity: true) } stepResult = sqlite3_step(statement) } if !batch.isEmpty { - continuation.yield(.rows(batch)) + spill.deliver(batch) } do { try validateCompletion(stepResult, on: db) } catch { - continuation.finish(throwing: error) + spill.finish(throwing: error) return } - continuation.finish() + spill.finish() } private func validateCompletion(_ stepResult: Int32, on db: OpaquePointer) throws { diff --git a/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift b/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift index c2a30356ce..a31ae7d688 100644 --- a/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift +++ b/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift @@ -143,7 +143,13 @@ nonisolated private func freetdsRecordMessage(_ message: MSSQLServerMessage, for return } let key = freetdsConnectionKey(dbproc) + // db-lib hands a login's messages the DBPROCESS it is still building, which has no gate until it is adopted and is + // freed when the login fails, so the first refusal goes where a failed open reads its reason. + let refusesLogin = message.isError && freetdsGate(forKey: key) == nil freetdsErrors.withLock { state in + if refusesLogin, state.global.isEmpty { + state.global = message.text + } state.perConnection[key, default: FreeTDSDiagnostics()].record(message) } } @@ -342,7 +348,7 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { defer { dbloginfree(login) } try configure(login) - let opened: UnsafeMutablePointer? + let opened: (proc: UnsafeMutablePointer?, refusal: String) do { opened = try freetdsConfigFile.withEntry( entry, @@ -350,7 +356,6 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { givingUpWhen: { attempt.isSettled } ) { guard !attempt.isSettled else { throw CancellationError() } - freetdsClearError(for: nil) return try open(login, serverName: entry.name, deadline: deadline) } } catch MSSQLFreeTDSConfigError.nameInUse { @@ -358,8 +363,8 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { } catch let error as MSSQLFreeTDSConfigError { throw MSSQLCoreError.connectionFailed(error.localizedDescription) } - guard let proc = opened else { - throw openFailure() + guard let proc = opened.proc else { + throw openFailure(detail: opened.refusal) } return proc } @@ -373,14 +378,16 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { } } + // A refused login's reason lands in process-wide state the next login clears, so it is read under the same lock. private func open( _ login: UnsafeMutablePointer, serverName: String, deadline: MSSQLConnectDeadline - ) throws -> UnsafeMutablePointer? { + ) throws -> (proc: UnsafeMutablePointer?, refusal: String) { Self.loginStartLock.lock() defer { Self.loginStartLock.unlock() } - return try withKerberosEnvironmentIfNeeded(deadline: deadline) { + freetdsClearError(for: nil) + let proc = try withKerberosEnvironmentIfNeeded(deadline: deadline) { let timeoutMilliseconds = deadline.remainingMilliseconds guard timeoutMilliseconds > 0 else { throw MSSQLCoreError.connectionTimedOut(isKerberos: options.authMethod == .windows) @@ -389,6 +396,7 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { _ = dbsetlogintime(seconds) return dbopen(login, serverName) } + return (proc, proc == nil ? freetdsGetError(for: nil) : "") } private func discardKerberosCache() { @@ -426,8 +434,7 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { } } - private func openFailure() -> MSSQLCoreError { - let detail = freetdsGetError(for: nil) + private func openFailure(detail: String) -> MSSQLCoreError { let msg = detail.isEmpty ? "Check host, port, credentials, and TLS settings" : detail if let kind = MSSQLTLSClassifier.classifySSLError(detail) { return .tlsHandshakeFailed(kind: kind, serverMessage: detail) @@ -1000,7 +1007,7 @@ nonisolated private struct FreeTDSBatchReader { private mutating func readResultSet(columnCount: Int, errorsBefore: Int) throws -> ResultSetEnd { let descriptors = (1...columnCount).map(describeColumn) if case .stream(let continuation) = sink { - continuation.yield(.header(columns: descriptors)) + _ = PluginRowStream.deliverBlocking(.header(columns: descriptors), to: continuation, stops: readerStopped) headerStreamed = true } @@ -1123,21 +1130,26 @@ nonisolated private struct FreeTDSBatchReader { } } + // A reader that left or a Stop is handled at the top of the row loop. private func streamIfFull(_ rows: inout [[MSSQLRawCell]]) { guard case .stream(let continuation) = sink, rows.count >= MSSQLRowLimits.streamBatchSize else { return } - continuation.yield(.rows(rows)) + guard PluginRowStream.deliverBlocking(.rows(rows), to: continuation, stops: readerStopped) else { return } rows.removeAll(keepingCapacity: true) } private func streamRemaining(_ rows: inout [[MSSQLRawCell]]) { guard case .stream(let continuation) = sink, !rows.isEmpty else { return } - continuation.yield(.rows(rows)) + _ = PluginRowStream.deliverBlocking(.rows(rows), to: continuation, stops: readerStopped) rows.removeAll() } private func finishCountOnlyStream() { guard case .stream(let continuation) = sink, !headerStreamed, errorTotal == 0 else { return } - continuation.yield(.affectedRows(rowsAffected)) + _ = PluginRowStream.deliverBlocking(.affectedRows(rowsAffected), to: continuation, stops: readerStopped) + } + + private func readerStopped() -> Bool { + userCancelled() || consumerStopped() } private mutating func absorbMessages() { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift index 0f78551a21..16ed36aa7e 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift @@ -325,6 +325,7 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { .uuidCellText, .userManagement, .temporaryTables, + .columnDefaultMarks, ] } @@ -634,6 +635,26 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await boundedRead(query: query, rowCap: rowCap) + } + + // An external read with a cap ends the request one row past it, as the bounded read does, instead of reading up to + // the emergency ceiling and cutting here. Parameters ride in `sp_executesql`, which only the default path writes. + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + guard context.readOnly, context.origin == .external, parameters?.isEmpty ?? true, let cap = rowCap, cap > 0 else { + return try await PluginParameterBinder.withHostContext(context) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } + } + return try await boundedRead(query: query, rowCap: cap) + } + + private func boundedRead(query: String, rowCap: Int) async throws -> PluginQueryResult { guard let conn = freeTDSConn else { throw MSSQLPluginError.notConnected } @@ -699,9 +720,11 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { guard let conn = freeTDSConn else { return AsyncThrowingStream { $0.finish(throwing: MSSQLPluginError.notConnected) } } - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { - let coreStream = AsyncThrowingStream { coreContinuation in + let coreStream = AsyncThrowingStream( + bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches) + ) { coreContinuation in /// A plain `Task {}` inherits context but is not a child, so cancelling the /// outer task never reaches this one. The abort closure is what does. Task { @@ -720,21 +743,23 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } do { for try await element in coreStream { + let delivered: Bool switch element { case .header(let columns): - continuation.yield(.header(PluginStreamHeader( + delivered = await PluginRowStream.deliver(.header(PluginStreamHeader( columns: columns.map { $0.name }, columnTypeNames: columns.map { $0.type.canonicalName }, estimatedRowCount: nil - ))) + )), to: continuation) case .rows(let batch): let pluginRows: [PluginRow] = batch.map { row in row.map { $0.asPluginCell } } - continuation.yield(.rows(pluginRows)) + delivered = await PluginRowStream.deliver(.rows(pluginRows), to: continuation) case .affectedRows: - break + delivered = true } + guard delivered else { break } } continuation.finish() } catch let error as MSSQLCoreError { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift index 2eec458c6d..04ee0192ab 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift @@ -36,11 +36,12 @@ extension MSSQLPluginDriver { case .addColumn(let column): return MSSQLTableDefinitionSQL.refusal(for: column) case .modifyColumn(let old, let new): - let writesType = old.dataType != new.dataType || old.isNullable != new.isNullable + let alteration = Self.columnAlteration(from: old, to: new) return MSSQLTableDefinitionSQL.refusal( for: new, - writesType: writesType, - writesDefault: writesType || old.defaultValue != new.defaultValue + writesType: alteration.writesType, + writesDefault: alteration.writesType || old.defaultValue != new.defaultValue, + collation: alteration.writesType ? alteration.collation : nil ) default: return nil @@ -60,7 +61,8 @@ extension MSSQLPluginDriver { func generateModifyColumnSQL(table: String, oldColumn: PluginColumnDefinition, newColumn: PluginColumnDefinition) -> String? { let qt = mssqlQualifiedTable(table) var stmts: [String] = [] - let needsTypeChange = oldColumn.dataType != newColumn.dataType || oldColumn.isNullable != newColumn.isNullable + let alteration = Self.columnAlteration(from: oldColumn, to: newColumn) + let needsTypeChange = alteration.writesType let defaultChanged = oldColumn.defaultValue != newColumn.defaultValue // Rename column first so subsequent statements reference the correct name @@ -88,9 +90,14 @@ extension MSSQLPluginDriver { } if needsTypeChange { - guard MSSQLDefinitionText.isColumnType(newColumn.dataType) else { return nil } - let nullable = newColumn.isNullable ? "NULL" : "NOT NULL" - stmts.append("ALTER TABLE \(qt) ALTER COLUMN \(colName) \(newColumn.dataType) \(nullable)") + guard let alter = try? MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: qt, + column: newColumn.name, + type: newColumn.dataType, + collation: alteration.collation, + isNullable: newColumn.isNullable + ) else { return nil } + stmts.append(alter) } // The re-add mirrors the drop above. A type or nullability change drops the constraint too, @@ -104,6 +111,18 @@ extension MSSQLPluginDriver { return stmts.isEmpty ? nil : stmts.joined(separator: ";\n") } + // ALTER COLUMN also runs when only the collation changes, since it is the one statement that sets it. + private static func columnAlteration( + from old: PluginColumnDefinition, + to new: PluginColumnDefinition + ) -> (writesType: Bool, collation: String?) { + let collation = MSSQLSchemaQueries.alterColumnCollation( + type: new.dataType, collation: new.collation, previous: old.collation + ) + let changesCollation = collation.map { $0 != old.collation } ?? false + return (old.dataType != new.dataType || old.isNullable != new.isNullable || changesCollation, collation) + } + func generateDropColumnSQL(table: String, columnName: String) -> String? { "ALTER TABLE \(mssqlQualifiedTable(table)) DROP COLUMN \(quoteIdentifier(columnName))" } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift index 9bf02abdf9..98e731ebcd 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift @@ -64,7 +64,7 @@ extension MSSQLPluginDriver { func fetchViewRowEditing(view: String, schema: String?) async throws -> PluginViewRowEditing? { let schema = effectiveSchema(schema) - let factRows = try await viewCatalogRows(MSSQLViewSources.viewFactsQuery(schema: schema, view: view)) + let factRows = try await catalogRows(MSSQLViewSources.viewFactsQuery(schema: schema, view: view)) guard let facts = MSSQLViewSources.ViewFacts(rows: factRows) else { return nil } if let refusal = facts.refusal { return .readOnly(PluginViewReadOnlyReason(refusal)) } // Once the relation proved to be a view, a failed read must not let it pass for a table. @@ -77,7 +77,7 @@ extension MSSQLPluginDriver { private func rowEditing(throughView view: String, schema: String) async throws -> PluginViewRowEditing { let browse = MSSQLViewSources.BrowseFacts( - rows: try await viewCatalogRows(MSSQLViewSources.browseQuery(schema: schema, view: view)) + rows: try await catalogRows(MSSQLViewSources.browseQuery(schema: schema, view: view)) ) let baseTable: MSSQLViewSources.BaseTable switch browse.source { @@ -87,7 +87,7 @@ extension MSSQLPluginDriver { baseTable = table } - let probeRows = try await viewCatalogRows(MSSQLViewSources.probesQuery( + let probeRows = try await catalogRows(MSSQLViewSources.probesQuery( schema: schema, view: view, writableColumns: browse.writableColumns @@ -100,7 +100,7 @@ extension MSSQLPluginDriver { probeRows: probeRows ) let base = MSSQLViewSources.BaseFacts( - rows: try await viewCatalogRows(MSSQLViewSources.baseColumnsQuery(baseTable)) + rows: try await catalogRows(MSSQLViewSources.baseColumnsQuery(baseTable)) ) return PluginViewRowEditing.resolve( @@ -126,7 +126,7 @@ extension MSSQLPluginDriver { ) } - private func viewCatalogRows(_ query: String) async throws -> [[String?]] { + private func catalogRows(_ query: String) async throws -> [[String?]] { try await execute(query: query).rows.map { $0.map(\.asText) } } @@ -175,31 +175,18 @@ extension MSSQLPluginDriver { }.sorted { $0.name < $1.name } } - // Only the primary key's index is left out: `fetchTableDDL` never declares a `UNIQUE` constraint, so its index comes here. + // The primary key and UNIQUE constraints are declared in `fetchTableDDL`, which builds their indexes. What runs + // after the rows also turns back on the checks the source enforced without having checked its rows. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - // SQL Server allows both `]` and `'` in a name, so it is bracket-quoted and then quoted as a literal. - let objectLiteral = MSSQLStringLiteral.quoted( - MSSQLSchemaQueries.bracketed(schema: effectiveSchema(schema), table: table)) - let sql = """ - SELECT i.name, i.type_desc, i.is_unique, i.filter_definition, - c.name AS column_name, ic.is_included_column, ic.is_descending_key - FROM sys.indexes i - JOIN sys.index_columns ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id - JOIN sys.columns c - ON ic.object_id = c.object_id AND ic.column_id = c.column_id - WHERE i.object_id = OBJECT_ID(\(objectLiteral)) - AND i.name IS NOT NULL - AND i.is_primary_key = 0 - AND i.type IN (1, 2) - ORDER BY i.index_id, ic.is_included_column, ic.key_ordinal - """ - let result = try await execute(query: sql) + let resolvedSchema = effectiveSchema(schema) + let rows = try await catalogRows(MSSQLSchemaQueries.indexDefinitions(schema: resolvedSchema, table: table)) + let checks = try await catalogRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) do { - return try MSSQLSchemaQueries.indexStatements( - rows: result.rows.map { row in row.map { $0.asText } }, - schema: effectiveSchema(schema), - table: table) + return try MSSQLSchemaQueries.indexStatements(rows: rows, schema: resolvedSchema, table: table) + + MSSQLSchemaQueries.checkConstraintReenableStatements( + schema: resolvedSchema, table: table, checkConstraints: checks + ) } catch let refusal as MSSQLDefinitionRefusal { throw MSSQLPluginError(refusal) } @@ -266,6 +253,8 @@ extension MSSQLPluginDriver { return "DROP TRIGGER \(quoteIdentifier(resolved)).\(quoteIdentifier(name))" } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let result = try await execute(query: MSSQLSchemaQueries.allColumns(schema: effectiveSchema(schema))) var columnsByTable: [String: [PluginColumnInfo]] = [:] @@ -345,21 +334,35 @@ extension MSSQLPluginDriver { } let columns = try await columnRows(table: table, schema: schema) - let indexes = try await fetchIndexes(table: table, schema: schema) + let keyConstraints = try await catalogRows(MSSQLSchemaQueries.keyConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseKeyConstraintRow) + let checkConstraints = try await catalogRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) let foreignKeys = try await foreignKeyRows(table: table, schema: schema) + let collation = await databaseCollation(forTable: table) do { return try MSSQLSchemaQueries.createTableStatement( schema: resolvedSchema, table: table, columns: columns, - primaryKeyColumns: indexes.filter(\.isPrimary).flatMap(\.columns), - foreignKeys: foreignKeys + keyConstraints: keyConstraints, + checkConstraints: checkConstraints, + foreignKeys: foreignKeys, + databaseCollation: collation ) } catch let refusal as MSSQLDefinitionRefusal { throw MSSQLPluginError(refusal) } } + // Unread, every column states its collation, which keeps it rather than changing it. + private func databaseCollation(forTable table: String) async -> String? { + guard let result = try? await execute(query: MSSQLSchemaQueries.databaseCollation(forTable: table)) else { + return nil + } + return result.rows.first?[safe: 0]?.asText + } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { let resolvedSchema = effectiveSchema(schema) let module = try await storedModule(schema: resolvedSchema, name: view) @@ -493,6 +496,7 @@ extension MSSQLColumnRow { isPrimaryKey: isPrimaryKey, defaultValue: defaultValue, extra: isIdentity ? "IDENTITY" : nil, + collation: collation, identityKind: isIdentity ? .always : nil, isGenerated: isGenerated ) diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift index a195f9ef45..e6a7e184fc 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift @@ -14,9 +14,6 @@ import TableProSQLQuoting /// second row that happens to look the same. That bound is only safe when every column takes part in the match: a /// match with one left out can pick a different row, so such a change is refused instead. struct MSSQLStatementGenerator { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - let qualifiedTable: String let columns: [String] let primaryKeyColumns: [String] @@ -28,8 +25,8 @@ struct MSSQLStatementGenerator { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite] { + // In the order received, which is edit order: a row deleted to free a unique value goes before the row taking it. var writes: [PluginRowWrite] = [] - var deletes: [PluginRowWrite] = [] for change in changes { switch change.type { case .insert: @@ -38,19 +35,30 @@ struct MSSQLStatementGenerator { let statement = try insertStatement( values: values, rowIndex: change.rowIndex, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) case .update: guard let statement = try updateStatement(for: change) else { continue } - writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) + writes.append(PluginRowWrite( + statement: statement.statement, + parameters: statement.parameters, + rowIndices: [change.rowIndex], + expectedRowCount: 1 + )) case .delete: guard deletedRowIndices.contains(change.rowIndex), let statement = try deleteStatement(for: change) else { continue } - deletes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) + writes.append(PluginRowWrite( + statement: statement.statement, + parameters: statement.parameters, + rowIndices: [change.rowIndex], + expectedRowCount: 1 + )) } } - return writes + deletes + return writes } /// The statements alone, for a host that asks without a context. With no context there is nothing to refuse. @@ -72,10 +80,11 @@ struct MSSQLStatementGenerator { func insertStatement( values: [PluginCellValue], rowIndex: Int = 0, - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) throws -> (statement: String, parameters: [PluginCellValue]) { let written = zip(columns, values).enumerated() - .filter { $0.element.1 != Self.defaultMarker } + .filter { !defaultColumns.contains($0.offset) } .map { (index: $0.offset, column: $0.element.0, value: $0.element.1) } if let owned = written.first(where: { context.serverOwnedColumns.contains($0.column) }) { throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: Self.serverOwnedReason(owned.column)) @@ -106,7 +115,7 @@ struct MSSQLStatementGenerator { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cellChange -> String in let column = Self.quote(cellChange.columnName) - guard cellChange.newValue != Self.defaultMarker else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cellChange.columnIndex) else { return "\(column) = DEFAULT" } if let sql = Self.expression( cellChange.newValue, isMarked: expressionColumns.contains(cellChange.columnIndex) @@ -134,11 +143,12 @@ struct MSSQLStatementGenerator { primaryKeyColumns.isEmpty ? "TOP (1) " : "" } - /// The app holds a keyless write to the row it matched, and under `SET NOCOUNT ON` the server reports no count. - /// A SET inside `sp_executesql` reverts when the call returns. A bound statement already runs in one, so only an - /// unbound one, a row whose every column is NULL, is given its own. + /// The app holds every update and delete to the row it matched, and under `SET NOCOUNT ON` the server reports no + /// count. A SET inside `sp_executesql` reverts when the call returns. A bound statement already runs in one, so + /// only an unbound one, such as a keyless row whose every column is NULL, is given its own. Measured on SQL Server + /// 2022: the count is the rows matched, a row already holding the value included, and leaves out rows a trigger + /// writes (`scripts/probes/check-mssql-keyed-row-counts.sh`). private func countReported(_ sql: String, parameters: [PluginCellValue]) -> String { - guard primaryKeyColumns.isEmpty else { return sql } let counted = "SET NOCOUNT OFF; \(sql)" guard parameters.isEmpty else { return counted } return "EXEC sp_executesql \(MSSQLStringLiteral.quoted(counted))" diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift index bfb50d77a9..945552275e 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift @@ -144,10 +144,18 @@ enum MSSQLTableDefinitionSQL { return definition } - static func refusal(for column: PluginColumnDefinition, writesType: Bool = true, writesDefault: Bool = true) -> String? { + static func refusal( + for column: PluginColumnDefinition, + writesType: Bool = true, + writesDefault: Bool = true, + collation: String? = nil + ) -> String? { if writesType, !MSSQLDefinitionText.isColumnType(column.dataType) { return MSSQLDefinitionRefusal.columnType(column.name, column.dataType).reason } + if let collation, !MSSQLDefinitionText.isCollationName(collation) { + return MSSQLDefinitionRefusal.collation(column.name, collation).reason + } if writesDefault, let value = column.defaultValue, MSSQLDefinitionText.expression(value) == nil { return MSSQLDefinitionRefusal.columnDefault(column.name).reason } diff --git a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift index 2ee2f17664..8fc08e0fd3 100644 --- a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift +++ b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift @@ -54,13 +54,15 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable if !committed { PluginExportUtilities.rollbackAtomicWrite(at: tempURL) } } + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings for (index, table) in tables.enumerated() { try progress.checkCancellation() progress.setCurrentTable(table.qualifiedName, index: index + 1) - if settings.includesTableNames { + if options.includesTableNames { try fileHandle.write(contentsOf: "## \(table.qualifiedName)\n\n".toUTF8Data()) } - try await writeTable(table, dataSource: dataSource, to: fileHandle, progress: progress) + try await writeTable(table, options: options, dataSource: dataSource, to: fileHandle, progress: progress) try fileHandle.write(contentsOf: "\n".toUTF8Data()) } @@ -73,6 +75,7 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable private func writeTable( _ table: PluginExportTable, + options: MarkdownExportOptions, dataSource: any PluginExportDataSource, to fileHandle: FileHandle, progress: PluginExportProgress @@ -89,9 +92,9 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable columns = header.columns case .rows(let rows): for row in rows { - let cells = row.map { cellText($0) } + let cells = row.map { cellText($0, nullPlaceholder: options.nullPlaceholder) } progress.incrementRow() - guard settings.alignsColumns, !wroteHeader else { + guard options.alignsColumns, !wroteHeader else { if !wroteHeader { try writeHeader(columns, widths: nil, to: fileHandle) wroteHeader = true @@ -115,7 +118,7 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable } if !wroteHeader { - widths = settings.alignsColumns + widths = options.alignsColumns ? MarkdownTableRenderer.widths(header: columns, sample: buffered) : nil try writeHeader(columns, widths: widths, to: fileHandle) @@ -138,10 +141,10 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable ).toUTF8Data()) } - private func cellText(_ value: PluginCellValue) -> String { + private func cellText(_ value: PluginCellValue, nullPlaceholder: String) -> String { switch value { case .null: - return MarkdownTableRenderer.cell(settings.nullPlaceholder) + return MarkdownTableRenderer.cell(nullPlaceholder) case .bytes(let data): return MarkdownTableRenderer.cell("0x\(data.map { String(format: "%02X", $0) }.joined())") case .text(let text): diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift index 272d58c088..cbbbd6569a 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift @@ -11,11 +11,16 @@ import TableProPluginKit private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sendable { typealias InboundIn = ByteBuffer + private let closed: NIOLockedValueBox private var reader: MemcachedReplyReader? private var continuation: CheckedContinuation? private var failure: Error? private let lock = NIOLock() + init(closed: NIOLockedValueBox) { + self.closed = closed + } + func expect( _ shape: MemcachedReplyShape, lineLimit: Int, @@ -57,6 +62,7 @@ private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sen } func channelInactive(context: ChannelHandlerContext) { + closed.withLockedValue { $0 = true } finish(with: MemcachedError.connectionClosed) } @@ -124,6 +130,10 @@ actor MemcachedConnection { private var wireSlotIsTaken = false private var waitingForWireSlot: [CheckedContinuation] = [] private var requestTimeout: TimeAmount? + private nonisolated let closed = NIOLockedValueBox(false) + + // True once the channel closed under the connection, as a cancelled or timed-out request closes it. + nonisolated var hasClosed: Bool { closed.withLockedValue { $0 } } /// A reply that has not finished by then closes the connection, since the text protocol cannot skip it. func setRequestTimeout(seconds: Int) { @@ -138,7 +148,7 @@ actor MemcachedConnection { guard channel == nil else { return } let sslContext = try Self.makeSSLContext(endpoint.ssl) let serverHostname = endpoint.ssl.verifiesHostname ? endpoint.tlsServerName : nil - let replyHandler = MemcachedReplyHandler() + let replyHandler = MemcachedReplyHandler(closed: closed) let bootstrap = ClientBootstrap(group: NIOSingletons.posixEventLoopGroup) .connectTimeout(endpoint.connectTimeout) .channelOption(ChannelOptions.socketOption(.so_keepalive), value: 1) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift index 813c3d6258..155c700630 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift @@ -12,6 +12,7 @@ enum MemcachedError: Error, PluginDriverError { case server(MemcachedServerError) case command(MemcachedCommandError) case filter(MemcachedFilterRefusal) + case compareAndSwap(MemcachedCASRefusal) case refused(String) case unexpectedReply(String) @@ -35,6 +36,10 @@ enum MemcachedError: Error, PluginDriverError { return Self.describe(error) case .filter(let refusal): return Self.describe(refusal) + case .compareAndSwap(.changed): + return String(localized: "The item changed since its CAS value was read, so nothing was saved. Refresh to see its current value.") + case .compareAndSwap(.notFound): + return String(localized: "The item does not exist, so nothing was saved. Refresh to see the current keys.") case .refused(let message): return message case .unexpectedReply(let line): diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index cc1021e399..c86b077cb8 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -13,6 +13,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// Keys one listing reads before it stops, so a cache holding millions of items cannot exhaust memory. static let listingLimit = 1_000_000 private static let valuesPerRequest = 100 + private static let rowsPerStreamBatch = 1_000 private let endpoint: MemcachedEndpoint private let tunneledServerName: String? @@ -22,8 +23,9 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var keySource: MemcachedKeySource = .metadump private var queryTimeout = 0 private var shownExpiries: [String: MemcachedKnownExpiry] = [:] + private var shownItems: [String: MemcachedLoadedItem] = [:] private var serverClockOffset: Int64 = 0 - private static let shownExpiryLimit = 100_000 + private static let shownKeyLimit = 100_000 init(config: DriverConnectionConfig) { tunneledServerName = config.tunneledServerName @@ -40,7 +42,10 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var serverVersion: String? { lock.withLock { _serverVersion } } var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [] } + var capabilities: PluginCapabilities { [.columnDefaultMarks] } + + // A stopped request closes the channel, since the text protocol cannot skip its reply; the host then reconnects. + var hasLostConnection: Bool { lock.withLock { connection }?.hasClosed ?? false } func beginTransaction() async throws {} func commitTransaction() async throws {} @@ -155,7 +160,11 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if case .metadump = command { return try await metadumpResult(command, started: started) } - switch try await connection.send(command) { + let reply = try await connection.send(command) + if case .status(let line) = reply, let refusal = MemcachedCASRefusal(replyTo: command, status: line) { + throw MemcachedError.compareAndSwap(refusal) + } + switch reply { case .values(let values): let withCAS: Bool switch command { @@ -208,55 +217,93 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Browsing + private struct BrowsePage { + let entries: [MemcachedKeyEntry] + let serverTime: Int64 + // The listing stopped at `listingLimit`, so the page is drawn from part of the cache. + let isCut: Bool + } + private func browse(_ spec: MemcachedBrowseSpec, started: Date) async throws -> PluginQueryResult { let connection = try liveConnection() + let page = try await browsePage(spec, on: connection) + let keys = page.entries.map(\.key) + let values = try await values(of: keys, on: connection) + rememberLoadedItems(of: keys, values: values) + let rows = rows(of: page.entries, values: values, serverTime: page.serverTime) + return result( + columns: MemcachedColumn.allCases.map(\.rawValue), + types: MemcachedColumn.allCases.map(\.typeName), + rows: rows, started: started + ) + } + + private func browsePage(_ spec: MemcachedBrowseSpec, on connection: MemcachedConnection) async throws -> BrowsePage { let serverTime = try await Self.serverTime(on: connection) - let listed = try await listKeys().filter { !$0.isExpired(at: serverTime) } - let page = try spec.page(of: listed) + let listing = try await listKeys() + let page = try spec.page(of: listing.entries.filter { !$0.isExpired(at: serverTime) }) + lock.withLock { + serverClockOffset = MemcachedServerClock.offset( + serverTime: serverTime, localTime: Int64(Date().timeIntervalSince1970) + ) + if shownExpiries.count + page.count > Self.shownKeyLimit { shownExpiries.removeAll() } + for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } + } + return BrowsePage(entries: page, serverTime: serverTime, isCut: listing.isCut) + } + // `gets`, so each value comes with the unique a `cas` needs to write it back. + private func values(of keys: [String], on connection: MemcachedConnection) async throws -> [String: MemcachedValue] { var values: [String: MemcachedValue] = [:] - var start = page.startIndex - while start < page.endIndex { + var start = keys.startIndex + while start < keys.endIndex { try Task.checkCancellation() - let end = min(start + Self.valuesPerRequest, page.endIndex) - let keys = page[start ..< end].map(\.key) - if case .values(let read) = try await connection.send(.get(keys: keys, withCAS: false)) { + let end = min(start + Self.valuesPerRequest, keys.endIndex) + if case .values(let read) = try await connection.send(.get(keys: Array(keys[start ..< end]), withCAS: true)) { for value in read { values[value.key] = value } } start = end } + return values + } - lock.withLock { - serverClockOffset = MemcachedServerClock.offset( - serverTime: serverTime, localTime: Int64(Date().timeIntervalSince1970) - ) - if shownExpiries.count + page.count > Self.shownExpiryLimit { shownExpiries.removeAll() } - for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } - } - - let rows: [PluginRow] = page.compactMap { entry in + // A key removed since the listing has no value and no row. + private func rows( + of entries: some Collection, + values: [String: MemcachedValue], + serverTime: Int64 + ) -> [PluginRow] { + entries.compactMap { entry in guard let value = values[entry.key] else { return nil } return [ .text(entry.key), Self.cell(value.data), PluginCellValue.fromOptional(entry.secondsLeft(at: serverTime).map(String.init)), - .text(String(value.flags)), + Self.flagsCell(value.flags), ] } - return result( - columns: MemcachedColumn.allCases.map(\.rawValue), - types: MemcachedColumn.allCases.map(\.typeName), - rows: rows, started: started - ) } - private func listKeys() async throws -> [MemcachedKeyEntry] { + private func rememberLoadedItems(of keys: [String], values: [String: MemcachedValue]) { + lock.withLock { + if shownItems.count + keys.count > Self.shownKeyLimit { shownItems.removeAll() } + for key in keys { + shownItems[key] = values[key].flatMap { value in + value.unique.map { + MemcachedLoadedItem(unique: $0, flags: Self.flagsCell(value.flags), value: Self.cell(value.data)) + } + } + } + } + } + + private func listKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { if lock.withLock({ keySource }) == .metadump { do { return try await withListingConnection { connection in let reply = try await connection.send(.metadump(selector: "all"), lineLimit: Self.listingLimit) - guard case .lines(let lines, _) = reply else { return [] } - return lines.compactMap(MemcachedKeyListing.metadumpEntry) + guard case .lines(let lines, let truncated) = reply else { return ([], false) } + return (lines.compactMap(MemcachedKeyListing.metadumpEntry), truncated) } } catch let error as MemcachedServerError where error.kind != .busy && !error.needsAuthentication { Self.logger.info("metadump refused, listing with cachedump") @@ -266,18 +313,71 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return try await cachedumpKeys() } - private func cachedumpKeys() async throws -> [MemcachedKeyEntry] { + private func cachedumpKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { let connection = try liveConnection() - guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return [] } + guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return ([], false) } var entries: [MemcachedKeyEntry] = [] for slab in MemcachedKeyListing.slabClasses(fromItemStats: itemStats) { try Task.checkCancellation() let reply = try await connection.send(.stats(arguments: ["cachedump", String(slab), "0"])) guard case .lines(let lines, _) = reply else { continue } entries += lines.compactMap(MemcachedKeyListing.cachedumpEntry) - if entries.count >= Self.listingLimit { break } + if entries.count >= Self.listingLimit { return (Array(entries.prefix(Self.listingLimit)), true) } + } + return (entries, false) + } + + // MARK: - Streaming + + // An export reads every key, so its values are read and sent a batch at a time on a connection of their own, and + // stopping closes that connection, never the session's. + func streamRows(query: String) -> AsyncThrowingStream { + let statement = query.trimmingCharacters(in: .whitespacesAndNewlines) + let spec = MemcachedBrowseSpec.refusalMessage(statement) == nil ? MemcachedBrowseSpec.parse(statement) : nil + return PluginRowStream.makeBounded { continuation, abort in + let task = Task { + do { + if let spec { + try await self.streamBrowse(spec, into: continuation) + } else { + let result = try await self.execute(query: query) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: MemcachedError.wrap(error)) + } + } + abort.onAbort { task.cancel() } + } + } + + private func streamBrowse( + _ spec: MemcachedBrowseSpec, + into continuation: AsyncThrowingStream.Continuation + ) async throws { + let page = try await browsePage(spec, on: try liveConnection()) + var header = PluginStreamHeader( + columns: MemcachedColumn.allCases.map(\.rawValue), + columnTypeNames: MemcachedColumn.allCases.map(\.typeName) + ) + if page.isCut { header.rowLimit = page.entries.count } + continuation.yield(.header(header)) + try await withListingConnection { connection in + var start = page.entries.startIndex + while start < page.entries.endIndex { + try Task.checkCancellation() + let end = min(start + Self.rowsPerStreamBatch, page.entries.endIndex) + let batch = page.entries[start ..< end] + let values = try await self.values(of: batch.map(\.key), on: connection) + let rows = self.rows(of: batch, values: values, serverTime: page.serverTime) + if !rows.isEmpty { _ = await PluginRowStream.deliver(.rows(rows), to: continuation) } + start = end + } } - return entries } /// Opens a connection for one listing and closes it after, even when the listing stopped early with the rest @@ -344,6 +444,8 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ] } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { [MemcachedPlugin.keysTable: try await fetchColumns(table: MemcachedPlugin.keysTable, schema: schema)] } @@ -431,12 +533,13 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { - let offset = lock.withLock { serverClockOffset } - let expiries = lock.withLock { shownExpiries } + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } + let (offset, expiries, items) = lock.withLock { (serverClockOffset, shownExpiries, shownItems) } let generator = MemcachedStatementGenerator( columns: columns, now: { MemcachedServerClock.serverTime(localTime: Int64(Date().timeIntervalSince1970), offset: offset) }, - knownExpiry: { expiries[$0] } + knownExpiry: { expiries[$0] }, + loadedItem: { items[$0] } ) return try generator.generateRowWrites( from: changes, @@ -448,8 +551,14 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Helpers + // String(data:encoding:) drops a leading byte order mark, and a save that carries the value forward would too. private static func cell(_ data: Data) -> PluginCellValue { - String(data: data, encoding: .utf8).map(PluginCellValue.text) ?? .bytes(data) + let text = String(decoding: data, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(data) ? .text(text) : .bytes(data) + } + + private static func flagsCell(_ flags: UInt32) -> PluginCellValue { + .text(String(flags)) } private func result( diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift index a0e40fe779..78815b317e 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation import TableProMemcachedCore import TableProPluginKit @@ -7,13 +8,47 @@ struct MemcachedKnownExpiry: Sendable { let expiresAt: Int64? } -/// Turns grid edits into console commands: `set` for a new row or a new value or flags, `touch` for a TTL alone, -/// `delete` for a removed row. The key names the item, so it is never rewritten in place. +// The unique `gets` returned for a key, with a digest of the Flags and Value the grid showed. A later read, from +// another tab, holds a newer unique, which must not let this row's older Value or Flags be written back. +struct MemcachedLoadedItem: Sendable { + private let casUnique: UInt64 + private let digest: Data + + init(unique: UInt64, flags: PluginCellValue, value: PluginCellValue) { + casUnique = unique + digest = Self.digest(flags: flags, value: value) + } + + func unique(ifShowing flags: PluginCellValue?, _ value: PluginCellValue?) -> UInt64? { + guard let flags, let value, Self.digest(flags: flags, value: value) == digest else { return nil } + return casUnique + } + + private static func digest(flags: PluginCellValue, value: PluginCellValue) -> Data { + var hash = SHA256() + for cell in [flags, value] { + let tagged: (tag: UInt8, bytes: Data) + switch cell { + case .null: tagged = (0, Data()) + case .text(let text): tagged = (1, Data(text.utf8)) + case .bytes(let data): tagged = (2, data) + } + hash.update(data: [tagged.tag]) + withUnsafeBytes(of: UInt64(tagged.bytes.count).littleEndian) { hash.update(bufferPointer: $0) } + hash.update(data: tagged.bytes) + } + return Data(hash.finalize()) + } +} + +// Grid edits as console commands: `set` for a new row or one that writes both Value and Flags, `cas` for one that keeps +// the loaded Value or Flags, `touch` for a TTL alone, `delete` for a removed row. The key is never rewritten in place. struct MemcachedStatementGenerator { let columns: [String] /// The server's clock, so an absolute expiry is compared with the time the server keeps. var now: () -> Int64 = { Int64(Date().timeIntervalSince1970) } var knownExpiry: (String) -> MemcachedKnownExpiry? = { _ in nil } + var loadedItem: (String) -> MemcachedLoadedItem? = { _ in nil } func generateRowWrites( from changes: [PluginRowChange], @@ -82,27 +117,38 @@ struct MemcachedStatementGenerator { } edited[column] = cell.newValue } - if let ttl = edited[.ttl], edited[.value] == nil, edited[.flags] == nil { - let seconds = try number(ttl, of: .ttl, rowIndex: change.rowIndex) + guard let write = MemcachedItemEdit.write( + editsValue: edited[.value] != nil, editsFlags: edited[.flags] != nil, editsTTL: edited[.ttl] != nil, + unique: loadedItem(key)?.unique(ifShowing: original(.flags, of: change), original(.value, of: change)) + ) else { return nil } + switch write { + case .touch: + let seconds = try number(edited[.ttl], of: .ttl, rowIndex: change.rowIndex) return MemcachedStatements.touch(key: key, exptime: try exptime(forTTL: seconds, rowIndex: change.rowIndex)) + case .needsRefresh: + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "Refresh before changing this item, so the save can check that it has not changed since it was loaded.") + ) + case .set, .compareAndSwap: + break } - guard edited[.value] != nil || edited[.flags] != nil else { return nil } let exptime: Int64 if let ttl = edited[.ttl] { exptime = try self.exptime(forTTL: try number(ttl, of: .ttl, rowIndex: change.rowIndex), rowIndex: change.rowIndex) } else { exptime = try currentExptime(of: key, change: change) } - return setStatement( - key: key, - flags: try number(edited[.flags] ?? original(.flags, of: change), of: .flags, rowIndex: change.rowIndex) ?? 0, - exptime: exptime, - value: edited[.value] ?? original(.value, of: change) ?? .null - ) + let flags = try number(edited[.flags] ?? original(.flags, of: change), of: .flags, rowIndex: change.rowIndex) ?? 0 + let value = edited[.value] ?? original(.value, of: change) ?? .null + guard case .compareAndSwap(let unique) = write else { + return setStatement(key: key, flags: flags, exptime: exptime, value: value) + } + return casStatement(key: key, flags: flags, exptime: exptime, unique: unique, value: value) } - /// A `set` replaces the expiry too, so one that leaves TTL alone sends the time the item already ends at, - /// rather than the TTL the grid read, which has been counting down since. + // A `set` or `cas` replaces the expiry too, so one that leaves TTL alone sends the time the item already ends at, + // rather than the TTL the grid read, which has been counting down since. private func currentExptime(of key: String, change: PluginRowChange) throws -> Int64 { guard let known = knownExpiry(key) else { return try exptime(forTTL: try number(original(.ttl, of: change), of: .ttl, rowIndex: change.rowIndex), rowIndex: change.rowIndex) @@ -125,6 +171,14 @@ struct MemcachedStatementGenerator { return MemcachedStatements.set(key: key, flags: flags, exptime: exptime, value: value.asText ?? "") } + private func casStatement(key: String, flags: Int64, exptime: Int64, unique: UInt64, value: PluginCellValue) -> String { + let flags = UInt32(flags) + if case .bytes(let data) = value { + return MemcachedStatements.cas(key: key, flags: flags, exptime: exptime, unique: unique, bytes: data) + } + return MemcachedStatements.cas(key: key, flags: flags, exptime: exptime, unique: unique, value: value.asText ?? "") + } + private func exptime(forTTL ttl: Int64?, rowIndex: Int) throws -> Int64 { guard let exptime = MemcachedStatements.exptime(forTTL: ttl, now: now()) else { throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: String(localized: "This TTL is too large for Memcached.")) diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift index 3be35d7dd1..ad218e5240 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift @@ -9,6 +9,7 @@ import CLibMongoc import Foundation import os import OSLog +import TableProLogRedaction import TableProPluginKit #if canImport(CLibMongoc) @@ -417,11 +418,11 @@ nonisolated extension MongoDBConnection { func iterateCursorStreaming( cursor: OpaquePointer, - continuation: AsyncThrowingStream.Continuation, - streamState: MongoStreamState, + delivery: MongoStreamDelivery, census: () throws -> MongoFieldCensus? ) { - defer { cleanup(streamState) } + defer { cleanup(delivery.state) } + var batch = MongoStreamBatch() do { var docPtr: OpaquePointer? var sample: [[String: Any]] = [] @@ -429,53 +430,69 @@ nonisolated extension MongoDBConnection { var projection: MongoStreamProjection? var emitted = 0 var unannounced = Set() - while mongoc_cursor_next(cursor, &docPtr) { - if streamState.lock.withLock({ streamState.cancelled }) { throw CancellationError() } + var reachedLimit = false + while !reachedLimit, mongoc_cursor_next(cursor, &docPtr) { + if delivery.state.isCancelled { throw CancellationError() } try checkCancelled() guard let doc = docPtr else { continue } - let text = try canonicalText(of: doc) - guard let raw = try JSONSerialization.jsonObject(with: Data(text.utf8)) as? [String: Any], - let dict = Self.unwrapExtendedJson(raw) as? [String: Any] else { - throw PluginExportError.exportFailed(String(localized: "A document could not be read", bundle: .main)) - } - if projection == nil, sample.count >= MongoStreamProjection.sampleSize { - projection = try openStream( - sample: sample, texts: sampleTexts, census: try census(), continuation: continuation + // The whole cursor is read in one block on the queue, so what a document leaves autoreleased goes here, + // not when the read ends. + try autoreleasepool { + let text = try canonicalText(of: doc) + guard let raw = try JSONSerialization.jsonObject(with: Data(text.utf8)) as? [String: Any], + let dict = Self.unwrapExtendedJson(raw) as? [String: Any] else { + throw PluginExportError.exportFailed(String(localized: "A document could not be read", bundle: .main)) + } + if projection == nil, sample.count >= MongoStreamProjection.sampleSize { + projection = try openStream( + sample: sample, texts: sampleTexts, census: try census(), delivery: delivery + ) + emitted += sample.count + sample = [] + sampleTexts = [] + } + guard let projection else { + sample.append(dict) + sampleTexts.append(text) + return + } + let size = MongoLibbsonCodec.size(of: doc) + let document = try MongoExportDocument.read( + canonicalJSON: text, dictionary: dict, representation: uuidRepresentation, + bsonSizeUpperBound: size ) - emitted += sample.count - sample = [] - sampleTexts = [] - } - guard let projection else { - sample.append(dict) - sampleTexts.append(text) - continue - } - let document = try MongoExportDocument.read( - canonicalJSON: text, dictionary: dict, representation: uuidRepresentation, - bsonSizeUpperBound: MongoLibbsonCodec.size(of: doc) - ) - let row = BsonDocumentFlattener.flatten( - documents: [dict], columns: projection.columns, kinds: projection.kinds, - representation: uuidRepresentation, storedTexts: [text] - )[0] - continuation.yield(.rows([row], documents: [document])) - unannounced.formUnion(projection.unannouncedFields(in: dict)) - emitted += 1 - if emitted >= PluginRowLimits.emergencyMax { - logger.warning("Streamed result truncated at \(PluginRowLimits.emergencyMax) documents") - break + let row = BsonDocumentFlattener.flatten( + documents: [dict], columns: projection.columns, kinds: projection.kinds, + representation: uuidRepresentation, storedTexts: [text] + )[0] + batch.add(row, document: document, size: size) + if batch.isFull { + try delivery.send(batch.take()) + } + unannounced.formUnion(projection.unannouncedFields(in: dict)) + emitted += 1 + if emitted >= PluginRowLimits.emergencyMax { + logger.warning("Streamed result truncated at \(PluginRowLimits.emergencyMax) documents") + reachedLimit = true + } } } + if !batch.isEmpty { + try delivery.send(batch.take()) + } logUnannouncedFields(unannounced) var error = bson_error_t() if mongoc_cursor_error(cursor, &error) { throw makeError(error) } if projection == nil { - _ = try openStream(sample: sample, texts: sampleTexts, census: nil, continuation: continuation) + _ = try openStream(sample: sample, texts: sampleTexts, census: nil, delivery: delivery) } - continuation.finish() + delivery.finish() } catch { - continuation.finish(throwing: error) + // The rows read before the failure still reach the reader, as they did one document at a time. + if !batch.isEmpty, !(error is CancellationError) { + try? delivery.send(batch.take()) + } + delivery.finish(throwing: error) } } @@ -512,35 +529,306 @@ nonisolated extension MongoDBConnection { sample: [[String: Any]], texts: [String], census: MongoFieldCensus?, - continuation: AsyncThrowingStream.Continuation + delivery: MongoStreamDelivery ) throws -> MongoStreamProjection { let projection = MongoStreamProjection(sample: sample, census: census, representation: uuidRepresentation) let documents = try MongoExportDocument.read( MongoReadDocuments(dictionaries: sample, texts: texts), representation: uuidRepresentation ) - continuation.yield(.header(projection.header)) + var header = projection.header + // iterateCursorStreaming stops there, so a stream that reaches it was cut. + header.rowLimit = PluginRowLimits.emergencyMax + try delivery.send(.header(header)) if !sample.isEmpty { let rows = BsonDocumentFlattener.flatten( documents: sample, columns: projection.columns, kinds: projection.kinds, representation: uuidRepresentation, storedTexts: texts ) - continuation.yield(.rows(rows, documents: documents)) + try delivery.send(.rows(rows, documents: documents)) } return projection } - private func cleanup(_ state: MongoStreamState) { + // The cursor goes before the session it runs in, as libmongoc requires. + func cleanup(_ state: MongoStreamState) { state.lock.lock() let cur = state.cursor let col = state.collection + let session = state.session let alreadyDrained = state.drained state.drained = true state.cursor = nil state.collection = nil + state.session = nil state.lock.unlock() guard !alreadyDrained else { return } if let cur { mongoc_cursor_destroy(cur) } if let col { mongoc_collection_destroy(col) } + session?.end() + } + + // Runs on the connection's queue. Nil leaves the stream reading as fast as the server sends. + func startStreamSession(isFind: Bool) -> MongoStreamSession? { + guard let client, let policy = MongoCursorIdlePolicy(serverVersion: serverVersion(), isFind: isFind) else { + return nil + } + return MongoStreamSession.start(client: client, policy: policy, connection: self) + } +} + +// A paced stream's own session. While the reader holds a full buffer, `refreshSessions` keeps it alive, and with +// it the cursor: the manual's way to hold a cursor idle (cursor.noCursorTimeout(), "Refresh a Cursor with +// refreshSessions"). +nonisolated final class MongoStreamSession: @unchecked Sendable { + let session: OpaquePointer + let policy: MongoCursorIdlePolicy + let refreshInterval: TimeInterval + private let refreshCommand: OpaquePointer + private let connection: MongoDBConnection + private var nextRefresh = ContinuousClock.now + private var reportedRefreshFailure = false + + private init( + session: OpaquePointer, + policy: MongoCursorIdlePolicy, + refreshInterval: TimeInterval, + refreshCommand: OpaquePointer, + connection: MongoDBConnection + ) { + self.session = session + self.policy = policy + self.refreshInterval = refreshInterval + self.refreshCommand = refreshCommand + self.connection = connection + } + + // A deployment that answers no session, or refuses `refreshSessions` (DocumentDB, Cosmos DB), gets nil. + static func start( + client: OpaquePointer, + policy: MongoCursorIdlePolicy, + connection: MongoDBConnection + ) -> MongoStreamSession? { + guard let options = mongoc_session_opts_new() else { return nil } + defer { mongoc_session_opts_destroy(options) } + mongoc_session_opts_set_causal_consistency(options, false) + var error = bson_error_t() + guard let session = mongoc_client_start_session(client, options, &error) else { return nil } + guard let lsid = mongoc_client_session_get_lsid(session) else { + mongoc_client_session_destroy(session) + return nil + } + guard let command = bson_new() else { + mongoc_client_session_destroy(session) + return nil + } + let sessions = bson_new() + bson_append_document(sessions, "0", -1, lsid) + bson_append_array(command, "refreshSessions", -1, sessions) + bson_destroy(sessions) + let stream = MongoStreamSession( + session: session, + policy: policy, + refreshInterval: refreshInterval(client, connection: connection), + refreshCommand: command, + connection: connection + ) + do { + try stream.refresh(client) + } catch { + logger.info("The server refused refreshSessions, so this stream reads ahead of its reader") + stream.end() + return nil + } + stream.nextRefresh = .now + .seconds(stream.refreshInterval) + return stream + } + + // Timed from the last refresh, not from the last getMore: one getMore can fill many batches, and a slow reader + // takes them one wait at a time. A refresh that fails is tried again soon; a session that does expire ends the + // read with the cursor's own error. + func refreshIfDue(_ client: OpaquePointer) { + let now = ContinuousClock.now + guard now >= nextRefresh else { return } + do { + try refresh(client) + nextRefresh = now + .seconds(refreshInterval) + } catch { + if !reportedRefreshFailure { + reportedRefreshFailure = true + logger.warning( + "refreshSessions failed, trying again: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + } + nextRefresh = now + .seconds(min(30, refreshInterval)) + } + } + + private func refresh(_ client: OpaquePointer) throws { + let reply = bson_new() + defer { bson_destroy(reply) } + var error = bson_error_t() + guard mongoc_client_command_simple(client, "admin", refreshCommand, nil, reply, &error) else { + throw connection.makeError(error) + } + } + + func append(to options: OpaquePointer) throws { + var error = bson_error_t() + guard mongoc_client_session_append(session, options, &error) else { throw connection.makeError(error) } + if policy.needsNoCursorTimeout { + bson_append_bool(options, "noCursorTimeout", -1, true) + } + } + + func end() { + mongoc_client_session_destroy(session) + bson_destroy(refreshCommand) + } + + // A refresh reaches config.system.sessions at the server's next refresh job, five minutes apart by default, so it + // goes out well inside the session timeout: every 5 minutes against the default 30, a sixth of a shorter one. + private static func refreshInterval(_ client: OpaquePointer, connection: MongoDBConnection) -> TimeInterval { + var error = bson_error_t() + guard let server = mongoc_client_select_server(client, false, nil, &error) else { return 300 } + defer { mongoc_server_description_destroy(server) } + let hello = connection.bsonToDict(mongoc_server_description_hello_response(server)) + let minutes = (hello["logicalSessionTimeoutMinutes"] as? NSNumber)?.intValue ?? 30 + return TimeInterval(min(300, max(minutes, 1) * 10)) + } +} + +// How a stream's documents reach its reader. With a session, the cursor waits for room and the session is refreshed +// while it does. Without one the server cannot hold an idle cursor, so the cursor never waits: what the reader has +// no room for waits here, as it did when the stream was unbounded. +nonisolated final class MongoStreamDelivery { + let continuation: AsyncThrowingStream.Continuation + let state: MongoStreamState + private let session: MongoStreamSession? + private let client: OpaquePointer + private var waiting: [PluginExportStreamElement] = [] + private var firstWaiting = 0 + + init( + continuation: AsyncThrowingStream.Continuation, + state: MongoStreamState, + session: MongoStreamSession?, + client: OpaquePointer + ) { + self.continuation = continuation + self.state = state + self.session = session + self.client = client + } + + func send(_ element: PluginExportStreamElement) throws { + guard let session else { + waiting.append(element) + handOverWaiting() + return + } + session.refreshIfDue(client) + let delivered = PluginRowStream.deliverBlocking( + element, + to: continuation, + stops: { state.isCancelled }, + keepingAliveEvery: Self.refreshCheckInterval + ) { + session.refreshIfDue(client) + } + guard delivered else { throw CancellationError() } + } + + // What the reader has not taken yet goes out from a task of its own, so the connection's queue is free once the + // cursor is done, as it was when the stream was unbounded. + func finish(throwing error: Error? = nil) { + let rest = Array(waiting[firstWaiting...]) + waiting = [] + firstWaiting = 0 + let continuation = self.continuation + guard !rest.isEmpty else { + Self.end(continuation, throwing: error) + return + } + Task.detached { + for element in rest { + guard await PluginRowStream.deliver(element, to: continuation) else { break } + } + Self.end(continuation, throwing: error) + } + } + + private static func end( + _ continuation: AsyncThrowingStream.Continuation, + throwing error: Error? + ) { + if let error { + continuation.finish(throwing: error) + } else { + continuation.finish() + } + } + + private static let refreshCheckInterval: TimeInterval = 1 + + private func handOverWaiting() { + while firstWaiting < waiting.count { + guard case .enqueued = continuation.yield(waiting[firstWaiting]) else { break } + firstWaiting += 1 + } + if firstWaiting == waiting.count { + waiting.removeAll(keepingCapacity: true) + firstWaiting = 0 + } else if firstWaiting > waiting.count / 2 { + waiting.removeFirst(firstWaiting) + firstWaiting = 0 + } } } #endif + +// From 4.4.8 a session's cursors no longer expire after cursorTimeoutMillis (SERVER-6036). Before it only a find can +// wait, with noCursorTimeout; an aggregate has no way to. An unknown version waits for nothing. +nonisolated struct MongoCursorIdlePolicy: Sendable, Equatable { + let needsNoCursorTimeout: Bool + + init?(serverVersion: String?, isFind: Bool) { + guard let serverVersion else { return nil } + let parts = serverVersion.split(separator: ".").prefix(3).map { Int($0.prefix { $0.isNumber }) ?? 0 } + guard let major = parts.first, major > 0 else { return nil } + let release = (major, parts.count > 1 ? parts[1] : 0, parts.count > 2 ? parts[2] : 0) + let sessionCursorsWait = release >= (4, 4, 8) + guard sessionCursorsWait || isFind else { return nil } + needsNoCursorTimeout = !sessionCursorsWait + } +} + +// Documents go out a batch at a time; one element per document made a bounded buffer a handful of rows deep. +nonisolated struct MongoStreamBatch { + static let maximumDocuments = 1_000 + static let maximumBytes = 4 * 1_048_576 + + private var rows: [PluginRow] = [] + private var documents: [PluginExportDocument] = [] + private var bytes = 0 + + var isEmpty: Bool { rows.isEmpty } + + var isFull: Bool { + rows.count >= Self.maximumDocuments || bytes >= Self.maximumBytes + } + + mutating func add(_ row: PluginRow, document: PluginExportDocument, size: Int) { + rows.append(row) + documents.append(document) + bytes += size + } + + mutating func take() -> PluginExportStreamElement { + defer { + rows = [] + documents = [] + bytes = 0 + } + return .rows(rows, documents: documents) + } +} diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift index 2fe2d211a1..58222abc9c 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection.swift @@ -843,62 +843,20 @@ nonisolated final class MongoDBConnection: @unchecked Sendable { census: MongoFieldCensus.Request? ) -> AsyncThrowingStream { #if canImport(CLibMongoc) - let queue = self.queue - let streamState = MongoStreamState() - - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in - continuation.onTermination = { @Sendable _ in - streamState.lock.withLock { streamState.cancelled = true } - queue.async { - streamState.lock.lock() - let cur = streamState.cursor - let col = streamState.collection - let alreadyDrained = streamState.drained - streamState.drained = true - streamState.cursor = nil - streamState.collection = nil - streamState.lock.unlock() - guard !alreadyDrained else { return } - if let cur { mongoc_cursor_destroy(cur) } - if let col { mongoc_collection_destroy(col) } - } + cursorStream(database: database, collection: collection, isFind: true, census: census) { col, session in + guard let filterBson = self.jsonToBson(filter) else { + throw MongoDBError(code: 0, message: "Invalid JSON filter: \(filter)") } - - queue.async { [self] in - guard !isShuttingDown, let client = self.client else { - continuation.finish(throwing: MongoDBError.notConnected) - return - } - - do { - guard let filterBson = jsonToBson(filter) else { - throw MongoDBError(code: 0, message: "Invalid JSON filter: \(filter)") - } - defer { bson_destroy(filterBson) } - - let optsBson = jsonToBson(optionsJson) - defer { if let opts = optsBson { bson_destroy(opts) } } - - let col = try getCollection(client, database: database, collection: collection) - guard let cursor = mongoc_collection_find_with_opts(col, filterBson, optsBson, nil) else { - mongoc_collection_destroy(col) - throw MongoDBError(code: 0, message: "Failed to create find cursor") - } - - streamState.lock.lock() - streamState.cursor = cursor - streamState.collection = col - streamState.lock.unlock() - - iterateCursorStreaming( - cursor: cursor, continuation: continuation, streamState: streamState - ) { - try self.fieldCensus(census, client: client, database: database, collection: collection) - } - } catch { - continuation.finish(throwing: error) - } + defer { bson_destroy(filterBson) } + guard let optsBson = self.jsonToBson(optionsJson) ?? bson_new() else { + throw MongoDBError(code: 0, message: "Failed to build query options") } + defer { bson_destroy(optsBson) } + try session?.append(to: optsBson) + guard let cursor = mongoc_collection_find_with_opts(col, filterBson, optsBson, nil) else { + throw MongoDBError(code: 0, message: "Failed to create find cursor") + } + return cursor } #else return AsyncThrowingStream { $0.finish(throwing: MongoDBError.libmongocUnavailable) } @@ -913,66 +871,75 @@ nonisolated final class MongoDBConnection: @unchecked Sendable { census: MongoFieldCensus.Request? ) -> AsyncThrowingStream { #if canImport(CLibMongoc) + cursorStream(database: database, collection: collection, isFind: false, census: census) { col, session in + guard let pipelineBson = self.jsonToBson(pipeline) else { + throw MongoDBError(code: 0, message: "Invalid JSON pipeline: \(pipeline)") + } + defer { bson_destroy(pipelineBson) } + var optsBson: OpaquePointer? + if let optionsJson { + optsBson = self.jsonToBson(optionsJson) + } else { + let timeoutMS = self.queryTimeoutMS + if timeoutMS > 0 { + optsBson = self.jsonToBson("{\"maxTimeMS\": \(timeoutMS)}") + } + } + if session != nil, optsBson == nil { + optsBson = bson_new() + } + defer { if let opts = optsBson { bson_destroy(opts) } } + if let optsBson { + try session?.append(to: optsBson) + } + guard let cursor = mongoc_collection_aggregate(col, MONGOC_QUERY_NONE, pipelineBson, optsBson, nil) else { + throw MongoDBError(code: 0, message: "Failed to create aggregation cursor") + } + return cursor + } + #else + return AsyncThrowingStream { $0.finish(throwing: MongoDBError.libmongocUnavailable) } + #endif + } + + #if canImport(CLibMongoc) + // The cursor, its collection and the stream's session live and end inside one block on the connection's queue, so + // none of them can outlast the client that `disconnect` destroys behind it. A paced stream holds that queue while + // its reader waits, as every blocking driver does. + private func cursorStream( + database: String, + collection: String, + isFind: Bool, + census: MongoFieldCensus.Request?, + open: @escaping @Sendable (_ collection: OpaquePointer, _ session: MongoStreamSession?) throws -> OpaquePointer + ) -> AsyncThrowingStream { let queue = self.queue let streamState = MongoStreamState() - - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in continuation.onTermination = { @Sendable _ in streamState.lock.withLock { streamState.cancelled = true } - queue.async { - streamState.lock.lock() - let cur = streamState.cursor - let col = streamState.collection - let alreadyDrained = streamState.drained - streamState.drained = true - streamState.cursor = nil - streamState.collection = nil - streamState.lock.unlock() - guard !alreadyDrained else { return } - if let cur { mongoc_cursor_destroy(cur) } - if let col { mongoc_collection_destroy(col) } - } + queue.async { self.cleanup(streamState) } } queue.async { [self] in + defer { cleanup(streamState) } guard !isShuttingDown, let client = self.client else { continuation.finish(throwing: MongoDBError.notConnected) return } - do { - guard let pipelineBson = jsonToBson(pipeline) else { - throw MongoDBError(code: 0, message: "Invalid JSON pipeline: \(pipeline)") - } - defer { bson_destroy(pipelineBson) } - + let session = startStreamSession(isFind: isFind) + streamState.lock.withLock { streamState.session = session } let col = try getCollection(client, database: database, collection: collection) - - var optsBson: OpaquePointer? - if let optionsJson { - optsBson = jsonToBson(optionsJson) - } else { - let timeoutMS = queryTimeoutMS - if timeoutMS > 0 { - optsBson = jsonToBson("{\"maxTimeMS\": \(timeoutMS)}") - } - } - defer { if let opts = optsBson { bson_destroy(opts) } } - - guard let cursor = mongoc_collection_aggregate( - col, MONGOC_QUERY_NONE, pipelineBson, optsBson, nil - ) else { - mongoc_collection_destroy(col) - throw MongoDBError(code: 0, message: "Failed to create aggregation cursor") - } - - streamState.lock.lock() - streamState.cursor = cursor - streamState.collection = col - streamState.lock.unlock() + streamState.lock.withLock { streamState.collection = col } + let cursor = try open(col, session) + streamState.lock.withLock { streamState.cursor = cursor } iterateCursorStreaming( - cursor: cursor, continuation: continuation, streamState: streamState + cursor: cursor, + delivery: MongoStreamDelivery( + continuation: continuation, state: streamState, session: session, client: client + ) ) { try self.fieldCensus(census, client: client, database: database, collection: collection) } @@ -981,19 +948,24 @@ nonisolated final class MongoDBConnection: @unchecked Sendable { } } } - #else - return AsyncThrowingStream { $0.finish(throwing: MongoDBError.libmongocUnavailable) } - #endif } + #endif } nonisolated final class MongoStreamState: @unchecked Sendable { var cursor: OpaquePointer? var collection: OpaquePointer? + #if canImport(CLibMongoc) + var session: MongoStreamSession? + #endif var drained = false var cancelled = false let lock = NSLock() + + var isCancelled: Bool { + lock.withLock { cancelled } + } } // MARK: - BSON Helpers diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBNumberKinds.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBNumberKinds.swift new file mode 100644 index 0000000000..1e61cc450d --- /dev/null +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBNumberKinds.swift @@ -0,0 +1,123 @@ +import Foundation +import TableProNumberFormatting +import TableProPluginKit + +// The numeric type each top-level number of a read document held, by the document's row locator. One field can hold +// int32, int64, double and decimal128 side by side, and the column's majority kind would retype a cell on save. +nonisolated struct MongoDBNumberKinds: Sendable { + private var kindsByLocator: [String: [String: BsonValueKind]] = [:] + private(set) var count = 0 + + static let empty = MongoDBNumberKinds() + + static let cellLimit = 100_000 + + private static let numberKinds: Set = [.int32, .int64, .double, .decimal128] + + // A document holding no number is kept as an empty entry, so merging it clears what an earlier read recorded. + static func recording(_ read: MongoReadDocuments, representation: MongoDBUuidRepresentation) -> MongoDBNumberKinds { + var recorded = MongoDBNumberKinds() + for (document, text) in zip(read.dictionaries, read.texts) { + guard let locator = MongoDocumentIdentity.locator(inDocument: text) else { continue } + var kinds: [String: BsonValueKind] = [:] + for (field, value) in document where field != MongoDBCollectionDDL.idField { + let kind = BsonDocumentFlattener.valueKind(for: value, representation: representation) + if numberKinds.contains(kind) { kinds[field] = kind } + } + recorded.kindsByLocator[locator] = kinds + recorded.count += kinds.count + } + return recorded + } + + var isEmpty: Bool { kindsByLocator.isEmpty } + + func kinds(ofRow locator: String) -> [String: BsonValueKind]? { + kindsByLocator[locator] + } + + // A document read again replaces what was recorded for it, since its fields can have changed type or gone. + func merging(_ page: MongoDBNumberKinds) -> MongoDBNumberKinds { + var merged = self + for (locator, kinds) in page.kindsByLocator { + merged.count -= merged.kindsByLocator[locator]?.count ?? 0 + merged.kindsByLocator[locator] = kinds.isEmpty ? nil : kinds + merged.count += kinds.count + } + return merged + } +} + +// What a driver's reads recorded, by the same collection key as its other caches. Past `cellLimit` a collection starts +// over from the latest read, and a row no longer recorded is typed by its column. +nonisolated final class MongoDBNumberKindStore: @unchecked Sendable { + private let lock = NSLock() + private var byCollection: [String: MongoDBNumberKinds] = [:] + + func remember(_ read: MongoReadDocuments, representation: MongoDBUuidRepresentation, collection key: String) { + let page = MongoDBNumberKinds.recording(read, representation: representation) + guard !page.isEmpty else { return } + lock.withLock { + let merged = (byCollection[key] ?? .empty).merging(page) + byCollection[key] = merged.count > MongoDBNumberKinds.cellLimit ? MongoDBNumberKinds.empty.merging(page) : merged + } + } + + func kinds(ofRows rowLocators: [Int: String], collection key: String) -> [Int: [String: BsonValueKind]] { + guard let kinds = lock.withLock({ byCollection[key] }) else { return [:] } + return rowLocators.compactMapValues { kinds.kinds(ofRow: $0) } + } + + func forget(collectionsWhere isStale: (String) -> Bool) { + lock.withLock { byCollection = byCollection.filter { !isStale($0.key) } } + } +} + +nonisolated enum MongoDBDoubleText { + // `0.1` and `4` are doubles exactly as typed; `9007199254740993` is not, a double rounds it to ...992. + static func isExact(_ text: String) -> Bool { + guard let parsed = Double(text), parsed.isFinite, + let typed = DecimalDigits(text), let held = DecimalDigits(parsed.description) else { return false } + return typed == held + } + + // A JSON number as sign, significant digits and power of ten, so `4`, `4.0` and `0.4e1` compare equal. + private struct DecimalDigits: Equatable { + let isNegative: Bool + let significand: String + let exponent: Int + + init?(_ text: String) { + guard NumberText.isJSONNumberLiteral(text) else { return nil } + var rest = Substring(text) + let isNegative = rest.first == "-" + if isNegative { rest = rest.dropFirst() } + var exponent = 0 + if let marker = rest.firstIndex(where: { $0 == "e" || $0 == "E" }) { + guard let power = Int(rest[rest.index(after: marker)...]), power.magnitude < 1 << 32 else { return nil } + exponent = power + rest = rest[..] = [:] private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] private var fieldKindsByCollection: [String: MongoDBFieldKinds] = [:] + private let numberKinds = MongoDBNumberKindStore() private static let logger = Logger(subsystem: "com.TablePro", category: "MongoDBPluginDriver") @@ -35,7 +36,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se func quoteIdentifier(_ name: String) -> String { name } var capabilities: PluginCapabilities { - [.cancelQuery, .statementPolicy] + [.cancelQuery, .statementPolicy, .columnDefaultMarks] } func defaultExportQuery(table: String) -> String? { @@ -835,8 +836,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se insertedRowIndices: Set, context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { - var generator = writeGenerator(for: table, columns: columns, database: context.database) - generator.rowLocators = context.rowLocators + let generator = writeGenerator(for: table, columns: columns, database: context.database, rowLocators: context.rowLocators) return try generator.generateRowWrites( from: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices @@ -866,8 +866,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se absentCells: [Int: Set], context: PluginRowWriteContext ) -> [(statement: String, parameters: [PluginCellValue])]? { - var generator = writeGenerator(for: table, columns: columns, database: context.database) - generator.rowLocators = context.rowLocators + let generator = writeGenerator(for: table, columns: columns, database: context.database, rowLocators: context.rowLocators) return generator.generateRestore(rows: rows, absentCells: absentCells) } @@ -884,7 +883,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se return query + #".collation({"locale":"simple"})"# } - private func writeGenerator(for table: String, columns: [String], database: String? = nil) -> MongoDBStatementGenerator { + private func writeGenerator(for table: String, columns: [String], database: String?, rowLocators: [Int: String]) -> MongoDBStatementGenerator { let key = columnKindKey(table, database: database) return columnKindLock.withLock { let declared = declaredSchemasByCollection[key] ?? .empty @@ -894,6 +893,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se columnKinds: columnKindsByCollection[key] ?? [:], declaredKinds: declared.valueKinds, identityKinds: identityKindsByCollection[key], + rowLocators: rowLocators, readNumberKinds: numberKinds.kinds(ofRows: rowLocators, collection: key), fieldKinds: fieldKindsByCollection[key] ?? .empty, binarySubtypes: binarySubtypesByCollection[key] ?? .empty, declaredBinaryFields: Self.binaryFields(declaredBy: declared), @@ -933,7 +933,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } let timeout = conn.queryTimeoutMS - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in let work = Task { do { switch try await runtime.exportPlan(for: trimmed, database: db, access: access, shell: shell) { @@ -965,7 +965,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se do { for try await element in inner { try Task.checkCancellation() - continuation.yield(element) + guard await PluginRowStream.deliver(element, to: continuation) else { break } } } catch { throw MongoScriptStatementFailure.carrying( @@ -974,7 +974,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } case .result(let outcome): if let switched = outcome.databaseSwitch { self.currentDb = switched } - self.yieldMaterialised(outcome, into: continuation) + await self.deliverMaterialised(outcome, to: continuation) } continuation.finish() } catch { @@ -1014,10 +1014,10 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se /// Hands over a statement that had already run by the time the export asked, rather than /// running it a second time. Nested values render the way a streamed cursor renders them, so /// one export never mixes two spellings. - private func yieldMaterialised( + private func deliverMaterialised( _ outcome: MongoScriptStatementResult, - into continuation: AsyncThrowingStream.Continuation - ) { + to continuation: AsyncThrowingStream.Continuation + ) async { let result = MongoScriptResultBuilder.result( for: outcome, startTime: Date(), @@ -1031,13 +1031,12 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } ) if !result.columns.isEmpty { - continuation.yield(.header(PluginStreamHeader( - columns: result.columns, - columnTypeNames: result.columnTypeNames - ))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + guard await PluginRowStream.deliver(.header(header), to: continuation) else { return } } if !result.rows.isEmpty { - continuation.yield(.rows(result.rows, documents: result.exportDocuments)) + _ = await PluginRowStream.deliver(.rows(result.rows, documents: result.exportDocuments), to: continuation) } } @@ -1073,6 +1072,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se rememberIdentityKinds(of: documents, collection: collection) rememberBinarySubtypes(of: documents, collection: collection) rememberFieldKinds(of: documents, collection: collection) + if !collection.isEmpty { numberKinds.remember(read, representation: uuidRepresentation, collection: columnKindKey(collection)) } } let unseen = MongoDBCollectionShape.declaredColumnsMissing(from: sampled, schema: declared) let columns = sampled + unseen @@ -1256,6 +1256,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se declaredSchemasByCollection = declaredSchemasByCollection.filter { !isStale($0.key) } identityKindsByCollection = identityKindsByCollection.filter { !isStale($0.key) } } + numberKinds.forget(collectionsWhere: isStale) } } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index 2729efa23b..b4082e663f 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -409,21 +409,26 @@ nonisolated struct MongoDBQueryBuilder { return (parts[0], parts[1]) } + // Equal values come back in any order, so each skip/limit page could repeat or miss documents. `_id` is unique and + // makes the order total (MongoDB `cursor.sort()`, Sort Consistency). private func buildSortDocument( sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String] ) -> String? { - guard !sortColumns.isEmpty else { return nil } - - let parts = sortColumns.compactMap { sortCol -> String? in + var parts: [String] = [] + var sortsById = false + for sortCol in sortColumns { guard sortCol.columnIndex >= 0, sortCol.columnIndex < columns.count, - Self.isPlainFieldName(columns[sortCol.columnIndex]) else { return nil } - let columnName = Self.escapeJsonString(columns[sortCol.columnIndex]) - let direction = sortCol.ascending ? 1 : -1 - return "\"\(columnName)\": \(direction)" + Self.isPlainFieldName(columns[sortCol.columnIndex]) else { continue } + let column = columns[sortCol.columnIndex] + sortsById = sortsById || column == MongoDBCollectionDDL.idField + parts.append("\"\(Self.escapeJsonString(column))\": \(sortCol.ascending ? 1 : -1)") } guard !parts.isEmpty else { return nil } + if !sortsById { + parts.append("\"\(MongoDBCollectionDDL.idField)\": 1") + } return "{\(parts.joined(separator: ", "))}" } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index e3ff61ce49..f366ee5e50 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -26,6 +26,8 @@ nonisolated struct MongoDBStatementGenerator { // By `PluginRowChange.rowIndex`. Display text cannot say which value an `_id` came from: a string and an ObjectId // with the same hex read the same. var rowLocators: [Int: String] = [:] + // By row index like `rowLocators`: the numeric type each top-level number of the row was read as. + var readNumberKinds: [Int: [String: BsonValueKind]] = [:] /// Every kind each field held in the documents read, which says whether a cell showing `{` or /// `[` holds a document or an array or a string that reads the same. var fieldKinds: MongoDBFieldKinds = .empty @@ -40,8 +42,6 @@ nonisolated struct MongoDBStatementGenerator { /// connection. var capabilities: () -> MongoDBCapabilities = { .unknown } - private static let defaultMarker = "__DEFAULT__" - /// Where a value written whole comes from. A cell carries only text or bytes, so this is all the /// generator knows about the type the value has to keep. private enum Provenance { @@ -133,7 +133,7 @@ nonisolated struct MongoDBStatementGenerator { let absent = change.absentColumns ?? [] let written = cells.filter { cell in let serverAssignsIdentity = cell.field == MongoDBCollectionDDL.idField && cell.value.isNull - return !serverAssignsIdentity && !isLeftOut(cell.value, isAbsent: absent.contains(cell.column)) + return !serverAssignsIdentity && !absent.contains(cell.column) && !change.isColumnDefault(cell.column) } let entries = try written.map { cell in let provenance: Provenance = typed[cell.field] == cell.value ? .typedIntoNewDocument : .copiedIntoNewDocument @@ -142,23 +142,19 @@ nonisolated struct MongoDBStatementGenerator { return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" } - private func isLeftOut(_ value: PluginCellValue, isAbsent: Bool) -> Bool { - isAbsent || value.asText == Self.defaultMarker - } - - /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a - /// deleted document put back. An `_id` the row holds is kept, typed the way the filters type it; - /// the grid leaves a new row's `_id` as DEFAULT, so the server generates it. - /// - /// The shell inserts through libmongoc, which refuses a document holding an empty key at any - /// depth, and a JavaScript object reads `__proto__` as its prototype and drops it. An update - /// carries both, so only a new document refuses them, before anything in the save is sent. - private func documentValueJson(_ value: PluginCellValue, field: String, provenance: Provenance) throws -> String { + // libmongoc refuses an inserted document holding an empty key at any depth, and JavaScript reads `__proto__` as the + // prototype and drops it. An update carries both, so only a new or restored document refuses them, before any write. + private func documentValueJson( + _ value: PluginCellValue, + field: String, + provenance: Provenance, + readKind: BsonValueKind? = nil + ) throws -> String { guard !field.isEmpty else { throw MongoDBWriteRefusal.emptyFieldNameInNewDocument } guard field != "__proto__" else { throw MongoDBWriteRefusal.prototypeFieldInNewDocument } let json = field == MongoDBCollectionDDL.idField ? try idValueJson(value) - : try valueJson(value, field: field, provenance: provenance) + : try valueJson(value, field: field, provenance: provenance, readKind: readKind) if opensContainer(json), case .text(let text) = value, parsedContainer(text)?.holdsEmptyKey == true { throw MongoDBWriteRefusal.emptyKeyInNewDocument(field: field) } @@ -167,12 +163,8 @@ nonisolated struct MongoDBStatementGenerator { // MARK: - Restore - /// Puts a deleted document back with the `_id` it had. - /// - /// A new row leaves `_id` to the server. Undoing a delete is the opposite requirement: a new - /// `_id` is a different document, and anything that referenced the old one still points at - /// nothing. A value that cannot be written refuses the restore rather than dropping the field. - /// A field the document did not have stays missing, and one that held null holds it again. + // An undone delete keeps its `_id`, since a new one is another document that nothing referencing the old one finds. + // A value that cannot be written refuses the restore rather than dropping the field. func generateRestore( rows: [[PluginCellValue]], absentCells: [Int: Set] = [:] @@ -184,12 +176,14 @@ nonisolated struct MongoDBStatementGenerator { guard idIndex < row.count else { throw MongoDBWriteRefusal.missingIdentity } guard let locator = rowLocators[rowIndex] else { throw MongoDBWriteRefusal.unlocatedRow } let absent = absentCells[rowIndex] ?? [] + let readKinds = readNumberKinds[rowIndex] ?? [:] let idField = MongoDBCollectionDDL.idField var entries = ["\(quotedKey(idField)): \(try Self.identityJson(locator: locator))"] for (index, value) in row.enumerated() where index != idIndex && index < columns.count { - guard !isLeftOut(value, isAbsent: absent.contains(index)) else { continue } + guard !absent.contains(index) else { continue } let field = columns[index] - entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field, provenance: .restored))") + let json = try documentValueJson(value, field: field, provenance: .restored, readKind: readKinds[field]) + entries.append("\(quotedKey(field)): \(json)") } let statement = "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" return (statement: statement, parameters: []) @@ -218,15 +212,16 @@ nonisolated struct MongoDBStatementGenerator { } } - /// A field the change names in `absentColumns` is removed with `$unset`; NULL is stored as null. private func updateStatement(for change: PluginRowChange) throws -> String? { guard !change.cellChanges.isEmpty else { return nil } let identity = try identityFilter(of: change) let removed = change.absentColumns ?? [] + let readKinds = readNumberKinds[change.rowIndex] ?? [:] let cellWrites = try change.cellChanges.map { try cellWrite( field: $0.columnName, from: $0.oldValue, to: $0.newValue, - removesField: removed.contains($0.columnIndex) + removesField: removed.contains($0.columnIndex), isDefault: change.isColumnDefault($0.columnIndex), + readKind: readKinds[$0.columnName] ) } let update = try updateDocument(for: cellWrites) @@ -237,21 +232,24 @@ nonisolated struct MongoDBStatementGenerator { field: String, from oldValue: PluginCellValue, to newValue: PluginCellValue, - removesField: Bool + removesField: Bool, + isDefault: Bool, + readKind: BsonValueKind? ) throws -> CellWrite { guard field != MongoDBCollectionDDL.idField else { throw MongoDBWriteRefusal.identityChanged } guard !removesField else { return .remove(field: field) } + guard !isDefault else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } switch newValue { case .null: return .whole(field: field, json: "null") case .bytes(let data): return .whole(field: field, json: try binaryJson(data, field: field, provenance: .edit(replacing: oldValue))) case .text(let text): - guard text != Self.defaultMarker else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } if let nested = try nestedEdit(of: field, from: oldValue, to: text) { return nested } - return .whole(field: field, json: try valueJson(newValue, field: field, provenance: .edit(replacing: oldValue))) + let json = try valueJson(newValue, field: field, provenance: .edit(replacing: oldValue), readKind: readKind) + return .whole(field: field, json: json) } } @@ -380,8 +378,12 @@ nonisolated struct MongoDBStatementGenerator { return type == "array" || type == "regex" } - /// A cell's value as the JSON the statement carries. - private func valueJson(_ value: PluginCellValue, field: String, provenance: Provenance) throws -> String { + private func valueJson( + _ value: PluginCellValue, + field: String, + provenance: Provenance, + readKind: BsonValueKind? = nil + ) throws -> String { switch value { case .null: return "null" @@ -394,7 +396,7 @@ nonisolated struct MongoDBStatementGenerator { guard !JSONTruncation.isIncompleteStructure(text) else { throw MongoDBWriteRefusal.truncatedValue(field: field) } - return try jsonValue(for: text, field: field, provenance: provenance) + return try jsonValue(for: text, field: field, provenance: provenance, readKind: readKind) } } @@ -584,18 +586,23 @@ nonisolated struct MongoDBStatementGenerator { return nsValue.length == 24 && value.allSatisfy { $0.isHexDigit } } - /// A cell's text as the value it stands for, in the field's type when that type is known. - /// - /// The statement is JavaScript the shell evaluates, so text is only ever pasted in when it is - /// strict JSON. A stored string that merely starts with `[` and ends with `]` would otherwise run - /// as code the moment its row is duplicated or its delete is undone. - private func jsonValue(for value: String, field: String, provenance: Provenance) throws -> String { + // The statement is JavaScript the shell evaluates, so text goes in only as strict JSON: a stored string reading + // `[...]` would otherwise run as code once its row is duplicated or its delete undone. + private func jsonValue( + for value: String, + field: String, + provenance: Provenance, + readKind: BsonValueKind? + ) throws -> String { if declaredKinds[field] == .string || spellsAnotherKindInAStringField(value, field: field) { return "\"\(escapeJsonString(value))\"" } if value == "true" || value == "false" || value == "null" { return value } + if let typed = readNumberJson(value, field: field, readKind: readKind, provenance: provenance) { + return typed + } if let typed = typedJson(value, kind: kind(of: field)) { return typed } @@ -611,6 +618,34 @@ nonisolated struct MongoDBStatementGenerator { return "\"\(escapeJsonString(value))\"" } + // A number keeps the type its cell was read as when the text is a value of that type, since a field can mix numeric + // types and the column's majority would retype it. A validator's declared kind is what the server enforces. + private func readNumberJson( + _ value: String, + field: String, + readKind: BsonValueKind?, + provenance: Provenance + ) -> String? { + guard declaredKinds[field] == nil, let readKind, readsAsNumber(provenance) else { return nil } + switch readKind { + case .int32, .int64, .decimal128: + return typedJson(value, kind: readKind) + case .double: + return MongoDBDoubleText.isExact(value) ? typedJson(value, kind: .double) : nil + default: + return nil + } + } + + private func readsAsNumber(_ provenance: Provenance) -> Bool { + switch provenance { + case .edit(replacing: .text), .restored: + return true + case .edit, .typedIntoNewDocument, .copiedIntoNewDocument: + return false + } + } + // A cell shows the string "90210" and the number 90210 as the same text. In a field that held // strings and never a number, boolean or null, text of that shape is a string like the rest. private func spellsAnotherKindInAStringField(_ value: String, field: String) -> Bool { diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift index 9b6fab6db0..eaa84845cb 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBFieldMetadata.swift @@ -41,7 +41,8 @@ nonisolated internal func mariaDBTypeName( switch typeRaw { case 0: return "DECIMAL" - case 1: return "TINYINT" + // A BOOLEAN column arrives as TINY with length 1. + case 1: return length == 1 ? "TINYINT(1)" : "TINYINT" case 2: return "SMALLINT" case 3: return "INT" case 4: return "FLOAT" @@ -56,7 +57,8 @@ nonisolated internal func mariaDBTypeName( case 13: return "YEAR" case 14: return "NEWDATE" case 15: return "VARCHAR" - case 16: return "BIT" + // The width is what tells a one-bit flag from a BIT(8) that reads as a number. + case 16: return length > 1 ? "BIT(\(length))" : "BIT" case 245: return "JSON" case 246: return "DECIMAL" case 247: return "ENUM" diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift index dc1c4b5612..847f4bf2b9 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift @@ -39,6 +39,10 @@ enum MySQLTransport: Sendable, Equatable { mysql_options(handle, MYSQL_OPT_PROTOCOL, &value) } + // An UPDATE then reports the rows it matched, not only those whose value changed, so a save can tell a row that + // already held the value from one that is gone. + static let clientFlags = UInt(CLIENT_FOUND_ROWS) + /// Connector/C takes the socket path only when the host is NULL or exactly `localhost`. func realConnect( _ handle: UnsafeMutablePointer, @@ -48,14 +52,14 @@ enum MySQLTransport: Sendable, Equatable { ) -> UnsafeMutablePointer? { switch self { case .tcp(let host, let port): - let connected = mysql_real_connect(handle, host, user, password, database, port, nil, 0) + let connected = mysql_real_connect(handle, host, user, password, database, port, nil, Self.clientFlags) // Every session and kill connection dials through here, and a session's socket outlives the connect. if connected != nil, !MySQLSocketKeepalive.enable(on: mysql_get_socket(handle)) { Self.logger.warning("TCP keepalive could not be set: errno \(errno, privacy: .public)") } return connected case .unixSocket(let path): - return mysql_real_connect(handle, "localhost", user, password, database, 0, path, 0) + return mysql_real_connect(handle, "localhost", user, password, database, 0, path, Self.clientFlags) } } } diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index edea147522..81662838f4 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -88,7 +88,19 @@ struct MariaDBPluginQueryResult { // MARK: - Connection Class final class MariaDBPluginConnection: @unchecked Sendable { - private var mysql: UnsafeMutablePointer? + // Under stateLock, which disconnect() clears it under. Only `queue` may call into the handle: the close is queued there. + private var _mysql: UnsafeMutablePointer? + private var _serverThreadId: UInt = 0 + + private var mysql: UnsafeMutablePointer? { + stateLock.withLock { _mysql } + } + + // Read from the snapshot taken at connect: Stop runs off `queue`, where the handle may already be freed. + private var connectedThreadId: UInt? { + stateLock.withLock { _mysql == nil ? nil : _serverThreadId } + } + private let queue = DispatchQueue(label: "com.TablePro.mariadb.plugin", qos: .userInitiated) /// Serial, and separate from `queue`, which the read being cancelled is sitting on. Serial so a @@ -136,6 +148,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { private var baselineSelectLimit: UInt64? private var hasCapturedBaselineSelectLimit = false + // Whether this session already waits long enough for a streaming reader. Only touched from `queue`. + private var hasStreamingWriteTimeout = false + /// Whether the connection that actually succeeded negotiated TLS. `.preferred` falls back to /// plaintext, so the configured mode does not say what the transport ended up being, and the /// `KILL` connection has to repeat what worked rather than what was asked for. @@ -342,9 +357,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { if let cachePath = kerberos?.cachePath { KerberosTicket.discard(cachePath) } - let handle = mysql + let handle = _mysql let cleanupQueue = queue - mysql = nil + _mysql = nil if let handle = handle { cleanupQueue.async { mysql_close(handle) @@ -389,9 +404,12 @@ final class MariaDBPluginConnection: @unchecked Sendable { self.appliedSelectLimit = nil self.baselineSelectLimit = nil self.hasCapturedBaselineSelectLimit = false + self.hasStreamingWriteTimeout = false + let threadId = mysql_thread_id(handle) self.stateLock.lock() - self.mysql = handle + self._mysql = handle + self._serverThreadId = threadId self._isConnected = true self._errorLanguage = session.errorLanguage self._sessionEndedByKill = false @@ -600,10 +618,10 @@ final class MariaDBPluginConnection: @unchecked Sendable { KerberosTicket.discard(cachePath) } - let handle = mysql - mysql = nil - stateLock.lock() + let handle = _mysql + _mysql = nil + _serverThreadId = 0 _isConnected = false stateLock.unlock() @@ -635,7 +653,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { let target = killTarget guard mayEndSession || !target.endsSession else { return } - guard let mysql = mysql, let statement = target.statement(threadId: mysql_thread_id(mysql)) else { return } + guard let threadId = connectedThreadId, let statement = target.statement(threadId: threadId) else { return } cancelQueue.async { [self] in guard deliverKill(statement) else { return } if target.endsSession { @@ -657,11 +675,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { recordKillDelivered(generation: generation) } - /// The server thread this connection is on, read before a statement goes out so a kill still - /// has somewhere to go once the handle itself is unusable. + // Read before a statement goes out, so a kill still has somewhere to go once the handle itself is unusable. internal var currentThreadId: UInt { - guard let mysql = self.mysql else { return 0 } - return mysql_thread_id(mysql) + connectedThreadId ?? 0 } /// The kill has to reach the server the query is running on, which means repeating the transport @@ -881,35 +897,36 @@ final class MariaDBPluginConnection: @unchecked Sendable { // MARK: - Query Execution - func executeQuery(_ query: String, rowCap: Int? = nil) async throws -> MariaDBPluginQueryResult { + func executeQuery(_ query: String, rowCap: Int? = nil, byteLimit: Int? = nil) async throws -> MariaDBPluginQueryResult { let queryToRun = String(query) return try await pluginDispatchAsync(on: queue) { [self] in guard !isShuttingDown else { throw MariaDBPluginError.notConnected } - return try executeQuerySync(queryToRun, rowCap: rowCap) + return try executeQuerySync(queryToRun, rowCap: rowCap, byteLimit: byteLimit) } } func executeParameterizedQuery( _ query: String, parameters: [PluginCellValue], - rowCap: Int? = nil + rowCap: Int? = nil, + byteLimit: Int? = nil ) async throws -> MariaDBPluginQueryResult { let queryToRun = String(query) let params = parameters return try await pluginDispatchAsync(on: queue) { [self] in guard !isShuttingDown else { throw MariaDBPluginError.notConnected } - return try executeParameterizedQuerySync(queryToRun, parameters: params, rowCap: rowCap) + return try executeParameterizedQuerySync(queryToRun, parameters: params, rowCap: rowCap, byteLimit: byteLimit) } } - // `sessionState` runs in this same block, so nothing queued behind it can open a transaction before the wrap. func executeReadOnly( _ query: String, parameters: [PluginCellValue]?, rowCap: Int?, - sessionState: @escaping @Sendable (Bool) -> PluginSessionTransactionState + byteLimit: Int?, + scope: MySQLReadOnlyTransaction.Scope ) async throws -> MariaDBPluginQueryResult { let queryToRun = String(query) let params = parameters @@ -917,20 +934,30 @@ final class MariaDBPluginConnection: @unchecked Sendable { return try await pluginDispatchAsync(on: queue) { [self] in guard !isShuttingDown else { throw MariaDBPluginError.notConnected } return try MySQLReadOnlyTransaction.run( - state: sessionState(isInTransaction), + state: scope.sessionState(isInTransaction), + reason: scope.reason, + runsInOpenTransaction: scope.runsInOpenTransaction, send: { _ = try executeQuerySync($0) }, statement: { - guard let params else { return try executeQuerySync(queryToRun, rowCap: rowCap) } - return try executeParameterizedQuerySync(queryToRun, parameters: params, rowCap: rowCap) + guard let params else { + return try executeQuerySync(queryToRun, rowCap: rowCap, byteLimit: byteLimit) + } + return try executeParameterizedQuerySync( + queryToRun, parameters: params, rowCap: rowCap, byteLimit: byteLimit + ) } ) } } - private func executeQuerySync(_ query: String, rowCap: Int? = nil) throws -> MariaDBPluginQueryResult { + private func executeQuerySync( + _ query: String, + rowCap: Int? = nil, + byteLimit: Int? = nil + ) throws -> MariaDBPluginQueryResult { try runStatement(query) { try self.runCancellableStatement { mysql, generation in - try self.runTextStatement(query, rowCap: rowCap, on: mysql, generation: generation) + try self.runTextStatement(query, rowCap: rowCap, byteLimit: byteLimit, on: mysql, generation: generation) } } } @@ -968,6 +995,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { private func runTextStatement( _ query: String, rowCap: Int?, + byteLimit: Int?, on mysql: UnsafeMutablePointer, generation: Int ) throws -> MariaDBPluginQueryResult { @@ -1034,6 +1062,8 @@ final class MariaDBPluginConnection: @unchecked Sendable { let maxRows = mysqlClampedRowCap(rowCap) ?? PluginRowLimits.emergencyMax let fetchLimit = maxRows == PluginRowLimits.emergencyMax ? maxRows : maxRows + 1 var serverSentMore = false + var bytes = 0 + var stoppedAtByteLimit = false var firstRowTime: TimeInterval? while let rowPtr = mysql_fetch_row(resultPtr) { @@ -1050,20 +1080,30 @@ final class MariaDBPluginConnection: @unchecked Sendable { break } - rows.append(textProtocolRow(rowPtr, lengths: mysql_fetch_lengths(resultPtr), columns: columns)) + let row = textProtocolRow(rowPtr, lengths: mysql_fetch_lengths(resultPtr), columns: columns) + rows.append(row) + if let byteLimit { + bytes += mysqlDecodedByteCount(of: row) + if bytes > byteLimit { + stoppedAtByteLimit = true + break + } + } } let outcome = mysqlBoundedFetchOutcome( fetchedRows: rows.count, rowCap: maxRows, - serverSentMore: serverSentMore + serverSentMore: serverSentMore, + stoppedAtByteLimit: stoppedAtByteLimit ) let truncated = outcome.isTruncated + let stoppedEarly = outcome.serverIgnoredLimit || stoppedAtByteLimit if truncated { - logger.warning("Result set truncated at \(maxRows) rows") + logger.warning("Result set truncated at \(outcome.keptRows) rows") rows.removeLast(rows.count - outcome.keptRows) } - if outcome.serverIgnoredLimit { + if stoppedEarly { if !sessionFlavor.dropsIdleSessionOnKillQuery { killStatementOnServer(on: mysql, generation: generation) } @@ -1079,9 +1119,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { let fetchErrno = mysql_errno(mysql) if fetchErrno != 0 { let error = noting(getError(), generation: generation) - if !isExpectedInterruption( - errno: fetchErrno, message: error.message, wasTruncated: outcome.serverIgnoredLimit - ) { + if !isExpectedInterruption(errno: fetchErrno, message: error.message, wasTruncated: stoppedEarly) { mysql_free_result(resultPtr) throw error } @@ -1091,7 +1129,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { do { try consumeTrailingResults(on: mysql, generation: generation) } catch let error as MariaDBPluginError where isExpectedInterruption( - errno: error.code, message: error.message, wasTruncated: outcome.serverIgnoredLimit + errno: error.code, message: error.message, wasTruncated: stoppedEarly ) { // The row cap's own kill stopped the routine after its first result set, the truncation already reported. } @@ -1195,6 +1233,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { metadata: UnsafeMutablePointer, columns: MySQLResultColumns, rowCap: Int? = nil, + byteLimit: Int?, generation: Int, sentAt: Date ) throws -> (rows: [[PluginCellValue]], isTruncated: Bool, firstRowTime: TimeInterval) { @@ -1234,6 +1273,8 @@ final class MariaDBPluginConnection: @unchecked Sendable { let maxRows = mysqlClampedRowCap(rowCap) ?? PluginRowLimits.emergencyMax let fetchLimit = maxRows == PluginRowLimits.emergencyMax ? maxRows : maxRows + 1 var serverSentMore = false + var bytes = 0 + var stoppedAtByteLimit = false /// `mysql_stmt_execute` returns once the server has answered with a header, which on an /// unbuffered statement can be long before the first tuple exists. Only a fetch that /// returns a row proves the server produced one. @@ -1279,20 +1320,30 @@ final class MariaDBPluginConnection: @unchecked Sendable { } } - rows.append(columns.row(encoding: connectionEncoding) { index in + let row = columns.row(encoding: connectionEncoding) { index in guard resultBinds[index].is_null?.pointee != 1 else { return nil } let length = Int(resultBinds[index].length?.pointee ?? 0) return UnsafeRawBufferPointer(start: resultBuffers[index], count: length) - }) + } + rows.append(row) + // `mysql_stmt_next_result` and `mysql_stmt_close` skip the rows left unread. + if let byteLimit { + bytes += mysqlDecodedByteCount(of: row) + if bytes > byteLimit { + stoppedAtByteLimit = true + break + } + } } let outcome = mysqlBoundedFetchOutcome( fetchedRows: rows.count, rowCap: maxRows, - serverSentMore: serverSentMore + serverSentMore: serverSentMore, + stoppedAtByteLimit: stoppedAtByteLimit ) if outcome.isTruncated { - logger.warning("Prepared statement result truncated at \(maxRows) rows") + logger.warning("Prepared statement result truncated at \(outcome.keptRows) rows") rows.removeLast(rows.count - outcome.keptRows) } @@ -1306,18 +1357,19 @@ final class MariaDBPluginConnection: @unchecked Sendable { private func executeParameterizedQuerySync( _ query: String, parameters: [PluginCellValue], - rowCap: Int? = nil + rowCap: Int? = nil, + byteLimit: Int? = nil ) throws -> MariaDBPluginQueryResult { guard flavor.preparesOnServer else { let inlined = flavor.isDatabend ? try DatabendLiteral.inline(query, parameters: parameters, serverVersion: serverVersion()) : try StarRocksLiteral.inline(query, parameters: parameters) - return try executeQuerySync(inlined, rowCap: rowCap) + return try executeQuerySync(inlined, rowCap: rowCap, byteLimit: byteLimit) } return try runStatement(query) { try self.runCancellableStatement { mysql, generation in try self.runPreparedStatement( - query, parameters: parameters, rowCap: rowCap, on: mysql, generation: generation + query, parameters: parameters, rowCap: rowCap, byteLimit: byteLimit, on: mysql, generation: generation ) } } @@ -1327,6 +1379,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { _ query: String, parameters: [PluginCellValue], rowCap: Int?, + byteLimit: Int?, on mysql: UnsafeMutablePointer, generation: Int ) throws -> MariaDBPluginQueryResult { @@ -1407,7 +1460,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { let fetchResult = try fetchResultSet( from: stmt, metadata: metadata, - columns: columns, rowCap: rowCap, generation: generation, sentAt: sentAt + columns: columns, rowCap: rowCap, byteLimit: byteLimit, generation: generation, sentAt: sentAt ) try consumeTrailingResults(of: stmt, generation: generation) @@ -1429,17 +1482,17 @@ final class MariaDBPluginConnection: @unchecked Sendable { // MARK: - Streaming Query - // With `readOnlySessionState` the whole stream runs inside `executeReadOnly`'s transaction. + // With `readOnly` the whole stream runs inside `executeReadOnly`'s transaction. func streamQuery( _ query: String, - readOnlySessionState: (@Sendable (Bool) -> PluginSessionTransactionState)? = nil + readOnly: MySQLReadOnlyTransaction.Scope? = nil ) -> AsyncThrowingStream { let queryToRun = String(query) /// The drain belongs to the producer. Enqueued from `onTermination` it would sit behind the /// producer on this one serial queue and run only after the whole result had been read, /// which is why the abort is a polled flag instead. - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in self.queue.async { [self] in do { let stream = { @@ -1451,13 +1504,15 @@ final class MariaDBPluginConnection: @unchecked Sendable { } } } - if let readOnlySessionState { + if let readOnly { guard !abort.isAborted else { continuation.finish() return } try MySQLReadOnlyTransaction.run( - state: readOnlySessionState(isInTransaction), + state: readOnly.sessionState(isInTransaction), + reason: readOnly.reason, + runsInOpenTransaction: readOnly.runsInOpenTransaction, send: { _ = try executeQuerySync($0) }, statement: stream ) @@ -1482,6 +1537,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { guard !abort.isAborted else { return } try reconcileSelectLimit(rowCap: nil, statement: queryToRun, on: mysql) + raiseStreamingWriteTimeout(on: mysql) let queryStatus = queryToRun.withCString { queryPtr in mysql_real_query(mysql, queryPtr, UInt(queryToRun.utf8.count)) @@ -1508,11 +1564,12 @@ final class MariaDBPluginConnection: @unchecked Sendable { binaryStringsAreText: binaryStringsAreText(in: queryToRun, flavor: sessionFlavor) ) - continuation.yield(.header(PluginStreamHeader( + let stops = { self.cancellationGate.isCancelled(generation) } + _ = PluginRowStream.deliverBlocking(.header(PluginStreamHeader( columns: columns.names, columnTypeNames: columns.typeNames, estimatedRowCount: nil - ))) + )), to: continuation, stops: stops) let batchSize = 5_000 var batch: [PluginRow] = [] @@ -1529,14 +1586,13 @@ final class MariaDBPluginConnection: @unchecked Sendable { } batch.append(textProtocolRow(rowPtr, lengths: mysql_fetch_lengths(resultPtr), columns: columns)) - if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // A reader that left or a Stop is handled at the top of the loop. + if batch.count >= batchSize, + PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) { batch.removeAll(keepingCapacity: true) } } - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } + let delivered = batch.isEmpty || PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) if mysql_errno(mysql) != 0 { let error = noting(getError(), generation: generation) @@ -1546,6 +1602,24 @@ final class MariaDBPluginConnection: @unchecked Sendable { mysql_free_result(resultPtr) try consumeTrailingResults(on: mysql, generation: generation) + guard delivered else { throw CancellationError() } + } + + // While the stream waits for its reader the server waits on the socket, and gives up after `net_write_timeout`, + // 60 seconds by default. 600 is the value MySQL Connector/J sets for a streaming result + // (`netTimeoutForStreamingResults`). Asked once per session: a server that refuses it keeps its own. + private func raiseStreamingWriteTimeout(on mysql: UnsafeMutablePointer) { + guard !hasStreamingWriteTimeout else { return } + hasStreamingWriteTimeout = true + let statement = "SET SESSION net_write_timeout = GREATEST(@@SESSION.net_write_timeout, 600)" + let status = statement.withCString { mysql_real_query(mysql, $0, UInt(statement.utf8.count)) } + if let discarded = mysql_store_result(mysql) { + mysql_free_result(discarded) + } + guard status == 0 else { + logger.warning("net_write_timeout not raised for a stream: \(self.errorMessage(from: mysql))") + return + } } // MARK: - Trailing Results diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift b/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift index 0d9061812f..aaf9fdf55d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift @@ -78,6 +78,21 @@ internal enum MySQLCheckConstraints { return "ALTER TABLE \(quotedTable) \(keyword) \(quotedName)" } + // CHECK_CONSTRAINTS has no TABLE_NAME here. TABLE_CONSTRAINTS also lists keys, whose names are per table, so + // without the type another table's key named like this table's check would match. + static func joinedCatalogQuery(databaseLiteral: String, tableLiteral: String) -> String { + """ + SELECT cc.CONSTRAINT_NAME, cc.CHECK_CLAUSE + FROM INFORMATION_SCHEMA.CHECK_CONSTRAINTS cc + JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + ON tc.CONSTRAINT_SCHEMA = cc.CONSTRAINT_SCHEMA + AND tc.CONSTRAINT_NAME = cc.CONSTRAINT_NAME + AND tc.CONSTRAINT_TYPE = 'CHECK' + WHERE cc.CONSTRAINT_SCHEMA = '\(databaseLiteral)' AND tc.TABLE_NAME = '\(tableLiteral)' + ORDER BY cc.CONSTRAINT_NAME + """ + } + static func parse(createTable sql: String) -> [PluginCheckConstraintInfo] { guard let body = MySQLCreateTableScanner.firstGroup(in: Substring(sql)) else { return [] } return MySQLCreateTableScanner.topLevelElements(of: body).compactMap(checkConstraint(in:)) diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift b/apple/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift index 9fc43e6dae..9075378c0b 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift @@ -10,7 +10,7 @@ import TableProPluginKit /// /// The app's curated table is the authority for its gates; these are the same facts, held here because a plugin /// cannot link the app's package, and `SQLLexicalFeatureMappingTests` holds the two equal. -enum MySQLLexicalFeatures { +nonisolated enum MySQLLexicalFeatures { /// MySQL 8.4.11 and MariaDB 11.8.9, measured. static let mySQL: SQLLexicalFeatures = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift index 092b8fff7b..6ba43b5c93 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift @@ -311,15 +311,7 @@ internal extension MySQLPluginDriver { ORDER BY CONSTRAINT_NAME """ } else { - query = """ - SELECT cc.CONSTRAINT_NAME, cc.CHECK_CLAUSE - FROM INFORMATION_SCHEMA.CHECK_CONSTRAINTS cc - JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc - ON tc.CONSTRAINT_SCHEMA = cc.CONSTRAINT_SCHEMA - AND tc.CONSTRAINT_NAME = cc.CONSTRAINT_NAME - WHERE cc.CONSTRAINT_SCHEMA = \'\(database)\' AND tc.TABLE_NAME = \'\(safeTable)\' - ORDER BY cc.CONSTRAINT_NAME - """ + query = MySQLCheckConstraints.joinedCatalogQuery(databaseLiteral: database, tableLiteral: safeTable) } let result = try await execute(ownStatement: query) return result.rows.compactMap { row in diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerObjects.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerObjects.swift new file mode 100644 index 0000000000..fb6ccc9b69 --- /dev/null +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerObjects.swift @@ -0,0 +1,143 @@ +// +// MySQLPluginDriver+ServerObjects.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProMySQLCore +import TableProPluginKit + +nonisolated internal enum MySQLEvents { + static let kindId = "event" + + static var kind: PluginServerObjectKind { + PluginServerObjectKind( + id: kindId, + displayName: String(localized: "Event"), + pluralDisplayName: String(localized: "Events"), + iconName: "calendar.badge.clock", + scope: .container + ) + } + + // TiDB lists `EVENTS` with no rows and OceanBase is not established, so only these two declare it. + static func isDeclared(flavor: MySQLServerFlavor, banner: String?) -> Bool { + guard flavor == .mysql || flavor == .mariadb else { return false } + return MySQLServerVersion.hasPartitionAndEventCatalogs(banner: banner, flavor: flavor) + } +} + +// Reads `MySQLObjectQueries.eventDetails` rows: name, type, status, schema, then the schedule columns. +nonisolated internal enum MySQLEventRows { + static func objects(from rows: [[PluginCellValue]], scheduler: String? = nil) -> [PluginServerObjectInfo] { + rows.compactMap { object(from: $0, scheduler: scheduler) } + } + + // `scheduler` is `@@global.event_scheduler`, read once beside the rows; nil leaves the attribute out. + static func object(from row: [PluginCellValue], scheduler: String? = nil) -> PluginServerObjectInfo? { + guard let name = text(row, 0) else { return nil } + let isRecurring = text(row, 1)?.uppercased() == "RECURRING" + let status = text(row, 2)?.uppercased() + var attributes: [PluginObjectAttribute] = [] + func add(_ label: String, _ value: String?) { + guard let value, !value.isEmpty else { return } + attributes.append(PluginObjectAttribute(label: label, value: value)) + } + add(String(localized: "Schedule"), schedule( + isRecurring: isRecurring, executeAt: text(row, 4), intervalValue: text(row, 5), intervalField: text(row, 6) + )) + add(String(localized: "Status"), status.map(statusLabel)) + add(String(localized: "Starts"), text(row, 7)) + add(String(localized: "Ends"), text(row, 8)) + add(String(localized: "On Completion"), text(row, 9)) + add(String(localized: "Last Executed"), text(row, 10)) + add(String(localized: "Definer"), text(row, 11)) + add(String(localized: "Comment"), text(row, 12)) + add(String(localized: "Event Scheduler"), scheduler) + return PluginServerObjectInfo( + kindId: MySQLEvents.kindId, + name: name, + schema: text(row, 3), + detail: isRecurring ? String(localized: "Recurring") : String(localized: "One Time"), + isEnabled: status.map { $0 == "ENABLED" }, + attributes: attributes + ) + } + + // The schedule in the clause `CREATE EVENT` takes, quoting a composite interval as the server does. + static func schedule(isRecurring: Bool, executeAt: String?, intervalValue: String?, intervalField: String?) -> String? { + guard isRecurring else { return executeAt.map { "AT '\($0)'" } } + guard let intervalValue, let intervalField else { return nil } + let value = intervalValue.allSatisfy(\.isNumber) ? intervalValue : "'\(intervalValue)'" + return "EVERY \(value) \(intervalField)" + } + + // 8.2 renamed SLAVESIDE_DISABLED to REPLICA_SIDE_DISABLED; both mean the replica does not run it. + static func statusLabel(_ status: String) -> String { + switch status { + case "ENABLED": return String(localized: "Enabled") + case "DISABLED": return String(localized: "Disabled") + case "SLAVESIDE_DISABLED", "REPLICA_SIDE_DISABLED": return String(localized: "Disabled on Replica") + default: return status + } + } + + // The catalog compares names without case, so an exact spelling wins over another the read also matched. + static func row(named name: String, in rows: [[PluginCellValue]]) -> [PluginCellValue]? { + rows.first { $0[safe: 0]?.asText == name } ?? rows.first + } + + private static func text(_ row: [PluginCellValue], _ index: Int) -> String? { + guard let value = row[safe: index]?.asText, !value.isEmpty else { return nil } + return value + } +} + +internal extension MySQLPluginDriver { + var serverObjectKinds: [PluginServerObjectKind] { + let identity = serverIdentity + return MySQLEvents.isDeclared(flavor: identity.flavor, banner: identity.banner) ? [MySQLEvents.kind] : [] + } + + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [PluginServerObjectInfo] { + let identity = serverIdentity + guard kindId == MySQLEvents.kindId, MySQLEvents.isDeclared(flavor: identity.flavor, banner: identity.banner) else { + return [] + } + let exactSchema = await schemaNamesAreCaseSensitive() + let result = try await execute( + ownStatement: MySQLObjectQueries.eventDetails(schema: effectiveSchema(schema), exactSchema: exactSchema) + ) + guard !result.rows.isEmpty else { return [] } + return MySQLEventRows.objects(from: result.rows, scheduler: await eventScheduler()) + } + + // The tab reads the event's row again, so its attributes follow an ALTER EVENT as its DDL does. + func fetchServerObject(_ object: PluginServerObjectInfo) async throws -> PluginServerObjectInfo { + guard object.kindId == MySQLEvents.kindId else { throw PluginObjectSourceError.unsupported(object.name) } + let exactSchema = await schemaNamesAreCaseSensitive() + let result = try await execute(ownStatement: MySQLObjectQueries.eventDetails( + schema: effectiveSchema(object.schema), name: object.name, exactSchema: exactSchema + )) + guard let row = MySQLEventRows.row(named: object.name, in: result.rows), + let fresh = MySQLEventRows.object(from: row, scheduler: await eventScheduler()) + else { throw PluginObjectSourceError.notFound(object.name) } + let definition = try await fetchEventDDL(PluginEventInfo(name: fresh.name, schema: fresh.schema)) + return PluginServerObjectInfo( + kindId: fresh.kindId, + name: fresh.name, + schema: fresh.schema, + identity: object.identity, + detail: fresh.detail, + isEnabled: fresh.isEnabled, + definition: definition, + attributes: fresh.attributes + ) + } + + // A server or proxy without the variable loses this one attribute, never the list. + private func eventScheduler() async -> String? { + guard let result = try? await execute(ownStatement: MySQLObjectQueries.eventScheduler) else { return nil } + return result.rows.first?[safe: 0]?.asText + } +} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index e63ca8032d..6da86b8e0d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -180,6 +180,9 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if flavor.enforcesReadOnlyTransactions { capabilities.insert(.readOnlyStatements) } + if holdsForServer(MySQLReadOnlyTransaction.isAvailable(banner:flavor:)) { + capabilities.insert(.enforcesReadOnlyMode) + } return capabilities } @@ -384,14 +387,23 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + try await runUserQuery(query: query, rowCap: rowCap, parameters: parameters, byteLimit: nil) + } + + private func runUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + byteLimit: Int? + ) async throws -> PluginQueryResult { let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } guard let parameters else { return try await retryingAfterStop { - try await self.executeWithReconnect(query: query, isRetry: false, rowCap: cap) + try await self.executeWithReconnect(query: query, isRetry: false, rowCap: cap, byteLimit: byteLimit) } } return try await retryingAfterStop { - try await self.executeParameterizedOnce(query: query, parameters: parameters, rowCap: cap) + try await self.executeParameterizedOnce(query: query, parameters: parameters, rowCap: cap, byteLimit: byteLimit) } } @@ -403,85 +415,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { try await retryingAfterStop { - try await self.executeParameterizedOnce(query: query, parameters: parameters, rowCap: nil) - } - } - - func executeUserQuery( - query: String, - rowCap: Int?, - parameters: [PluginCellValue]?, - context: PluginStatementContext - ) async throws -> PluginQueryResult { - guard runsReadOnly(context) else { - return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) - } - let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } - return try await retryingAfterStop { - try await self.executeReadOnly(query: query, parameters: parameters, rowCap: cap, isRetry: false) - } - } - - func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { - guard runsReadOnly(context) else { return try await execute(query: query) } - return try await retryingAfterStop { - try await self.executeReadOnly(query: query, parameters: nil, rowCap: nil, isRetry: false) - } - } - - func executeParameterized( - query: String, - parameters: [PluginCellValue], - context: PluginStatementContext - ) async throws -> PluginQueryResult { - guard runsReadOnly(context) else { return try await executeParameterized(query: query, parameters: parameters) } - return try await retryingAfterStop { - try await self.executeReadOnly(query: query, parameters: parameters, rowCap: nil, isRetry: false) - } - } - - private func runsReadOnly(_ context: PluginStatementContext) -> Bool { - MySQLReadOnlyTransaction.applies(to: context, flavor: flavor) - } - - // The lock comes from the footprint, because the status flags never show it. - private var readOnlySessionState: @Sendable (Bool) -> PluginSessionTransactionState { - { isInTransaction in - self.sessionLock.withLock { self.footprint.transactionState(isInTransaction: isInTransaction) } - } - } - - // Replays on a lost connection exactly when `executeWithReconnect` would. - private func executeReadOnly( - query: String, - parameters: [PluginCellValue]?, - rowCap: Int?, - isRetry: Bool - ) async throws -> PluginQueryResult { - let startTime = Date() - - let conn = try await requireConnection() - defer { endOperation(on: conn) } - noteActivity(query) - - do { - let result = try await conn.executeReadOnly( - query, - parameters: parameters, - rowCap: rowCap, - sessionState: readOnlySessionState - ) - return Self.pluginResult(result, startedAt: startTime) - } catch let error as MariaDBPluginError where !isRetry - && parameters == nil - && mysqlConnectionLossMayReplay(code: error.code, outlastedSocketTimeout: error.outlastedSocketTimeout) - && mayReplay(query) { - try await reconnect() - return try await executeReadOnly(query: query, parameters: nil, rowCap: rowCap, isRetry: true) - } catch { - noteFailure(query) - await refreshTransactionFlag(after: error, on: conn) - throw error + try await self.executeParameterizedOnce(query: query, parameters: parameters, rowCap: nil, byteLimit: nil) } } @@ -499,7 +433,8 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private func executeParameterizedOnce( query: String, parameters: [PluginCellValue], - rowCap: Int? + rowCap: Int?, + byteLimit: Int? ) async throws -> PluginQueryResult { let conn = try await requireConnection() defer { endOperation(on: conn) } @@ -507,7 +442,9 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let startTime = Date() let result: MariaDBPluginQueryResult do { - result = try await conn.executeParameterizedQuery(query, parameters: parameters, rowCap: rowCap) + result = try await conn.executeParameterizedQuery( + query, parameters: parameters, rowCap: rowCap, byteLimit: byteLimit + ) } catch { noteFailure(query) await refreshTransactionFlag(after: error, on: conn) @@ -562,6 +499,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { query: String, isRetry: Bool, rowCap: Int? = nil, + byteLimit: Int? = nil, countsAsActivity: Bool = true ) async throws -> PluginQueryResult { let startTime = Date() @@ -573,7 +511,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } do { - let result = try await conn.executeQuery(query, rowCap: rowCap) + let result = try await conn.executeQuery(query, rowCap: rowCap, byteLimit: byteLimit) return Self.pluginResult(result, startedAt: startTime) } catch let error as MariaDBPluginError where !isRetry && mysqlConnectionLossMayReplay(code: error.code, outlastedSocketTimeout: error.outlastedSocketTimeout) @@ -583,6 +521,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { query: query, isRetry: true, rowCap: rowCap, + byteLimit: byteLimit, countsAsActivity: countsAsActivity ) } catch { @@ -819,7 +758,8 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { mariadbConnection?.disconnect() mariadbConnection = nil sessionLock.withLock { footprint.reset() } - try await connect() + // A minted IAM token is good for 15 minutes, and this sign-in can come long after the first. + try await open(password: try await config.passwordForNewSignIn()) if let seconds = sessionLock.withLock({ appliedQueryTimeoutSeconds }) { try await applyQueryTimeout(seconds) } @@ -1083,12 +1023,12 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming func streamRows(query: String) -> AsyncThrowingStream { - streamRows(query: query, readOnlySessionState: nil) + streamRows(query: query, readOnly: nil) } func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - guard runsReadOnly(context) else { return streamRows(query: query) } - return streamRows(query: query, readOnlySessionState: readOnlySessionState) + guard let scope = readOnlyScope(context) else { return streamRows(query: query) } + return streamRows(query: query, readOnly: scope) } /// The bridge task is retained and cancelled from `onTermination`. Without that, a consumer @@ -1097,17 +1037,18 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// nothing ever cancels the task awaiting it. private func streamRows( query: String, - readOnlySessionState: (@Sendable (Bool) -> PluginSessionTransactionState)? + readOnly: MySQLReadOnlyTransaction.Scope? ) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in + // Bounded like the connection's own stream, or this relay would read it dry into an unbounded buffer. + AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in let task = Task { do { try await retryingAfterStop { let conn = try await requireConnection() defer { self.endOperation(on: conn) } noteActivity(query) - for try await element in conn.streamQuery(query, readOnlySessionState: readOnlySessionState) { - continuation.yield(element) + for try await element in conn.streamQuery(query, readOnly: readOnly) { + guard await PluginRowStream.deliver(element, to: continuation) else { break } } } continuation.finish() @@ -1492,3 +1433,133 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { flavor.hasMySQLObjectCatalogs ? ["SET FOREIGN_KEY_CHECKS=1"] : nil } } + +// MARK: - Statement Context + +// An app read marked `readOnly` alone, a catalog read or a dashboard tile, runs as it is. `enforcesReadOnly` from any +// origin, and `readOnly` from an external one, run in `START TRANSACTION READ ONLY` on a flavor that enforces it. +extension MySQLPluginDriver { + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + guard let scope = readOnlyScope(context) else { + return try await runUserQuery( + query: query, rowCap: rowCap, parameters: parameters, byteLimit: context.resultByteLimit + ) + } + return try await readOnlyRead( + query: query, parameters: parameters, rowCap: rowCap, byteLimit: context.resultByteLimit, scope: scope + ) + } + + func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { + guard let scope = readOnlyScope(context) else { + return try await runUserQuery(query: query, rowCap: nil, parameters: nil, byteLimit: context.resultByteLimit) + } + return try await readOnlyRead( + query: query, parameters: nil, rowCap: nil, byteLimit: context.resultByteLimit, scope: scope + ) + } + + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + guard let scope = readOnlyScope(context) else { + return try await runUserQuery( + query: query, rowCap: nil, parameters: parameters, byteLimit: context.resultByteLimit + ) + } + return try await readOnlyRead( + query: query, parameters: parameters, rowCap: nil, byteLimit: context.resultByteLimit, scope: scope + ) + } + + // A context the read-only mode does not cover goes back to the host as PluginKit's default sends it, so TiDB, + // OceanBase, Databend and StarRocks never run an `enforcesReadOnly` read without the mode. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + guard let scope = readOnlyScope(context) else { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await runUserQuery( + query: query, rowCap: rowCap, parameters: nil, byteLimit: context.resultByteLimit + ) + } + return try await readOnlyRead( + query: query, parameters: nil, rowCap: rowCap, byteLimit: context.resultByteLimit, scope: scope + ) + } + + private func readOnlyScope(_ context: PluginStatementContext) -> MySQLReadOnlyTransaction.Scope? { + let identity = serverIdentity + guard let reason = MySQLReadOnlyTransaction.reason(for: context, flavor: identity.flavor, banner: identity.banner) + else { return nil } + // The lock comes from the footprint, because the status flags never show it. + return MySQLReadOnlyTransaction.Scope( + reason: reason, + runsInOpenTransaction: context.enforcesReadOnly && !context.callsUnlistedFunction, + sessionState: { isInTransaction in + self.sessionLock.withLock { self.footprint.transactionState(isInTransaction: isInTransaction) } + } + ) + } + + private func readOnlyRead( + query: String, + parameters: [PluginCellValue]?, + rowCap: Int?, + byteLimit: Int?, + scope: MySQLReadOnlyTransaction.Scope + ) async throws -> PluginQueryResult { + let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } + return try await retryingAfterStop { + try await self.executeReadOnly( + query: query, parameters: parameters, rowCap: cap, byteLimit: byteLimit, scope: scope, isRetry: false + ) + } + } + + // Replays on a lost connection exactly when `executeWithReconnect` would. + private func executeReadOnly( + query: String, + parameters: [PluginCellValue]?, + rowCap: Int?, + byteLimit: Int?, + scope: MySQLReadOnlyTransaction.Scope, + isRetry: Bool + ) async throws -> PluginQueryResult { + let startTime = Date() + + let conn = try await requireConnection() + defer { endOperation(on: conn) } + noteActivity(query) + + do { + let result = try await conn.executeReadOnly( + query, parameters: parameters, rowCap: rowCap, byteLimit: byteLimit, scope: scope + ) + return Self.pluginResult(result, startedAt: startTime) + } catch let error as MariaDBPluginError where !isRetry + && parameters == nil + && mysqlConnectionLossMayReplay(code: error.code, outlastedSocketTimeout: error.outlastedSocketTimeout) + && mayReplay(query) { + try await reconnect() + return try await executeReadOnly( + query: query, parameters: nil, rowCap: rowCap, byteLimit: byteLimit, scope: scope, isRetry: true + ) + } catch { + noteFailure(query) + await refreshTransactionFlag(after: error, on: conn) + throw error + } + } +} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction+Statement.swift b/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction+Statement.swift new file mode 100644 index 0000000000..906e361373 --- /dev/null +++ b/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction+Statement.swift @@ -0,0 +1,80 @@ +// +// MySQLReadOnlyTransaction+Statement.swift +// MySQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +extension MySQLReadOnlyTransaction { + // The session state is read on the statement's own queue, so nothing queued behind it can open a transaction first. + struct Scope: Sendable { + let reason: Reason + let runsInOpenTransaction: Bool + let sessionState: @Sendable (Bool) -> PluginSessionTransactionState + } + + static var sessionNotIdle: MariaDBPluginError { + MariaDBPluginError(code: 0, message: notIdleMessage(.externalClients), sqlState: nil) + } + + static var safeModeSessionNotIdle: MariaDBPluginError { + MariaDBPluginError(code: 0, message: notIdleMessage(.safeMode), sqlState: nil) + } + + // An external read stays wrapped on a server too old for the mode, where the begin fails and nothing runs: no host + // check stands behind it at a writable level. Safe Mode's function check stands behind Read-Only. + static func reason(for context: PluginStatementContext, flavor: MySQLServerFlavor, banner: String? = nil) -> Reason? { + guard flavor.enforcesReadOnlyTransactions else { return nil } + if context.readOnly, context.origin == .external { return .externalClients } + if context.enforcesReadOnly, isAvailable(banner: banner, flavor: flavor) { return .safeMode } + return nil + } + + static func applies(to context: PluginStatementContext, flavor: MySQLServerFlavor) -> Bool { + reason(for: context, flavor: flavor) != nil + } + + // `START TRANSACTION` would commit the user's open transaction and release their `LOCK TABLES`, so a session that + // holds either runs a statement the host checked for unlisted function calls as it is. A `SET autocommit = 0` + // session holds a transaction from its first statement on. + static func run( + state: PluginSessionTransactionState, + reason: Reason, + runsInOpenTransaction: Bool = false, + send: (String) throws -> Void, + statement: () throws -> Result + ) throws -> Result { + guard state == .idle else { + guard runsInOpenTransaction else { throw notIdle(reason) } + return try statement() + } + do { + try send(begin) + } catch { + try? send(end) + throw error + } + let result: Result + do { + result = try statement() + } catch { + try? send(end) + throw refusal(from: error, reason: reason) + } + try send(end) + return result + } + + static func notIdle(_ reason: Reason) -> MariaDBPluginError { + switch reason { + case .externalClients: sessionNotIdle + case .safeMode: safeModeSessionNotIdle + } + } + + static func refusal(from error: Error, reason: Reason) -> Error { + guard let failure = error as? MariaDBPluginError, failure.code == refusalCode else { return error } + return MariaDBPluginError(code: failure.code, message: refusalMessage(reason), sqlState: failure.sqlState) + } +} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction.swift b/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction.swift index 2527e87292..7a0fe12b41 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction.swift @@ -4,9 +4,16 @@ // import Foundation -import TableProPluginKit -internal enum MySQLReadOnlyTransaction { +// The iOS MySQL driver compiles this file too; the plugin's statement handling is in +// MySQLReadOnlyTransaction+Statement.swift. +nonisolated internal enum MySQLReadOnlyTransaction { + // Who asked for the read-only mode, which is what a refusal names. + enum Reason: Sendable, Equatable { + case externalClients + case safeMode + } + // Not the `/*!50605 ... */` spelling: a server older than 5.6.5 would run that as a plain read-write transaction. static let begin = "START TRANSACTION READ ONLY" static let end = "ROLLBACK" @@ -14,52 +21,29 @@ internal enum MySQLReadOnlyTransaction { // ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION. static let refusalCode: UInt32 = 1_792 - static var sessionNotIdle: MariaDBPluginError { - MariaDBPluginError( - code: 0, - message: String( - localized: "This connection has a transaction open. Commit or roll it back in TablePro, then try again." - ), - sqlState: nil - ) - } - - static func applies(to context: PluginStatementContext, flavor: MySQLServerFlavor) -> Bool { - context.readOnly && context.origin == .external && flavor.enforcesReadOnlyTransactions + // MySQL 5.5.62 and MariaDB 5.5.64 answer `begin` with 1064 (measured), so an older server runs a read as it is. + static func isAvailable(banner: String?, flavor: MySQLServerFlavor) -> Bool { + flavor.enforcesReadOnlyTransactions && MySQLServerVersion.hasReadOnlyTransactions(banner: banner, flavor: flavor) } - // `START TRANSACTION` would commit the user's open transaction and release their `LOCK TABLES`. - static func run( - state: PluginSessionTransactionState, - send: (String) throws -> Void, - statement: () throws -> Result - ) throws -> Result { - guard state == .idle else { throw sessionNotIdle } - do { - try send(begin) - } catch { - try? send(end) - throw error - } - let result: Result - do { - result = try statement() - } catch { - try? send(end) - throw refusal(from: error) + static func notIdleMessage(_ reason: Reason) -> String { + switch reason { + case .externalClients: + String(localized: "This connection has a transaction open. Commit or roll it back in TablePro, then try again.") + // Read-Only offers no Commit, so the way out is a rollback. + case .safeMode: + String( + localized: "Safe Mode is set to Read-Only for this connection, and it has a transaction open. Roll it back, then try again." + ) } - try send(end) - return result } - static func refusal(from error: Error) -> Error { - guard let failure = error as? MariaDBPluginError, failure.code == refusalCode else { return error } - return MariaDBPluginError( - code: failure.code, - message: String( - localized: "This connection is read only for external clients, and the database refused the statement." - ), - sqlState: failure.sqlState - ) + static func refusalMessage(_ reason: Reason) -> String { + switch reason { + case .externalClients: + String(localized: "This connection is read only for external clients, and the database refused the statement.") + case .safeMode: + String(localized: "Safe Mode is set to Read-Only for this connection, and the database refused the statement.") + } } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLSelectLimitStatement.swift b/apple/Plugins/MySQLDriverPlugin/MySQLSelectLimitStatement.swift index eaf152013a..606703975d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLSelectLimitStatement.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLSelectLimitStatement.swift @@ -121,15 +121,28 @@ internal func mysqlStatementReturnsAtMostOneRow(_ sql: String) -> Bool { return mysqlBodyMatches(String(body[fromRange.lowerBound...]), "^FROM\\s+DUAL\\s*$") } +// A read that stopped at the byte limit keeps the row that crossed it, so the host sees its own limit was reached. internal func mysqlBoundedFetchOutcome( fetchedRows: Int, rowCap: Int, - serverSentMore: Bool + serverSentMore: Bool, + stoppedAtByteLimit: Bool = false ) -> MySQLBoundedFetchOutcome { - let truncated = serverSentMore || fetchedRows > rowCap - return MySQLBoundedFetchOutcome( - keptRows: truncated ? rowCap : fetchedRows, - isTruncated: truncated, + MySQLBoundedFetchOutcome( + keptRows: min(fetchedRows, rowCap), + isTruncated: serverSentMore || stoppedAtByteLimit || fetchedRows > rowCap, serverIgnoredLimit: serverSentMore ) } + +// Text as UTF-8, bytes as they are, null as nothing: never more than the host counts for the same row, so the read never +// stops before the host's own budget is full. +internal func mysqlDecodedByteCount(of row: [PluginCellValue]) -> Int { + row.reduce(0) { total, cell in + switch cell { + case .null: total + case .text(let text): total + text.utf8.count + case .bytes(let data): total + data.count + } + } +} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLServerVersion.swift b/apple/Plugins/MySQLDriverPlugin/MySQLServerVersion.swift index 06b49ebf21..0908a55b5e 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLServerVersion.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLServerVersion.swift @@ -101,6 +101,19 @@ nonisolated internal enum MySQLServerVersion { flavor.isMariaDB && isAtLeast((10, 2, 7), banner: banner) } + // MySQL 5.6.5 (WL #5968) and MariaDB 10.0.0 added `START TRANSACTION READ ONLY`. No other flavor is held to it, as + // `MySQLServerFlavor.enforcesReadOnlyTransactions` says. + static func hasReadOnlyTransactions(banner: String?, flavor: MySQLServerFlavor) -> Bool { + switch flavor { + case .mysql: + return !isLegacy(below: (5, 6, 5), banner: banner, flavor: flavor) + case .mariadb: + return !isLegacy(below: (10, 0, 0), banner: banner, flavor: flavor) + case .tidb, .oceanbase, .databend, .starrocks: + return false + } + } + // A binary default arrives as its bytes from MySQL 5.7.44, MariaDB and OceanBase, and as `0x` hex from MySQL 8.0.11 to 8.4.13. static func reportsBinaryDefaultAsHex(banner: String?, flavor: MySQLServerFlavor) -> Bool { switch flavor { diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift b/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift index 8f55556a51..9e386e5dc9 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift @@ -29,7 +29,7 @@ import TableProPluginKit /// Every ambiguity resolves toward "the session is holding something". The cost of that is only /// that a connection keeps its slot, which is what happens today; the cost of the opposite is /// destroying a user's transaction or temporary table without telling them. -struct MySQLSessionFootprint: Equatable { +nonisolated struct MySQLSessionFootprint: Equatable { private(set) var hasOpenTransaction = false private(set) var hasTemporaryTables = false private(set) var hasUserVariables = false diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLTemporaryTableNames.swift b/apple/Plugins/MySQLDriverPlugin/MySQLTemporaryTableNames.swift index 8dc474070a..73b7298922 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLTemporaryTableNames.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLTemporaryTableNames.swift @@ -8,7 +8,7 @@ import Foundation // MySQL 8.4.11 lists a session's temporary tables nowhere an ordinary user can read (`INNODB_TEMP_TABLE_INFO` needs // PROCESS), so the names come from the statement text. Only the table part is kept: `SHOW CREATE TABLE name` on the // session resolves it in the browsed database and answers `CREATE TEMPORARY TABLE` for one. -enum MySQLTemporaryTableNames { +nonisolated enum MySQLTemporaryTableNames { static func created(by statement: String) -> String? { var reader = Reader(statement) guard reader.accept("CREATE") else { return nil } @@ -53,7 +53,7 @@ enum MySQLTemporaryTableNames { } } -private enum NameToken: Equatable { +nonisolated private enum NameToken: Equatable { case word(String) case quoted(String) case dot @@ -62,7 +62,7 @@ private enum NameToken: Equatable { } // A string literal is one `other` token, so a keyword inside one is never read. -private struct Reader { +nonisolated private struct Reader { private let scalars: [Unicode.Scalar] private var index = 0 private var executableCommentDepth = 0 diff --git a/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift b/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift new file mode 100644 index 0000000000..52bb160f9a --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift @@ -0,0 +1,38 @@ +// +// MariaDBStopThreadIdTests.swift +// MySQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// Stop runs off the statement queue, where a disconnect's queued close can free the handle under it, so the kill's +// thread id is the one the handshake gave, held beside the handle and cleared with it. +@Suite(.timeLimit(.minutes(1))) +struct MariaDBStopThreadIdTests { + @Test("The kill's thread id is the session's own, and none is left once the session is closed") + func threadIdFollowsTheSession() async throws { + let server = try PlaintextMySQLServer(asksForCleartext: false) + defer { server.close() } + async let session = server.serveOneSession() + let connection = MariaDBPluginConnection( + transport: .tcp(host: "127.0.0.1", port: UInt32(server.port)), + user: "app", + password: "app-password", + database: "", + sslConfig: SSLConfiguration(mode: .disabled), + enableCleartextPlugin: false + ) + + try await connection.connect() + let signedIn = connection.currentThreadId + connection.disconnect() + let closed = connection.currentThreadId + _ = await session + + // The fake server's handshake names connection 1. + #expect(signedIn == 1) + #expect(closed == 0) + } +} diff --git a/apple/Plugins/MySQLDriverTests/MariaDBTypeNameTests.swift b/apple/Plugins/MySQLDriverTests/MariaDBTypeNameTests.swift index f029fa39ab..f4cdb34e79 100644 --- a/apple/Plugins/MySQLDriverTests/MariaDBTypeNameTests.swift +++ b/apple/Plugins/MySQLDriverTests/MariaDBTypeNameTests.swift @@ -47,6 +47,15 @@ struct MariaDBTypeNameTests { #expect(resolve(typeRaw: 255) == "GEOMETRY") } + @Test("TINY of length 1 is TINYINT(1), and a BIT wider than one bit names its width") + func widthsTheClassifierReads() { + #expect(resolve(typeRaw: 1, length: 1) == "TINYINT(1)") + #expect(resolve(typeRaw: 1, length: 4) == "TINYINT") + #expect(resolve(typeRaw: 16, length: 1) == "BIT") + #expect(resolve(typeRaw: 16, length: 8) == "BIT(8)") + #expect(resolve(typeRaw: 16, length: 64) == "BIT(64)") + } + @Test("Unknown type code returns UNKNOWN") func unknownType() { #expect(resolve(typeRaw: 999) == "UNKNOWN") diff --git a/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift b/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift index 1830f44e54..cb6820610a 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift @@ -156,4 +156,12 @@ struct MySQLCheckConstraintsTests { quotedTable: "`t`", quotedName: "`c`", banner: "10.6.28-MariaDB", flavor: .mariadb ) == "ALTER TABLE `t` DROP CONSTRAINT `c`") } + + // Measured on MySQL 8.4.13: without the type, table b's UNIQUE key named like table a's check listed a's check. + @Test("The joined catalog read matches only check constraints") + func joinedCatalogQueryMatchesChecksOnly() { + let query = MySQLCheckConstraints.joinedCatalogQuery(databaseLiteral: "shop", tableLiteral: "b") + #expect(query.contains("AND tc.CONSTRAINT_TYPE = 'CHECK'\n")) + #expect(query.contains("WHERE cc.CONSTRAINT_SCHEMA = 'shop' AND tc.TABLE_NAME = 'b'\n")) + } } diff --git a/apple/Plugins/MySQLDriverTests/MySQLEventServerObjectTests.swift b/apple/Plugins/MySQLDriverTests/MySQLEventServerObjectTests.swift new file mode 100644 index 0000000000..3704a927b3 --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MySQLEventServerObjectTests.swift @@ -0,0 +1,144 @@ +// +// MySQLEventServerObjectTests.swift +// MySQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct MySQLEventServerObjectTests { + private func row(_ cells: String?...) -> [PluginCellValue] { + cells.map(PluginCellValue.fromOptional) + } + + private func attribute(_ object: PluginServerObjectInfo?, _ label: String) -> String? { + object?.attributes.first { $0.label == label }?.value + } + + @Test("MySQL and MariaDB declare Events; the other flavors sharing the driver declare none") + func kindsPerFlavor() { + #expect(MySQLEvents.isDeclared(flavor: .mysql, banner: "8.0.36")) + #expect(MySQLEvents.isDeclared(flavor: .mariadb, banner: "10.11.6-MariaDB")) + #expect(MySQLEvents.isDeclared(flavor: .mysql, banner: nil)) + #expect(!MySQLEvents.isDeclared(flavor: .tidb(version: nil), banner: "8.0.11-TiDB-v7.5.0")) + #expect(!MySQLEvents.isDeclared(flavor: .databend, banner: nil)) + #expect(!MySQLEvents.isDeclared(flavor: .oceanbase(version: nil), banner: nil)) + #expect(!MySQLEvents.isDeclared(flavor: .starrocks, banner: nil)) + } + + @Test("A server older than 5.1.6 has no events catalog and declares no Events") + func kindsFollowTheEventsFloor() { + #expect(!MySQLEvents.isDeclared(flavor: .mysql, banner: "5.0.96-log")) + #expect(!MySQLEvents.isDeclared(flavor: .mysql, banner: "4.1.22-standard")) + #expect(MySQLEvents.isDeclared(flavor: .mysql, banner: "5.1.6")) + } + + @Test("The Events kind is listed per database, under its own symbol") + func eventKindDescriptor() { + let kind = MySQLEvents.kind + #expect(kind.id == "event") + #expect(kind.scope == .container) + #expect(kind.iconName == "calendar.badge.clock") + #expect(kind.pluralDisplayName == String(localized: "Events")) + } + + @Test("A recurring event reads its interval as the schedule and counts as enabled") + func recurringEvent() { + let object = MySQLEventRows.object( + from: row( + "nightly_cleanup", "RECURRING", "ENABLED", "shop", + nil, "1", "DAY", "2026-01-01 03:00:00", nil, + "PRESERVE", "2026-11-10 03:00:00", "app@%", "Purges the log" + ), + scheduler: "ON" + ) + + #expect(object?.kindId == "event") + #expect(object?.name == "nightly_cleanup") + #expect(object?.schema == "shop") + #expect(object?.identity == nil) + #expect(object?.detail == String(localized: "Recurring")) + #expect(object?.isEnabled == true) + #expect(attribute(object, String(localized: "Schedule")) == "EVERY 1 DAY") + #expect(attribute(object, String(localized: "Status")) == String(localized: "Enabled")) + #expect(attribute(object, String(localized: "Starts")) == "2026-01-01 03:00:00") + #expect(attribute(object, String(localized: "Ends")) == nil) + #expect(attribute(object, String(localized: "On Completion")) == "PRESERVE") + #expect(attribute(object, String(localized: "Last Executed")) == "2026-11-10 03:00:00") + #expect(attribute(object, String(localized: "Definer")) == "app@%") + #expect(attribute(object, String(localized: "Comment")) == "Purges the log") + #expect(attribute(object, String(localized: "Event Scheduler")) == "ON") + } + + @Test("A one-time event reads its run time as the schedule") + func oneTimeEvent() { + let object = MySQLEventRows.object( + from: row( + "close_quarter", "ONE TIME", "DISABLED", "shop", + "2026-12-31 23:59:00", nil, nil, nil, nil, + "NOT PRESERVE", nil, "root@localhost", nil + ), + scheduler: "OFF" + ) + + #expect(object?.detail == String(localized: "One Time")) + #expect(object?.isEnabled == false) + #expect(attribute(object, String(localized: "Schedule")) == "AT '2026-12-31 23:59:00'") + #expect(attribute(object, String(localized: "Status")) == String(localized: "Disabled")) + #expect(attribute(object, String(localized: "Last Executed")) == nil) + #expect(attribute(object, String(localized: "Event Scheduler")) == "OFF") + } + + @Test("An event disabled on a replica says so under both spellings the server has used") + func replicaDisabledEvent() { + for status in ["SLAVESIDE_DISABLED", "REPLICA_SIDE_DISABLED"] { + let object = MySQLEventRows.object(from: row( + "sync_totals", "RECURRING", status, "shop", nil, "1", "HOUR", nil, nil, "PRESERVE", nil, nil, nil + )) + #expect(object?.isEnabled == false, "\(status)") + #expect(attribute(object, String(localized: "Status")) == String(localized: "Disabled on Replica"), "\(status)") + } + } + + @Test("A composite interval is quoted the way CREATE EVENT spells it") + func compositeInterval() { + #expect(MySQLEventRows.schedule(isRecurring: true, executeAt: nil, intervalValue: "1:30", intervalField: "HOUR_MINUTE") + == "EVERY '1:30' HOUR_MINUTE") + #expect(MySQLEventRows.schedule(isRecurring: true, executeAt: nil, intervalValue: nil, intervalField: "DAY") == nil) + #expect(MySQLEventRows.schedule(isRecurring: false, executeAt: nil, intervalValue: "1", intervalField: "DAY") == nil) + } + + @Test("A scheduler the server would not report leaves only that attribute out") + func missingSchedulerDropsOneAttribute() { + let object = MySQLEventRows.object(from: row( + "nightly", "RECURRING", "ENABLED", "shop", nil, "1", "DAY", nil, nil, "PRESERVE", nil, "app@%", nil + )) + + #expect(attribute(object, String(localized: "Event Scheduler")) == nil) + #expect(attribute(object, String(localized: "Schedule")) == "EVERY 1 DAY") + #expect(attribute(object, String(localized: "Definer")) == "app@%") + } + + @Test("A re-read picks the row spelled exactly as the tab names it") + func rereadPicksTheExactSpelling() { + let rows = [row("Nightly", "RECURRING", "ENABLED", "shop"), row("nightly", "ONE TIME", "DISABLED", "shop")] + + #expect(MySQLEventRows.row(named: "nightly", in: rows)?.first == PluginCellValue.fromOptional("nightly")) + #expect(MySQLEventRows.row(named: "NIGHTLY", in: rows)?.first == PluginCellValue.fromOptional("Nightly")) + #expect(MySQLEventRows.row(named: "nightly", in: []) == nil) + } + + @Test("Four columns alone still map, and a row with no name is dropped") + func shortAndNamelessRows() { + let objects = MySQLEventRows.objects(from: [ + row("legacy", "RECURRING", "ENABLED", "shop"), + row(nil, "RECURRING", "ENABLED", "shop") + ]) + + #expect(objects.count == 1) + #expect(objects.first?.name == "legacy") + #expect(objects.first?.isEnabled == true) + #expect(attribute(objects.first, String(localized: "Schedule")) == nil) + } +} diff --git a/apple/Plugins/MySQLDriverTests/MySQLFoundRowsTests.swift b/apple/Plugins/MySQLDriverTests/MySQLFoundRowsTests.swift new file mode 100644 index 0000000000..ccd89fa473 --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MySQLFoundRowsTests.swift @@ -0,0 +1,33 @@ +import CMariaDB +import Foundation +import TableProPluginKit +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct MySQLFoundRowsTests { + // A save holds a keyed UPDATE to the one row it matched, which only reads right when the server counts a row + // that already held the value: scripts/probes/check-mysql-found-rows.sh measures that against real servers. + @Test("A session signs in asking for CLIENT_FOUND_ROWS, so an UPDATE counts the rows it matched") + func signInAsksForFoundRows() async throws { + let server = try PlaintextMySQLServer(asksForCleartext: false) + defer { server.close() } + async let session = server.serveOneSession() + let connection = MariaDBPluginConnection( + transport: .tcp(host: "127.0.0.1", port: UInt32(server.port)), + user: "app", + password: "app-password", + database: "", + sslConfig: SSLConfiguration(mode: .disabled), + enableCleartextPlugin: false + ) + + try await connection.connect() + connection.disconnect() + let received = await session.received + + // The first packet is the sign-in: a 4-byte header, then the client's capability flags, little-endian. + try #require(received.count >= 8) + let flags = received[4..<8].enumerated().reduce(UInt32(0)) { $0 | UInt32($1.element) << (8 * $1.offset) } + #expect(flags & UInt32(CLIENT_FOUND_ROWS) != 0) + } +} diff --git a/apple/Plugins/MySQLDriverTests/MySQLReadOnlyTransactionTests.swift b/apple/Plugins/MySQLDriverTests/MySQLReadOnlyTransactionTests.swift index 2d42d93758..9c90e0f14b 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLReadOnlyTransactionTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLReadOnlyTransactionTests.swift @@ -20,14 +20,20 @@ struct MySQLReadOnlyTransactionTests { private static let statement = "SELECT * FROM orders" - private static func context(readOnly: Bool, origin: PluginStatementOrigin) -> PluginStatementContext { + private static func context( + readOnly: Bool, + origin: PluginStatementOrigin, + enforcesReadOnly: Bool = false + ) -> PluginStatementContext { var context = PluginStatementContext() context.readOnly = readOnly context.origin = origin + context.enforcesReadOnly = enforcesReadOnly return context } private static let externalRead = context(readOnly: true, origin: .external) + private static let safeModeRead = context(readOnly: false, origin: .app, enforcesReadOnly: true) private static func driver(variant: String? = nil) -> MySQLPluginDriver { MySQLPluginDriver(config: DriverConnectionConfig( @@ -36,11 +42,15 @@ struct MySQLReadOnlyTransactionTests { )) } + private static let nonEnforcingVariants = [ + MySQLServerFlavor.oceanbaseVariant, MySQLServerFlavor.databendVariant, MySQLServerFlavor.starrocksVariant, + ] + @Test("An external read runs between a plain START TRANSACTION READ ONLY and a ROLLBACK") func wrapsTheStatement() throws { let wire = Wire() - let rows = try MySQLReadOnlyTransaction.run(state: .idle, send: wire.send) { + let rows = try MySQLReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) return 3 } @@ -58,7 +68,7 @@ struct MySQLReadOnlyTransactionTests { let wire = Wire() let error = #expect(throws: MariaDBPluginError.self) { - try MySQLReadOnlyTransaction.run(state: state, send: wire.send) { + try MySQLReadOnlyTransaction.run(state: state, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } @@ -67,6 +77,24 @@ struct MySQLReadOnlyTransactionTests { #expect(error?.message == MySQLReadOnlyTransaction.sessionNotIdle.message) } + @Test( + "At Read-Only a session that is not idle is refused before anything is sent, naming Safe Mode", + arguments: [PluginSessionTransactionState.inTransaction, .holdsSessionLocks, .abortedTransaction, .unknown] + ) + func safeModeRefusesUnlessIdle(state: PluginSessionTransactionState) { + let wire = Wire() + + let error = #expect(throws: MariaDBPluginError.self) { + try MySQLReadOnlyTransaction.run(state: state, reason: .safeMode, send: wire.send) { + try wire.send(Self.statement) + } + } + + #expect(wire.sent.isEmpty) + #expect(error?.message == MySQLReadOnlyTransaction.safeModeSessionNotIdle.message) + #expect(error?.message != MySQLReadOnlyTransaction.sessionNotIdle.message) + } + @Test("A session holding LOCK TABLES is refused although the server reports no transaction") func tableLockIsNotIdle() { let wire = Wire() @@ -76,6 +104,7 @@ struct MySQLReadOnlyTransactionTests { #expect(throws: MariaDBPluginError.self) { try MySQLReadOnlyTransaction.run( state: footprint.transactionState(isInTransaction: false), + reason: .externalClients, send: wire.send ) { try wire.send(Self.statement) @@ -92,7 +121,7 @@ struct MySQLReadOnlyTransactionTests { wire.failing[Self.statement] = MariaDBPluginError(code: 1_792, message: serverMessage, sqlState: "25006") let error = #expect(throws: MariaDBPluginError.self) { - try MySQLReadOnlyTransaction.run(state: .idle, send: wire.send) { + try MySQLReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } @@ -100,7 +129,28 @@ struct MySQLReadOnlyTransactionTests { #expect(wire.sent == ["START TRANSACTION READ ONLY", Self.statement, "ROLLBACK"]) #expect(error?.code == 1_792) #expect(error?.sqlState == "25006") - #expect(error?.message != serverMessage) + #expect(error?.message == MySQLReadOnlyTransaction.refusalMessage(.externalClients)) + } + + // A stored function that writes fails inside the read-only transaction with 1792, which is the EX-1 case. + @Test("At Read-Only error 1792 names Safe Mode, and the rollback still goes out") + func safeModeRefusalNamesSafeMode() { + let wire = Wire() + let call = "SELECT purge_orders()" + wire.failing[call] = MariaDBPluginError( + code: 1_792, message: "Cannot execute statement in a READ ONLY transaction.", sqlState: "25006" + ) + + let error = #expect(throws: MariaDBPluginError.self) { + try MySQLReadOnlyTransaction.run(state: .idle, reason: .safeMode, send: wire.send) { + try wire.send(call) + } + } + + #expect(wire.sent == ["START TRANSACTION READ ONLY", call, "ROLLBACK"]) + #expect(error?.code == 1_792) + #expect(error?.message == MySQLReadOnlyTransaction.refusalMessage(.safeMode)) + #expect(error?.message != MySQLReadOnlyTransaction.refusalMessage(.externalClients)) } @Test("Any other failure keeps its own message, and the rollback still goes out") @@ -110,7 +160,7 @@ struct MySQLReadOnlyTransactionTests { wire.failing[Self.statement] = missing let error = #expect(throws: MariaDBPluginError.self) { - try MySQLReadOnlyTransaction.run(state: .idle, send: wire.send) { + try MySQLReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } @@ -128,7 +178,7 @@ struct MySQLReadOnlyTransactionTests { ) let error = #expect(throws: MariaDBPluginError.self) { - try MySQLReadOnlyTransaction.run(state: .idle, send: wire.send) { + try MySQLReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } @@ -137,7 +187,54 @@ struct MySQLReadOnlyTransactionTests { #expect(error?.code == 1_064) } - @Test("Only an external read on MySQL or MariaDB is wrapped", arguments: [ + // A `SET autocommit = 0` session always holds a transaction, so Read-Only could otherwise run nothing there. + @Test( + "At Read-Only a session that is not idle runs a statement with no unlisted call as it is", + arguments: [PluginSessionTransactionState.inTransaction, .holdsSessionLocks, .abortedTransaction, .unknown] + ) + func runsInTheOpenTransaction(state: PluginSessionTransactionState) throws { + let wire = Wire() + + let rows = try MySQLReadOnlyTransaction.run( + state: state, reason: .safeMode, runsInOpenTransaction: true, send: wire.send + ) { + try wire.send(Self.statement) + return 3 + } + + #expect(rows == 3) + #expect(wire.sent == [Self.statement]) + } + + @Test("An idle session still takes the read-only transaction for a statement with no unlisted call") + func idleSessionStillWraps() throws { + let wire = Wire() + + try MySQLReadOnlyTransaction.run(state: .idle, reason: .safeMode, runsInOpenTransaction: true, send: wire.send) { + try wire.send(Self.statement) + } + + #expect(wire.sent == ["START TRANSACTION READ ONLY", Self.statement, "ROLLBACK"]) + } + + // Measured on MySQL 5.5.62 and MariaDB 5.5.64: `START TRANSACTION READ ONLY` is error 1064 there. Read-Only falls + // to Safe Mode's function check; an external read keeps failing at the begin, as nothing else checks it. + @Test("A server before the READ ONLY characteristic runs a Read-Only read as it is", arguments: [ + ("5.5.62", MySQLServerFlavor.mysql, false), + ("5.6.4-m7", .mysql, false), + ("5.6.5", .mysql, true), + ("8.4.13", .mysql, true), + ("5.5.64-MariaDB-1~trusty", .mariadb, false), + ("10.0.38-MariaDB", .mariadb, true), + ("11.4.13-MariaDB-ubu2404", .mariadb, true), + ]) + func olderServerIsNotWrapped(banner: String, flavor: MySQLServerFlavor, wraps: Bool) { + #expect(MySQLReadOnlyTransaction.isAvailable(banner: banner, flavor: flavor) == wraps) + #expect((MySQLReadOnlyTransaction.reason(for: Self.safeModeRead, flavor: flavor, banner: banner) != nil) == wraps) + #expect(MySQLReadOnlyTransaction.reason(for: Self.externalRead, flavor: flavor, banner: banner) == .externalClients) + } + + @Test("Only an external read or a Read-Only statement on MySQL or MariaDB is wrapped", arguments: [ (MySQLServerFlavor.mysql, true), (.mariadb, true), (.tidb(version: MySQLEngineVersion(major: 8, minor: 5, patch: 1)), false), @@ -147,17 +244,60 @@ struct MySQLReadOnlyTransactionTests { ]) func wrapsOnlyEnforcingFlavors(flavor: MySQLServerFlavor, wraps: Bool) { #expect(MySQLReadOnlyTransaction.applies(to: Self.externalRead, flavor: flavor) == wraps) + #expect(MySQLReadOnlyTransaction.applies(to: Self.safeModeRead, flavor: flavor) == wraps) #expect(!MySQLReadOnlyTransaction.applies(to: PluginStatementContext(), flavor: flavor)) #expect(!MySQLReadOnlyTransaction.applies(to: Self.context(readOnly: true, origin: .app), flavor: flavor)) } - @Test("The driver declares read-only statements only for a flavor that enforces them") + @Test("The reason follows who asked: an external read names external clients, anything else at Read-Only names Safe Mode") + func reasonFollowsTheContext() { + var catalogRead = Self.context(readOnly: true, origin: .app) + catalogRead.readsCatalog = true + + #expect(MySQLReadOnlyTransaction.reason(for: Self.externalRead, flavor: .mysql) == .externalClients) + #expect(MySQLReadOnlyTransaction.reason(for: Self.safeModeRead, flavor: .mariadb) == .safeMode) + #expect( + MySQLReadOnlyTransaction.reason( + for: Self.context(readOnly: true, origin: .external, enforcesReadOnly: true), flavor: .mysql + ) == .externalClients + ) + #expect( + MySQLReadOnlyTransaction.reason( + for: Self.context(readOnly: false, origin: .external, enforcesReadOnly: true), flavor: .mysql + ) == .safeMode + ) + #expect(MySQLReadOnlyTransaction.reason(for: catalogRead, flavor: .mysql) == nil) + } + + @Test("The driver declares both read-only bits only for a flavor that enforces them") func capabilityFollowsFlavor() { - #expect(Self.driver().capabilities.contains(.readOnlyStatements)) - for variant in [ - MySQLServerFlavor.oceanbaseVariant, MySQLServerFlavor.databendVariant, MySQLServerFlavor.starrocksVariant, - ] { - #expect(!Self.driver(variant: variant).capabilities.contains(.readOnlyStatements), "\(variant)") + let capabilities = Self.driver().capabilities + #expect(capabilities.contains(.readOnlyStatements)) + #expect(capabilities.contains(.enforcesReadOnlyMode)) + for variant in Self.nonEnforcingVariants { + let other = Self.driver(variant: variant).capabilities + #expect(!other.contains(.readOnlyStatements), "\(variant)") + #expect(!other.contains(.enforcesReadOnlyMode), "\(variant)") } } + + // Nil sends the host back to a call that takes the context; none of these touch the connection. + @Test("A bounded read the read-only mode cannot cover goes back to the host") + func boundedReadGoesBackWhenNotCovered() async throws { + for variant in Self.nonEnforcingVariants { + let answer = try await Self.driver(variant: variant).executeBoundedQuery( + query: Self.statement, rowCap: 10, context: Self.safeModeRead + ) + #expect(answer == nil, "\(variant)") + } + + var catalogRead = Self.context(readOnly: true, origin: .app) + catalogRead.readsCatalog = true + #expect(try await Self.driver().executeBoundedQuery(query: Self.statement, rowCap: 10, context: catalogRead) == nil) + #expect( + try await Self.driver().executeBoundedQuery( + query: Self.statement, rowCap: 10, context: Self.context(readOnly: true, origin: .app) + ) == nil + ) + } } diff --git a/apple/Plugins/MySQLDriverTests/MySQLSelectLimitTests.swift b/apple/Plugins/MySQLDriverTests/MySQLSelectLimitTests.swift index b58dcb6868..24c6b5e607 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLSelectLimitTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLSelectLimitTests.swift @@ -3,6 +3,7 @@ // TableProTests // +import Foundation import TableProPluginKit import Testing @@ -106,6 +107,32 @@ struct MySQLSelectLimitTests { #expect(outcome.isTruncated == false) } + @Test("A read stopped at the byte limit keeps the row that crossed it and reports truncated") + func byteLimitKeepsTheCrossingRow() { + let outcome = mysqlBoundedFetchOutcome( + fetchedRows: 3, rowCap: 1_000, serverSentMore: false, stoppedAtByteLimit: true + ) + #expect(outcome.keptRows == 3) + #expect(outcome.isTruncated == true) + #expect(outcome.serverIgnoredLimit == false) + } + + @Test("A byte stop on the row past the cap still drops that row") + func byteLimitOnTheExtraRowDropsIt() { + let outcome = mysqlBoundedFetchOutcome( + fetchedRows: 1_001, rowCap: 1_000, serverSentMore: false, stoppedAtByteLimit: true + ) + #expect(outcome.keptRows == 1_000) + #expect(outcome.isTruncated == true) + } + + @Test("A row's size is its text as UTF-8 and its bytes, with null counting nothing") + func decodedByteCountMeasuresTheCells() { + let row: [PluginCellValue] = [.null, .text("héllo"), .bytes(Data(count: 4)), .text("")] + #expect(mysqlDecodedByteCount(of: row) == 10) + #expect(mysqlDecodedByteCount(of: []) == 0) + } + /// Installing the cap costs a `SET`, and a `SET` resets what `ROW_COUNT()` reports, so /// `UPDATE t SET ...; SELECT ROW_COUNT()` answered 0 instead of the UPDATE's count. Measured on /// MariaDB 12.3.2 and MySQL 8.4.11. diff --git a/apple/Plugins/MySQLDriverTests/MySQLServerVersionTests.swift b/apple/Plugins/MySQLDriverTests/MySQLServerVersionTests.swift index db606c0d9b..fb79fabb9f 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLServerVersionTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLServerVersionTests.swift @@ -15,6 +15,22 @@ struct MySQLServerVersionTests { #expect(!MySQLServerVersion.hasGenerationExpression(banner: "unknown", flavor: .mysql)) } + @Test("READ ONLY transactions start at MySQL 5.6.5 and MariaDB 10.0, an unreadable banner is not an old server, and no other flavor has them") + func readOnlyTransactionFloor() { + #expect(!MySQLServerVersion.hasReadOnlyTransactions(banner: "5.6.4-log", flavor: .mysql)) + #expect(MySQLServerVersion.hasReadOnlyTransactions(banner: "5.6.5", flavor: .mysql)) + #expect(MySQLServerVersion.hasReadOnlyTransactions(banner: "8.4.13", flavor: .mysql)) + #expect(!MySQLServerVersion.hasReadOnlyTransactions(banner: "5.5.64-MariaDB", flavor: .mariadb)) + #expect(MySQLServerVersion.hasReadOnlyTransactions(banner: "10.0.0-MariaDB", flavor: .mariadb)) + #expect(MySQLServerVersion.hasReadOnlyTransactions(banner: "11.4.13-MariaDB", flavor: .mariadb)) + #expect(MySQLServerVersion.hasReadOnlyTransactions(banner: nil, flavor: .mysql)) + #expect(MySQLServerVersion.hasReadOnlyTransactions(banner: "unknown", flavor: .mariadb)) + #expect(!MySQLServerVersion.hasReadOnlyTransactions(banner: "8.0.11-TiDB-v8.5.1", flavor: .tidb(version: nil))) + #expect(!MySQLServerVersion.hasReadOnlyTransactions(banner: "5.7.25", flavor: .oceanbase(version: nil))) + #expect(!MySQLServerVersion.hasReadOnlyTransactions(banner: "8.0.0", flavor: .databend)) + #expect(!MySQLServerVersion.hasReadOnlyTransactions(banner: "8.0.33", flavor: .starrocks)) + } + /// The other direction: a gate that picks legacy syntax has to hear a version before it does, /// because the legacy statements are a 1064 on MySQL 8. @Test("isKnownBelow answers false for a banner it cannot read") diff --git a/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift b/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift index 970a601ae8..ce469d21a0 100644 --- a/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift +++ b/apple/Plugins/Neo4jDriverPlugin/Neo4jConnection.swift @@ -381,7 +381,7 @@ actor Neo4jConnection { _ statement: BoltRequest, batchSize: Int64, interruptible: Bool, - receive: @Sendable (_ keys: [String], _ records: [[PackStreamValue]], _ isFirst: Bool) -> Bool + receive: @Sendable (_ keys: [String], _ records: [[PackStreamValue]], _ isFirst: Bool) async -> Bool ) async throws { try await withTurn(interruptible: interruptible) { wire, since in let replies = try await Self.exchange([statement, .pull(count: batchSize)], on: wire, cancellableSince: since) @@ -391,7 +391,7 @@ actor Neo4jConnection { var batch = replies[1] var isFirst = true while true { - let wanted = receive(keys, batch.records, isFirst) + let wanted = await receive(keys, batch.records, isFirst) isFirst = false guard batch.hasMore else { return } guard wanted, !Task.isCancelled else { diff --git a/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift b/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift index 4bca23eae3..c07018f4d6 100644 --- a/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift +++ b/apple/Plugins/Neo4jDriverPlugin/Neo4jPluginDriver.swift @@ -236,7 +236,7 @@ final class Neo4jPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // Pulls a batch at a time, so an export never holds the whole result. func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.stream(query, context: context, into: continuation) @@ -245,7 +245,7 @@ final class Neo4jPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish(throwing: error) } } - continuation.onTermination = { _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -278,8 +278,7 @@ final class Neo4jPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.yield(.header(PluginStreamHeader(columns: table.columns, columnTypeNames: table.typeNames))) } guard !table.rows.isEmpty else { return true } - if case .terminated = continuation.yield(.rows(Self.cells(table.rows))) { return false } - return true + return await PluginRowStream.deliver(.rows(Self.cells(table.rows)), to: continuation) } } diff --git a/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift b/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift index 37d91df4fa..92029328e5 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift @@ -4,59 +4,120 @@ // import Foundation +import TableProOracleCore import TableProPluginKit import TableProSQLQuoting -/// Writes the values for the host's `?` placeholders into an Oracle statement. -/// -/// A value a literal can carry is written as the literal the PluginKit default has always written, so a statement the -/// host builds (an import, a copy, an MCP write, a query parameter) reads exactly as it did: numeric text unquoted, -/// other text quoted, NULL as `NULL`. What no literal can carry is bound as a positional `:1 ... :n`, numbered in -/// order: bytes, because Oracle has no `X'..'` literal (ORA-03046), and text over the literal limit (ORA-01704). -/// -/// Only a `?` that is a placeholder is replaced. One inside a string literal, an alternative-quoted `q'[...]'` -/// literal (any delimiter, with or without the national `n` prefix), a double-quoted name such as `"Paid?"`, or a -/// `--` or `/* */` comment is text, and replacing it would change the statement and shift every value after it. +// Writes the values for the host's `?` placeholders into an Oracle statement. Each value is written for the type of +// the column it goes into when the host names one, the same way a grid save writes it; else for the kind the user +// gave the parameter; else as the PluginKit default writes untyped text. What no literal can carry is bound as a +// positional `:1 ... :n`, numbered in order: bytes, because Oracle has no `X'..'` literal (ORA-03046), and text over +// the literal limit (ORA-01704). No session NLS setting is changed: a quoted decimal fails with ORA-01722 under +// `NLS_NUMERIC_CHARACTERS = ',.'`, so a number goes in as a validated literal. +// +// Only a `?` that is a placeholder is replaced. One inside a string literal, an alternative-quoted `q'[...]'` literal +// (any delimiter, with or without the national `n` prefix), a double-quoted name such as `"Paid?"`, or a `--` or +// `/* */` comment is text, and replacing it would change the statement and shift every value after it. internal enum OracleBindPlaceholders { /// The longest string literal Oracle parses, in bytes of the value it holds rather than of its doubled quotes /// (measured on 23ai with `MAX_STRING_SIZE = STANDARD`: 4,000 bytes parse, 4,001 fail with ORA-01704). static let maxLiteralBytes = 4_000 - /// The statement with every placeholder replaced, and the values left to bind, in `:n` order. A placeholder with - /// no parameter left stays `?`, as the PluginKit default leaves it. - static func substituting(_ sql: String, parameters: [PluginCellValue]) -> (sql: String, binds: [PluginCellValue]) { + // A placeholder with no parameter left stays `?`, as the PluginKit default leaves it. Throws for a BFILE value that + // is not a `BFILENAME` call, which no SQL writes. + static func substituting( + _ sql: String, + parameters: [PluginCellValue], + kinds: [PluginColumnKind?]? = nil, + typeNames: [String?]? = nil, + writesBooleans: Bool = false + ) throws -> (sql: String, binds: [PluginCellValue]) { var binds: [PluginCellValue] = [] - let rewritten = replacingPlaceholders(in: sql) { index in + let rewritten = try replacingPlaceholders(in: sql) { index in guard index < parameters.count else { return nil } let value = parameters[index] - if let literal = literal(for: value) { return literal } - binds.append(value) - return ":\(binds.count)" + guard let written = Self.operand( + for: value, + kind: kinds.flatMap { $0.indices.contains(index) ? $0[index] : nil }, + typeName: typeNames.flatMap { $0.indices.contains(index) ? $0[index] : nil }, + writesBooleans: writesBooleans, + bind: { + binds.append(value) + return ":\(binds.count)" + } + ) else { + throw OraclePluginError(core: .queryFailed(OracleValueKind.unwritableBFileReason)) + } + return written } return (rewritten, binds) } - /// The literal for a value, or nil when it has to be bound. NUL characters are dropped as the PluginKit default - /// drops them. A quote is doubled scalar by scalar, so one carrying a combining mark cannot end the literal. - static func literal(for value: PluginCellValue) -> String? { + static func operand( + for value: PluginCellValue, + kind: PluginColumnKind?, + typeName: String?, + writesBooleans: Bool, + bind: () -> String + ) -> String? { + if let typeName { + return OracleValueKind(typeName: typeName).operand(for: value, bind: bind) + } switch value { case .null: return "NULL" case .bytes: - return nil + return bind() case .text(let text): - return PluginNumericLiteral.isValid(text) ? text : quotedLiteral(text) + return literal(for: text, kind: kind, writesBooleans: writesBooleans) ?? bind() + } + } + + // Nil when the text is over the literal limit and has to be bound. Text the kind does not fit is quoted, for the + // server to refuse. `TRUE` and `FALSE` parse only where SQL has a BOOLEAN type, 23ai on. + static func literal(for text: String, kind: PluginColumnKind?, writesBooleans: Bool) -> String? { + guard let kind else { + // No numeric column reads as `00123` or `-007`, so that text is a code or a zip and stays a string. + if PluginNumericLiteral.isValid(text), !opensWithPaddingZero(text) { return text } + return quotedLiteral(text) + } + switch kind { + case .integer, .decimal: + if OracleNumericLiteral.isValid(text) { return text } + case .boolean: + guard writesBooleans else { break } + switch PluginSQLLiteral.booleanSynonym(for: text, kind: kind) { + case .isTrue?: return "TRUE" + case .isFalse?: return "FALSE" + default: break + } + case .text, .other: + break + @unknown default: + break } + return quotedLiteral(text) } - // Nil when the text is over the literal limit and has to be bound. + // Nil when the text is over the literal limit and has to be bound. NUL characters are dropped as the PluginKit + // default drops them. A quote is doubled scalar by scalar, so one carrying a combining mark cannot end the literal. static func quotedLiteral(_ text: String) -> String? { let kept = text.unicodeScalars.filter { $0 != "\0" } guard String(kept).utf8.count <= maxLiteralBytes else { return nil } return SQLQuoting.stringLiteral(text) } - private static func replacingPlaceholders(in sql: String, with replacement: (Int) -> String?) -> String { + // `007`, `-0042`: an integer part that opens with 0 and goes on with another digit. + private static func opensWithPaddingZero(_ text: String) -> Bool { + let unsigned = text.unicodeScalars.drop { $0 == "-" || $0 == "+" } + guard unsigned.first == "0", let next = unsigned.dropFirst().first else { return false } + return ("0"..."9").contains(next) + } + + private static func replacingPlaceholders( + in sql: String, + with replacement: (Int) throws -> String? + ) rethrows -> String { let units = Array(sql.utf16) var output = "" var placeholderIndex = 0 @@ -67,7 +128,7 @@ internal enum OracleBindPlaceholders { index = skippedEnd(in: units, from: index) continue } - let replaced = replacement(placeholderIndex) + let replaced = try replacement(placeholderIndex) placeholderIndex += 1 guard let text = replaced else { index += 1 diff --git a/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift b/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift index 07a77e1231..df7ff44f6b 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift @@ -4,24 +4,32 @@ // import Foundation +import TableProOracleCore import TableProSQLQuoting -/// The indexes an Oracle dump has to recreate, and the `CREATE INDEX` that recreates them. -/// -/// Every index is taken, constraint-backed ones included, because this driver's `fetchTableDDL` -/// writes column definitions and nothing else: no primary key, no unique constraint. Excluding -/// them would have left an Oracle dump with no uniqueness anywhere, so a restore would accept -/// duplicate rows the source rejected. +// The indexes an Oracle dump recreates after the rows, and the `CREATE INDEX` that recreates each. An index an enabled +// PRIMARY KEY or UNIQUE constraint owns is left out: `fetchTableDDL` writes the constraint, which makes its own index, +// and a second index on the same columns fails with ORA-01408. A disabled or NOVALIDATE constraint is written disabled +// and makes none, so an index it kept is written, and a NOVALIDATE one is enabled on it after the rows. internal enum OracleIndexStatements { - // NORMAL and BITMAP only: a function-based index keeps its key in a LONG column OracleNIO cannot decode. + // NORMAL and BITMAP only: a function-based index keeps its key in a LONG column OracleNIO cannot decode. A + // constraint row with no `INDEX_OWNER` is read as naming an index of its own schema. internal static func query(schema: String, table: String) -> String { """ SELECT i.INDEX_NAME, i.UNIQUENESS, i.INDEX_TYPE, ic.COLUMN_NAME, ic.DESCEND - FROM SYS.ALL_INDEXES i - JOIN SYS.ALL_IND_COLUMNS ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER + FROM \(OracleDictionary.allIndexes) i + JOIN \(OracleDictionary.allIndColumns) ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER WHERE i.TABLE_NAME = \(SQLQuoting.stringLiteral(table)) AND i.OWNER = \(SQLQuoting.stringLiteral(schema)) AND i.INDEX_TYPE IN ('NORMAL', 'BITMAP') + AND NOT EXISTS ( + SELECT 1 FROM \(OracleDictionary.allConstraints) c + WHERE c.CONSTRAINT_TYPE IN ('P', 'U') + AND c.STATUS = 'ENABLED' + AND c.VALIDATED = 'VALIDATED' + AND c.INDEX_NAME = i.INDEX_NAME + AND NVL(c.INDEX_OWNER, c.OWNER) = i.OWNER + ) ORDER BY i.INDEX_NAME, ic.COLUMN_POSITION """ } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index 0b5c15591c..c3764f36da 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -215,6 +215,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .cellExpressions, .uuidCellText, .userManagement, + .columnDefaultMarks, ] } @@ -377,11 +378,25 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return pluginResult(of: query, result, executionTime: Date().timeIntervalSince(startTime)) } - /// Values a literal can carry stay literals, so the host's statements read as they always have; only bytes and - /// text over the literal limit are bound. See ``OracleBindPlaceholders``. func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await executeParameterized(query: query, parameters: parameters, context: PluginStatementContext()) + } + + // Values a literal can carry stay literals; only bytes and text over the literal limit are bound. The context + // says the type of each value's column, or the kind the user gave it (`OracleBindPlaceholders`). + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { guard !parameters.isEmpty else { return try await execute(query: query) } - let statement = OracleBindPlaceholders.substituting(query, parameters: parameters) + let statement = try OracleBindPlaceholders.substituting( + query, + parameters: parameters, + kinds: context.parameterKinds, + typeNames: context.parameterTypeNames, + writesBooleans: core?.serverRelease?.hasBooleanType ?? false + ) guard !statement.binds.isEmpty else { return try await execute(query: statement.sql) } guard let core else { throw OraclePluginError(core: .notConnected) } let startTime = Date() @@ -395,6 +410,37 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return pluginResult(of: query, result, executionTime: Date().timeIntervalSince(startTime)) } + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters, context: PluginStatementContext()) + } + + // Overridden so the kinds the user gave the parameters reach the binder; the PluginKit default drops the context. + // A capped result keeps its column hints, so a DATE column still reads with its time. + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + let raw: PluginQueryResult + if let parameters { + raw = try await executeParameterized(query: query, parameters: parameters, context: context) + } else { + raw = try await execute(query: query, context: context) + } + guard let cap = rowCap, cap > 0, raw.rows.count > cap else { return raw } + return PluginQueryResult( + columns: raw.columns, + columnTypeNames: raw.columnTypeNames, + rows: Array(raw.rows.prefix(cap)), + rowsAffected: raw.rowsAffected, + timing: raw.timing, + isTruncated: true, + statusMessage: raw.statusMessage, + columnMeta: raw.columnMeta + ) + } + /// An anonymous block reports no row count of its own; whatever oracle-nio counted belongs to the statements /// inside it. private func pluginResult( @@ -444,37 +490,34 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming - /// The stream header has no field for classification hints, so they are added to the collected result. + // The stream header has no field for classification hints, so they are added to the collected result. func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { - let result = try await boundedQueryFromStream(query: query, rowCap: rowCap) + // The collector needs one row past the cap to tell a full result from a truncated one. + let cap = max(rowCap, 1) + let result = try await PluginBoundedStream.collect( + rowStream(query: query, rowBudget: cap < Int.max ? cap + 1 : nil), + rowCap: cap, + startedAt: Date() + ) try await reportCompilationErrors(of: query) return result.withOracleClassificationHints() } func streamRows(query: String) -> AsyncThrowingStream { + rowStream(query: query, rowBudget: nil) + } + + private func rowStream(query: String, rowBudget: Int?) -> AsyncThrowingStream { guard let core else { return AsyncThrowingStream { $0.finish(throwing: OraclePluginError(core: .notConnected)) } } + let batching = OracleRowBatcher(rowsPerBatch: OracleRowBatcher.batchRows, rowBudget: rowBudget) - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in + // A plain `Task {}` is not a child of this one, so only the explicit cancel on abort stops oracle-nio. let streamTask = Task { - /// The core read runs in its own unstructured task, which the outer one cannot - /// cancel: a plain `Task {}` inherits context but is not a child. Cancelling it - /// explicitly on abort is what stops oracle-nio, whose own loop already checks - /// cancellation once it is reachable. - let coreStream = AsyncThrowingStream { coreContinuation in - let coreTask = Task { - do { - try await core.streamQuery(query, continuation: coreContinuation) - } catch { - coreContinuation.finish(throwing: error) - } - } - abort.onAbort { coreTask.cancel() } - coreContinuation.onTermination = { @Sendable _ in coreTask.cancel() } - } do { - for try await element in coreStream { + try await core.streamQuery(query, batching: batching, onElement: { element in switch element { case .header(let columns): continuation.yield(.header(PluginStreamHeader( @@ -482,13 +525,13 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { columnTypeNames: columns.map(\.typeName) ))) case .rows(let batch): - continuation.yield(.rows(batch.map { $0.map(\.asPluginCell) })) + _ = await PluginRowStream.deliver(.rows(batch.map { $0.map(\.asPluginCell) }), to: continuation) case .affectedRows: // The plugin stream carries rows only. The app sends writes and DDL through // `execute`, which reports the count, and bounds only reads through this stream. break } - } + }) continuation.finish() } catch let error as OracleCoreError { continuation.finish(throwing: error.asPluginError) @@ -577,11 +620,19 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { let owner = effectiveSchema(schema) let result = try await rawQuery(OracleIndexStatements.query(schema: owner, table: table)) - return OracleIndexStatements.render( + let indexes = OracleIndexStatements.render( rows: result.rows.map { row in row.map { $0.stringValue } }, schema: owner, table: table, quote: quoteIdentifier) + let constraints = try await rawQuery( + OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: serverRelease()) + ) + return indexes + (try OracleTableDDL.afterRowsStatements( + qualifiedTable: "\(quoteIdentifier(owner)).\(quoteIdentifier(table))", + constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), + quote: quoteIdentifier + )) } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { @@ -626,6 +677,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { OracleObjectQueries.dropTrigger(name: name, schema: schema, currentSchema: _currentSchema) } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let result = try await rawQuery( OracleSchemaQueries.allColumns(schema: effectiveSchema(schema), release: serverRelease()) @@ -697,9 +750,14 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { if !view.rows.isEmpty { return try await viewStatement(from: view, schema: owner, view: table) } + let columns = try await fetchColumns(table: table, schema: schema) + let constraints = try await rawQuery( + OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: serverRelease()) + ) return try OracleTableDDL.createTable( qualifiedTable: "\(quoteIdentifier(owner)).\(quoteIdentifier(table))", - columns: try await fetchColumns(table: table, schema: schema), + columns: columns, + constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), quote: quoteIdentifier ) } diff --git a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift index 13d859d092..14ee2761f0 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift @@ -11,7 +11,8 @@ import TableProSQLQuoting /// /// A keyed row is found by its primary key alone. A keyless row is found by every column it has, bounded with /// `ROWNUM = 1` so a save never touches a second row that looks the same, and a change whose match would need a -/// column that cannot be compared is refused rather than sent without it. +/// column that cannot be compared is refused rather than sent without it. An update or delete reports the rows it +/// matches: measured on 23ai, the count takes in a row already holding the value and leaves out rows a trigger writes. /// /// Every value is written for the type the host reports for its column, so nothing depends on the session's NLS /// settings. Measured on 23ai: a DATE or TIMESTAMP value is converted with an explicit mask, because the same text @@ -20,16 +21,21 @@ import TableProSQLQuoting /// because a NULL bind into an object-type column fails with ORA-00932; and text for an NCHAR, NVARCHAR2 or NCLOB /// column is `UNISTR`, because a literal or bind passes through the database character set (``OracleNationalText``). internal struct OracleRowWriter { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - /// A keyed delete carries at most this many key values. One statement with 32,767 binds took 30 seconds on 23ai /// and one with 1,000 took 0.07, and a statement that outlives the query timeout closes the session. static let maxValuesPerDelete = 1_000 static let dateMask = "YYYY-MM-DD HH24:MI:SS" static let timestampMask = "YYYY-MM-DD HH24:MI:SS.FF" - static let timestampWithTimeZoneMask = "YYYY-MM-DD HH24:MI:SS.FFTZH:TZM" + // A cell reads `….ffffff+02:00`, the offset straight after the fraction, or `….ffffff Europe/Paris`, a region after + // a blank. TZH:TZM reads the first as written and TZR the second. + static let timestampWithOffsetMask = "YYYY-MM-DD HH24:MI:SS.FFTZH:TZM" + static let timestampWithTimeZoneMask = "YYYY-MM-DD HH24:MI:SS.FF TZR" + + static func timestampWithTimeZoneMask(for text: String?) -> String { + let zone = text?.split(separator: " ").last.flatMap(\.first) + return zone?.isLetter == true ? timestampWithTimeZoneMask : timestampWithOffsetMask + } let qualifiedTable: String let columns: [String] @@ -56,7 +62,8 @@ internal struct OracleRowWriter { let write = try insert( values: values, rowIndex: change.rowIndex, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) else { continue } try flushDeletes() writes.append(write) @@ -80,7 +87,8 @@ internal struct OracleRowWriter { private func insert( values: [PluginCellValue], rowIndex: Int, - expressionColumns: Set + expressionColumns: Set, + defaultColumns: Set ) throws -> PluginRowWrite? { guard let firstColumn = columns.first else { return nil } var names: [String] = [] @@ -89,7 +97,7 @@ internal struct OracleRowWriter { for (index, (column, value)) in zip(columns, values).enumerated() where !context.serverOwnedColumns.contains(column) { names.append(Self.quote(column)) - if value == Self.defaultMarker { + if defaultColumns.contains(index) { placeholders.append("DEFAULT") } else if let expression = Self.expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(expression) @@ -117,7 +125,7 @@ internal struct OracleRowWriter { var parameters: [PluginCellValue] = [] let assignments = try change.cellChanges.map { cell -> String in let column = Self.quote(cell.columnName) - guard cell.newValue != Self.defaultMarker else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cell.columnIndex) else { return "\(column) = DEFAULT" } if let expression = Self.expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) { return "\(column) = \(expression)" } @@ -127,7 +135,7 @@ internal struct OracleRowWriter { let match = try rowMatch(for: change, parameters: ¶meters) let bound = primaryKeyColumns.isEmpty ? " AND ROWNUM = 1" : "" let statement = "UPDATE \(qualifiedTable) SET \(assignments.joined(separator: ", ")) WHERE \(match)\(bound)" - return PluginRowWrite(statement: statement, parameters: parameters, rowIndices: [change.rowIndex]) + return PluginRowWrite(statement: statement, parameters: parameters, rowIndices: [change.rowIndex], expectedRowCount: 1) } private func deleteWrites(for changes: [PluginRowChange]) throws -> [PluginRowWrite] { @@ -138,7 +146,8 @@ internal struct OracleRowWriter { return PluginRowWrite( statement: "DELETE FROM \(qualifiedTable) WHERE \(match) AND ROWNUM = 1", parameters: parameters, - rowIndices: [change.rowIndex] + rowIndices: [change.rowIndex], + expectedRowCount: 1 ) } } @@ -153,7 +162,8 @@ internal struct OracleRowWriter { return PluginRowWrite( statement: "DELETE FROM \(qualifiedTable) WHERE \(matches.joined(separator: " OR "))", parameters: parameters, - rowIndices: chunk.map(\.rowIndex) + rowIndices: chunk.map(\.rowIndex), + expectedRowCount: chunk.count ) } } @@ -233,10 +243,7 @@ internal struct OracleRowWriter { // MARK: - Values - /// Text for a column of a known type that is not a number is quoted here: left as `?`, it would reach the - /// placeholder writer, which writes numeric-looking text unquoted, so `00123` would be stored in a VARCHAR2 as - /// `123` and a keyless match on it would compare the column as a number. Text over the literal limit stays `?` and - /// is bound. A column of no known type keeps the plain `?`. + // A column of no known type keeps the plain `?`, which the placeholder writer fills in. private func sql( for value: PluginCellValue, column: String, @@ -248,37 +255,13 @@ internal struct OracleRowWriter { parameters.append(value) return "?" } - let kind = OracleValueKind(typeName: typeName) - if kind == .bfile, !value.isNull { - return try bfileSQL(for: value, rowIndex: rowIndex) - } - switch value { - case .null: - return "NULL" - case .bytes: + guard let operand = OracleValueKind(typeName: typeName).operand(for: value, bind: { parameters.append(value) return "?" - case .text(let text): - if let literal = kind.inlineSQL(for: text) { return literal } - if !kind.isNumeric, let quoted = OracleBindPlaceholders.quotedLiteral(text) { - return kind.converting(quoted) - } - parameters.append(value) - return kind.converting("?") + }) else { + throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: OracleValueKind.unwritableBFileReason) } - } - - /// A BFILE reads as the `BFILENAME('DIR', 'file')` call that makes it, and only that call writes one back: the - /// text quoted or bound fails with ORA-00932. Anything else in a BFILE cell, such as the `` placeholder, - /// is refused rather than sent. - private func bfileSQL(for value: PluginCellValue, rowIndex: Int) throws -> String { - guard let text = value.asText, let call = OracleBFileText.call(from: text) else { - throw PluginRowWriteRefusal( - rowIndex: rowIndex, - reason: String(format: String(localized: "Values of type %@ can't be written."), "BFILE") - ) - } - return call + return operand } // MARK: - Helpers @@ -346,20 +329,46 @@ internal enum OracleValueKind: Equatable { } } + static var unwritableBFileReason: String { + String(format: String(localized: "Values of type %@ can't be written."), "BFILE") + } + + // A literal where one carries the value, else `bind()`'s placeholder in the conversion the type needs. Text that + // is not a number is quoted here: the placeholder writer would write `00123` bare, and a VARCHAR2 would store 123. + // Nil for a BFILE value that is not the `BFILENAME('DIR', 'file')` call it reads as, the only text that writes + // one back (anything else fails with ORA-00932). + func operand(for value: PluginCellValue, bind: () -> String) -> String? { + if self == .bfile, !value.isNull { + return value.asText.flatMap(OracleBFileText.call(from:)) + } + switch value { + case .null: + return "NULL" + case .bytes: + return bind() + case .text(let text): + if let literal = inlineSQL(for: text) { return literal } + if !isNumeric, let quoted = OracleBindPlaceholders.quotedLiteral(text) { + return converting(quoted, text: text) + } + return converting(bind(), text: text) + } + } + /// A text operand, a literal or `?`, as the column's type takes it. /// /// A TIMESTAMP WITH LOCAL TIME ZONE value reads as the session's wall clock with that instant's offset, which /// `TO_TIMESTAMP` rejects (ORA-01830). `TO_TIMESTAMP_TZ` takes it, and text without an offset in the session's /// zone, so the value round-trips through SET and a keyless match whatever the session and database zones are /// (measured on 23ai with the session at `+07:00` and at `America/New_York` against a `+00:00` database). - func converting(_ operand: String) -> String { + func converting(_ operand: String, text: String? = nil) -> String { switch self { case .date: return "TO_DATE(\(operand), '\(OracleRowWriter.dateMask)')" case .timestamp: return "TO_TIMESTAMP(\(operand), '\(OracleRowWriter.timestampMask)')" case .timestampWithTimeZone, .timestampWithLocalTimeZone: - return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask)')" + return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask(for: text))')" case .number, .binaryFloat, .binaryDouble, .bfile, .nationalText, .nationalLOB, .other: return operand } diff --git a/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift b/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift index 036c2ad204..215d9e4ee4 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift @@ -17,15 +17,60 @@ import TableProSQLGrammar /// `GENERATED ALWAYS AS (...) VIRTUAL` or `DEFAULT`, then `NOT NULL`. `NOT NULL` before `DEFAULT` fails with /// ORA-03076, and a type folded to upper case names no type at all when it is a quoted mixed-case object type. internal enum OracleTableDDL { + // The keys and checks follow the columns. A NOT NULL the column definition writes is not written again as a check. static func createTable( qualifiedTable: String, columns: [PluginColumnInfo], + constraints: [OracleTableConstraint] = [], quote: (String) -> String ) throws -> String { - let definitions = try columns.map { " " + (try columnDefinition($0, quote: quote)) } + let notNullColumns = Set(columns.filter { !$0.isNullable }.map(\.name)) + let columnDefinitions = try columns.map { try columnDefinition($0, quote: quote) } + let written = constraints.filter { !$0.restatesNotNull(of: notNullColumns) } + // A check added after the rows is refused here, with the others, rather than in the index phase. + for check in written where check.isAddedAfterRows { + _ = try checkCondition(of: check) + } + let constraintClauses = try written.filter { !$0.isAddedAfterRows }.map { try constraintClause($0, quote: quote) } + let definitions = (columnDefinitions + constraintClauses).map { " " + $0 } return "CREATE TABLE \(qualifiedTable) (\n" + definitions.joined(separator: ",\n") + "\n);" } + // A check's catalog text goes in only as one expression, as a default does. + private static func constraintClause(_ constraint: OracleTableConstraint, quote: (String) -> String) throws -> String { + guard constraint.kind == .check else { return constraint.clause(quote: quote) } + var written = constraint + written.condition = try checkCondition(of: constraint) + return written.clause(quote: quote) + } + + private static func checkCondition(of constraint: OracleTableConstraint) throws -> String { + let condition = SQLExpressionFragment.strippingComments(constraint.condition ?? "", databaseTypeId: "Oracle") + guard !constraint.conditionMayBeCut, !condition.isEmpty, isSingleExpression(condition) else { + throw OraclePluginError(core: .queryFailed(String( + format: String(localized: "The condition of check constraint %@ cannot be written as SQL."), + constraint.name + ))) + } + return condition + } + + // What runs after the rows: a NOVALIDATE constraint is enabled again without checking them, and one with a + // generated name is enabled by what it is, or added if it is a check. + static func afterRowsStatements( + qualifiedTable: String, + constraints: [OracleTableConstraint], + quote: (String) -> String + ) throws -> [String] { + try constraints.compactMap { constraint in + if let statement = constraint.enableAfterRowsStatement(qualifiedTable: qualifiedTable, quote: quote) { + return statement + } + guard constraint.isAddedAfterRows else { return nil } + return "ALTER TABLE \(qualifiedTable) ADD CHECK (\(try checkCondition(of: constraint))) ENABLE NOVALIDATE" + } + } + /// `TEXT_VC` is the view's query alone, so its column names go in the header: a view declared /// `v (public_id) AS SELECT internal_id ...` replayed without them exposes `INTERNAL_ID`, and one /// over an unaliased expression cannot be recreated at all. `DBMS_METADATA` writes the list too. diff --git a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift index 90eeda01e8..2495d396c6 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift @@ -59,10 +59,12 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP progress: PluginExportProgress ) async throws -> ExportFormatResult { guard !tables.isEmpty else { return ExportFormatResult() } + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings var warnings: [String] = [] try await ParquetTableFiles.write(tables, destination: destination, progress: progress) { table, fileURL in - warnings += try await writeTable(table, dataSource: dataSource, to: fileURL, progress: progress) + warnings += try await writeTable(table, options: options, dataSource: dataSource, to: fileURL, progress: progress) } if tables.count > 1 { @@ -76,6 +78,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP private func writeTable( _ table: PluginExportTable, + options: ParquetExportOptions, dataSource: any PluginExportDataSource, to fileURL: URL, progress: PluginExportProgress @@ -123,8 +126,8 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP } let fallbacks = try staging.copyToParquet( fileURL: fileURL, - compression: settings.compression.duckDBValue, - rowGroupSize: settings.rowGroupSize + compression: options.compression.duckDBValue, + rowGroupSize: options.rowGroupSize ) return fallbacks.map { fallback in String( diff --git a/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift b/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift index 6ab10742af..c716dab7fa 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift @@ -25,6 +25,10 @@ public enum ParquetTypeMapper { if dateTypes.contains(base) { return enginesWhereDateCarriesTime.contains(databaseTypeId) ? "TIMESTAMP" : "DATE" } + // A cast to TIMESTAMP or TIME drops the offset, so 12:00+07 would read as 12:00 UTC. + if isZoned(typeName) || base == "timestamptz" || base == "timetz" { + return timeTypes.contains(base) ? "TIMETZ" : "TIMESTAMPTZ" + } if timestampTypes.contains(base) { return "TIMESTAMP" } if timeTypes.contains(base) { return "TIME" } if binaryTypes.contains(base) { return "BLOB" } @@ -49,6 +53,12 @@ public enum ParquetTypeMapper { words(typeName).contains("unsigned") } + // Oracle's `with local time zone` counts: the driver writes it with the session's offset. + private static func isZoned(_ typeName: String) -> Bool { + let words = words(typeName).filter { $0 != "local" } + return words.indices.dropLast(2).contains { words[$0] == "with" && words[$0 + 1] == "time" && words[$0 + 2] == "zone" } + } + // `bit` and `bit(1)` hold one flag. `bit(8)` and `bit varying` hold a string of them. private static func isSingleBit(_ typeName: String) -> Bool { guard !words(typeName).contains("varying") else { return false } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift index 0ea3e26c4a..16a768935d 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift @@ -32,6 +32,8 @@ final class CockroachPluginDriver: LibPQBackedDriver, @unchecked Sendable { .schemaCompare, .dataCompare, .readOnlyStatements, + .enforcesReadOnlyMode, + .copyFromStandardInput, ] } @@ -79,6 +81,8 @@ final class CockroachPluginDriver: LibPQBackedDriver, @unchecked Sendable { return result.rows.compactMap { Self.mapColumnRow($0, includesTableName: false) } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let resolvedSchema = schema ?? core.currentSchema let query = CockroachRelationSQL.columnsQuery(schema: resolvedSchema, table: nil) diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQCappedCursorRead.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCappedCursorRead.swift new file mode 100644 index 0000000000..865cec303d --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCappedCursorRead.swift @@ -0,0 +1,100 @@ +// +// LibPQCappedCursorRead.swift +// PostgreSQLDriverPlugin +// + +import Foundation + +// A capped read for a session that cannot cancel: inside the user's transaction block a cancel aborts the block, and +// a single-connection server takes none. Single-row mode only changes how libpq hands rows over, so the server would +// still send every row. Through a cursor, FETCH sends the cap's rows and MOVE runs the rest without sending them, so +// every row's side effects still happen, as they did when the tail was drained. +internal enum LibPQCappedCursorRead { + enum Scope: Equatable { + // A savepoint holds the cursor, so a refused DECLARE leaves the user's block as it was. + case insideTransaction + // DECLARE needs a transaction block, so the read opens and commits one of its own. + case ownTransaction + } + + struct Plan: Equatable { + let scope: Scope + // Its own round trip: the server parses a whole query string before it runs any of it, so a syntax error in + // the user's text would otherwise leave no savepoint to roll back to. + let opening: String + let statement: String + // Undoes the attempt, so a refused cursor can run again the plain way. + let recovery: String + let fetchCount: Int + } + + static let cursorName = "tablepro_capped_read" + + private static let cursorableKeywords: Set = ["select", "with", "values", "table"] + + // DECLARE refuses INTO and a data-modifying WITH, and a rolled-back stream savepoint would drop the locks of FOR + // UPDATE and FOR SHARE. A false match costs nothing but the plain read. + private static let excludedWords: Set = ["into", "insert", "update", "delete", "merge", "share"] + + static func plan(for query: String, rowCap: Int, scope: Scope) -> Plan? { + guard rowCap > 0, rowCap < Int(Int32.max), let body = cursorBody(of: query) else { return nil } + let fetchCount = rowCap + 1 + let opening: String + let closing: String + let recovery: String + switch scope { + case .insideTransaction: + opening = "SAVEPOINT \(cursorName)" + closing = "RELEASE SAVEPOINT \(cursorName)" + recovery = "ROLLBACK TO SAVEPOINT \(cursorName); RELEASE SAVEPOINT \(cursorName)" + case .ownTransaction: + opening = "BEGIN" + closing = "COMMIT" + recovery = "ROLLBACK" + } + // The body sits on lines of its own, so a trailing `--` comment cannot swallow the FETCH. + let statement = [ + "DECLARE \(cursorName) NO SCROLL CURSOR FOR", + body, + ";", + "FETCH FORWARD \(fetchCount) FROM \(cursorName);", + "MOVE FORWARD ALL FROM \(cursorName);", + "CLOSE \(cursorName);", + closing, + ].joined(separator: "\n") + return Plan(scope: scope, opening: opening, statement: statement, recovery: recovery, fetchCount: fetchCount) + } + + // One SELECT, VALUES or TABLE statement, without its trailing semicolons. Any other semicolon, even one inside a + // literal or a comment, sends the text the plain way rather than risk a second statement inside the cursor. + static func cursorBody(of query: String) -> String? { + var body = Substring(query) + while let last = body.last, last.isWhitespace || last == ";" { + body = body.dropLast() + } + guard !body.isEmpty, !body.contains(";") else { return nil } + guard let keyword = leadingKeyword(of: body), cursorableKeywords.contains(keyword) else { return nil } + let words = body.lowercased().split { !($0.isLetter || $0.isNumber || $0 == "_" || $0 == "$") } + guard !words.contains(where: { excludedWords.contains(String($0)) }) else { return nil } + return String(body) + } + + private static func leadingKeyword(of text: Substring) -> String? { + var rest = text[...] + while let first = rest.first { + if first.isWhitespace || first == "(" { + rest = rest.dropFirst() + } else if rest.hasPrefix("--") { + guard let lineEnd = rest.firstIndex(where: \.isNewline) else { return nil } + rest = rest[lineEnd...] + } else if rest.hasPrefix("/*") { + guard let commentEnd = rest.range(of: "*/") else { return nil } + rest = rest[commentEnd.upperBound...] + } else { + break + } + } + let word = rest.prefix { $0.isLetter } + return word.isEmpty ? nil : word.lowercased() + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift index 04bb56d2ea..7cd8bc1d45 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift @@ -6,6 +6,7 @@ import Foundation import TableProPluginKit import TableProSQLQuoting +import TableProTLSClientIdentity internal struct LibPQConnectTimeout: Equatable, Sendable { static let defaultMilliseconds = 10_000 @@ -46,9 +47,8 @@ internal struct LibPQConnectTimeout: Equatable, Sendable { } } -/// One monotonic wall-clock budget for libpq's TCP/TLS/authentication and the bootstrap queries -/// that make the session usable. Keeping the absolute instant here prevents every catalog probe -/// from receiving the full timeout again. +// One absolute instant for the TCP, TLS and sign-in steps and the bootstrap queries, so no catalog probe gets the full +// timeout again. internal struct LibPQConnectDeadline: Equatable, Sendable { private let expiresAtMicroseconds: Int64 @@ -67,6 +67,13 @@ internal struct LibPQConnectDeadline: Equatable, Sendable { } } +// Where libpq takes the roots it checks the server's certificate against. +internal enum LibPQRootCertificateSource: Equatable, Sendable { + case caFile(String) + case libpqDefault + case systemTrustAnchors +} + internal enum LibPQConnectionString { static let clientEncoding = "UTF8" @@ -75,11 +82,10 @@ internal enum LibPQConnectionString { static let sessionApplicationName = "TablePro" static let metadataApplicationName = "TablePro Metadata" - /// Left to the kernel, macOS sends the first keepalive after 7,200 s of idle, while an AWS - /// Network Load Balancer drops an idle flow after 350 s and an Azure Load Balancer silently after - /// 4 minutes. These keep a parked session's flow open and let the kernel declare a dead peer - /// about 90 s after the last traffic. `tcp_user_timeout` is left out: macOS has no - /// `TCP_USER_TIMEOUT`, so libpq accepts it and changes nothing. + // Left to the kernel, macOS sends the first keepalive after 7,200 s of idle, while an AWS Network Load Balancer drops + // an idle flow after 350 s and an Azure Load Balancer silently after 4 minutes. These keep a parked session's flow + // open and find a dead peer about 90 s after the last traffic. No `tcp_user_timeout`: macOS has no + // `TCP_USER_TIMEOUT`, so libpq accepts it and changes nothing. static let keepaliveParameters: [(String, String)] = [ ("keepalives", "1"), ("keepalives_idle", "60"), @@ -98,7 +104,8 @@ internal enum LibPQConnectionString { applicationName: String? = nil, connectTimeoutSeconds: Int? = nil, kerberos: LibPQKerberosSignIn? = nil, - serverName: String? = nil + serverName: String? = nil, + systemTrustAnchorsPath: String? = nil ) throws -> String { var parameters: [(String, String)] = [] if let serverName, !serverName.isEmpty, serverName != host, isAddressLiteral(host) { @@ -142,6 +149,8 @@ internal enum LibPQConnectionString { } if sslConfig.verifiesCertificate, !sslConfig.caCertificatePath.isEmpty { parameters.append(("sslrootcert", sslConfig.caCertificatePath)) + } else if sslConfig.verifiesHostname, let systemTrustAnchorsPath { + parameters.append(("sslrootcert", systemTrustAnchorsPath)) } if !sslConfig.clientCertificatePath.isEmpty { parameters.append(("sslcert", sslConfig.clientCertificatePath)) @@ -181,6 +190,29 @@ internal enum LibPQConnectionString { // Nothing can exist under a device file, so libpq finds no file there and says nothing. static let unreadablePassfile = "/dev/null/pgpass" + // Nil when the mode checks no certificate. Verify CA never takes this Mac's roots: it would accept any server with a + // public certificate, which libpq refuses for sslrootcert=system. A `~/.postgresql/root.crt` keeps libpq's default. + static func rootCertificateSource( + for sslConfig: SSLConfiguration, + host: String, + environment: [String: String] = ProcessInfo.processInfo.environment, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> LibPQRootCertificateSource? { + guard sslConfig.verifiesCertificate else { return nil } + guard sslConfig.caCertificatePath.isEmpty else { return .caFile(sslConfig.caCertificatePath) } + return LibPQSystemTrustRule.usesSystemTrustAnchors( + verifiesHostname: sslConfig.verifiesHostname, + caCertificatePath: sslConfig.caCertificatePath, + host: host, + environment: environment, + fileExists: fileExists + ) ? .systemTrustAnchors : .libpqDefault + } + + static func defaultRootCertificatePath(environment: [String: String]) -> String { + LibPQSystemTrustRule.defaultRootCertificatePath(environment: environment) + } + private static func isAddressLiteral(_ host: String) -> Bool { var ipv4 = in_addr() var ipv6 = in6_addr() @@ -191,16 +223,15 @@ internal enum LibPQConnectionString { purpose == "metadata" ? metadataApplicationName : sessionApplicationName } - /// libpq applies `fallback_application_name` over an `application_name` set in `options`, so a - /// name the user already gave the session there has to keep the fallback out entirely. + // libpq applies `fallback_application_name` over an `application_name` set in `options`, so a name the user gave the + // session there keeps the fallback out. static func namesApplication(_ options: String?) -> Bool { guard let options else { return false } return settingNames(in: options).contains("application_name") } - /// The setting each `-c name=value`, `-cname=value` and `--name=value` in `options` assigns, spelled - /// the way the server reads a name: in any case, with `-` and `_` as the same character. Only the - /// names, because a value such as `search_path=application_name` assigns something else. + // Spelled the way the server reads a name: any case, `-` and `_` alike. Only the names, because a value such as + // `search_path=application_name` assigns something else. static func settingNames(in options: String) -> [String] { let tokens = options.split(whereSeparator: \.isWhitespace).map(String.init) var names: [String] = [] diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift new file mode 100644 index 0000000000..37c02a0267 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift @@ -0,0 +1,233 @@ +// +// LibPQCopyIn.swift +// PostgreSQLDriverPlugin +// + +import CLibPQ +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore + +/// Sends a `COPY ... FROM STDIN` its rows the way psql 18 sends a script's rows: the data as `CopyData`, never the +/// `\.` line, then `CopyDone`. Measured with libpq 17.4 against PostgreSQL 17.11 and 18.6 +/// (`apple/scripts/probes/check-postgresql-copy-in.sh`). +/// +/// The connection is nonblocking from the statement to its result and every wait is a slice, so a Stop is seen while +/// the server takes no data. A Stop ends the rows with `CopyFail`; when the server has not answered a slice later +/// (a row lock wait, or rows a proxy still holds), one cancel request follows from this thread, so it has reached the +/// server before the results are read and cannot stop the statement after the COPY. +nonisolated enum LibPQCopyIn { + struct Sent { + // The copy's own result, or nil when libpq had none to give. The caller clears it. + let result: OpaquePointer? + // Why the rows stopped before their end, when they did. + let interruption: Interruption? + } + + enum Interruption { + case cancelled + case binaryFormat + case source(Error) + case connection + } + + // Read on the connection's queue only, for one COPY. + final class Stop { + private let requested: () -> Bool + private let closing: () -> Bool + private let sendCancelRequest: () -> Void + private var sentCancelRequest = false + + init(requested: @escaping () -> Bool, closing: @escaping () -> Bool, sendCancelRequest: @escaping () -> Void) { + self.requested = requested + self.closing = closing + self.sendCancelRequest = sendCancelRequest + } + + var isRequested: Bool { requested() } + var isClosing: Bool { closing() } + + func cancelServerOnce() { + guard !sentCancelRequest else { return } + sentCancelRequest = true + sendCancelRequest() + } + } + + // psql's own CopyFail reasons, which the server echoes as "COPY from stdin failed: ". + static let cancelledReason = "canceled by user" + static let sourceFailedReason = "aborted because of read failure" + static let binaryReason = "binary COPY data cannot come from a script" + + static var binaryRefusal: String { + String(localized: "A script cannot hold binary COPY data, so no rows were sent. Dump the rows as text or CSV.") + } + + // PQputCopyData takes an `int` length and grows its buffer to hold a whole message. + private static let sliceLength = 1 << 20 + private static let waitMicroseconds: Int64 = 100_000 + + /// The statement's result as `PQexec` gives it: the last one, or the first that starts a copy. Nil when the + /// connection was lost. The connection stays nonblocking only after `PGRES_COPY_IN`, for `send`. + static func begin(_ statement: String, conn: OpaquePointer, stop: Stop) -> OpaquePointer? { + guard PQsetnonblocking(conn, 1) == 0, statement.withCString({ PQsendQuery(conn, $0) }) == 1 else { + _ = PQsetnonblocking(conn, 0) + return nil + } + var last: OpaquePointer? + // Nothing to retract while the statement starts, so a Stop sends the cancel request at once. + while awaitResult(conn, stop: stop, patience: 0) == .ready, let next = PQgetResult(conn) { + PQclear(last) + last = next + let status = PQresultStatus(next) + if status == PGRES_COPY_IN || status == PGRES_COPY_OUT || status == PGRES_COPY_BOTH + || PQstatus(conn) == CONNECTION_BAD { + break + } + } + if last.map({ PQresultStatus($0) != PGRES_COPY_IN }) ?? true { + _ = PQsetnonblocking(conn, 0) + } + return last + } + + /// After `begin` answered `PGRES_COPY_IN`. Returns with the connection blocking again. + static func send(_ rows: PluginCopyData, format: LibPQCopyFormat, conn: OpaquePointer, stop: Stop) -> Sent { + // A script holds text, and psql would send the rest of the file as the binary stream. + let interruption: Interruption? = format == .binary ? .binaryFormat : sendPieces(rows, conn: conn, stop: stop) + let ended: Int32 + let stopped: Bool + switch interruption { + case nil: + ended = PQputCopyEnd(conn, nil) + stopped = false + case .connection: + ended = -1 + stopped = false + case .cancelled: + ended = cancelledReason.withCString { PQputCopyEnd(conn, $0) } + stopped = true + case .binaryFormat: + ended = binaryReason.withCString { PQputCopyEnd(conn, $0) } + stopped = false + case .source(let error): + stopped = isStop(error) + ended = (stopped ? cancelledReason : sourceFailedReason).withCString { PQputCopyEnd(conn, $0) } + } + // A CopyFail can still be answered on its own, so the cancel request waits a slice; after CopyDone it cannot. + let answered = ended == 1 + && awaitResult(conn, stop: stop, patience: interruption == nil ? 0 : 1, stopped: stopped) == .ready + _ = PQsetnonblocking(conn, 0) + guard answered else { + return Sent(result: nil, interruption: interruption ?? .connection) + } + return Sent(result: finalResult(conn), interruption: interruption) + } + + private static func sendPieces(_ rows: PluginCopyData, conn: OpaquePointer, stop: Stop) -> Interruption? { + do { + while true { + if stop.isRequested { return .cancelled } + guard let piece = try rows.nextPiece() else { return nil } + guard put(piece, conn: conn) else { return .connection } + switch flush(conn, stop: stop) { + case .sent: continue + case .stopped: return .cancelled + case .lost: return .connection + } + } + } catch { + return .source(error) + } + } + + // An import's Stop reaches here as its source's throw, and the server log should read it as one. + private static func isStop(_ error: Error) -> Bool { + error is PluginImportCancellationError || error is CancellationError + } + + // Nonblocking, so each slice only queues it: `flush` sends it. + private static func put(_ piece: Data, conn: OpaquePointer) -> Bool { + piece.withUnsafeBytes { raw in + guard let base = raw.baseAddress?.assumingMemoryBound(to: CChar.self) else { return true } + var offset = 0 + while offset < raw.count { + let length = min(sliceLength, raw.count - offset) + guard PQputCopyData(conn, base + offset, Int32(length)) == 1 else { return false } + offset += length + } + return true + } + } + + private enum Flush { + case sent + case stopped + case lost + } + + private static func flush(_ conn: OpaquePointer, stop: Stop) -> Flush { + while true { + let flushed = PQflush(conn) + guard flushed >= 0, !stop.isClosing, PQstatus(conn) != CONNECTION_BAD else { return .lost } + if flushed == 0 { return .sent } + if stop.isRequested { return .stopped } + guard wait(conn, forWrite: true) else { return .lost } + // The server's notices, so it never blocks writing to us while we wait to write to it. + _ = PQconsumeInput(conn) + } + } + + private enum Answer { + case ready + case lost + } + + // Until libpq holds a result and has sent everything queued. A proxy can take every byte while the server reads + // none, so this is the wait that tells such a server. `patience` is how many slices a Stop waits before its + // cancel request goes. + private static func awaitResult(_ conn: OpaquePointer, stop: Stop, patience: Int, stopped: Bool = false) -> Answer { + var unanswered = 0 + while true { + let flushed = PQflush(conn) + guard flushed >= 0, !stop.isClosing else { return .lost } + if flushed == 0, PQisBusy(conn) == 0 { return .ready } + if stopped || stop.isRequested { + if unanswered >= patience { stop.cancelServerOnce() } + unanswered += 1 + } + guard wait(conn, forWrite: flushed == 1) else { return .lost } + // A failed read leaves the connection bad, and PQgetResult then gives the reason. + if PQconsumeInput(conn) == 0, PQisBusy(conn) == 0 { return .ready } + } + } + + private static func wait(_ conn: OpaquePointer, forWrite: Bool) -> Bool { + let socket = PQsocket(conn) + guard socket >= 0 else { return false } + let deadline = pg_usec_time_t(Int64(PQgetCurrentTimeUSec()) + waitMicroseconds) + return PQsocketPoll(socket, 1, forWrite ? 1 : 0, deadline) >= 0 || errno == EINTR + } + + // While libpq still answers PGRES_COPY_IN the copy is not over: end it again, as psql does. The first other + // result is the copy's; the rest are drained so the connection takes the next statement. + private static func finalResult(_ conn: OpaquePointer) -> OpaquePointer? { + var copyResult: OpaquePointer? + var endsAgain = 0 + while let next = PQgetResult(conn) { + guard PQresultStatus(next) != PGRES_COPY_IN else { + PQclear(next) + endsAgain += 1 + guard endsAgain <= 3 else { break } + _ = "trying to exit copy mode".withCString { PQputCopyEnd(conn, $0) } + continue + } + if copyResult == nil { + copyResult = next + } else { + PQclear(next) + } + } + return copyResult + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift index 9b6c67a479..db64ba3433 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift @@ -246,24 +246,67 @@ final class LibPQDriverCore: @unchecked Sendable { return try await executeTransactionScopedRead(path + statement) } - private func executeTransactionScopedRead(_ statement: String) async throws -> PluginQueryResult { - let pqConn = try connection() + // The server runs one Query's statements as one implicit transaction, so one `SET LOCAL` covers every read. + func executeCatalogReads(_ statements: [String]) async -> [Result] { + guard pinsCatalogSearchPath, statements.count > 1, let pqConn = libpqConnection else { + return await catalogReadsOneByOne(statements) + } let startTime = Date() - let result = try await pqConn.executeTransactionScopedRead(statement) - return PluginQueryResult( + let answered: [Result] + do { + let text = PostgreSQLCatalogSearchPath.pinned + statements.joined(separator: ";\n") + guard let batch = try await pqConn.executeConnectStatements(text) else { + return await catalogReadsOneByOne(statements) + } + // The first result is the pinned path's own. + answered = Array(batch.dropFirst()) + } catch { + // A round trip that failed fails the connect in `finishConnecting`, so no read is sent again. + return statements.map { _ in .failure(error) } + } + let executionTime = Date().timeIntervalSince(startTime) + var reads: [Result] = [] + for (index, statement) in statements.enumerated() { + guard index < answered.count else { + reads.append(await catalogReadResult(statement)) + continue + } + reads.append(answered[index].map { Self.pluginResult($0, executionTime: executionTime) }) + } + return reads + } + + private func catalogReadsOneByOne(_ statements: [String]) async -> [Result] { + var reads: [Result] = [] + for statement in statements { + reads.append(await catalogReadResult(statement)) + } + return reads + } + + private func catalogReadResult(_ statement: String) async -> Result { + do { + return .success(try await executeCatalogRead(statement)) + } catch { + return .failure(error) + } + } + + private static func pluginResult(_ result: LibPQPluginQueryResult, executionTime: TimeInterval) -> PluginQueryResult { + PluginQueryResult( columns: result.columns, columnTypeNames: result.columnTypeNames, rows: result.rows, rowsAffected: result.affectedRows, - executionTime: Date().timeIntervalSince(startTime), + executionTime: executionTime, isTruncated: result.isTruncated ) } - func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + private func executeTransactionScopedRead(_ statement: String) async throws -> PluginQueryResult { let pqConn = try connection() let startTime = Date() - let result = try await pqConn.executeParameterizedQuery(query, parameters: parameters) + let result = try await pqConn.executeTransactionScopedRead(statement) return PluginQueryResult( columns: result.columns, columnTypeNames: result.columnTypeNames, @@ -274,10 +317,10 @@ final class LibPQDriverCore: @unchecked Sendable { ) } - func executeReadOnly(query: String, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { let pqConn = try connection() let startTime = Date() - let result = try await pqConn.executeReadOnlyStatement(query, parameters: parameters) + let result = try await pqConn.executeParameterizedQuery(query, parameters: parameters) return PluginQueryResult( columns: result.columns, columnTypeNames: result.columnTypeNames, @@ -288,33 +331,75 @@ final class LibPQDriverCore: @unchecked Sendable { ) } - // The statement an external caller approved, sent as `executeReadOnly`, `executeParameterized` or - // `executeSingleStatement` sends it, with the relation column each result column reads. + func executeReadOnly( + query: String, + parameters: [PluginCellValue]?, + scope: LibPQReadOnlyTransaction.Scope + ) async throws -> PluginQueryResult { + let pqConn = try connection() + let startTime = Date() + let result = try await pqConn.executeReadOnlyStatement(query, parameters: parameters, scope: scope) + return Self.answer(result, since: startTime) + } + + func executeBoundedReadOnly( + query: String, + parameters: [PluginCellValue]?, + bound: LibPQReadBound, + scope: LibPQReadOnlyTransaction.Scope + ) async throws -> PluginQueryResult { + let pqConn = try connection() + let startTime = Date() + let result = try await pqConn.boundedReadOnlyStatement(query, parameters: parameters, bound: bound, scope: scope) + return Self.answer(result, since: startTime) + } + + // The statement an external caller approved, sent as `executeReadOnly`, `executeBoundedReadOnly`, + // `executeParameterized` or `executeSingleStatement` sends it, with the relation column each result column reads. + // Only a read-only read takes `bound`: stopping a write would cancel what the caller approved to run to its end. func executeExternalStatement( query: String, parameters: [PluginCellValue]?, - readOnly: Bool + readOnly: LibPQReadOnlyTransaction.Scope?, + bound: LibPQReadBound? ) async throws -> PluginQueryResult { let pqConn = try connection() let startTime = Date() let result: LibPQPluginQueryResult - if readOnly { - result = try await pqConn.executeReadOnlyStatement(query, parameters: parameters) + if let readOnly, let bound { + result = try await pqConn.boundedReadOnlyStatement(query, parameters: parameters, bound: bound, scope: readOnly) + } else if let readOnly { + result = try await pqConn.executeReadOnlyStatement(query, parameters: parameters, scope: readOnly) } else if let parameters { result = try await pqConn.executeParameterizedQuery(query, parameters: parameters) } else { result = try await pqConn.executeSingleStatement(query) } - var answer = PluginQueryResult( + var answer = Self.answer(result, since: startTime) + answer.columnOrigins = await columnOrigins(for: result.columnSources) + return answer + } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + readOnly: LibPQReadOnlyTransaction.Reason? + ) async throws -> PluginQueryResult { + let pqConn = try connection() + let startTime = Date() + let result = try await pqConn.copyFromStandardInput(statement, rows: rows, readOnly: readOnly) + return Self.answer(result, since: startTime) + } + + private static func answer(_ result: LibPQPluginQueryResult, since startTime: Date) -> PluginQueryResult { + PluginQueryResult( columns: result.columns, columnTypeNames: result.columnTypeNames, rows: result.rows, rowsAffected: result.affectedRows, - executionTime: Date().timeIntervalSince(startTime), + timing: PluginQueryTiming(total: Date().timeIntervalSince(startTime), firstRow: result.firstRowTime), isTruncated: result.isTruncated ) - answer.columnOrigins = await columnOrigins(for: result.columnSources) - return answer } // Unpinned (Redshift, CockroachDB) the lookup would run unscoped inside the user's transaction, so it reports none. @@ -344,7 +429,10 @@ final class LibPQDriverCore: @unchecked Sendable { ) } - func streamRows(query: String, readOnly: Bool = false) -> AsyncThrowingStream { + func streamRows( + query: String, + readOnly: LibPQReadOnlyTransaction.Scope? = nil + ) -> AsyncThrowingStream { guard let pqConn = libpqConnection else { return AsyncThrowingStream { $0.finish(throwing: LibPQPluginError.notConnected) } } @@ -433,6 +521,23 @@ extension LibPQBackedDriver { core.sessionLexicalState } + // Every libpq engine here reaches this through one conformance, so a sibling that does not declare the capability + // (Redshift, whose COPY reads only its cloud sources) refuses here. + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + guard capabilities.contains(.copyFromStandardInput) else { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + return try await core.copyFromStandardInput( + statement: statement, + rows: rows, + readOnly: LibPQReadOnlyTransaction.reason(for: context, capabilities: capabilities) + ) + } + /// The new name must be bare. Every libpq engine here rejects a qualified one, because this /// statement renames in place and never moves the object; `SET SCHEMA` is the separate verb. /// @@ -529,20 +634,31 @@ extension LibPQBackedDriver { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { + let readOnly = readOnlyScope(context) if context.origin == .external { let answer = try await core.executeExternalStatement( - query: query, parameters: parameters, readOnly: runsReadOnly(context) + query: query, + parameters: parameters, + readOnly: readOnly, + bound: LibPQReadBound.ifSet(rowCap: rowCap, byteLimit: context.resultByteLimit) ) return Self.capped(answer, rowCap: rowCap) } - guard runsReadOnly(context) else { + guard let readOnly else { return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } - return Self.capped(try await core.executeReadOnly(query: query, parameters: parameters), rowCap: rowCap) + let answer = try await readOnlyRead( + query: query, parameters: parameters, rowCap: rowCap, context: context, scope: readOnly + ) + return Self.capped(answer, rowCap: rowCap) } + // An app read the host marked `readOnly` alone, a catalog read among them, has no read-only reason and keeps its pin. + // One that also sets `enforcesReadOnly` runs read-only and unpinned. func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { - if runsReadOnly(context) { return try await core.executeReadOnly(query: query, parameters: nil) } + if let readOnly = readOnlyScope(context) { + return try await readOnlyRead(query: query, parameters: nil, rowCap: nil, context: context, scope: readOnly) + } // The catalog entry runs its text as a multi-statement string, so it takes only the app's own SQL. if context.readsCatalog, context.origin == .app { return try await catalogRead(query) } return try await execute(query: query) @@ -553,12 +669,28 @@ extension LibPQBackedDriver { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - guard runsReadOnly(context) else { return try await executeParameterized(query: query, parameters: parameters) } - return try await core.executeReadOnly(query: query, parameters: parameters) + guard let readOnly = readOnlyScope(context) else { + return try await executeParameterized(query: query, parameters: parameters) + } + return try await readOnlyRead(query: query, parameters: parameters, rowCap: nil, context: context, scope: readOnly) } - private func runsReadOnly(_ context: PluginStatementContext) -> Bool { - LibPQReadOnlyTransaction.applies(to: context, capabilities: capabilities) + private func readOnlyScope(_ context: PluginStatementContext) -> LibPQReadOnlyTransaction.Scope? { + LibPQReadOnlyTransaction.scope(for: context, capabilities: capabilities) + } + + // Bounded at the source when the caller set a row cap or a byte limit, whole otherwise. + private func readOnlyRead( + query: String, + parameters: [PluginCellValue]?, + rowCap: Int?, + context: PluginStatementContext, + scope: LibPQReadOnlyTransaction.Scope + ) async throws -> PluginQueryResult { + guard let bound = LibPQReadBound.ifSet(rowCap: rowCap, byteLimit: context.resultByteLimit) else { + return try await core.executeReadOnly(query: query, parameters: parameters, scope: scope) + } + return try await core.executeBoundedReadOnly(query: query, parameters: parameters, bound: bound, scope: scope) } private static func capped(_ raw: PluginQueryResult, rowCap: Int?) -> PluginQueryResult { @@ -579,13 +711,33 @@ extension LibPQBackedDriver { try await core.executeBoundedQuery(query: query, rowCap: rowCap) } + // A context the read-only mode does not cover goes back to the host as PluginKit's default sends it, so Redshift, + // which declares neither bit, never runs an `enforcesReadOnly` read without the mode. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + guard let readOnly = readOnlyScope(context) else { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await executeBoundedQuery(query: query, rowCap: rowCap) + } + let bound = LibPQReadBound(rowCap: rowCap, byteLimit: context.resultByteLimit) + guard context.origin == .app else { + return try await core.executeExternalStatement(query: query, parameters: nil, readOnly: readOnly, bound: bound) + } + return try await core.executeBoundedReadOnly(query: query, parameters: nil, bound: bound, scope: readOnly) + } + func streamRows(query: String) -> AsyncThrowingStream { core.streamRows(query: query) } func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - guard runsReadOnly(context) else { return streamRows(query: query) } - return core.streamRows(query: query, readOnly: true) + guard let readOnly = readOnlyScope(context) else { return streamRows(query: query) } + return core.streamRows(query: query, readOnly: readOnly) } func cancelQuery() throws { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift new file mode 100644 index 0000000000..d353374ac2 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift @@ -0,0 +1,65 @@ +// +// LibPQParameterBuffers.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +// The C buffers `PQexecParams` and `PQsendQueryParams` read the parameters from: text as a C string, bytes in binary +// format, null as a null pointer. +internal enum LibPQParameterBuffers { + // libpq reads the buffers only while the call sends them, so they live for `body` alone. + static func with( + _ parameters: [PluginCellValue], + _ body: (_ values: [UnsafePointer?], _ lengths: UnsafePointer?, _ formats: UnsafePointer?) throws -> T + ) throws -> T { + var values: [UnsafePointer?] = [] + var lengths: [Int32] = [] + var formats: [Int32] = [] + var allocations: [UnsafeMutableRawPointer] = [] + defer { + for pointer in allocations { + free(pointer) + } + } + values.reserveCapacity(parameters.count) + lengths.reserveCapacity(parameters.count) + formats.reserveCapacity(parameters.count) + + for parameter in parameters { + switch parameter { + case .null: + values.append(nil) + lengths.append(0) + formats.append(0) + case .text(let text): + guard let cString = strdup(text) else { + throw LibPQPluginError(message: "Failed to allocate parameter buffer", sqlState: nil, detail: nil) + } + allocations.append(UnsafeMutableRawPointer(cString)) + values.append(UnsafePointer(cString)) + lengths.append(0) + formats.append(0) + case .bytes(let data): + let byteCount = data.count + guard let raw = malloc(max(byteCount, 1)) else { + throw LibPQPluginError(message: "Failed to allocate parameter buffer", sqlState: nil, detail: nil) + } + allocations.append(raw) + if byteCount > 0 { + data.copyBytes(to: raw.assumingMemoryBound(to: UInt8.self), count: byteCount) + } + values.append(UnsafePointer(raw.assumingMemoryBound(to: CChar.self))) + lengths.append(Int32(byteCount)) + formats.append(1) + } + } + + return try lengths.withUnsafeBufferPointer { lengthsBuffer in + try formats.withUnsafeBufferPointer { formatsBuffer in + try body(values, lengthsBuffer.baseAddress, formatsBuffer.baseAddress) + } + } + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift index d2e7ea8c04..af6cf4a197 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift @@ -87,6 +87,13 @@ internal extension LibPQPluginConnection { return LibPQCellDecoding.value(from: UnsafeRawBufferPointer(start: valuePtr, count: length), oid: oid) } + // The one row a single-row mode result holds. + static func singleRow(of result: OpaquePointer, oids: [UInt32]) -> [PluginCellValue] { + oids.enumerated().map { column in + decodeCell(from: result, row: 0, column: Int32(column.offset), oid: column.element) + } + } + private func parseRows( from result: OpaquePointer, columns: [String], diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index f818f69baf..9c37479dd3 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -15,6 +15,7 @@ import TableProCoreTypes import TableProKerberos import TableProPluginKit import TableProPostgreSQLCore +import TableProTLSClientIdentity private final class LibPQCancelRequest: @unchecked Sendable { private let cancelObject: OpaquePointer @@ -32,11 +33,11 @@ private final class LibPQCancelRequest: @unchecked Sendable { // MARK: - Connection Class -/// libpq allows one thread at a time on a `PGconn`, so every libpq call on `conn` runs on `queue`, -/// and a member that other files can reach and that makes such a call opens with -/// `preconditionOnQueue()`. `stateLock` guards `conn`, `serverMessages` and the underscored state, -/// is never held across a libpq call, and is taken in this file only. Cancelling a running query -/// is the one path that calls libpq off the queue. +// libpq allows one thread at a time on a `PGconn`, so every libpq call on `conn` runs on `queue`, +// and a member that other files can reach and that makes such a call opens with +// `preconditionOnQueue()`. `stateLock` guards `conn`, `serverMessages` and the underscored state, +// is never held across a libpq call, and is taken in this file only. Cancelling a running query +// is the one path that calls libpq off the queue. final class LibPQPluginConnection: @unchecked Sendable { static let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "LibPQPluginConnection") @@ -75,11 +76,16 @@ final class LibPQPluginConnection: @unchecked Sendable { private var _lastTransactionState: LibPQTransactionState = .idle private var _hasLostConnection = false private var _interruptsBlockedCallOnDisconnect = false + // A copy-in sends its own cancel request (LibPQCopyIn), so none outlives it. + private var copyInRunning = false private var serverMessages: Unmanaged? private var _standardConformingStrings = true - /// Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. + // Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. private var activeConnectDeadline: LibPQConnectDeadline? private var bootstrapTransportFailed = false + // Queue-confined. Nil until a probe, sent while no block was open, showed whether the server takes savepoints: + // inside the user's block a refused SAVEPOINT aborts the block, so none is sent there until this is true. + private var takesSavepoints: Bool? var isConnected: Bool { stateLock.lock() @@ -201,7 +207,10 @@ final class LibPQPluginConnection: @unchecked Sendable { KerberosTicket.discard(cachePath) } } - let conninfo = try connectionString() + // libpq reads the roots file while it polls, and every reconnect writes a new one. + let trustAnchors = try systemTrustAnchorsFile() + defer { trustAnchors?.remove() } + let conninfo = try connectionString(systemTrustAnchorsPath: trustAnchors?.url.path) let connection = try signingIn(until: deadline) { () throws -> OpaquePointer in guard let connection = conninfo.withCString({ PQconnectStart($0) }) else { throw LibPQPluginError.connectionFailed @@ -239,6 +248,18 @@ final class LibPQPluginConnection: @unchecked Sendable { adopted = true } + // A failure fails the connect: without sslrootcert libpq would look for a file that is not there. + private func systemTrustAnchorsFile() throws -> SystemTrustAnchorFile? { + guard LibPQConnectionString.rootCertificateSource(for: sslConfig, host: host) == .systemTrustAnchors else { + return nil + } + do { + return try SystemTrustAnchors.shared.writeFile() + } catch { + throw LibPQPluginError(message: error.localizedDescription, sqlState: nil, detail: nil) + } + } + // PQconnectStart can already acquire the GSS credential, so the whole start sits in the scope. private func signingIn(until deadline: LibPQConnectDeadline, _ body: () throws -> T) throws -> T { guard let kerberos else { return try body() } @@ -254,8 +275,8 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// The core calls this only after schema and flavor-specific catalog probes. Until then every - /// statement uses libpq's nonblocking API and the original absolute connect deadline. + // The core calls this only after schema and flavor-specific catalog probes. Until then every + // statement uses libpq's nonblocking API and the original absolute connect deadline. func finishConnecting() async throws { try await pluginDispatchAsync(on: queue) { [self] in try checkConnectCancellation() @@ -322,9 +343,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// `PGRES_POLLING_*` only says whether the socket wants a read or a write, so it cannot tell - /// a TLS handshake from an authentication exchange. `PQstatus` can, and reading it costs one - /// pointer dereference per poll slice. + // `PGRES_POLLING_*` only says whether the socket wants a read or a write, so it cannot tell + // a TLS handshake from an authentication exchange. `PQstatus` can, and reading it costs one + // pointer dereference per poll slice. private func reportHandshakeStage( of connection: OpaquePointer, last: inout ConnStatusType?, @@ -351,6 +372,9 @@ final class LibPQPluginConnection: @unchecked Sendable { private func connectionError(from connection: OpaquePointer) -> Error { let error = getError(from: connection) + if let missingRoots = LibPQSSLMapping.missingRootCertificateError(error.message, sslConfig: sslConfig) { + return missingRoots + } if let sslError = LibPQSSLClassifier.classifySSLError(error.message) { return sslError } @@ -368,11 +392,14 @@ final class LibPQPluginConnection: @unchecked Sendable { deadline: LibPQConnectDeadline ) throws { logUnexpectedClientEncoding(of: connection) - try runSessionSetupStatement( - LibPQStringConformance.enableStatement, - on: connection, - deadline: deadline - ) + // Every server since 8.2 reports the setting and PgBouncer tracks it, so a reported `on` needs no round trip. + if reportedStandardConformingStrings(on: connection) != true { + try runSessionSetupStatement( + LibPQStringConformance.enableStatement, + on: connection, + deadline: deadline + ) + } let standardConformingStrings = try ( reportedStandardConformingStrings(on: connection) ?? queriedStandardConformingStrings(on: connection, deadline: deadline) @@ -446,29 +473,45 @@ final class LibPQPluginConnection: @unchecked Sendable { return LibPQStringConformance.isOn(String(cString: value)) } - /// Executes a bootstrap statement without ever entering libpq's blocking `PQexec` path. A - /// statement can produce more than one result, so this mirrors `PQexec` by returning the last - /// one while still polling every byte under the same connect deadline. + // Executes a bootstrap statement without ever entering libpq's blocking `PQexec` path. A + // statement can produce more than one result, so this mirrors `PQexec` by returning the last + // one while still polling every byte under the same connect deadline. private func bootstrapResult( for statement: String, on connection: OpaquePointer, deadline: LibPQConnectDeadline ) throws -> OpaquePointer { + // One element: only the last result is kept, and no result at all throws. + try bootstrapResults(for: statement, on: connection, deadline: deadline, keepsOnlyLast: true)[0] + } + + private func bootstrapResults( + for statements: String, + on connection: OpaquePointer, + deadline: LibPQConnectDeadline, + keepsOnlyLast: Bool + ) throws -> [OpaquePointer] { do { - return try uncheckedBootstrapResult(for: statement, on: connection, deadline: deadline) + return try uncheckedBootstrapResults( + for: statements, + on: connection, + deadline: deadline, + keepsOnlyLast: keepsOnlyLast + ) } catch { bootstrapTransportFailed = true throw error } } - private func uncheckedBootstrapResult( - for statement: String, + private func uncheckedBootstrapResults( + for statements: String, on connection: OpaquePointer, - deadline: LibPQConnectDeadline - ) throws -> OpaquePointer { + deadline: LibPQConnectDeadline, + keepsOnlyLast: Bool + ) throws -> [OpaquePointer] { try checkBootstrapState(connection, deadline: deadline) - let sent = statement.withCString { PQsendQuery(connection, $0) } + let sent = statements.withCString { PQsendQuery(connection, $0) } guard sent != 0 else { throw connectionError(from: connection) } while true { @@ -479,10 +522,10 @@ final class LibPQPluginConnection: @unchecked Sendable { try pollBootstrapSocket(connection, reads: false, writes: true, deadline: deadline) } - var finalResult: OpaquePointer? - var returnsFinalResult = false + var results: [OpaquePointer] = [] + var returnsResults = false defer { - if !returnsFinalResult, let finalResult { PQclear(finalResult) } + if !returnsResults { results.forEach { PQclear($0) } } } while true { while PQisBusy(connection) != 0 { @@ -493,13 +536,13 @@ final class LibPQPluginConnection: @unchecked Sendable { } guard let result = PQgetResult(connection) else { break } - if let finalResult { PQclear(finalResult) } - finalResult = result + if keepsOnlyLast, let earlier = results.popLast() { PQclear(earlier) } + results.append(result) } - guard let finalResult else { throw connectionError(from: connection) } - returnsFinalResult = true - return finalResult + guard !results.isEmpty else { throw connectionError(from: connection) } + returnsResults = true + return results } private func pollBootstrapSocket( @@ -579,7 +622,7 @@ final class LibPQPluginConnection: @unchecked Sendable { Self.logger.warning("standard_conforming_strings is off; string literals escape backslashes") } - private func connectionString() throws -> String { + private func connectionString(systemTrustAnchorsPath: String?) throws -> String { try LibPQConnectionString.build( host: host, port: port, @@ -591,7 +634,8 @@ final class LibPQPluginConnection: @unchecked Sendable { applicationName: applicationName, connectTimeoutSeconds: connectTimeout.nativeSeconds, kerberos: kerberos, - serverName: serverName + serverName: serverName, + systemTrustAnchorsPath: systemTrustAnchorsPath ) } @@ -669,7 +713,7 @@ final class LibPQPluginConnection: @unchecked Sendable { // MARK: - Query Cancellation func cancelCurrentQuery() { - guard cancellationGate.cancel() != nil else { return } + guard cancellationGate.cancel() != nil, !stateLock.withLock({ copyInRunning }) else { return } requestServerCancelOnCurrentConnection() } @@ -697,20 +741,18 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// Runs a read whose statement changes session settings with `SET LOCAL`, and confines those - /// settings to the read whether or not the session has a transaction block open. - /// - /// With no block open, the settings and the read are one implicit transaction that ends with it. - /// Inside a block they would outlast the read, so a savepoint scopes them and is rolled back - /// straight after. The status check and every statement run in one block on this connection's - /// queue. Split into separate dispatches, another caller sharing the session could run between - /// the savepoint and its rollback, and the rollback silently undid that caller's work. PGlite has - /// no pooled connection, so a query tab and the metadata reads share one session. - /// - /// The `ROLLBACK TO` is its own `PQexec`, because `PQexec` returns only a string's last result - /// and the rows are the read's. A rollback that fails after a read that succeeded is reported - /// rather than the rows: the session would otherwise keep the read's settings for the rest of - /// the user's transaction. + // Runs a read whose statement changes session settings with `SET LOCAL`, and confines those + // settings to the read whether or not the session has a transaction block open. + // With no block open, the settings and the read are one implicit transaction that ends with it. + // Inside a block they would outlast the read, so a savepoint scopes them and is rolled back + // straight after. The status check and every statement run in one block on this connection's + // queue. Split into separate dispatches, another caller sharing the session could run between + // the savepoint and its rollback, and the rollback silently undid that caller's work. PGlite has + // no pooled connection, so a query tab and the metadata reads share one session. + // The `ROLLBACK TO` is its own `PQexec`, because `PQexec` returns only a string's last result + // and the rows are the read's. A rollback that fails after a read that succeeded is reported + // rather than the rows: the session would otherwise keep the read's settings for the rest of + // the user's transaction. func executeTransactionScopedRead(_ statement: String) async throws -> LibPQPluginQueryResult { let statementToRun = String(statement) @@ -736,9 +778,20 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// A catalog query the driver sends on its own while reading a result, on the connection's queue, scoped the way - /// `executeTransactionScopedRead` scopes one. The caller clears the result. Inside an aborted block it hands back - /// the savepoint's own `25P02` failure. + // Nil once the connect has finished, when a block of the user's may be open: the caller then scopes each read alone. + func executeConnectStatements(_ statements: String) async throws -> [Result]? { + let statementsToRun = String(statements) + + return try await pluginDispatchAsync(on: queue) { [self] () throws -> [Result]? in + guard !isShuttingDown else { throw LibPQPluginError.notConnected } + guard let deadline = activeConnectDeadline else { return nil } + return try executeConnectStatementsSync(statementsToRun, deadline: deadline) + } + } + + // A catalog query the driver sends on its own while reading a result, on the connection's queue, scoped the way + // `executeTransactionScopedRead` scopes one. The caller clears the result. Inside an aborted block it hands back + // the savepoint's own `25P02` failure. func pinnedCatalogSideQueryOnQueue(_ statement: String, conn: OpaquePointer) -> OpaquePointer? { preconditionOnQueue() guard pinsCatalogSearchPath else { return statement.withCString { PQexec(conn, $0) } } @@ -755,7 +808,8 @@ final class LibPQPluginConnection: @unchecked Sendable { // One dispatch for the state check, the transaction and the statement, as `executeTransactionScopedRead` does. func executeReadOnlyStatement( _ statement: String, - parameters: [PluginCellValue]? + parameters: [PluginCellValue]?, + scope: LibPQReadOnlyTransaction.Scope ) async throws -> LibPQPluginQueryResult { let statementToRun = String(statement) let params = parameters @@ -765,6 +819,7 @@ final class LibPQPluginConnection: @unchecked Sendable { if let ended = sessionEndedBeforeSending(conn) { throw ended } return try LibPQReadOnlyTransaction.run( state: transactionStateOnQueue().sessionTransactionState, + scope: scope, send: { try runReadOnlyControl($0, on: conn) } ) { guard let params else { return try executeQuerySync(statementToRun, singleStatement: true) } @@ -788,11 +843,11 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let scopedReadSavepoint = "SAVEPOINT tablepro_scoped_read" private static let scopedReadRollback = "ROLLBACK TO SAVEPOINT tablepro_scoped_read; RELEASE SAVEPOINT tablepro_scoped_read" - /// Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the - /// user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it - /// succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` - /// gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` - /// refuses in the same dispatch, so no transaction can open between the check and the statement. + // Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the + // user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it + // succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` + // gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` + // refuses in the same dispatch, so no transaction can open between the check and the statement. func executeSessionSetting( _ statement: String, parameters: [PluginCellValue], @@ -826,19 +881,18 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let sessionSettingRollback = "ROLLBACK TO SAVEPOINT tablepro_session_setting; RELEASE SAVEPOINT tablepro_session_setting" - /// Whether the session is inside a transaction block, including one a failed statement has - /// aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` - /// may not be used from two threads at once, and the lock guards the pointer rather than the call. + // Whether the session is inside a transaction block, including one a failed statement has + // aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` + // may not be used from two threads at once, and the lock guards the pointer rather than the call. private func isInsideTransactionBlockOnQueue() -> Bool { let state = transactionStateOnQueue() return state == .inTransaction || state == .inError } - /// What the session has open, from the `ReadyForQuery` status libpq keeps from the last reply. - /// - /// No round trip and no server work: measured against PostgreSQL 17.11, a million calls took - /// 1.5ms, and the answer is local enough to survive a backend another session terminated - /// (`PQtransactionStatus` still reported `INTRANS` with `PQstatus` OK). + // What the session has open, from the `ReadyForQuery` status libpq keeps from the last reply. + // No round trip and no server work: measured against PostgreSQL 17.11, a million calls took + // 1.5ms, and the answer is local enough to survive a backend another session terminated + // (`PQtransactionStatus` still reported `INTRANS` with `PQstatus` OK). func transactionState() async -> LibPQTransactionState { do { return try await pluginDispatchAsync(on: queue) { [self] in @@ -855,10 +909,10 @@ final class LibPQPluginConnection: @unchecked Sendable { return Self.transactionState(PQtransactionStatus(conn)) } - /// libpq has no round trip that sends no statement, so the socket is read first: that sees a - /// server that closed the session, and inside a transaction block it is the whole check - /// (`LibPQSessionCheck.sendsStatement`). A statement the server refuses still proves the - /// backend answered, so it does not fail the ping. + // libpq has no round trip that sends no statement, so the socket is read first: that sees a + // server that closed the session, and inside a transaction block it is the whole check + // (`LibPQSessionCheck.sendsStatement`). A statement the server refuses still proves the + // backend answered, so it does not fail the ping. func ping() async throws { try await pluginDispatchAsync(on: queue) { [self] in guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } @@ -899,6 +953,28 @@ final class LibPQPluginConnection: @unchecked Sendable { } } + // One dispatch holds the connection from the COPY to its last result: a PQexec sent in between would end the copy. + func copyFromStandardInput( + _ statement: String, + rows: PluginCopyData, + readOnly: LibPQReadOnlyTransaction.Reason? + ) async throws -> LibPQPluginQueryResult { + let statementToRun = String(statement) + + return try await pluginDispatchAsync(on: queue) { [self] in + guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } + guard let readOnly else { return try copyFromStandardInputSync(statementToRun, rows: rows, conn: conn) } + if let ended = sessionEndedBeforeSending(conn) { throw ended } + return try LibPQReadOnlyTransaction.run( + state: transactionStateOnQueue().sessionTransactionState, + reason: readOnly, + send: { try runReadOnlyControl($0, on: conn) } + ) { + try copyFromStandardInputSync(statementToRun, rows: rows, conn: conn) + } + } + } + // MARK: - Server Information func serverVersion() -> String? { @@ -942,6 +1018,38 @@ final class LibPQPluginConnection: @unchecked Sendable { return decoded } + // One simple Query: the server answers each statement in order and runs none after the first that fails. + private func executeConnectStatementsSync( + _ statements: String, + deadline: LibPQConnectDeadline + ) throws -> [Result] { + guard !isShuttingDown, let connection = connectionHandle else { + throw LibPQPluginError.notConnected + } + + let generation = cancellationGate.beginQuery() + defer { cancellationGate.endQuery(generation) } + defer { refreshStandardConformingStrings(from: connection) } + + let cancelsOutput = cancelsAbandonedOutput(connection) + if let ended = sessionEndedBeforeSending(connection) { throw ended } + let results = try bootstrapResults(for: statements, on: connection, deadline: deadline, keepsOnlyLast: false) + recordTransactionState(of: connection) + let decoded = results.map { result in + Result { + try queryResult( + from: result, + connection: connection, + generation: generation, + cancelsOutput: cancelsOutput, + allowsSideQueries: false + ) + } + } + try checkBootstrapState(connection, deadline: deadline) + return decoded + } + private func executeQuerySync(_ query: String, singleStatement: Bool = false) throws -> LibPQPluginQueryResult { let conn = connectionHandle @@ -975,6 +1083,78 @@ final class LibPQPluginConnection: @unchecked Sendable { ) } + // A statement that is not a copy-in, the server's refusal of the COPY among them, reads as any other result. + private func copyFromStandardInputSync( + _ statement: String, + rows: PluginCopyData, + conn: OpaquePointer + ) throws -> LibPQPluginQueryResult { + // Set around the whole generation, so no Stop finds it running without the flag. + stateLock.withLock { copyInRunning = true } + let generation = cancellationGate.beginQuery() + defer { + cancellationGate.endQuery(generation) + stateLock.withLock { copyInRunning = false } + } + defer { refreshStandardConformingStrings(from: conn) } + + // PQexec drained them itself; PQsendQuery refuses while any are left. + let cancelsOutput = cancelsAbandonedOutput(conn) + finishAbandonedResults(conn, cancellingOutput: cancelsOutput) + if let ended = sessionEndedBeforeSending(conn) { throw ended } + if cancellationGate.isCancelled(generation) { throw CancellationError() } + let stop = LibPQCopyIn.Stop( + requested: { [cancellationGate] in cancellationGate.isCancelled(generation) }, + closing: { [self] in isShuttingDown }, + sendCancelRequest: { [self] in sendServerCancelAndWait(on: conn) } + ) + guard let start = LibPQCopyIn.begin(statement, conn: conn, stop: stop) else { + throw lostConnection(getError(from: conn), on: conn, sent: true) + } + recordTransactionState(of: conn) + guard let copy = LibPQCopyState.copy(of: start), copy.direction == .copyIn else { + return try queryResult( + from: start, + connection: conn, + generation: generation, + cancelsOutput: cancelsOutput, + allowsSideQueries: true + ) + } + PQclear(start) + + let sent = LibPQCopyIn.send(rows, format: copy.format, conn: conn, stop: stop) + recordTransactionState(of: conn) + guard let result = sent.result else { + throw lostConnection(getError(from: conn), on: conn, sent: true) + } + defer { PQclear(result) } + switch sent.interruption { + case .cancelled: + throw CancellationError() + case .source(let error): + throw error + case .binaryFormat: + throw LibPQPluginError(message: LibPQCopyIn.binaryRefusal, sqlState: nil, detail: nil) + case .connection, nil: + break + } + guard PQresultStatus(result) == PGRES_COMMAND_OK else { + let error = getResultError(from: result) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + throw lostConnection(error, on: conn, sent: true) + } + return LibPQPluginQueryResult( + columns: [], + columnOids: [], + columnTypeNames: [], + rows: [], + affectedRows: getAffectedRows(from: result), + commandTag: getCommandTag(from: result), + isTruncated: false + ) + } + private func queryResult( from result: OpaquePointer, connection conn: OpaquePointer, @@ -1021,13 +1201,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// Reads at most `rowCap` rows through libpq's single-row mode, then cancels the statement and - /// drains the connection instead of pulling the rest of the result across the socket. - /// - /// One row past the cap is read so `isTruncated` can tell "exactly `rowCap` rows" from "more - /// rows exist". The cancel is not an optimization: libpq drains an abandoned result inside the - /// next `PQexec`, charging that cost to the user's next statement, and the backend stays active - /// holding its snapshot until it can finish writing to the client. + // Reads at most `rowCap` rows, plus one to tell "exactly `rowCap`" from "more", then stops the server producing + // the rest. The stop is not an optimization: libpq drains an abandoned result inside the next `PQexec`, and the + // backend holds its snapshot until it has written the whole result to the client. private func boundedQuerySync(_ query: String, rowCap: Int) throws -> LibPQPluginQueryResult { let conn = connectionHandle @@ -1039,19 +1215,27 @@ final class LibPQPluginConnection: @unchecked Sendable { defer { cancellationGate.endQuery(generation) } defer { refreshStandardConformingStrings(from: conn) } - /// Started before the drain, so a result the previous statement abandoned is charged to the - /// time before the first row rather than appearing as this query's row transfer. + // Started before the drain, so a result the previous statement abandoned is charged to the time before the + // first row rather than to this query's row transfer. let sentAt = Date() - /// Cancelling a statement inside a transaction block puts the transaction into the aborted - /// state, and every later command fails until ROLLBACK. Reading the tail of one result costs - /// less than throwing away the transaction the user opened, so the cancel is withheld here - /// and the connection is drained instead. + // A cancel inside a transaction block aborts the block the user opened, so there the statement is read + // through a cursor, and where it cannot be the connection is drained instead. let cancelsOutput = cancelsAbandonedOutput(conn) let suppressCancel = !cancelsOutput finishAbandonedResults(conn, cancellingOutput: cancelsOutput) if let ended = sessionEndedBeforeSending(conn) { throw ended } + // A Stop during one of the driver's own round trips is spent there, so each is followed by a check before + // the user's statement goes out. + probeSavepointsWhileIdle(conn) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + if let plan = cappedCursorPlan(for: query, rowCap: rowCap, conn: conn), + let read = try cappedCursorRead(plan, rowCap: rowCap, conn: conn, generation: generation, sentAt: sentAt) { + return read + } + if cancellationGate.isCancelled(generation) { throw CancellationError() } + let localQuery = String(query) let sendOk = localQuery.withCString { queryPtr in PQsendQuery(conn, queryPtr) @@ -1164,79 +1348,179 @@ final class LibPQPluginConnection: @unchecked Sendable { return PostGISTypes.rendering(bounded, types: typeNames.postgisTypes) } - private func executeParameterizedQuerySync(_ query: String, parameters: [PluginCellValue]) throws -> LibPQPluginQueryResult { - let conn = connectionHandle + private static let savepointProbe = + "BEGIN; SAVEPOINT tablepro_probe; RELEASE SAVEPOINT tablepro_probe; ROLLBACK" - guard !isShuttingDown, let conn else { - throw LibPQPluginError.notConnected + // Unpinned connections (Redshift, CockroachDB) never get a driver savepoint inside the user's block. A PG-wire + // server that refuses one is found here, outside any block, where the refusal costs one round trip. + private func probeSavepointsWhileIdle(_ conn: OpaquePointer) { + guard pinsCatalogSearchPath, takesSavepoints == nil, PQtransactionStatus(conn) == PQTRANS_IDLE, + let result = Self.savepointProbe.withCString({ PQexec(conn, $0) }) + else { return } + let accepted = PQresultStatus(result) == PGRES_COMMAND_OK + let cancelled = !accepted && getResultError(from: result).sqlState == "57014" + PQclear(result) + if PQtransactionStatus(conn) != PQTRANS_IDLE, let rollback = "ROLLBACK".withCString({ PQexec(conn, $0) }) { + PQclear(rollback) } + recordTransactionState(of: conn) + guard PQstatus(conn) == CONNECTION_OK, !cancelled else { return } + takesSavepoints = accepted + } + + private func cappedCursorPlan(for query: String, rowCap: Int, conn: OpaquePointer) -> LibPQCappedCursorRead.Plan? { + guard pinsCatalogSearchPath else { return nil } + let scope: LibPQCappedCursorRead.Scope + switch PQtransactionStatus(conn) { + case PQTRANS_INTRANS where takesSavepoints == true: + scope = .insideTransaction + case PQTRANS_IDLE where suppressServerSideCancel: + scope = .ownTransaction + default: + return nil + } + return LibPQCappedCursorRead.plan(for: query, rowCap: rowCap, scope: scope) + } - let generation = cancellationGate.beginQuery() - defer { cancellationGate.endQuery(generation) } - defer { refreshStandardConformingStrings(from: conn) } - - let cancelsOutput = cancelsAbandonedOutput(conn) - var paramValues: [UnsafePointer?] = [] - var paramLengths: [Int32] = [] - var paramFormats: [Int32] = [] - var allocations: [UnsafeMutableRawPointer] = [] + // Nil when the caller should run the statement the plain way: after a refused DECLARE, undone before the statement + // ran, so the plain run reports any error of its own in the user's text. A failure after DECLARE is the + // statement's own and leaves the user's block aborted, as the plain read does. + private func cappedCursorRead( + _ plan: LibPQCappedCursorRead.Plan, + rowCap: Int, + conn: OpaquePointer, + generation: Int, + sentAt: Date + ) throws -> LibPQPluginQueryResult? { + guard let opened = plan.opening.withCString({ PQexec(conn, $0) }) else { + throw lostConnection(getError(from: conn), on: conn, sent: true) + } + let openingFailure = PQresultStatus(opened) == PGRES_COMMAND_OK ? nil : getResultError(from: opened) + PQclear(opened) + recordTransactionState(of: conn) + if let openingFailure { + guard PQstatus(conn) == CONNECTION_OK else { throw lostConnection(openingFailure, on: conn, sent: true) } + if cancellationGate.isCancelled(generation) { throw CancellationError() } + takesSavepoints = false + if PQtransactionStatus(conn) == PQTRANS_INERROR { throw openingFailure } + return nil + } + if cancellationGate.isCancelled(generation) { + _ = undoCappedCursorRead(plan, conn: conn) + throw CancellationError() + } - defer { - for ptr in allocations { - free(ptr) - } + let sendOk = plan.statement.withCString { PQsendQuery(conn, $0) } + guard sendOk != 0 else { + throw lostConnection(getError(from: conn), on: conn, sent: false) + } + guard PQsetSingleRowMode(conn) != 0 else { + _ = finishPendingResults(conn, cancellingOutput: false) + recordTransactionState(of: conn) + throw LibPQPluginError(message: "Failed to enter single-row mode", sqlState: nil, detail: nil) } - paramValues.reserveCapacity(parameters.count) - paramLengths.reserveCapacity(parameters.count) - paramFormats.reserveCapacity(parameters.count) + var metadata: ColumnMetadata? + var rows: [[PluginCellValue]] = [] + rows.reserveCapacity(min(plan.fetchCount, 10_000)) + var firstRowTime: TimeInterval? + var declared = false + var failure: LibPQPluginError? - for param in parameters { - switch param { - case .null: - paramValues.append(nil) - paramLengths.append(0) - paramFormats.append(0) - case .text(let str): - guard let cStr = strdup(str) else { - throw LibPQPluginError(message: "Failed to allocate parameter buffer", sqlState: nil, detail: nil) + while let result = PQgetResult(conn) { + let status = PQresultStatus(result) + if status == PGRES_SINGLE_TUPLE || status == PGRES_TUPLES_OK { + if firstRowTime == nil { firstRowTime = Date().timeIntervalSince(sentAt) } + let columns = metadata ?? readColumnMetadata(from: result) + metadata = columns + for rowIndex in 0.. 0 { - data.copyBytes(to: raw.assumingMemoryBound(to: UInt8.self), count: byteCount) + throw CancellationError() + } + } + recordTransactionState(of: conn) + + if let failure { + guard PQstatus(conn) == CONNECTION_OK else { throw lostConnection(failure, on: conn, sent: true) } + let cancelled = cancellationGate.isCancelled(generation) + if !declared, !cancelled { + guard undoCappedCursorRead(plan, conn: conn) else { + if cancellationGate.isCancelled(generation) { throw CancellationError() } + throw failure } - paramValues.append(UnsafePointer(raw.assumingMemoryBound(to: CChar.self))) - paramLengths.append(Int32(byteCount)) - paramFormats.append(1) + return nil } + if plan.scope == .ownTransaction { _ = undoCappedCursorRead(plan, conn: conn) } + if cancelled { throw CancellationError() } + throw failure } + if cancellationGate.isCancelled(generation) { throw CancellationError() } - if let ended = sessionEndedBeforeSending(conn) { throw ended } + let truncated = rows.count > rowCap + if truncated { rows.removeLast() } + let resolvedMetadata = metadata.map { resolvingUnknownTypes($0, conn: conn) } + let bounded = LibPQPluginQueryResult( + columns: resolvedMetadata?.columns ?? [], + columnOids: resolvedMetadata?.columnOids ?? [], + columnTypeNames: resolvedMetadata?.columnTypeNames ?? [], + rows: rows, + affectedRows: 0, + commandTag: nil, + isTruncated: truncated, + firstRowTime: firstRowTime ?? Date().timeIntervalSince(sentAt) + ) + return PostGISTypes.rendering(bounded, types: typeNames.postgisTypes) + } + + private func undoCappedCursorRead(_ plan: LibPQCappedCursorRead.Plan, conn: OpaquePointer) -> Bool { + guard let result = plan.recovery.withCString({ PQexec(conn, $0) }) else { return false } + defer { PQclear(result) } + recordTransactionState(of: conn) + return PQresultStatus(result) == PGRES_COMMAND_OK + } + + private func executeParameterizedQuerySync(_ query: String, parameters: [PluginCellValue]) throws -> LibPQPluginQueryResult { + let conn = connectionHandle + + guard !isShuttingDown, let conn else { + throw LibPQPluginError.notConnected + } + + let generation = cancellationGate.beginQuery() + defer { cancellationGate.endQuery(generation) } + defer { refreshStandardConformingStrings(from: conn) } + let cancelsOutput = cancelsAbandonedOutput(conn) let localQuery = String(query) - let result: OpaquePointer? = localQuery.withCString { queryPtr in - paramLengths.withUnsafeBufferPointer { lengthsBuf in - paramFormats.withUnsafeBufferPointer { formatsBuf in - PQexecParams( - conn, - queryPtr, - Int32(parameters.count), - nil, - paramValues, - lengthsBuf.baseAddress, - formatsBuf.baseAddress, - 0 - ) - } + let result: OpaquePointer? = try LibPQParameterBuffers.with(parameters) { values, lengths, formats in + if let ended = sessionEndedBeforeSending(conn) { throw ended } + return localQuery.withCString { queryPtr in + PQexecParams(conn, queryPtr, Int32(parameters.count), nil, values, lengths, formats, 0) } } @@ -1281,9 +1565,9 @@ final class LibPQPluginConnection: @unchecked Sendable { // MARK: - Pending Results - /// A statement the user did not get to see is worse than a slow one, so a COPY ended by any - /// drain is reported rather than swallowed: `INSERT INTO t VALUES (1); COPY t FROM STDIN` used - /// to come back as "INSERT 0 1" with the COPY discarded. + // A statement the user did not get to see is worse than a slow one, so a COPY ended by any + // drain is reported rather than swallowed: `INSERT INTO t VALUES (1); COPY t FROM STDIN` used + // to come back as "INSERT 0 1" with the COPY discarded. private func abandonedCopyError(_ outcome: LibPQDrainOutcome, generation: Int) -> Error? { guard let copy = outcome.abandonedCopy else { return nil } if cancellationGate.isCancelled(generation) { return CancellationError() } @@ -1305,9 +1589,9 @@ final class LibPQPluginConnection: @unchecked Sendable { LibPQPluginError(message: copy.direction.unsupportedMessage, sqlState: nil, detail: nil) } - /// Reading `COPY TO STDOUT` to its end defeats the row cap the bounded read exists for, so the - /// statement is cancelled first wherever a cancel is safe. Inside a transaction block it is not, - /// because a cancel aborts the transaction the user opened. + // Reading `COPY TO STDOUT` to its end defeats the row cap the bounded read exists for, so the + // statement is cancelled first wherever a cancel is safe. Inside a transaction block it is not, + // because a cancel aborts the transaction the user opened. private func cancelsAbandonedOutput(_ conn: OpaquePointer) -> Bool { !suppressServerSideCancel && PQtransactionStatus(conn) != PQTRANS_INTRANS } @@ -1337,13 +1621,16 @@ final class LibPQPluginConnection: @unchecked Sendable { return finishPendingResults(conn, cancellingOutput: false) } - /// The abort is polled by the producer rather than acted on from `onTermination`, because both - /// run on one serial queue: a drain enqueued from the handler sits behind the producer and runs - /// only once the whole result has been read, which is no abort at all. - func streamQuery(_ query: String, readOnly: Bool = false) -> AsyncThrowingStream { + // The abort is polled by the producer rather than acted on from `onTermination`, because both + // run on one serial queue: a drain enqueued from the handler sits behind the producer and runs + // only once the whole result has been read, which is no abort at all. + func streamQuery( + _ query: String, + readOnly: LibPQReadOnlyTransaction.Scope? = nil + ) -> AsyncThrowingStream { let queryToRun = String(query) - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in self.queue.async { [self] in let handle = connectionHandle @@ -1356,15 +1643,15 @@ final class LibPQPluginConnection: @unchecked Sendable { defer { cancellationGate.endQuery(generation) } defer { refreshStandardConformingStrings(from: conn) } - /// The consumer can go away before this block is scheduled, in which case the - /// query is never sent at all. + // The consumer can go away before this block is scheduled, in which case the + // query is never sent at all. guard !abort.isAborted else { continuation.finish() return } - /// Read before the query goes out: once it is in flight the status is - /// PQTRANS_ACTIVE, and the transaction this guard exists for is invisible. + // Read before the query goes out: once it is in flight the status is + // PQTRANS_ACTIVE, and the transaction this guard exists for is invisible. let cancelsOutput = cancelsAbandonedOutput(conn) finishAbandonedResults(conn, cancellingOutput: cancelsOutput) if let ended = sessionEndedBeforeSending(conn) { @@ -1381,10 +1668,11 @@ final class LibPQPluginConnection: @unchecked Sendable { cancelsOutput: cancelsOutput ) do { - if readOnly { + if let readOnly { // The extended protocol takes one statement, so no `COMMIT; INSERT` can follow the read. try LibPQReadOnlyTransaction.run( state: transactionStateOnQueue().sessionTransactionState, + scope: readOnly, send: { try runReadOnlyControl($0, on: conn) } ) { try streamResults(stream, singleStatement: true) @@ -1430,7 +1718,17 @@ final class LibPQPluginConnection: @unchecked Sendable { let continuation = run.continuation let generation = run.generation let cancelsOutput = run.cancelsOutput - let suppressCancel = !cancelsOutput + // Inside the user's block a cancel aborts the block, so a savepoint holds the read: a reader that goes away + // cancels it and rolls back to the savepoint instead of draining the rest of the result. The abandoned read + // then keeps nothing it did, as a cancelled statement outside a block. + if !singleStatement { probeSavepointsWhileIdle(conn) } + if run.abort.isAborted || cancellationGate.isCancelled(generation) { throw CancellationError() } + let holdsSavepoint = !singleStatement && !cancelsOutput && openStreamSavepoint(for: run.query, conn: conn) + let suppressCancel = !cancelsOutput && !holdsSavepoint + if holdsSavepoint, run.abort.isAborted || cancellationGate.isCancelled(generation) { + if let aborted = endStreamSavepoint(conn, undoing: true) { throw aborted } + throw CancellationError() + } let sendOk = run.query.withCString { queryPtr in singleStatement @@ -1444,6 +1742,7 @@ final class LibPQPluginConnection: @unchecked Sendable { if PQsetSingleRowMode(conn) == 0 { _ = cancelAndDrain(conn, suppressCancel: suppressCancel) + if holdsSavepoint, let aborted = endStreamSavepoint(conn, undoing: true) { throw aborted } throw LibPQPluginError(message: "Failed to enter single-row mode", sqlState: nil, detail: nil) } @@ -1453,6 +1752,7 @@ final class LibPQPluginConnection: @unchecked Sendable { let batchSize = 5_000 var batch: [PluginRow] = [] batch.reserveCapacity(batchSize) + let stops = { self.cancellationGate.isCancelled(generation) } while let result = PQgetResult(conn) { let status = PQresultStatus(result) @@ -1460,7 +1760,7 @@ final class LibPQPluginConnection: @unchecked Sendable { if !headerSent, Self.carriesRowDescription(status) { let described = streamHeader(from: result) columnOids = described.columnOids - continuation.yield(.header(described.header)) + _ = PluginRowStream.deliverBlocking(.header(described.header), to: continuation, stops: stops) headerSent = true } @@ -1480,16 +1780,17 @@ final class LibPQPluginConnection: @unchecked Sendable { PQclear(result) batch.append(row) - if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // A reader that left or a Stop is handled just below. + if batch.count >= batchSize, + PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) { batch.removeAll(keepingCapacity: true) } if run.abort.isAborted || cancellationGate.isCancelled(generation) { - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } _ = cancelAndDrain(conn, suppressCancel: suppressCancel) + // A Stop leaves the block aborted, as it does for any statement; only a reader that left is undone. + let undoing = !cancellationGate.isCancelled(generation) + if holdsSavepoint, let aborted = endStreamSavepoint(conn, undoing: undoing) { throw aborted } throw CancellationError() } } else if status == PGRES_TUPLES_OK { @@ -1514,14 +1815,13 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } + let delivered = batch.isEmpty || PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) let outcome = finishPendingResults(conn, cancellingOutput: cancelsOutput) recordTransactionState(of: conn) - /// The header went out with the first row, so this stream keeps what it said; - /// the lookup is for the results that follow. + if holdsSavepoint, let aborted = endStreamSavepoint(conn, undoing: false) { throw aborted } + // The header went out with the first row, so this stream keeps what it said; the lookup is for the results + // that follow. let missing = typeNames.unresolvedOids(in: columnOids) if !missing.isEmpty { learnTypeNames(for: missing, conn: conn) @@ -1530,6 +1830,52 @@ final class LibPQPluginConnection: @unchecked Sendable { if let abandoned = abandonedCopyError(outcome, generation: generation) { throw abandoned } + guard delivered else { throw CancellationError() } + } + + private static let streamSavepoint = "SAVEPOINT tablepro_stream_read" + private static let streamSavepointRelease = "RELEASE SAVEPOINT tablepro_stream_read" + private static let streamSavepointRollback = + "ROLLBACK TO SAVEPOINT tablepro_stream_read; RELEASE SAVEPOINT tablepro_stream_read" + + // Only one read statement, since rolling back to the savepoint would undo a write, a lock or a second statement. + // Unpinned connections (Redshift, CockroachDB) never get a driver savepoint inside the user's block. + private func openStreamSavepoint(for query: String, conn: OpaquePointer) -> Bool { + guard pinsCatalogSearchPath, !suppressServerSideCancel, takesSavepoints == true, + PQtransactionStatus(conn) == PQTRANS_INTRANS, + LibPQCappedCursorRead.cursorBody(of: query) != nil, + let result = Self.streamSavepoint.withCString({ PQexec(conn, $0) }) + else { return false } + defer { PQclear(result) } + recordTransactionState(of: conn) + guard PQresultStatus(result) == PGRES_COMMAND_OK else { + takesSavepoints = false + return false + } + return true + } + + // Undoing rolls back to the savepoint whatever state the cancel left, so an abandoned read keeps nothing. Without + // it, a read that ended keeps its work and one a failure aborted stays aborted. The error is a release or rollback + // the server refused, which leaves the user's block aborted: a late cancel can land on it, and the block's state + // is reported then rather than found by the next statement. + private func endStreamSavepoint(_ conn: OpaquePointer, undoing: Bool) -> LibPQPluginError? { + let status = PQtransactionStatus(conn) + let statement: String + if undoing, status == PQTRANS_INTRANS || status == PQTRANS_INERROR { + statement = Self.streamSavepointRollback + } else if status == PQTRANS_INTRANS { + statement = Self.streamSavepointRelease + } else { + return nil + } + guard let result = statement.withCString({ PQexec(conn, $0) }) else { return getError(from: conn) } + defer { PQclear(result) } + recordTransactionState(of: conn) + guard PQresultStatus(result) != PGRES_COMMAND_OK else { return nil } + let refused = getResultError(from: result) + Self.logger.error("A stream savepoint was refused, so the transaction block is aborted") + return refused } // MARK: - Private Helpers @@ -1554,12 +1900,12 @@ final class LibPQPluginConnection: @unchecked Sendable { private func sessionEndedBeforeSending(_ conn: OpaquePointer) -> LibPQConnectionLostError? { recordTransactionState(of: conn) - /// All three calls earn their place, measured against a terminated backend on 9.1.24 and - /// 17.11. One `PQconsumeInput` leaves the status `CONNECTION_OK`, so a single read never - /// sees the loss; the second one turns it `CONNECTION_BAD`. `PQisBusy` never moves the - /// status, but it is what parses the buffered message: without it every closed-session - /// case loses the server's FATAL and its SQLSTATE and reports libpq's own "server closed - /// the connection unexpectedly" instead. + // All three calls earn their place, measured against a terminated backend on 9.1.24 and + // 17.11. One `PQconsumeInput` leaves the status `CONNECTION_OK`, so a single read never + // sees the loss; the second one turns it `CONNECTION_BAD`. `PQisBusy` never moves the + // status, but it is what parses the buffered message: without it every closed-session + // case loses the server's FATAL and its SQLSTATE and reports libpq's own "server closed + // the connection unexpectedly" instead. if PQstatus(conn) == CONNECTION_OK { _ = PQconsumeInput(conn) _ = PQisBusy(conn) @@ -1639,3 +1985,293 @@ extension LibPQPluginConnection { requestServerCancelOnCurrentConnection() } } + +// MARK: - Bounded Read-Only Read + +// Outside the class body, which is at its length limit. An extension in this file still reaches its private members. +extension LibPQPluginConnection { + // The call waits for it, unlike the queued requests: PQcancel returns once the server has the request, so it + // cannot reach a statement the call sends later. PQcancel has no timeout of its own, so the wait ends at the + // connect timeout or a disconnect. + fileprivate func sendServerCancelAndWait(on connection: OpaquePointer) { + guard !suppressServerSideCancel, let cancelObject = PQgetCancel(connection) else { return } + let request = LibPQCancelRequest(cancelObject) + let delivered = DispatchSemaphore(value: 0) + Self.cancellationQueue.async { + request.perform() + delivered.signal() + } + let deadline = DispatchTime.now() + .milliseconds(connectTimeout.milliseconds) + while delivered.wait(timeout: .now() + .milliseconds(100)) == .timedOut { + guard !isShuttingDown, DispatchTime.now() < deadline else { return } + } + } + + // `executeReadOnlyStatement` bounded at the source: the statement stops at `bound`, so no more than that is ever + // held. Stopping ends only the read-only transaction this opened. Unknown column types are looked up after its + // ROLLBACK, since a cancel can leave it aborted. + func boundedReadOnlyStatement( + _ statement: String, + parameters: [PluginCellValue]?, + bound: LibPQReadBound, + scope: LibPQReadOnlyTransaction.Scope + ) async throws -> LibPQPluginQueryResult { + let statementToRun = String(statement) + let params = parameters ?? [] + + return try await pluginDispatchAsync(on: queue) { [self] in + guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } + if let ended = sessionEndedBeforeSending(conn) { throw ended } + let read = try LibPQReadOnlyTransaction.run( + state: transactionStateOnQueue().sessionTransactionState, + scope: scope, + send: { try runReadOnlyControl($0, on: conn) } + ) { + if suppressServerSideCancel, + let cursorRead = try cursorBoundedReadSync(statementToRun, parameters: params, bound: bound, conn: conn) { + return cursorRead + } + return try boundedReadSync(statementToRun, parameters: params, bound: bound, conn: conn) + } + let metadata = read.metadata.map { resolvingUnknownTypes($0, conn: conn) } + var result = PostGISTypes.rendering( + LibPQPluginQueryResult( + columns: metadata?.columns ?? [], + columnOids: metadata?.columnOids ?? [], + columnTypeNames: metadata?.columnTypeNames ?? [], + rows: read.rows, + affectedRows: read.affectedRows, + commandTag: read.commandTag, + isTruncated: read.isTruncated, + firstRowTime: read.firstRowTime + ), + types: typeNames.postgisTypes + ) + result.columnSources = read.columnSources + return result + } + } + + private struct BoundedRead { + var metadata: ColumnMetadata? + var columnSources: [PostgreSQLColumnOrigins.Source?] = [] + var rows: [[PluginCellValue]] = [] + var affectedRows = 0 + var commandTag: String? + var isTruncated = false + var firstRowTime: TimeInterval? + } + + // Runs inside the read-only transaction `boundedReadOnlyStatement` opened. The session reports a block open, but + // the block is this read's own and ends in ROLLBACK, so the server is cancelled wherever it takes a cancel, which + // `cancelsAbandonedOutput` would refuse inside a block the user opened. A server that takes none reaches this only + // for a statement no cursor can hold or whose DECLARE it refused, and its rest is drained. The column sources come + // from the first row description, as every single-row result carries one. + private func boundedReadSync( + _ query: String, + parameters: [PluginCellValue], + bound: LibPQReadBound, + conn: OpaquePointer + ) throws -> BoundedRead { + let generation = cancellationGate.beginQuery() + defer { cancellationGate.endQuery(generation) } + defer { refreshStandardConformingStrings(from: conn) } + + let sentAt = Date() + let suppressCancel = suppressServerSideCancel + let localQuery = String(query) + let sent: Int32 = try LibPQParameterBuffers.with(parameters) { values, lengths, formats in + if let ended = sessionEndedBeforeSending(conn) { throw ended } + return localQuery.withCString { queryPtr in + PQsendQueryParams(conn, queryPtr, Int32(parameters.count), nil, values, lengths, formats, 0) + } + } + guard sent != 0 else { + throw lostConnection(getError(from: conn), on: conn, sent: false) + } + guard PQsetSingleRowMode(conn) != 0 else { + _ = cancelAndDrain(conn, suppressCancel: suppressCancel) + recordTransactionState(of: conn) + throw LibPQPluginError(message: "Failed to enter single-row mode", sqlState: nil, detail: nil) + } + + var read = BoundedRead() + read.rows.reserveCapacity(min(bound.rowCap, 10_000)) + var bytes = 0 + var stopped = false + var failedStatement: LibPQPluginError? + var pendingError: Error? + + while let result = PQgetResult(conn) { + let status = PQresultStatus(result) + if read.firstRowTime == nil { read.firstRowTime = Date().timeIntervalSince(sentAt) } + if read.metadata == nil, Self.carriesRowDescription(status) { + read.metadata = readColumnMetadata(from: result) + read.columnSources = PostgreSQLColumnOrigins.sources(of: result) + } + + if status == PGRES_SINGLE_TUPLE { + let row = Self.singleRow(of: result, oids: read.metadata?.columnOids ?? []) + PQclear(result) + bytes += LibPQReadBound.byteCount(of: row) + read.rows.append(row) + + if cancellationGate.isCancelled(generation) { + _ = cancelAndDrain(conn, suppressCancel: suppressCancel) + recordTransactionState(of: conn) + throw CancellationError() + } + if bound.isReached(rows: read.rows.count, bytes: bytes) { + stopped = true + break + } + continue + } + + if status == PGRES_TUPLES_OK { + PQclear(result) + continue + } + + if status == PGRES_COMMAND_OK { + read.affectedRows = getAffectedRows(from: result) + read.commandTag = getCommandTag(from: result) + PQclear(result) + continue + } + + if let copy = LibPQCopyState.copy(of: result) { + pendingError = Self.unsupportedCopyError(copy) + PQclear(result) + break + } + + failedStatement = getResultError(from: result) + PQclear(result) + break + } + + let outcome = stopped + ? cancelAndDrain(conn, suppressCancel: suppressCancel) + : finishPendingResults(conn, cancellingOutput: !suppressCancel) + recordTransactionState(of: conn) + + if let failedStatement { + if cancellationGate.isCancelled(generation) { throw CancellationError() } + throw lostConnection(failedStatement, on: conn, sent: true) + } + if let pendingError { + if cancellationGate.isCancelled(generation) { throw CancellationError() } + throw pendingError + } + if cancellationGate.isCancelled(generation) { throw CancellationError() } + if let abandoned = abandonedCopyError(outcome, generation: generation) { throw abandoned } + + read.isTruncated = stopped + read.rows = bound.kept(read.rows) + return read + } + + // A server that takes no cancel (PGlite has one connection) would make `boundedReadSync` read every row after the + // stop. There the statement goes through `LibPQBoundedCursor` and is fetched a batch at a time, so no row past the + // stop is produced. A savepoint holds the DECLARE: a refused one is undone, and nil sends the statement the plain + // way, which reports its own error in the user's text. + private func cursorBoundedReadSync( + _ query: String, + parameters: [PluginCellValue], + bound: LibPQReadBound, + conn: OpaquePointer + ) throws -> BoundedRead? { + guard bound.rowCap < Int(Int32.max), let body = LibPQCappedCursorRead.cursorBody(of: query) else { return nil } + let generation = cancellationGate.beginQuery() + defer { cancellationGate.endQuery(generation) } + defer { refreshStandardConformingStrings(from: conn) } + let sentAt = Date() + + try runReadOnlyControl(LibPQBoundedCursor.savepoint, on: conn) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + let declaration = LibPQBoundedCursor.declaration(of: body) + let declared: OpaquePointer? = try LibPQParameterBuffers.with(parameters) { values, lengths, formats in + declaration.withCString { PQexecParams(conn, $0, Int32(parameters.count), nil, values, lengths, formats, 0) } + } + guard let declared else { throw lostConnection(getError(from: conn), on: conn, sent: true) } + let refused = PQresultStatus(declared) == PGRES_COMMAND_OK ? nil : getResultError(from: declared) + PQclear(declared) + recordTransactionState(of: conn) + if let refused { + guard PQstatus(conn) == CONNECTION_OK else { throw lostConnection(refused, on: conn, sent: true) } + try runReadOnlyControl(LibPQBoundedCursor.savepointRollback, on: conn) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + return nil + } + + var read = BoundedRead() + var bytes = 0 + while true { + let count = bound.nextFetchCount(rows: read.rows.count, bytes: bytes) + let batch = try fetchCursorBatch( + count: count, bound: bound, read: &read, bytes: &bytes, sentAt: sentAt, generation: generation, conn: conn + ) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + if batch.stopped { + read.isTruncated = true + break + } + if batch.received < count { break } + } + read.rows = bound.kept(read.rows) + return read + } + + // One FETCH in single-row mode. A stop inside it drains only the rest of this batch, since the server takes no + // cancel; the cursor keeps everything after it unproduced. + private func fetchCursorBatch( + count: Int, + bound: LibPQReadBound, + read: inout BoundedRead, + bytes: inout Int, + sentAt: Date, + generation: Int, + conn: OpaquePointer + ) throws -> (received: Int, stopped: Bool) { + let sendOk = LibPQBoundedCursor.fetch(count).withCString { PQsendQuery(conn, $0) } + guard sendOk != 0 else { throw lostConnection(getError(from: conn), on: conn, sent: false) } + guard PQsetSingleRowMode(conn) != 0 else { + _ = finishPendingResults(conn, cancellingOutput: false) + recordTransactionState(of: conn) + throw LibPQPluginError(message: "Failed to enter single-row mode", sqlState: nil, detail: nil) + } + + var received = 0 + var stopped = false + var failure: LibPQPluginError? + while let result = PQgetResult(conn) { + let status = PQresultStatus(result) + if read.firstRowTime == nil { read.firstRowTime = Date().timeIntervalSince(sentAt) } + if read.metadata == nil, Self.carriesRowDescription(status) { + read.metadata = readColumnMetadata(from: result) + read.columnSources = PostgreSQLColumnOrigins.sources(of: result) + } + if status == PGRES_SINGLE_TUPLE { + let row = Self.singleRow(of: result, oids: read.metadata?.columnOids ?? []) + PQclear(result) + received += 1 + bytes += LibPQReadBound.byteCount(of: row) + read.rows.append(row) + if cancellationGate.isCancelled(generation) { break } + if bound.isReached(rows: read.rows.count, bytes: bytes) { + stopped = true + break + } + continue + } + if status != PGRES_TUPLES_OK, status != PGRES_COMMAND_OK { failure = getResultError(from: result) } + PQclear(result) + if failure != nil { break } + } + _ = finishPendingResults(conn, cancellingOutput: false) + recordTransactionState(of: conn) + if let failure { throw lostConnection(failure, on: conn, sent: true) } + return (received, stopped) + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadBound.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadBound.swift new file mode 100644 index 0000000000..2969124f51 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadBound.swift @@ -0,0 +1,82 @@ +// +// LibPQReadBound.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +// Where a read-only read stops: past `rowCap` rows, the extra one telling "exactly" from "more", or at the first row that +// takes the decoded rows past `byteLimit`. That row is kept, so the host sees its own limit was reached. +struct LibPQReadBound: Equatable, Sendable { + let rowCap: Int + let byteLimit: Int? + + init(rowCap: Int, byteLimit: Int?) { + self.rowCap = max(rowCap, 1) + self.byteLimit = byteLimit.map { max($0, 0) } + } + + // Nil when neither bound is set, and the caller reads the result whole as it always has. + static func ifSet(rowCap: Int?, byteLimit: Int?) -> LibPQReadBound? { + let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } + guard cap != nil || byteLimit != nil else { return nil } + return LibPQReadBound(rowCap: cap ?? PluginRowLimits.emergencyMax, byteLimit: byteLimit) + } + + func isReached(rows: Int, bytes: Int) -> Bool { + if rows > rowCap { return true } + guard let byteLimit else { return false } + return bytes > byteLimit + } + + // The row that only told "more" goes; a row that crossed the byte limit within the cap stays. + func kept(_ rows: [[PluginCellValue]]) -> [[PluginCellValue]] { + rows.count > rowCap ? Array(rows.prefix(rowCap)) : rows + } + + static let largestFetch = 1_000 + + // How many rows the next cursor FETCH asks for, given what was read so far. Without a byte limit, all the cap still + // allows. With one, a single row first, then as many as the rows so far say still fit: a FETCH cannot be cancelled + // where the cursor is used, so a batch the limit stops has the rest of its rows drained. + func nextFetchCount(rows: Int, bytes: Int) -> Int { + let remaining = max(rowCap - rows + 1, 1) + guard let byteLimit else { return remaining } + guard rows > 0 else { return 1 } + let average = max(bytes / rows, 1) + let fitting = max(byteLimit - bytes, 0) / average + 1 + return min(remaining, fitting, Self.largestFetch) + } + + // The decoded size, at most what the host counts for the same row, so the read never stops before the host's own + // budget is full. + static func byteCount(of row: [PluginCellValue]) -> Int { + row.reduce(0) { total, cell in + switch cell { + case .null: total + case .text(let text): total + text.utf8.count + case .bytes(let data): total + data.count + } + } + } +} + +// The cursor a bounded read-only read goes through where the server takes no cancel. It lives in the read's own +// read-only transaction, whose ROLLBACK closes it, so the rows after the stop are never produced or sent. A FETCH result +// describes the cursor query's own columns, so the column origins survive. +enum LibPQBoundedCursor { + static let name = "tablepro_bounded_read" + // A round trip of its own, so a DECLARE the server refuses is undone and the statement can run the plain way. + static let savepoint = "SAVEPOINT \(name)" + static let savepointRollback = "ROLLBACK TO SAVEPOINT \(name)" + + // The body on lines of its own, so a trailing `--` comment in it stays a comment. + static func declaration(of body: String) -> String { + "DECLARE \(name) NO SCROLL CURSOR FOR\n\(body)\n" + } + + static func fetch(_ count: Int) -> String { + "FETCH FORWARD \(count) FROM \(name)" + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction+Statement.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction+Statement.swift new file mode 100644 index 0000000000..e7a913a03b --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction+Statement.swift @@ -0,0 +1,99 @@ +// +// LibPQReadOnlyTransaction+Statement.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit + +extension LibPQReadOnlyTransaction { + struct Scope: Sendable, Equatable { + let reason: Reason + let runsInOpenTransaction: Bool + } + + static var sessionNotIdle: LibPQPluginError { + LibPQPluginError(message: notIdleMessage(.externalClients), sqlState: nil, detail: nil) + } + + static var safeModeSessionNotIdle: LibPQPluginError { + LibPQPluginError(message: notIdleMessage(.safeMode), sqlState: nil, detail: nil) + } + + static func reason(for context: PluginStatementContext, capabilities: PluginCapabilities) -> Reason? { + if context.readOnly, context.origin == .external, capabilities.contains(.readOnlyStatements) { + return .externalClients + } + if context.enforcesReadOnly, capabilities.contains(.enforcesReadOnlyMode) { + return .safeMode + } + return nil + } + + static func applies(to context: PluginStatementContext, capabilities: PluginCapabilities) -> Bool { + reason(for: context, capabilities: capabilities) != nil + } + + static func scope(for context: PluginStatementContext, capabilities: PluginCapabilities) -> Scope? { + reason(for: context, capabilities: capabilities).map { reason in + Scope(reason: reason, runsInOpenTransaction: context.enforcesReadOnly && !context.callsUnlistedFunction) + } + } + + static func run( + state: PluginSessionTransactionState, + scope: Scope, + send: (String) throws -> Void, + statement: () throws -> Result + ) throws -> Result { + try run( + state: state, + reason: scope.reason, + runsInOpenTransaction: scope.runsInOpenTransaction, + send: send, + statement: statement + ) + } + + // Inside an open transaction the wrap would make the user's own transaction read-only, so a session holding one runs + // a statement the host checked for unlisted function calls in it as it is. + static func run( + state: PluginSessionTransactionState, + reason: Reason, + runsInOpenTransaction: Bool = false, + send: (String) throws -> Void, + statement: () throws -> Result + ) throws -> Result { + guard state == .idle else { + guard runsInOpenTransaction else { throw notIdle(reason) } + return try statement() + } + do { + try send(begin) + } catch { + try? send(end) + throw error + } + let result: Result + do { + result = try statement() + } catch { + try? send(end) + throw refusal(from: error, reason: reason) + } + try send(end) + return result + } + + static func notIdle(_ reason: Reason) -> LibPQPluginError { + switch reason { + case .externalClients: sessionNotIdle + case .safeMode: safeModeSessionNotIdle + } + } + + static func refusal(from error: Error, reason: Reason) -> Error { + guard let failure = error as? LibPQPluginError, isRefusal(sqlState: failure.sqlState) else { return error } + return LibPQPluginError(message: refusalMessage(reason), sqlState: failure.sqlState, detail: failure.message) + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction.swift index f4806f707b..9d497da505 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction.swift @@ -4,9 +4,16 @@ // import Foundation -import TableProPluginKit -internal enum LibPQReadOnlyTransaction { +// The iOS PostgreSQL driver compiles this file too; the plugin's statement handling is in +// LibPQReadOnlyTransaction+Statement.swift. +nonisolated internal enum LibPQReadOnlyTransaction { + // Who asked for the read-only mode, which is what a refusal names. + enum Reason: Sendable, Equatable { + case externalClients + case safeMode + } + // `SELECT 1` takes the snapshot, after which the server refuses any statement that makes the transaction writable. static let begin = "START TRANSACTION READ ONLY; SELECT 1" // Not COMMIT, so a `SET`, `set_config` or `NOTIFY` the statement ran is undone too. @@ -15,55 +22,29 @@ internal enum LibPQReadOnlyTransaction { // `cannot execute ... in a read-only transaction`, and `cannot run inside a transaction block` for VACUUM. static let refusalStates: Set = ["25006", "25001"] - static var sessionNotIdle: LibPQPluginError { - LibPQPluginError( - message: String( - localized: "This connection has a transaction open. Commit or roll it back in TablePro, then try again." - ), - sqlState: nil, - detail: nil - ) - } - - static func applies(to context: PluginStatementContext, capabilities: PluginCapabilities) -> Bool { - context.readOnly && context.origin == .external && capabilities.contains(.readOnlyStatements) + static func isRefusal(sqlState: String?) -> Bool { + guard let sqlState else { return false } + return refusalStates.contains(sqlState) } - // Inside an open transaction the wrap would make the user's own transaction read-only. - static func run( - state: PluginSessionTransactionState, - send: (String) throws -> Void, - statement: () throws -> Result - ) throws -> Result { - guard state == .idle else { throw sessionNotIdle } - do { - try send(begin) - } catch { - try? send(end) - throw error - } - let result: Result - do { - result = try statement() - } catch { - try? send(end) - throw refusal(from: error) + static func notIdleMessage(_ reason: Reason) -> String { + switch reason { + case .externalClients: + String(localized: "This connection has a transaction open. Commit or roll it back in TablePro, then try again.") + // Read-Only offers no Commit, so the way out is a rollback. + case .safeMode: + String( + localized: "Safe Mode is set to Read-Only for this connection, and it has a transaction open. Roll it back, then try again." + ) } - try send(end) - return result } - static func refusal(from error: Error) -> Error { - guard let failure = error as? LibPQPluginError, - let sqlState = failure.sqlState, - refusalStates.contains(sqlState) - else { return error } - return LibPQPluginError( - message: String( - localized: "This connection is read only for external clients, and the database refused the statement." - ), - sqlState: sqlState, - detail: failure.message - ) + static func refusalMessage(_ reason: Reason) -> String { + switch reason { + case .externalClients: + String(localized: "This connection is read only for external clients, and the database refused the statement.") + case .safeMode: + String(localized: "Safe Mode is set to Read-Only for this connection, and the database refused the statement.") + } } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift index 366ba4f142..b263b12cce 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift @@ -11,4 +11,21 @@ enum LibPQSSLMapping { case .verifyIdentity: return "verify-full" } } + + // libpq's own text suggests sslrootcert=system, which TablePro does not offer and libpq refuses under verify-ca. + static func missingRootCertificateError(_ message: String, sslConfig: SSLConfiguration) -> LibPQPluginError? { + guard sslConfig.mode == .verifyCa, sslConfig.caCertificatePath.isEmpty else { return nil } + let lowered = message.lowercased() + guard lowered.contains("root certificate file"), lowered.contains("does not exist") else { return nil } + return LibPQPluginError( + message: String( + localized: """ + Verify CA needs the server's CA certificate. Choose it under SSL, or use Verify Identity to check \ + the server against this Mac's trusted certificates. + """ + ), + sqlState: nil, + detail: nil + ) + } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift index 39684d302e..cbf6456bb1 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift @@ -22,8 +22,9 @@ final class PGlitePluginDriver: PostgreSQLPluginDriver { super.init(config: config, singleConnectionMode: true) } + // pglite-socket 0.2.19 (PGlite 0.5.16) never reads CopyData: a COPY FROM STDIN, psql's too, loses the connection. override var capabilities: PluginCapabilities { - super.capabilities.subtracting(.cancelQuery) + super.capabilities.subtracting([.cancelQuery, .copyFromStandardInput]) } // pldebugger is a server C extension and needs a second session; PGlite has neither. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift index 08c52f7978..71c957c861 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift @@ -74,7 +74,8 @@ public enum PostgreSQLCommentStatements { return rows.compactMap { statement(target: target, row: $0) } } - private static func statement(target: String, row: [String?]) -> String? { + // `target` is already qualified and quoted. + static func statement(target: String, row: [String?]) -> String? { guard row.count >= 3, let relkind = row[0], let keyword = PostgreSQLRelationSQL.commentKeyword(forRelkind: relkind), diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLExtensionQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLExtensionQueries.swift new file mode 100644 index 0000000000..228f666e0d --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLExtensionQueries.swift @@ -0,0 +1,97 @@ +// +// PostgreSQLExtensionQueries.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore + +nonisolated enum PostgreSQLExtensions { + static let kindId = "extension" + + static var kind: PluginServerObjectKind { + PluginServerObjectKind( + id: kindId, + displayName: String(localized: "Extension"), + pluralDisplayName: String(localized: "Extensions"), + iconName: "puzzlepiece.extension", + scope: .database + ) + } + + static func isDeclared(capabilities: PostgreSQLCapabilities) -> Bool { + capabilities.hasExtensionsCatalog + } +} + +// Every column a 9.1 server has; the default version reads control files on disk, so only one object asks for it. +nonisolated enum PostgreSQLExtensionQueries { + private static let projection = """ + SELECT e.oid::pg_catalog.text, e.extname, n.nspname, e.extversion, e.extrelocatable, + pg_catalog.pg_get_userbyid(e.extowner), + pg_catalog.obj_description(e.oid, 'pg_extension') + FROM pg_catalog.pg_extension e + JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace + """ + + static let list = projection + "\nORDER BY e.extname" + + static func one(oid: UInt32) -> String { + projection + "\nWHERE e.oid = \(oid)" + } + + static func defaultVersion(name: String) -> String { + """ + SELECT default_version FROM pg_catalog.pg_available_extensions + WHERE name = \(PostgreSQLObjectQueries.quoteLiteral(name)) + """ + } + + // pg_dump's own form: it names the schema and intentionally leaves the version out. + static func definition(name: String, schema: String, comment: String?) -> String { + var text = "CREATE EXTENSION IF NOT EXISTS \(PostgreSQLObjectQueries.quoteIdentifier(name)) " + + "WITH SCHEMA \(PostgreSQLObjectQueries.quoteIdentifier(schema));" + if let comment, !comment.isEmpty { + text += "\n\nCOMMENT ON EXTENSION \(PostgreSQLObjectQueries.quoteIdentifier(name)) " + + "IS \(PostgreSQLObjectQueries.quoteLiteral(comment));" + } + return text + } + + static func object(from row: [PluginCellValue], defaultVersion: String? = nil) -> PluginServerObjectInfo? { + guard let oid = text(row, 0), UInt32(oid) != nil, let name = text(row, 1) else { return nil } + let schema = text(row, 2) + let version = text(row, 3) + var attributes: [PluginObjectAttribute] = [] + func add(_ label: String, _ value: String?) { + guard let value, !value.isEmpty else { return } + attributes.append(PluginObjectAttribute(label: label, value: value)) + } + add(String(localized: "Version"), version) + add(String(localized: "Default Version"), defaultVersion) + add(String(localized: "Schema"), schema) + add(String(localized: "Owner"), text(row, 5)) + add(String(localized: "Relocatable"), text(row, 4).map { isTrue($0) ? String(localized: "Yes") : String(localized: "No") }) + add(String(localized: "Comment"), text(row, 6)) + return PluginServerObjectInfo( + kindId: PostgreSQLExtensions.kindId, + name: name, + schema: schema, + identity: oid, + detail: version, + definition: schema.map { definition(name: name, schema: $0, comment: text(row, 6)) }, + attributes: attributes + ) + } + + // A boolean cell arrives as `true`/`false` once decoded, and as `t`/`f` from a text read. + private static func isTrue(_ value: String) -> Bool { + value == "true" || value == "t" + } + + private static func text(_ row: [PluginCellValue], _ index: Int) -> String? { + guard let value = row[safe: index]?.asText, !value.isEmpty else { return nil } + return value + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+EnumTypes.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+EnumTypes.swift index 9eb8f64f03..d9209bf74e 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+EnumTypes.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+EnumTypes.swift @@ -15,9 +15,9 @@ private let enumProbeLogger = Logger( ) extension PostgreSQLPluginDriver { - func probeEnumOids() async { + func applyEnumProbe(_ read: Result) { do { - let result = try await core.executeCatalogRead(PostgreSQLSchemaQueries.enumTypeOidQuery) + let result = try read.get() let rows = result.rows.map { row in row.map(\.asText) } core.mergeCatalogTypeNames(PostgreSQLCatalogTypeNames.enumProbeNames(rows: rows)) } catch { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Extensions.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Extensions.swift new file mode 100644 index 0000000000..64243d9326 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Extensions.swift @@ -0,0 +1,38 @@ +// +// PostgreSQLPluginDriver+Extensions.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore + +extension PostgreSQLPluginDriver { + // `serverObjectKinds` is dynamically dispatched, so a subclass that declares none reads nothing here. + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [PluginServerObjectInfo] { + guard kindId == PostgreSQLExtensions.kindId, serverObjectKinds.contains(where: { $0.id == kindId }) else { + return [] + } + let result = try await catalogRead(PostgreSQLExtensionQueries.list) + return result.rows.compactMap { PostgreSQLExtensionQueries.object(from: $0) } + } + + // Read again by the oid this driver handed out, which a rename or a schema move keeps; anything + // else is refused rather than widened into a lookup by name. + func fetchServerObject(_ object: PluginServerObjectInfo) async throws -> PluginServerObjectInfo { + guard object.kindId == PostgreSQLExtensions.kindId else { throw PluginObjectSourceError.unsupported(object.name) } + guard let identity = object.identity, let oid = UInt32(identity) else { + throw PluginObjectSourceError.notFound(object.name) + } + let result = try await catalogRead(PostgreSQLExtensionQueries.one(oid: oid)) + guard let row = result.rows.first(where: { $0[safe: 0]?.asText == String(oid) }), + let name = row[safe: 1]?.asText + else { throw PluginObjectSourceError.notFound(object.name) } + let defaultVersion = try? await catalogRead(PostgreSQLExtensionQueries.defaultVersion(name: name)) + guard let fresh = PostgreSQLExtensionQueries.object( + from: row, + defaultVersion: defaultVersion?.rows.first?[safe: 0]?.asText + ) else { throw PluginObjectSourceError.notFound(object.name) } + return fresh + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+ServerSupport.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+ServerSupport.swift index b8f8db1e45..3a32db50c5 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+ServerSupport.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+ServerSupport.swift @@ -27,9 +27,9 @@ internal extension PostgreSQLPluginDriver { PostgreSQLVersionedStatements.refusal(for: operation, capabilities: versionedCapabilities) } - func probeSessionFacts() async { + func applySessionFactsProbe(_ read: Result) { do { - let result = try await core.executeCatalogRead(PostgreSQLSessionFacts.probeQuery) + let result = try read.get() let row = result.rows.first?.map(\.asText) ?? [] let facts = PostgreSQLSessionFacts(probeRow: row) sessionFacts.withLock { $0 = facts } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift index 332e0df495..67de451b85 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift @@ -51,7 +51,9 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { .schemaCompare, .dataCompare, .temporaryTables, - .readOnlyStatements + .readOnlyStatements, + .enforcesReadOnlyMode, + .copyFromStandardInput, ] } @@ -59,6 +61,11 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { var canDebugRoutines: Bool { true } var canFollowNotifications: Bool { true } + // In the class body so KingbaseES, whose catalog is not established, can answer none. + var serverObjectKinds: [PluginServerObjectKind] { + PostgreSQLExtensions.isDeclared(capabilities: catalogCapabilities) ? [PostgreSQLExtensions.kind] : [] + } + init(config: DriverConnectionConfig, singleConnectionMode: Bool = false) { self.core = LibPQDriverCore(config: config, singleConnectionMode: singleConnectionMode) } @@ -74,16 +81,22 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { /// Runs inside the connect's deadline and cancellation, before the connection is handed out. func probeAfterConnect() async throws { - await probeSessionFacts() - await probeCatalogPresence() - await probePostgisOids() - await probeEnumOids() - await probeVectorTypes() + let probes: [(statement: String, apply: (Result) -> Void)] = [ + (PostgreSQLSessionFacts.probeQuery, applySessionFactsProbe), + (PostgreSQLCatalogPresence.probeQuery, applyCatalogPresenceProbe), + (PostGISTypes.probeQuery, applyPostgisProbe), + (PostgreSQLSchemaQueries.enumTypeOidQuery, applyEnumProbe), + (PgvectorTypes.probeQuery, applyVectorTypesProbe) + ] + let reads = await core.executeCatalogReads(probes.map { $0.statement }) + for (probe, read) in zip(probes, reads) { + probe.apply(read) + } } - private func probeCatalogPresence() async { + private func applyCatalogPresenceProbe(_ read: Result) { do { - let result = try await core.executeCatalogRead(PostgreSQLCatalogPresence.probeQuery) + let result = try read.get() let relationNames = result.rows.compactMap { $0.first?.asText } catalogPresence = PostgreSQLCatalogPresence(relationNames: relationNames) } catch { @@ -91,18 +104,18 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } } - private func probePostgisOids() async { + private func applyPostgisProbe(_ read: Result) { do { - let result = try await core.executeCatalogRead(PostGISTypes.probeQuery) + let result = try read.get() core.setPostgisOidMap(PostGISTypes.names(fromProbeRows: result.rows.map { row in row.map(\.asText) })) } catch { Self.logger.debug("PostGIS type probe failed; spatial values stay as hex: \(error.localizedDescription)") } } - private func probeVectorTypes() async { + private func applyVectorTypesProbe(_ read: Result) { do { - let result = try await core.executeCatalogRead(PgvectorTypes.probeQuery) + let result = try read.get() let rows = result.rows.map { row in row.map(\.asText) } core.mergeCatalogTypeNames(PgvectorTypes.names(fromProbeRows: rows)) } catch { @@ -446,16 +459,44 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { var parts = columnDefs parts.append(contentsOf: constraints) - let shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.rows.map { $0.map(\.asText) }) + var shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.rows.map { $0.map(\.asText) }) var partitions: [PostgreSQLTableDDL.Partition] = [] + var partitionIndexes: [String] = [] + var partitionComments: [String] = [] if shape.partitionKey != nil { let tree = try await catalogRead(PostgreSQLTableDDL.partitionTreeQuery(schema: resolvedSchema, table: table)) partitions = tree.rows.compactMap { PostgreSQLTableDDL.partition(from: $0.map(\.asText)) } + if !partitions.isEmpty { + async let elementRows = catalogRead(PostgreSQLTableDDL.partitionElementsQuery( + schema: resolvedSchema, table: table, scope: .below, capabilities: versionedCapabilities + )) + async let indexRows = catalogRead(PostgreSQLTableDDL.partitionIndexesQuery(schema: resolvedSchema, table: table)) + async let commentRows = catalogRead(PostgreSQLTableDDL.partitionCommentsQuery(schema: resolvedSchema, table: table)) + let elements = PostgreSQLTableDDL.elementsByPartition(try await elementRows.rows.map { $0.map(\.asText) }) + partitions = partitions.map { partition in + var partition = partition + partition.elements = elements[partition.name] ?? [] + return partition + } + partitionIndexes = try await indexRows.rows.compactMap { $0.first?.asText } + partitionComments = PostgreSQLTableDDL.partitionComments(try await commentRows.rows.map { $0.map(\.asText) }) + } + } else if shape.partitionOf != nil { + let elementRows = try await catalogRead(PostgreSQLTableDDL.partitionElementsQuery( + schema: resolvedSchema, table: table, scope: .itself, capabilities: versionedCapabilities + )) + shape.partitionElements = PostgreSQLTableDDL.elementsByPartition(elementRows.rows.map { $0.map(\.asText) }) + .values.first ?? [] } let quotedSchema = quoteIdentifier(resolvedSchema) return PostgreSQLTableDDL.script( - name: "\(quotedSchema).\(quotedTable)", definitions: parts, shape: shape, partitions: partitions + name: "\(quotedSchema).\(quotedTable)", + definitions: parts, + shape: shape, + partitions: partitions, + partitionIndexes: partitionIndexes, + partitionComments: partitionComments ) } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift index e4728a714a..347c088d4d 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift @@ -7,13 +7,15 @@ import Foundation import TableProPostgreSQLCore // What a table's CREATE statement says outside its column list. The table listing hides the partitions of a -// partitioned table, so the parent's script creates them: without them a restore has nowhere to put a row. +// partitioned table, so the parent's script creates them, with what each holds of its own: without them a restore has +// nowhere to put a row, and a partition's own check, key, default and index would be lost. enum PostgreSQLTableDDL { struct Shape: Equatable { var isUnlogged = false var partitionKey: String? var partitionOf: String? var partitionBound: String? + var partitionElements: [String] = [] } struct Partition: Equatable { @@ -22,6 +24,14 @@ enum PostgreSQLTableDDL { let bound: String let partitionKey: String? let isUnlogged: Bool + var foreignServer: String? + var foreignOptions: String? + var elements: [String] = [] + } + + enum PartitionScope { + case below + case itself } // `relispartition`, `relpartbound` and `pg_get_partkeydef` are PostgreSQL 10. @@ -47,39 +57,183 @@ enum PostgreSQLTableDDL { """ } - // Every partition below the table, a parent before its own partitions. A foreign table attached as a partition - // is left out: CREATE TABLE cannot make one. + // Every partition below the table, a parent before its own partitions, foreign table partitions with their server + // and options. static func partitionTreeQuery(schema: String, table: String) -> String { """ - WITH RECURSIVE parts AS ( - SELECT i.inhrelid AS relid, i.inhparent AS parent, 1 AS depth - FROM pg_catalog.pg_inherits i - JOIN pg_catalog.pg_class root ON root.oid = i.inhparent - JOIN pg_catalog.pg_namespace rn ON rn.oid = root.relnamespace - WHERE root.relname = \(PostgreSQLObjectQueries.quoteLiteral(table)) - AND rn.nspname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) - AND root.relkind = 'p' - UNION ALL - SELECT i.inhrelid, i.inhparent, parts.depth + 1 - FROM pg_catalog.pg_inherits i - JOIN parts ON parts.relid = i.inhparent - ) + \(partitionRelations(schema: schema, table: table, scope: .below)) SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), pg_catalog.quote_ident(pn.nspname) || '.' || pg_catalog.quote_ident(p.relname), pg_catalog.pg_get_expr(c.relpartbound, c.oid), CASE WHEN c.relkind = 'p' THEN pg_catalog.pg_get_partkeydef(c.oid) END, - c.relpersistence::text + c.relpersistence::text, + pg_catalog.quote_ident(fs.srvname), + (SELECT pg_catalog.string_agg( + pg_catalog.quote_ident(o.option_name) || ' ' || pg_catalog.quote_literal(o.option_value), + ', ') + FROM pg_catalog.pg_options_to_table(ft.ftoptions) o) FROM parts JOIN pg_catalog.pg_class c ON c.oid = parts.relid JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace JOIN pg_catalog.pg_class p ON p.oid = parts.parent JOIN pg_catalog.pg_namespace pn ON pn.oid = p.relnamespace + LEFT JOIN pg_catalog.pg_foreign_table ft ON ft.ftrelid = c.oid AND c.relkind = 'f' + LEFT JOIN pg_catalog.pg_foreign_server fs ON fs.oid = ft.ftserver WHERE c.relispartition - AND c.relkind IN ('r', 'p') + AND c.relkind IN ('r', 'p', 'f') ORDER BY parts.depth, cn.nspname, c.relname """ } + // What a partition holds of its own, as `PARTITION OF` element list entries: a default or NOT NULL its parent's + // column lacks, and a key or check it declares itself. The server copies the parent's into every new partition. + static func partitionElementsQuery( + schema: String, + table: String, + scope: PartitionScope, + capabilities: PostgreSQLCapabilities + ) -> String { + let notGenerated = capabilities.hasGeneratedColumns ? "a.attgenerated = ''" : "true" + let ownDefault = """ + d.oid IS NOT NULL AND (pd.oid IS NULL + OR pg_catalog.pg_get_expr(d.adbin, d.adrelid) IS DISTINCT FROM pg_catalog.pg_get_expr(pd.adbin, pd.adrelid)) + """ + return """ + \(partitionRelations(schema: schema, table: table, scope: scope)) + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), elements.element + FROM ( + SELECT parts.relid, parts.depth, 0 AS kind, a.attnum::int AS position, + pg_catalog.quote_ident(a.attname) || ' WITH OPTIONS' + || CASE WHEN \(ownDefault) THEN ' DEFAULT ' || pg_catalog.pg_get_expr(d.adbin, d.adrelid) ELSE '' END + || CASE WHEN a.attnotnull AND NOT pa.attnotnull THEN ' NOT NULL' ELSE '' END AS element + FROM parts + JOIN pg_catalog.pg_attribute a ON a.attrelid = parts.relid AND a.attnum > 0 AND NOT a.attisdropped + JOIN pg_catalog.pg_attribute pa + ON pa.attrelid = parts.parent AND pa.attname = a.attname AND NOT pa.attisdropped + LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + LEFT JOIN pg_catalog.pg_attrdef pd ON pd.adrelid = pa.attrelid AND pd.adnum = pa.attnum + WHERE \(notGenerated) + AND ((\(ownDefault)) OR (a.attnotnull AND NOT pa.attnotnull)) + UNION ALL + SELECT parts.relid, parts.depth, 1, + CASE con.contype WHEN 'p' THEN 0 WHEN 'u' THEN 1 WHEN 'c' THEN 2 ELSE 3 END, + 'CONSTRAINT ' || pg_catalog.quote_ident(con.conname) || ' ' + || pg_catalog.pg_get_constraintdef(con.oid, true) + FROM parts + JOIN pg_catalog.pg_constraint con ON con.conrelid = parts.relid + WHERE con.contype IN ('p', 'u', 'c', 'x') + AND con.conislocal + AND con.coninhcount = 0 + ) AS elements + JOIN pg_catalog.pg_class c ON c.oid = elements.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + ORDER BY elements.depth, cn.nspname, c.relname, elements.kind, elements.position, elements.element + """ + } + + // An index a partition has of its own: no parent index it is attached to, and no constraint of the partition's + // behind it, which the element list already declares. + static func partitionIndexesQuery(schema: String, table: String) -> String { + """ + \(partitionRelations(schema: schema, table: table, scope: .below)) + SELECT pg_catalog.pg_get_indexdef(i.indexrelid) + FROM parts + JOIN pg_catalog.pg_index i ON i.indrelid = parts.relid + JOIN pg_catalog.pg_class ic ON ic.oid = i.indexrelid + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + WHERE i.indisvalid + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_inherits ih WHERE ih.inhrelid = i.indexrelid) + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint con + WHERE con.conrelid = parts.relid AND con.conindid = i.indexrelid AND con.contype IN ('p', 'u', 'x')) + ORDER BY parts.depth, cn.nspname, c.relname, ic.relname + """ + } + + // Rows are the partition, then what `PostgreSQLCommentStatements.catalogQuery` projects. A foreign table partition + // is not created, so its comments are left out too. + static func partitionCommentsQuery(schema: String, table: String) -> String { + """ + \(partitionRelations(schema: schema, table: table, scope: .below)) + SELECT target, relkind, attname, description + FROM ( + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname) AS target, + parts.depth, c.relkind::text AS relkind, NULL::text AS attname, + pg_catalog.obj_description(c.oid, 'pg_class') AS description, 0 AS attnum + FROM parts + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + WHERE c.relkind <> 'f' + AND pg_catalog.obj_description(c.oid, 'pg_class') IS NOT NULL + UNION ALL + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), + parts.depth, c.relkind::text, a.attname::text, + pg_catalog.col_description(c.oid, a.attnum), a.attnum::int + FROM parts + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + WHERE c.relkind <> 'f' + AND pg_catalog.col_description(c.oid, a.attnum) IS NOT NULL + ) AS partition_comments + ORDER BY depth, target, attnum + """ + } + + private static func partitionRelations(schema: String, table: String, scope: PartitionScope) -> String { + let nameLiteral = PostgreSQLObjectQueries.quoteLiteral(table) + let schemaLiteral = PostgreSQLObjectQueries.quoteLiteral(schema) + switch scope { + case .below: + return """ + WITH RECURSIVE parts AS ( + SELECT i.inhrelid AS relid, i.inhparent AS parent, 1 AS depth + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class root ON root.oid = i.inhparent + JOIN pg_catalog.pg_namespace rn ON rn.oid = root.relnamespace + WHERE root.relname = \(nameLiteral) + AND rn.nspname = \(schemaLiteral) + AND root.relkind = 'p' + UNION ALL + SELECT i.inhrelid, i.inhparent, parts.depth + 1 + FROM pg_catalog.pg_inherits i + JOIN parts ON parts.relid = i.inhparent + ) + """ + case .itself: + return """ + WITH parts AS ( + SELECT c.oid AS relid, i.inhparent AS parent, 0 AS depth + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + JOIN pg_catalog.pg_inherits i ON i.inhrelid = c.oid + WHERE c.relname = \(nameLiteral) + AND n.nspname = \(schemaLiteral) + AND c.relispartition + ) + """ + } + } + + // Rows from `partitionElementsQuery`, by the partition's qualified name. + static func elementsByPartition(_ rows: [[String?]]) -> [String: [String]] { + var grouped: [String: [String]] = [:] + for row in rows { + guard row.count >= 2, let name = row[0], let element = row[1], !element.isEmpty else { continue } + grouped[name, default: []].append(element) + } + return grouped + } + + static func partitionComments(_ rows: [[String?]]) -> [String] { + rows.compactMap { row in + guard let target = row.first ?? nil, !target.isEmpty else { return nil } + return PostgreSQLCommentStatements.statement(target: target, row: Array(row.dropFirst())) + } + } + // A server that reports version 10 without these catalog functions fails the read; the table is then plain. static func shape(fromRows rows: [[String?]]?) -> Shape { rows?.first.map(shape(from:)) ?? Shape() @@ -105,14 +259,27 @@ enum PostgreSQLTableDDL { let bound = row[2], !bound.isEmpty else { return nil } let key = row[3].flatMap { $0.isEmpty ? nil : $0 } - return Partition(name: name, parent: parent, bound: bound, partitionKey: key, isUnlogged: row[4] == "u") + func text(_ index: Int) -> String? { + guard row.indices.contains(index), let value = row[index], !value.isEmpty else { return nil } + return value + } + return Partition( + name: name, + parent: parent, + bound: bound, + partitionKey: key, + isUnlogged: row[4] == "u", + foreignServer: text(5), + foreignOptions: text(6) + ) } static func createTable(name: String, definitions: [String], shape: Shape) -> String { let partitionBy = shape.partitionKey.map { " PARTITION BY \($0)" } ?? "" if let parent = shape.partitionOf, let bound = shape.partitionBound { let isUnlogged = shape.isUnlogged && shape.partitionKey == nil - return "\(keyword(isUnlogged: isUnlogged)) \(name) PARTITION OF \(parent) \(bound)\(partitionBy);" + let elements = elementList(shape.partitionElements) + return "\(keyword(isUnlogged: isUnlogged)) \(name) PARTITION OF \(parent)\(elements) \(bound)\(partitionBy);" } // PostgreSQL 18 refuses an unlogged partitioned table, and before it the flag did nothing on the parent. let isUnlogged = shape.isUnlogged && shape.partitionKey == nil @@ -121,24 +288,51 @@ enum PostgreSQLTableDDL { + "\n)\(partitionBy);" } + // A foreign table partition is written as a comment. The export reads its rows through the parent, and replayed + // into a partition on that server they would be written to the remote table a second time. static func createPartition(_ partition: Partition) -> String { - createTable( + if let server = partition.foreignServer { + let options = partition.foreignOptions.map { " OPTIONS (\($0))" } ?? "" + let statement = "CREATE FOREIGN TABLE \(partition.name) PARTITION OF \(partition.parent)" + + "\(elementList(partition.elements)) \(partition.bound) SERVER \(server)\(options);" + let note = "\(partition.name) is a foreign table partition on server \(server), left out so a restore writes" + + " nothing\nto that server. Rows exported for it have no partition until it is created:\n" + // Every line break the server's lexer ends a comment at, a carriage return included, starts a new comment. + return (note + statement).components(separatedBy: .newlines).map { "-- " + $0 }.joined(separator: "\n") + } + return createTable( name: partition.name, definitions: [], shape: Shape( isUnlogged: partition.isUnlogged, partitionKey: partition.partitionKey, partitionOf: partition.parent, - partitionBound: partition.bound + partitionBound: partition.bound, + partitionElements: partition.elements ) ) } - static func script(name: String, definitions: [String], shape: Shape, partitions: [Partition]) -> String { - ([createTable(name: name, definitions: definitions, shape: shape)] + partitions.map(createPartition)) + // A partition's own indexes and comments follow every partition, since one may name a partition created later. + static func script( + name: String, + definitions: [String], + shape: Shape, + partitions: [Partition], + partitionIndexes: [String] = [], + partitionComments: [String] = [] + ) -> String { + let indexes = partitionIndexes.map { indexCoveringPartitions($0) + ";" } + let comments = partitionComments.map { $0 + ";" } + return ([createTable(name: name, definitions: definitions, shape: shape)] + partitions.map(createPartition) + + indexes + comments) .joined(separator: "\n\n") } + private static func elementList(_ elements: [String]) -> String { + elements.isEmpty ? "" : " (\n " + elements.joined(separator: ",\n ") + "\n)" + } + // `pg_get_indexdef` writes ON ONLY for an index over a partitioned table, and replayed that builds nothing on the // partitions and leaves the index invalid. Without it the server builds each partition's index itself. static func indexCoveringPartitions(_ definition: String) -> String { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift index 55db6d0359..8b7c359b62 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift @@ -224,6 +224,8 @@ final class RedshiftPluginDriver: LibPQBackedDriver, @unchecked Sendable { } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let resolvedSchema = schema ?? core.currentSchema if isExternalSchema(resolvedSchema) { diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQBoundedReadOnlyTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQBoundedReadOnlyTests.swift new file mode 100644 index 0000000000..e994340eb2 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQBoundedReadOnlyTests.swift @@ -0,0 +1,361 @@ +// +// LibPQBoundedReadOnlyTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// A scripted server sends every row at once and, where the answer says so, holds the statement open until a cancel +// arrives, so a read that does not stop and cancel would end only after the server gives up waiting. Where the server +// takes no cancel, `CursorTable` counts the rows the cursor's FETCHes were sent. +@Suite(.timeLimit(.minutes(1))) +struct LibPQBoundedReadOnlyTests { + private static let events = "SELECT * FROM events" + + private static let columns = [ + LibPQScriptedServer.Column(name: "id", table: 16_400, attribute: 1, type: 23), + LibPQScriptedServer.Column(name: "body", table: 16_400, attribute: 3, type: 25), + ] + + private static func rows(_ count: Int, width: Int = 10) -> [[String?]] { + (0.. UInt8 { + let controls = [LibPQReadOnlyTransaction.begin, LibPQReadOnlyTransaction.end, "BEGIN"] + let last = (earlier + [query]).last { controls.contains($0) } + guard let last, last != LibPQReadOnlyTransaction.end else { return LibPQScriptedServer.idle } + return LibPQScriptedServer.inTransaction + } + + private static func server(answering answer: LibPQScriptedServer.Answer) throws -> LibPQScriptedServer { + try LibPQScriptedServer( + status: { query, earlier in Self.status(query, earlier) }, + answer: { query in query == Self.events ? answer : nil } + ) + } + + private static func connected( + to server: LibPQScriptedServer, + suppressingCancel: Bool = false + ) async throws -> LibPQPluginConnection { + let connection = LibPQPluginConnection( + host: "127.0.0.1", + port: server.port, + user: "postgres", + password: nil, + database: "postgres", + suppressServerSideCancel: suppressingCancel + ) + try await connection.connect() + try await connection.finishConnecting() + return connection + } + + private static func read( + _ connection: LibPQPluginConnection, + rowCap: Int, + byteLimit: Int? = nil, + parameters: [PluginCellValue]? = nil, + query: String = LibPQBoundedReadOnlyTests.events + ) async throws -> LibPQPluginQueryResult { + try await connection.boundedReadOnlyStatement( + query, + parameters: parameters, + bound: LibPQReadBound(rowCap: rowCap, byteLimit: byteLimit), + scope: LibPQReadOnlyTransaction.Scope(reason: .safeMode, runsInOpenTransaction: false) + ) + } + + @Test("A read past the row cap keeps the cap, cancels the rest and still rolls back its transaction") + func stopsAtTheRowCap() async throws { + let server = try Self.server(answering: .init(columns: Self.columns, rows: Self.rows(50), waitsForCancel: true)) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + let before = server.queries.count + + let result = try await Self.read(connection, rowCap: 3) + + let sent = Array(server.queries.dropFirst(before)) + #expect(result.rows.count == 3) + #expect(result.rows.first == [.text("0"), .text("xxxxxxxxxx")]) + #expect(result.isTruncated) + #expect(server.cancelRequests == 1) + #expect(sent == [LibPQReadOnlyTransaction.begin, Self.events, LibPQReadOnlyTransaction.end]) + } + + @Test("A read that passes the byte limit stops there and keeps the row that crossed it") + func stopsAtTheByteLimit() async throws { + let wide = Self.rows(50, width: 100) + let server = try Self.server(answering: .init(columns: Self.columns, rows: wide, waitsForCancel: true)) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let result = try await Self.read(connection, rowCap: 1_000, byteLimit: 250) + + #expect(result.rows.count == 3) + #expect(result.isTruncated) + #expect(server.cancelRequests == 1) + #expect(server.queries.last == LibPQReadOnlyTransaction.end) + } + + @Test("A result within the cap is read to its end with no cancel") + func shortResultReadsWhole() async throws { + let server = try Self.server(answering: .init(columns: Self.columns, rows: Self.rows(2))) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + let before = server.queries.count + + let result = try await Self.read(connection, rowCap: 3, byteLimit: 1_000) + + let sent = Array(server.queries.dropFirst(before)) + #expect(result.rows.count == 2) + #expect(!result.isTruncated) + #expect(server.cancelRequests == 0) + #expect(sent == [LibPQReadOnlyTransaction.begin, Self.events, LibPQReadOnlyTransaction.end]) + } + + // Serves the cursor's FETCHes from a table and counts every row it sent. + private final class CursorTable: @unchecked Sendable { + private static let fetchPrefix = "FETCH FORWARD " + private let lock = NSLock() + private let rows: [[String?]] + private var position = 0 + + init(_ rows: [[String?]]) { + self.rows = rows + } + + var served: Int { + lock.withLock { position } + } + + func answer(_ query: String) -> LibPQScriptedServer.Answer? { + guard query.hasPrefix(Self.fetchPrefix), + let count = Int(query.dropFirst(Self.fetchPrefix.count).prefix { $0.isNumber }) + else { return nil } + return lock.withLock { + let batch = Array(rows[position.. LibPQScriptedServer { + try LibPQScriptedServer( + status: { query, earlier in Self.status(query, earlier) }, + answer: { query in table.answer(query) } + ) + } + + private static let declaredEvents = LibPQBoundedCursor.declaration(of: LibPQBoundedReadOnlyTests.events) + + // PGlite has one connection and takes no cancel request. + @Test("Without a server cancel the read goes through a cursor and no row past the cap is sent") + func cursorStopsAtTheRowCap() async throws { + let table = CursorTable(Self.rows(50)) + let server = try Self.cursorServer(table) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, suppressingCancel: true) + defer { connection.disconnect() } + let before = server.queries.count + + let result = try await Self.read(connection, rowCap: 3) + + let sent = Array(server.queries.dropFirst(before)) + #expect(result.rows.count == 3) + #expect(result.isTruncated) + #expect(table.served == 4) + #expect(server.cancelRequests == 0) + #expect(result.columnSources == [ + PostgreSQLColumnOrigins.Source(relation: 16_400, attribute: 1), + PostgreSQLColumnOrigins.Source(relation: 16_400, attribute: 3), + ]) + #expect(sent == [ + LibPQReadOnlyTransaction.begin, + LibPQBoundedCursor.savepoint, + Self.declaredEvents, + LibPQBoundedCursor.fetch(4), + LibPQReadOnlyTransaction.end, + ]) + } + + @Test("Through a cursor, the byte limit sizes each fetch, so the read stops with no row left to drain") + func cursorStopsAtTheByteLimit() async throws { + let table = CursorTable(Self.rows(50, width: 100)) + let server = try Self.cursorServer(table) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, suppressingCancel: true) + defer { connection.disconnect() } + let before = server.queries.count + + let result = try await Self.read(connection, rowCap: 1_000, byteLimit: 250) + + let sent = Array(server.queries.dropFirst(before)) + #expect(result.rows.count == 3) + #expect(result.isTruncated) + #expect(table.served == 3) + #expect(sent.filter { $0.hasPrefix("FETCH") } == [LibPQBoundedCursor.fetch(1), LibPQBoundedCursor.fetch(2)]) + #expect(sent.last == LibPQReadOnlyTransaction.end) + } + + @Test("Through a cursor, a result within the cap ends with the first short fetch") + func cursorReadsAShortResultWhole() async throws { + let table = CursorTable(Self.rows(2)) + let server = try Self.cursorServer(table) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, suppressingCancel: true) + defer { connection.disconnect() } + + let result = try await Self.read(connection, rowCap: 3) + + #expect(result.rows.count == 2) + #expect(!result.isTruncated) + #expect(server.queries.filter { $0.hasPrefix("FETCH") } == [LibPQBoundedCursor.fetch(4)]) + } + + @Test("A DECLARE the server refuses is rolled back to its savepoint, and the statement runs the plain way") + func refusedDeclarationRunsThePlainWay() async throws { + let refused = LibPQScriptedServer.Failure(sqlState: "42601", message: "syntax error at or near \"DECLARE\"") + let server = try LibPQScriptedServer( + status: { query, earlier in Self.status(query, earlier) }, + answer: { query in + if query.hasPrefix("DECLARE") { return LibPQScriptedServer.Answer(failure: refused) } + return query == Self.events ? LibPQScriptedServer.Answer(columns: Self.columns, rows: Self.rows(2)) : nil + } + ) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, suppressingCancel: true) + defer { connection.disconnect() } + let before = server.queries.count + + let result = try await Self.read(connection, rowCap: 3) + + let sent = Array(server.queries.dropFirst(before)) + #expect(result.rows.count == 2) + #expect(sent == [ + LibPQReadOnlyTransaction.begin, + LibPQBoundedCursor.savepoint, + Self.declaredEvents, + LibPQBoundedCursor.savepointRollback, + Self.events, + LibPQReadOnlyTransaction.end, + ]) + } + + // `LibPQCappedCursorRead.cursorBody` refuses any semicolon, even one inside a literal. + @Test("Without a server cancel, a statement no cursor can hold still stops at the cap and drains the rest") + func drainsWhatNoCursorCanHold() async throws { + let marked = "SELECT ';' AS mark FROM events" + let server = try LibPQScriptedServer( + status: { query, earlier in Self.status(query, earlier) }, + answer: { query in + query == marked ? LibPQScriptedServer.Answer(columns: Self.columns, rows: Self.rows(50)) : nil + } + ) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, suppressingCancel: true) + defer { connection.disconnect() } + let before = server.queries.count + + let result = try await Self.read(connection, rowCap: 3, query: marked) + + #expect(result.rows.count == 3) + #expect(result.isTruncated) + #expect(server.cancelRequests == 0) + #expect(Array(server.queries.dropFirst(before)) == [ + LibPQReadOnlyTransaction.begin, marked, LibPQReadOnlyTransaction.end, + ]) + } + + @Test("Each column keeps the relation column the first row description named, so masking by origin still works") + func keepsColumnSources() async throws { + let columns = Self.columns + [LibPQScriptedServer.Column(name: "total")] + let server = try Self.server(answering: .init(columns: columns, rows: [["1", "a", "2"]])) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let result = try await Self.read(connection, rowCap: 10) + + #expect(result.columns == ["id", "body", "total"]) + #expect(result.columnSources == [ + PostgreSQLColumnOrigins.Source(relation: 16_400, attribute: 1), + PostgreSQLColumnOrigins.Source(relation: 16_400, attribute: 3), + nil, + ]) + } + + @Test("A write the server refuses at Read-Only names Safe Mode, and the rollback still goes out") + func refusalNamesSafeMode() async throws { + let refused = LibPQScriptedServer.Failure( + sqlState: "25006", message: "cannot execute DELETE in a read-only transaction" + ) + let server = try Self.server(answering: .init(failure: refused)) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let error = await #expect(throws: LibPQPluginError.self) { + _ = try await Self.read(connection, rowCap: 10) + } + + #expect(error?.sqlState == "25006") + #expect(error?.message == LibPQReadOnlyTransaction.refusalMessage(.safeMode)) + #expect(error?.detail?.contains("read-only transaction") == true) + #expect(server.queries.last == LibPQReadOnlyTransaction.end) + } + + @Test("A session holding a transaction is refused at Read-Only before anything is sent") + func refusesAnOpenTransaction() async throws { + let server = try Self.server(answering: .init(columns: Self.columns, rows: Self.rows(2))) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + _ = try await connection.executeQuery("BEGIN") + let before = server.queries.count + + let error = await #expect(throws: LibPQPluginError.self) { + _ = try await Self.read(connection, rowCap: 10) + } + + #expect(error?.message == LibPQReadOnlyTransaction.safeModeSessionNotIdle.message) + #expect(server.queries.count == before) + } + + @Test("Parameters go out with the statement, a null as a null") + func bindsParameters() async throws { + let server = try Self.server(answering: .init()) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + _ = try await Self.read( + connection, + rowCap: 10, + parameters: [.text("7"), .null], + query: "SELECT * FROM events WHERE id = $1 AND note = $2" + ) + + #expect(server.boundParameters.last == ["7", nil]) + #expect(server.queries.last == LibPQReadOnlyTransaction.end) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQCappedCursorReadTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQCappedCursorReadTests.swift new file mode 100644 index 0000000000..5f8497a740 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQCappedCursorReadTests.swift @@ -0,0 +1,92 @@ +// +// LibPQCappedCursorReadTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import Testing + +struct LibPQCappedCursorReadTests { + @Test("Inside a transaction the cap plus one row are sent and the rest run on the server unsent") + func insideTransactionFetchesCapPlusOne() throws { + let plan = try #require(LibPQCappedCursorRead.plan( + for: "SELECT * FROM events", rowCap: 1_000, scope: .insideTransaction + )) + + #expect(plan.fetchCount == 1_001) + #expect(plan.opening == "SAVEPOINT tablepro_capped_read") + #expect(plan.statement == """ + DECLARE tablepro_capped_read NO SCROLL CURSOR FOR + SELECT * FROM events + ; + FETCH FORWARD 1001 FROM tablepro_capped_read; + MOVE FORWARD ALL FROM tablepro_capped_read; + CLOSE tablepro_capped_read; + RELEASE SAVEPOINT tablepro_capped_read + """) + #expect(plan.recovery == "ROLLBACK TO SAVEPOINT tablepro_capped_read; RELEASE SAVEPOINT tablepro_capped_read") + } + + @Test("Outside a transaction the cursor gets a transaction of its own") + func ownTransactionOpensAndCommits() throws { + let plan = try #require(LibPQCappedCursorRead.plan(for: "VALUES (1)", rowCap: 10, scope: .ownTransaction)) + + #expect(plan.opening == "BEGIN") + #expect(plan.statement.hasPrefix("DECLARE tablepro_capped_read NO SCROLL CURSOR FOR\nVALUES (1)\n;")) + #expect(plan.statement.hasSuffix( + "FETCH FORWARD 11 FROM tablepro_capped_read;\nMOVE FORWARD ALL FROM tablepro_capped_read;\nCLOSE tablepro_capped_read;\nCOMMIT" + )) + #expect(plan.recovery == "ROLLBACK") + } + + @Test("A trailing line comment ends before the FETCH") + func trailingLineCommentKeepsTheFetch() throws { + let plan = try #require(LibPQCappedCursorRead.plan( + for: "SELECT 1 -- tail", rowCap: 5, scope: .insideTransaction + )) + + #expect(plan.statement.contains("SELECT 1 -- tail\n;\nFETCH FORWARD 6")) + } + + @Test( + "One read statement goes through the cursor", + arguments: [ + ("SELECT * FROM t; \n", "SELECT * FROM t"), + ("select a from t;;", "select a from t"), + ("-- note\n(SELECT 1) UNION (SELECT 2)", "-- note\n(SELECT 1) UNION (SELECT 2)"), + ("/* c */ WITH x AS (SELECT 1) SELECT * FROM x", "/* c */ WITH x AS (SELECT 1) SELECT * FROM x"), + ("TABLE accounts", "TABLE accounts"), + ("SELECT updated_at, shared_by FROM t", "SELECT updated_at, shared_by FROM t"), + ] + ) + func readStatementIsCursorable(query: String, body: String) { + #expect(LibPQCappedCursorRead.cursorBody(of: query) == body) + } + + @Test( + "A statement a cursor would change or refuse runs the plain way", + arguments: [ + "SELECT 1; SELECT 2", + "SELECT ';' AS semicolon", + "SELECT * FROM t FOR UPDATE", + "SELECT * FROM t FOR NO KEY UPDATE SKIP LOCKED", + "SELECT * FROM t FOR KEY SHARE", + "SELECT * INTO copy FROM t", + "WITH gone AS (DELETE FROM t RETURNING *) SELECT * FROM gone", + "INSERT INTO t VALUES (1) RETURNING *", + "EXPLAIN SELECT 1", + "SHOW search_path", + "/* unterminated SELECT 1", + "", + " ; ", + ] + ) + func otherStatementsAreNotCursorable(query: String) { + #expect(LibPQCappedCursorRead.cursorBody(of: query) == nil) + } + + @Test("A cap FETCH cannot spell runs the plain way", arguments: [0, -1, Int(Int32.max)]) + func capOutsideFetchRangeHasNoPlan(rowCap: Int) { + #expect(LibPQCappedCursorRead.plan(for: "SELECT 1", rowCap: rowCap, scope: .insideTransaction) == nil) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift index ff3a6b976b..a38c092d50 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift @@ -14,7 +14,8 @@ struct LibPQConnectionStringTests { sslConfig: SSLConfiguration = SSLConfiguration(), options: String? = nil, applicationName: String? = nil, - connectTimeoutSeconds: Int? = nil + connectTimeoutSeconds: Int? = nil, + systemTrustAnchorsPath: String? = nil ) throws -> String { try LibPQConnectionString.build( host: "db.example.com", @@ -25,7 +26,8 @@ struct LibPQConnectionStringTests { sslConfig: sslConfig, options: options, applicationName: applicationName, - connectTimeoutSeconds: connectTimeoutSeconds + connectTimeoutSeconds: connectTimeoutSeconds, + systemTrustAnchorsPath: systemTrustAnchorsPath ) } @@ -260,6 +262,98 @@ struct LibPQConnectionStringTests { #expect(verifying.contains("sslrootcert='/ca.pem'")) } + @Test("Verify Identity with no CA file checks the server against this Mac's trusted roots") + func verifyIdentityWithoutACAUsesTheSystemAnchors() throws { + let conninfo = try build( + sslConfig: SSLConfiguration(mode: .verifyIdentity), + systemTrustAnchorsPath: "/tmp/roots.pem" + ) + + #expect(conninfo.contains("sslmode='verify-full'")) + #expect(conninfo.contains("sslrootcert='/tmp/roots.pem'")) + } + + @Test("A chosen CA file wins over this Mac's trusted roots") + func aCAFileWinsOverTheSystemAnchors() throws { + let conninfo = try build( + sslConfig: SSLConfiguration(mode: .verifyIdentity, caCertificatePath: "/ca.pem"), + systemTrustAnchorsPath: "/tmp/roots.pem" + ) + + #expect(conninfo.contains("sslrootcert='/ca.pem'")) + #expect(!conninfo.contains("/tmp/roots.pem")) + } + + @Test("Verify CA never checks the server against this Mac's trusted roots") + func verifyCAWithoutACAGetsNoSystemAnchors() throws { + let conninfo = try build(sslConfig: SSLConfiguration(mode: .verifyCa), systemTrustAnchorsPath: "/tmp/roots.pem") + + #expect(conninfo.contains("sslmode='verify-ca'")) + #expect(!conninfo.contains("sslrootcert")) + } + + @Test("A mode that checks no certificate gets no root file", arguments: [SSLMode.disabled, .preferred, .required]) + func requiredAndPreferredNeverGetSystemAnchors(mode: SSLMode) throws { + let conninfo = try build(sslConfig: SSLConfiguration(mode: mode), systemTrustAnchorsPath: "/tmp/roots.pem") + + #expect(!conninfo.contains("sslrootcert")) + } + + @Test("The roots come from the CA file, then libpq's own root.crt, then this Mac's trusted roots") + func rootCertificateSourceOrder() { + let home = ["HOME": "/Users/test"] + let rootCrt = "/Users/test/.postgresql/root.crt" + func source( + _ mode: SSLMode, + ca: String = "", + host: String = "db.example.com", + rootCrtExists: Bool = false + ) -> LibPQRootCertificateSource? { + LibPQConnectionString.rootCertificateSource( + for: SSLConfiguration(mode: mode, caCertificatePath: ca), + host: host, + environment: home, + fileExists: { rootCrtExists && $0 == rootCrt } + ) + } + + #expect(source(.verifyIdentity, ca: "/ca.pem", rootCrtExists: true) == .caFile("/ca.pem")) + #expect(source(.verifyCa, ca: "/ca.pem") == .caFile("/ca.pem")) + #expect(source(.verifyIdentity, rootCrtExists: true) == .libpqDefault) + #expect(source(.verifyIdentity) == .systemTrustAnchors) + #expect(source(.verifyIdentity, host: "/tmp") == .libpqDefault) + #expect(source(.verifyCa) == .libpqDefault) + #expect(source(.required) == nil) + #expect(source(.preferred) == nil) + #expect(source(.disabled) == nil) + } + + @Test("A PGSSLROOTCERT the user set, or an empty host, leaves the roots to libpq") + func namedRootFileAndDefaultSocketStayWithLibpq() { + let config = SSLConfiguration(mode: .verifyIdentity) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "db.example.com", environment: ["HOME": "/Users/test", "PGSSLROOTCERT": "/etc/corp-ca.pem"], + fileExists: { _ in false } + ) == .libpqDefault) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "", environment: ["HOME": "/Users/test"], fileExists: { _ in false } + ) == .libpqDefault) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "db.example.com", environment: ["HOME": "/Users/test", "PGSSLROOTCERT": ""], + fileExists: { _ in false } + ) == .systemTrustAnchors) + } + + @Test("libpq's own root file is found under HOME, and under the account's home when HOME is unset or empty") + func defaultRootCertificateFollowsHome() { + let fallback = NSHomeDirectory() + "/.postgresql/root.crt" + + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: ["HOME": "/Users/test"]) + == "/Users/test/.postgresql/root.crt") + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: ["HOME": ""]) == fallback) + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: [:]) == fallback) + } + @Test("A client certificate and key reach libpq as sslcert and sslkey") func clientCertificateIsSent() throws { let conninfo = try build(sslConfig: SSLConfiguration( @@ -330,8 +424,8 @@ struct LibPQConnectionStringTests { #expect(try !build(applicationName: "").contains("application_name")) } - /// Measured on libpq 17: a fallback name replaces an `application_name` set in `options`, so it - /// has to stay out whenever the user named the session there, in any of the server's spellings. + // Measured on libpq 17: a fallback name replaces an `application_name` set in `options`, so it + // has to stay out whenever the user named the session there, in any of the server's spellings. @Test( "A name the user gives the session in the connection options keeps the fallback out", arguments: [ diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQCopyInTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQCopyInTests.swift new file mode 100644 index 0000000000..0feba75642 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQCopyInTests.swift @@ -0,0 +1,283 @@ +// +// LibPQCopyInTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct LibPQCopyInTests { + private static let statement = "COPY public.t (a, b) FROM stdin" + + private struct SourceFailure: Error, Equatable {} + + private final class Pieces: @unchecked Sendable { + private let lock = NSLock() + private var remaining: [Data] + private var read = 0 + private let failsAfter: Int? + private let onRead: @Sendable (Int) -> Void + + init(_ pieces: [String], failsAfter: Int? = nil, onRead: @escaping @Sendable (Int) -> Void = { _ in }) { + remaining = pieces.map { Data($0.utf8) } + self.failsAfter = failsAfter + self.onRead = onRead + } + + var piecesRead: Int { + lock.withLock { read } + } + + var data: PluginCopyData { + PluginCopyData { [self] in try next() } + } + + private func next() throws -> Data? { + let (count, piece) = try lock.withLock { () -> (Int, Data?) in + if let failsAfter, read == failsAfter { throw SourceFailure() } + read += 1 + return (read, remaining.isEmpty ? nil : remaining.removeFirst()) + } + onRead(count) + return piece + } + } + + private static func connected(to server: LibPQScriptedServer) async throws -> LibPQPluginConnection { + let connection = LibPQPluginConnection( + host: "127.0.0.1", port: server.port, user: "postgres", password: nil, database: "postgres" + ) + try await connection.connect() + try await connection.finishConnecting() + return connection + } + + private static func server(_ copyIn: LibPQScriptedServer.CopyIn = .init()) throws -> LibPQScriptedServer { + let server = try LibPQScriptedServer(answer: { query in + query.hasPrefix("COPY") ? LibPQScriptedServer.Answer(copyIn: copyIn) : nil + }) + server.start() + return server + } + + @Test("The rows reach the server byte for byte in the order read, and the result counts them") + func rowsArriveWhole() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n2\tt", "wo\n3\t\\N\n", "4\tcaf\u{e9} \u{1F600}\r\n"]) + let result = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + + #expect(server.copyData == Data("1\tone\n2\ttwo\n3\t\\N\n4\tcaf\u{e9} \u{1F600}\r\n".utf8)) + #expect(result.affectedRows == 4) + #expect(result.commandTag == "COPY 4") + #expect(server.copyFailMessages.isEmpty) + #expect(server.queries.filter { $0.hasPrefix("COPY") } == [Self.statement]) + } + + @Test("No rows still ends the copy, as a COPY of zero rows") + func noRows() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let result = try await connection.copyFromStandardInput(Self.statement, rows: Pieces([]).data, readOnly: nil) + + #expect(server.copyData.isEmpty) + #expect(result.affectedRows == 0) + } + + @Test("A source that fails ends the copy with CopyFail and its own error comes back, not the server's echo") + func sourceFailureEndsTheCopy() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n", "2\ttwo\n"], failsAfter: 1) + await #expect(throws: SourceFailure.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(server.copyData == Data("1\tone\n".utf8)) + #expect(server.copyFailMessages == ["aborted because of read failure"]) + _ = try await connection.executeQuery("SELECT 1") + } + + @Test("An import's Stop thrown by the source ends the copy as cancelled by the user and comes back as itself") + func importStopEndsTheCopy() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let rows = PluginCopyData { throw PluginImportCancellationError() } + await #expect(throws: PluginImportCancellationError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: rows, readOnly: nil) + } + #expect(server.copyFailMessages == ["canceled by user"]) + } + + @Test("A Stop between pieces ends the copy with CopyFail and reports cancelled") + func stopEndsTheCopy() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n", "2\ttwo\n", "3\tthree\n"]) { count in + if count == 1 { connection.cancelCurrentQuery() } + } + await #expect(throws: CancellationError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(server.copyData == Data("1\tone\n".utf8)) + #expect(server.copyFailMessages == ["canceled by user"]) + #expect(pieces.piecesRead == 1) + _ = try await connection.executeQuery("SELECT 1") + #expect(server.cancelRequests == 0) + } + + @Test("A Stop while the server takes no data ends the copy with one cancel request, and the session goes on") + func stopReachesAServerThatTakesNoData() async throws { + let server = try Self.server(.init(takesNoDataUntilCancelled: true)) + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n", "2\ttwo\n"]) { count in + if count == 1 { connection.cancelCurrentQuery() } + } + await #expect(throws: CancellationError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(server.cancelRequests == 1) + #expect(server.copyFailMessages == ["canceled by user"]) + _ = try await connection.executeQuery("SELECT 1") + #expect(server.cancelRequests == 1) + } + + @Test("A Stop while the COPY waits to start sends the cancel request at once and reads no piece") + func stopBeforeTheCopyStarts() async throws { + let stop = StopBox() + let server = try LibPQScriptedServer( + onQuery: { query in if query.hasPrefix("COPY") { stop.press() } }, + answer: { query in query.hasPrefix("COPY") ? LibPQScriptedServer.Answer(waitsForCancel: true) : nil } + ) + server.start() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + stop.connection = connection + + let pieces = Pieces(["1\tone\n"]) + await #expect(throws: CancellationError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(pieces.piecesRead == 0) + #expect(server.cancelRequests == 1) + _ = try await connection.executeQuery("SELECT 1") + } + + private final class StopBox: @unchecked Sendable { + private let lock = NSLock() + private weak var _connection: LibPQPluginConnection? + + var connection: LibPQPluginConnection? { + get { lock.withLock { _connection } } + set { lock.withLock { _connection = newValue } } + } + + func press() { + connection?.cancelCurrentQuery() + } + } + + @Test("A binary COPY is refused with CopyFail before any piece is read") + func binaryIsRefused() async throws { + let server = try Self.server(.init(binary: true)) + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["PGCOPY"]) + await #expect(throws: LibPQPluginError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(pieces.piecesRead == 0) + #expect(server.copyData.isEmpty) + #expect(server.copyFailMessages == ["binary COPY data cannot come from a script"]) + } + + @Test("A row the server refuses fails the COPY with the server's error after every piece was sent") + func serverRefusalComesBack() async throws { + let server = try Self.server(.init(failure: .init(sqlState: "22P02", message: "invalid input syntax"))) + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + await #expect { + _ = try await connection.copyFromStandardInput(Self.statement, rows: Pieces(["x\tbad\n"]).data, readOnly: nil) + } throws: { error in + (error as? LibPQPluginError)?.sqlState == "22P02" + } + _ = try await connection.executeQuery("SELECT 1") + } + + @Test("A statement that answers no copy reads as any other, and no piece is read") + func notACopyReadsAsAStatement() async throws { + let server = try LibPQScriptedServer(answer: { _ in + LibPQScriptedServer.Answer(failure: .init(sqlState: "42P01", message: "relation does not exist")) + }) + server.start() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n"]) + await #expect { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } throws: { error in + (error as? LibPQPluginError)?.sqlState == "42P01" + } + #expect(pieces.piecesRead == 0) + } + + @Test("Under Safe Mode's Read-Only the COPY goes inside a read-only transaction that is rolled back") + func readOnlyWrapsTheCopy() async throws { + let server = try LibPQScriptedServer( + status: { query, _ in + query.hasPrefix("START TRANSACTION READ ONLY") || query.hasPrefix("COPY") + ? LibPQScriptedServer.inTransaction + : LibPQScriptedServer.idle + }, + answer: { query in + guard query.hasPrefix("COPY") else { return nil } + return LibPQScriptedServer.Answer(failure: .init( + sqlState: "25006", message: "cannot execute COPY FROM in a read-only transaction" + )) + } + ) + server.start() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n"]) + await #expect(throws: (any Error).self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: .safeMode) + } + + #expect(Array(server.queries.suffix(3)) == ["START TRANSACTION READ ONLY; SELECT 1", Self.statement, "ROLLBACK"]) + #expect(pieces.piecesRead == 0) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQReadBoundTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQReadBoundTests.swift new file mode 100644 index 0000000000..593cbdcb0f --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQReadBoundTests.swift @@ -0,0 +1,83 @@ +// +// LibPQReadBoundTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct LibPQReadBoundTests { + @Test("No row cap and no byte limit leaves the read whole") + func noBoundReadsWhole() { + #expect(LibPQReadBound.ifSet(rowCap: nil, byteLimit: nil) == nil) + #expect(LibPQReadBound.ifSet(rowCap: 0, byteLimit: nil) == nil) + #expect(LibPQReadBound.ifSet(rowCap: -5, byteLimit: nil) == nil) + } + + @Test("A byte limit alone bounds the rows at the emergency maximum") + func byteLimitAloneKeepsTheEmergencyCap() { + let bound = LibPQReadBound.ifSet(rowCap: nil, byteLimit: 4_096) + + #expect(bound == LibPQReadBound(rowCap: PluginRowLimits.emergencyMax, byteLimit: 4_096)) + } + + @Test("The read stops at the row past the cap") + func stopsPastTheCap() { + let bound = LibPQReadBound(rowCap: 3, byteLimit: nil) + + #expect(!bound.isReached(rows: 3, bytes: .max)) + #expect(bound.isReached(rows: 4, bytes: 0)) + } + + @Test("The read stops at the first row that takes it past the byte limit, not at the one that reaches it") + func stopsPastTheByteLimit() { + let bound = LibPQReadBound(rowCap: 1_000, byteLimit: 250) + + #expect(!bound.isReached(rows: 2, bytes: 250)) + #expect(bound.isReached(rows: 3, bytes: 251)) + } + + @Test("Only the row that told 'more' is dropped; a row that crossed the byte limit is kept") + func keepsTheCrossingRow() { + let bound = LibPQReadBound(rowCap: 3, byteLimit: 10) + let three: [[PluginCellValue]] = [[.text("a")], [.text("b")], [.text("c")]] + let four = three + [[.text("d")]] + + #expect(bound.kept(three) == three) + #expect(bound.kept(four) == three) + } + + @Test("Without a byte limit a cursor fetches all the cap still allows, the row that tells 'more' included") + func fetchCountWithoutByteLimit() { + let bound = LibPQReadBound(rowCap: 3, byteLimit: nil) + + #expect(bound.nextFetchCount(rows: 0, bytes: 0) == 4) + #expect(bound.nextFetchCount(rows: 2, bytes: 9_999) == 2) + } + + @Test("With a byte limit a cursor fetches one row, then as many as the rows so far say still fit") + func fetchCountWithByteLimit() { + let bound = LibPQReadBound(rowCap: 1_000, byteLimit: 250) + + #expect(bound.nextFetchCount(rows: 0, bytes: 0) == 1) + #expect(bound.nextFetchCount(rows: 1, bytes: 101) == 2) + #expect(bound.nextFetchCount(rows: 2, bytes: 250) == 1) + } + + @Test("A fetch never passes the cap or the largest batch") + func fetchCountIsCapped() { + let roomy = LibPQReadBound(rowCap: 100_000, byteLimit: 1 << 30) + let nearTheCap = LibPQReadBound(rowCap: 3, byteLimit: 1 << 30) + + #expect(roomy.nextFetchCount(rows: 1, bytes: 1) == LibPQReadBound.largestFetch) + #expect(nearTheCap.nextFetchCount(rows: 2, bytes: 2) == 2) + } + + @Test("A row's size is its text in UTF-8 and its bytes, with nulls free") + func measuresDecodedSize() { + let row: [PluginCellValue] = [.text("é"), .bytes(Data([1, 2, 3])), .null, .text("ab")] + + #expect(LibPQReadBound.byteCount(of: row) == 2 + 3 + 0 + 2) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQReadOnlyTransactionTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQReadOnlyTransactionTests.swift index e3066bc12c..91786656ca 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQReadOnlyTransactionTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQReadOnlyTransactionTests.swift @@ -20,14 +20,20 @@ struct LibPQReadOnlyTransactionTests { private static let statement = "SELECT * FROM orders" - private static func context(readOnly: Bool, origin: PluginStatementOrigin) -> PluginStatementContext { + private static func context( + readOnly: Bool, + origin: PluginStatementOrigin, + enforcesReadOnly: Bool = false + ) -> PluginStatementContext { var context = PluginStatementContext() context.readOnly = readOnly context.origin = origin + context.enforcesReadOnly = enforcesReadOnly return context } private static let externalRead = context(readOnly: true, origin: .external) + private static let safeModeRead = context(readOnly: false, origin: .app, enforcesReadOnly: true) private static let config = DriverConnectionConfig(host: "", port: 0, username: "", password: "", database: "") @@ -35,7 +41,7 @@ struct LibPQReadOnlyTransactionTests { func wrapsTheStatement() throws { let wire = Wire() - let rows = try LibPQReadOnlyTransaction.run(state: .idle, send: wire.send) { + let rows = try LibPQReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) return 3 } @@ -53,7 +59,7 @@ struct LibPQReadOnlyTransactionTests { let wire = Wire() let error = #expect(throws: LibPQPluginError.self) { - try LibPQReadOnlyTransaction.run(state: state, send: wire.send) { + try LibPQReadOnlyTransaction.run(state: state, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } @@ -62,6 +68,24 @@ struct LibPQReadOnlyTransactionTests { #expect(error?.message == LibPQReadOnlyTransaction.sessionNotIdle.message) } + @Test( + "At Read-Only a session that is not idle is refused before anything is sent, and told to roll back", + arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction, .holdsSessionLocks, .unknown] + ) + func safeModeRefusesUnlessIdle(state: PluginSessionTransactionState) { + let wire = Wire() + + let error = #expect(throws: LibPQPluginError.self) { + try LibPQReadOnlyTransaction.run(state: state, reason: .safeMode, send: wire.send) { + try wire.send(Self.statement) + } + } + + #expect(wire.sent.isEmpty) + #expect(error?.message == LibPQReadOnlyTransaction.safeModeSessionNotIdle.message) + #expect(error?.message != LibPQReadOnlyTransaction.sessionNotIdle.message) + } + @Test("A write the read-only transaction refuses comes back as a read-only refusal, and the rollback still goes out", arguments: ["25006", "25001"]) func refusalIsReported(sqlState: String) { @@ -70,14 +94,33 @@ struct LibPQReadOnlyTransactionTests { wire.failing[Self.statement] = LibPQPluginError(message: serverMessage, sqlState: sqlState, detail: nil) let error = #expect(throws: LibPQPluginError.self) { - try LibPQReadOnlyTransaction.run(state: .idle, send: wire.send) { + try LibPQReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } #expect(wire.sent == ["START TRANSACTION READ ONLY; SELECT 1", Self.statement, "ROLLBACK"]) #expect(error?.sqlState == sqlState) - #expect(error?.message != serverMessage) + #expect(error?.message == LibPQReadOnlyTransaction.refusalMessage(.externalClients)) + #expect(error?.detail == serverMessage) + } + + @Test("A write refused at Read-Only names Safe Mode and keeps the server's text") + func safeModeRefusalNamesSafeMode() { + let wire = Wire() + let serverMessage = "ERROR: cannot execute DELETE in a read-only transaction" + wire.failing[Self.statement] = LibPQPluginError(message: serverMessage, sqlState: "25006", detail: nil) + + let error = #expect(throws: LibPQPluginError.self) { + try LibPQReadOnlyTransaction.run(state: .idle, reason: .safeMode, send: wire.send) { + try wire.send(Self.statement) + } + } + + #expect(wire.sent == ["START TRANSACTION READ ONLY; SELECT 1", Self.statement, "ROLLBACK"]) + #expect(error?.sqlState == "25006") + #expect(error?.message == LibPQReadOnlyTransaction.refusalMessage(.safeMode)) + #expect(error?.message != LibPQReadOnlyTransaction.refusalMessage(.externalClients)) #expect(error?.detail == serverMessage) } @@ -88,7 +131,7 @@ struct LibPQReadOnlyTransactionTests { wire.failing[Self.statement] = missing let error = #expect(throws: LibPQPluginError.self) { - try LibPQReadOnlyTransaction.run(state: .idle, send: wire.send) { + try LibPQReadOnlyTransaction.run(state: .idle, reason: .safeMode, send: wire.send) { try wire.send(Self.statement) } } @@ -103,7 +146,7 @@ struct LibPQReadOnlyTransactionTests { wire.failing[LibPQReadOnlyTransaction.begin] = LibPQPluginError(message: "refused", sqlState: "0A000", detail: nil) #expect(throws: LibPQPluginError.self) { - try LibPQReadOnlyTransaction.run(state: .idle, send: wire.send) { + try LibPQReadOnlyTransaction.run(state: .idle, reason: .externalClients, send: wire.send) { try wire.send(Self.statement) } } @@ -111,11 +154,47 @@ struct LibPQReadOnlyTransactionTests { #expect(!wire.sent.contains(Self.statement)) } - @Test("Only an external read on a driver that declares read-only statements is wrapped") - func wrapsOnlyExternalReads() { + @Test( + "At Read-Only a session that is not idle runs a statement with no unlisted call in its transaction as it is", + arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction, .holdsSessionLocks, .unknown] + ) + func runsInTheOpenTransaction(state: PluginSessionTransactionState) throws { + let wire = Wire() + var checked = Self.safeModeRead + checked.callsUnlistedFunction = false + let scope = try #require(LibPQReadOnlyTransaction.scope(for: checked, capabilities: .enforcesReadOnlyMode)) + + let rows = try LibPQReadOnlyTransaction.run(state: state, scope: scope, send: wire.send) { + try wire.send(Self.statement) + return 3 + } + + #expect(rows == 3) + #expect(wire.sent == [Self.statement]) + } + + @Test("Only a Read-Only statement the host found no unlisted call in may run in an open transaction") + func openTransactionNeedsTheCheck() { + var checked = Self.safeModeRead + checked.callsUnlistedFunction = false + var external = Self.externalRead + external.callsUnlistedFunction = false + let capabilities: PluginCapabilities = [.readOnlyStatements, .enforcesReadOnlyMode] + + #expect(LibPQReadOnlyTransaction.scope(for: checked, capabilities: capabilities)?.runsInOpenTransaction == true) + #expect( + LibPQReadOnlyTransaction.scope(for: Self.safeModeRead, capabilities: capabilities)?.runsInOpenTransaction + == false + ) + #expect(LibPQReadOnlyTransaction.scope(for: external, capabilities: capabilities)?.runsInOpenTransaction == false) + } + + @Test("An external read and an app read at Read-Only are wrapped; the app's own read-only marks are not") + func wrapsOnlyExternalAndSafeModeReads() { let capabilities = PostgreSQLPluginDriver(config: Self.config).capabilities - #expect(LibPQReadOnlyTransaction.applies(to: Self.externalRead, capabilities: capabilities)) + #expect(LibPQReadOnlyTransaction.reason(for: Self.externalRead, capabilities: capabilities) == .externalClients) + #expect(LibPQReadOnlyTransaction.reason(for: Self.safeModeRead, capabilities: capabilities) == .safeMode) #expect(!LibPQReadOnlyTransaction.applies(to: PluginStatementContext(), capabilities: capabilities)) #expect(!LibPQReadOnlyTransaction.applies( to: Self.context(readOnly: true, origin: .app), capabilities: capabilities @@ -125,7 +204,27 @@ struct LibPQReadOnlyTransactionTests { )) } - @Test("PostgreSQL, PGlite and CockroachDB declare read-only statements") + // `executeCatalogRead` marks its read `readOnly` from the app with `readsCatalog`; a wrap would drop the pin. + @Test("An app catalog read keeps its pin, because only Read-Only wraps an app read") + func catalogReadIsNotWrapped() { + let capabilities = PostgreSQLPluginDriver(config: Self.config).capabilities + var catalogRead = Self.context(readOnly: true, origin: .app) + catalogRead.readsCatalog = true + + #expect(LibPQReadOnlyTransaction.reason(for: catalogRead, capabilities: capabilities) == nil) + } + + @Test("An external read keeps the external clients' wording when Read-Only is also set") + func externalReasonWinsForAnExternalRead() { + let capabilities = PostgreSQLPluginDriver(config: Self.config).capabilities + let both = Self.context(readOnly: true, origin: .external, enforcesReadOnly: true) + let writeAtReadOnly = Self.context(readOnly: false, origin: .external, enforcesReadOnly: true) + + #expect(LibPQReadOnlyTransaction.reason(for: both, capabilities: capabilities) == .externalClients) + #expect(LibPQReadOnlyTransaction.reason(for: writeAtReadOnly, capabilities: capabilities) == .safeMode) + } + + @Test("PostgreSQL, PGlite and CockroachDB declare read-only statements and honor Read-Only") func declaringDrivers() { let drivers: [any PluginDatabaseDriver] = [ PostgreSQLPluginDriver(config: Self.config), @@ -135,16 +234,29 @@ struct LibPQReadOnlyTransactionTests { for driver in drivers { #expect(driver.capabilities.contains(.readOnlyStatements), "\(type(of: driver))") + #expect(driver.capabilities.contains(.enforcesReadOnlyMode), "\(type(of: driver))") #expect(LibPQReadOnlyTransaction.applies(to: Self.externalRead, capabilities: driver.capabilities)) + #expect(LibPQReadOnlyTransaction.applies(to: Self.safeModeRead, capabilities: driver.capabilities)) } } // Redshift documents READ ONLY, but nobody has measured whether it refuses a write there. - @Test("Redshift runs an external read as it always has") + @Test("Redshift runs an external read as it always has, and declares no Read-Only mode") func redshiftDoesNotWrap() { let redshift = RedshiftPluginDriver(config: Self.config) #expect(!redshift.capabilities.contains(.readOnlyStatements)) + #expect(!redshift.capabilities.contains(.enforcesReadOnlyMode)) #expect(!LibPQReadOnlyTransaction.applies(to: Self.externalRead, capabilities: redshift.capabilities)) + #expect(!LibPQReadOnlyTransaction.applies(to: Self.safeModeRead, capabilities: redshift.capabilities)) + } + + @Test("Redshift hands a bounded read at Read-Only back to the host instead of running it without the mode") + func redshiftReturnsBoundedReadAtReadOnly() async throws { + let redshift = RedshiftPluginDriver(config: Self.config) + + let result = try await redshift.executeBoundedQuery(query: Self.statement, rowCap: 10, context: Self.safeModeRead) + + #expect(result == nil) } } diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift index c20d11494d..4c5a1a3633 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift @@ -32,4 +32,33 @@ struct LibPQSSLMappingTests { func verifyIdentity() { #expect(LibPQSSLMapping.sslmode(for: .verifyIdentity) == "verify-full") } + + private static let missingRootMessage = """ + root certificate file "/Users/test/.postgresql/root.crt" does not exist + Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable \ + server certificate verification. + """ + + @Test("Verify CA with no CA file names the missing CA and the mode that needs none, never sslrootcert=system") + func verifyCAWithoutACAExplainsTheMissingCA() throws { + let error = try #require( + LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: SSLConfiguration(mode: .verifyCa)) + ) + + #expect(error.message.contains("Verify CA needs the server's CA certificate")) + #expect(error.message.contains("Verify Identity")) + #expect(!error.message.contains("sslrootcert")) + } + + @Test("libpq's text is kept for any other mode, for a chosen CA file, and for any other failure") + func otherFailuresKeepLibpqText() { + let otherModes = [SSLMode.verifyIdentity, .required, .preferred].map { SSLConfiguration(mode: $0) } + for sslConfig in otherModes { + #expect(LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: sslConfig) == nil) + } + let withCA = SSLConfiguration(mode: .verifyCa, caCertificatePath: "/ca.pem") + #expect(LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: withCA) == nil) + let refused = "connection to server at \"db\" failed: FATAL: password authentication failed for user \"app\"" + #expect(LibPQSSLMapping.missingRootCertificateError(refused, sslConfig: SSLConfiguration(mode: .verifyCa)) == nil) + } } diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift new file mode 100644 index 0000000000..976411bfdc --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift @@ -0,0 +1,485 @@ +// +// LibPQScriptedServer.swift +// PostgreSQLDriverTests +// + +import Darwin +import Foundation + +// A loopback PostgreSQL server for one plaintext session. It refuses SSL and GSS encryption, accepts any startup +// without a password, records each simple query and each statement the extended protocol parses, runs `onQuery` before +// it answers, and reports the transaction status `status` picks from the query and the ones before it. A simple query +// or a parsed statement gets what `answer` returns for it; without one, each statement of a simple query gets an empty +// completion, except that a statement holding `failingStatement` gets an error and the ones after it nothing. Cancel +// requests arrive on connections of their own: each is counted, and a statement waiting for one then fails as a +// cancelled statement does. A query answered with `copyIn` starts a copy-in: the server keeps every CopyData byte, and +// answers CopyDone with `COPY ` (or the scripted failure) and CopyFail with 57014, as PostgreSQL does, or +// takes no data until a cancel request arrives. +final class LibPQScriptedServer: @unchecked Sendable { + struct SetupFailure: Error {} + + struct Column: Sendable { + let name: String + var table: UInt32 = 0 + var attribute: Int16 = 0 + var type: UInt32 = 25 + } + + struct Failure: Sendable { + let sqlState: String + let message: String + } + + struct Answer: Sendable { + var columns: [Column] = [] + var rows: [[String?]] = [] + // After the rows, wait for a cancel request and fail with 57014. A statement nobody cancels completes. + var waitsForCancel = false + var failure: Failure? + var copyIn: CopyIn? + } + + struct CopyIn: Sendable { + var binary = false + // Sent after CopyDone in place of the completion, as a row the server refuses is reported. + var failure: Failure? + // Reads nothing until a cancel request arrives, then fails with 57014 and passes over the rest of the copy, + // as a server waiting for a row lock does. + var takesNoDataUntilCancelled = false + } + + static let idle = UInt8(ascii: "I") + static let inTransaction = UInt8(ascii: "T") + static let failedTransaction = UInt8(ascii: "E") + + private static let protocolVersion: UInt32 = 196_608 + private static let sslRequest: UInt32 = 80_877_103 + private static let gssEncryptionRequest: UInt32 = 80_877_104 + private static let cancelRequest: UInt32 = 80_877_102 + private static let pollMilliseconds: Int32 = 10_000 + private static let cancelWait: DispatchTimeInterval = .seconds(5) + private static let parameters: [(String, String)] = [ + ("server_version", "17.4"), + ("server_encoding", "UTF8"), + ("client_encoding", "UTF8"), + ("standard_conforming_strings", "on"), + ("DateStyle", "ISO, MDY"), + ("integer_datetimes", "on"), + ("TimeZone", "UTC"), + ] + + let port: Int + private let listener: Int32 + private let status: @Sendable (_ query: String, _ earlier: [String]) -> UInt8 + private let onQuery: @Sendable (String) -> Void + private let answer: @Sendable (String) -> Answer? + private let cancelArrived = DispatchSemaphore(value: 0) + private let failingStatement: String? + private let lock = NSLock() + private var received: [String] = [] + private var bindings: [[String?]] = [] + private var cancels = 0 + private var isClosed = false + private var copied = Data() + private var copyFailures: [String] = [] + + var queries: [String] { + lock.withLock { received } + } + + // Each Bind's parameters as text, nil for a null. + var boundParameters: [[String?]] { + lock.withLock { bindings } + } + + var cancelRequests: Int { + lock.withLock { cancels } + } + + var copyData: Data { + lock.withLock { copied } + } + + // Each CopyFail's message. + var copyFailMessages: [String] { + lock.withLock { copyFailures } + } + + init( + status: @escaping @Sendable (_ query: String, _ earlier: [String]) -> UInt8 = { _, _ in LibPQScriptedServer.idle }, + onQuery: @escaping @Sendable (String) -> Void = { _ in }, + answer: @escaping @Sendable (String) -> Answer? = { _ in nil }, + failingStatement: String? = nil + ) throws { + self.status = status + self.onQuery = onQuery + self.answer = answer + self.failingStatement = failingStatement + let socketFD = socket(AF_INET, SOCK_STREAM, 0) + guard socketFD >= 0 else { throw SetupFailure() } + var option: Int32 = 1 + setsockopt(socketFD, SOL_SOCKET, SO_REUSEADDR, &option, socklen_t(MemoryLayout.size)) + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_addr.s_addr = inet_addr("127.0.0.1") + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(socketFD, $0, socklen_t(MemoryLayout.size)) + } + } + var boundAddress = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &boundAddress) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(socketFD, $0, &length) } + } + guard bound == 0, Darwin.listen(socketFD, 4) == 0, named == 0 else { + Darwin.close(socketFD) + throw SetupFailure() + } + listener = socketFD + port = Int(boundAddress.sin_port.bigEndian) + } + + func start() { + DispatchQueue(label: "com.TablePro.tests.scripted-postgresql").async { self.runSession() } + } + + func close() { + lock.withLock { isClosed = true } + Darwin.close(listener) + } + + private struct Statement { + let query: String + let earlier: [String] + let answer: Answer + } + + private func runSession() { + guard let client = acceptClient() else { return } + defer { Darwin.close(client) } + DispatchQueue(label: "com.TablePro.tests.scripted-postgresql.cancel").async { self.acceptCancelRequests() } + guard startUp(client) else { return } + + var statement: Statement? + var failed = false + while let type = receive(client, 1)?.first, let body = receiveBody(client) { + var reply: [UInt8] = [] + switch type { + case UInt8(ascii: "Q"): + let query = Self.cString(body, from: 0).text + let earlier = record(query) + onQuery(query) + var simpleFailed = false + let scripted = answer(query) + if let copyIn = scripted?.copyIn { + guard copy(copyIn, on: client, failed: &simpleFailed) else { return } + } else if let scripted { + if scripted.failure == nil, !scripted.columns.isEmpty { + guard send(client, Self.rowDescription(scripted.columns)) else { return } + } + guard execute(scripted, on: client, failed: &simpleFailed) else { return } + } else { + reply = answers(to: query) + } + var state = status(query, earlier) + if simpleFailed, state == Self.inTransaction { state = Self.failedTransaction } + reply += Self.message("Z", [state]) + case UInt8(ascii: "P"): + let query = Self.cString(body, from: Self.cString(body, from: 0).next).text + let earlier = record(query) + onQuery(query) + statement = Statement(query: query, earlier: earlier, answer: answer(query) ?? Answer()) + if !failed { reply = Self.message("1", []) } + case UInt8(ascii: "B"): + lock.withLock { bindings.append(Self.parameters(ofBind: body)) } + if !failed { reply = Self.message("2", []) } + case UInt8(ascii: "D"): + guard !failed, let statement else { break } + let columns = statement.answer.failure == nil ? statement.answer.columns : [] + reply = columns.isEmpty ? Self.message("n", []) : Self.rowDescription(columns) + case UInt8(ascii: "E"): + guard !failed, let statement else { break } + guard execute(statement.answer, on: client, failed: &failed) else { return } + case UInt8(ascii: "S"): + var state = statement.map { status($0.query, $0.earlier) } ?? Self.idle + if failed, state == Self.inTransaction { state = Self.failedTransaction } + reply = Self.message("Z", [state]) + statement = nil + failed = false + case UInt8(ascii: "H"): + break + case UInt8(ascii: "C"): + if !failed { reply = Self.message("3", []) } + default: + return + } + guard reply.isEmpty || send(client, reply) else { return } + } + } + + private func record(_ query: String) -> [String] { + lock.withLock { () -> [String] in + defer { received.append(query) } + return received + } + } + + // The rows go out before any wait, as a server streams them, so the client can stop and cancel while it waits. + private func execute(_ answer: Answer, on client: Int32, failed: inout Bool) -> Bool { + if let failure = answer.failure { + failed = true + return send(client, Self.errorResponse(failure)) + } + guard send(client, answer.rows.flatMap(Self.dataRow)) else { return false } + if answer.waitsForCancel, cancelArrived.wait(timeout: .now() + Self.cancelWait) == .success { + failed = true + return send(client, Self.errorResponse(Failure( + sqlState: "57014", message: "canceling statement due to user request" + ))) + } + return send(client, Self.message("C", Array("SELECT \(answer.rows.count)".utf8) + [0])) + } + + // CopyInResponse with every column in the copy's format, then CopyData until CopyDone or CopyFail. + private func copy(_ copyIn: CopyIn, on client: Int32, failed: inout Bool) -> Bool { + let format: UInt8 = copyIn.binary ? 1 : 0 + guard send(client, Self.message("G", [format] + Self.bytes(UInt16(1)) + Self.bytes(UInt16(format)))) else { + return false + } + if copyIn.takesNoDataUntilCancelled { + guard cancelArrived.wait(timeout: .now() + Self.cancelWait) == .success else { return false } + failed = true + guard send(client, Self.errorResponse(Failure( + sqlState: "57014", message: "canceling statement due to user request" + ))) else { return false } + while let type = receive(client, 1)?.first, let body = receiveBody(client) { + if type == UInt8(ascii: "f") { lock.withLock { copyFailures.append(Self.cString(body, from: 0).text) } } + if type == UInt8(ascii: "c") || type == UInt8(ascii: "f") { return true } + } + return false + } + var lines = 0 + while let type = receive(client, 1)?.first, let body = receiveBody(client) { + switch type { + case UInt8(ascii: "d"): + lines += body.filter { $0 == UInt8(ascii: "\n") }.count + lock.withLock { copied.append(contentsOf: body) } + case UInt8(ascii: "c"): + if let failure = copyIn.failure { + failed = true + return send(client, Self.errorResponse(failure)) + } + return send(client, Self.message("C", Array("COPY \(lines)".utf8) + [0])) + case UInt8(ascii: "f"): + let reason = Self.cString(body, from: 0).text + lock.withLock { copyFailures.append(reason) } + failed = true + return send(client, Self.errorResponse(Failure( + sqlState: "57014", message: "COPY from stdin failed: \(reason)" + ))) + default: + return false + } + } + return false + } + + private func acceptCancelRequests() { + while !lock.withLock({ isClosed }) { + var pollFD = pollfd(fd: listener, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, 100) >= 0 else { return } + guard pollFD.revents & Int16(POLLIN) != 0, !lock.withLock({ isClosed }) else { continue } + let client = accept(listener, nil, nil) + guard client >= 0 else { continue } + if let packet = receiveBody(client), packet.count == 12, Self.uint32(packet.prefix(4)) == Self.cancelRequest { + lock.withLock { cancels += 1 } + cancelArrived.signal() + } + Darwin.close(client) + } + } + + private func answers(to query: String) -> [UInt8] { + let statements = Self.statements(in: query) + var reply: [UInt8] = [] + for statement in statements.isEmpty ? [query] : statements { + if let failingStatement, statement.contains(failingStatement) { + return reply + Self.errorResponse(Failure(sqlState: "42501", message: "permission denied")) + } + reply += Self.message("C", Array("SELECT 0".utf8) + [0]) + } + return reply + } + + // Splits on each `;` outside a quoted literal or identifier, which is enough for the statements the driver sends. + private static func statements(in query: String) -> [String] { + var statements: [String] = [] + var current = "" + var quote: Character? + for character in query { + if character == ";", quote == nil { + statements.append(current) + current = "" + continue + } + if character == quote { + quote = nil + } else if quote == nil, character == "'" || character == "\"" { + quote = character + } + current.append(character) + } + statements.append(current) + return statements + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + } + + private func startUp(_ client: Int32) -> Bool { + while let packet = receiveBody(client), packet.count >= 4 { + let code = Self.uint32(packet.prefix(4)) + if code == Self.sslRequest || code == Self.gssEncryptionRequest { + guard send(client, [UInt8(ascii: "N")]) else { return false } + continue + } + guard code == Self.protocolVersion else { return false } + var reply = Self.message("R", Self.bytes(UInt32(0))) + for (name, value) in Self.parameters { + reply += Self.message("S", Array(name.utf8) + [0] + Array(value.utf8) + [0]) + } + reply += Self.message("K", Self.bytes(UInt32(4_242)) + Self.bytes(UInt32(7_777))) + reply += Self.message("Z", [Self.idle]) + return send(client, reply) + } + return false + } + + private static func rowDescription(_ columns: [Column]) -> [UInt8] { + var body = bytes(UInt16(columns.count)) + for column in columns { + body += Array(column.name.utf8) + [0] + body += bytes(column.table) + bytes(UInt16(bitPattern: column.attribute)) + bytes(column.type) + body += bytes(UInt16(bitPattern: -1)) + bytes(UInt32(bitPattern: -1)) + bytes(UInt16(0)) + } + return message("T", body) + } + + private static func dataRow(_ row: [String?]) -> [UInt8] { + var body = bytes(UInt16(row.count)) + for value in row { + guard let value else { + body += bytes(UInt32(bitPattern: -1)) + continue + } + body += bytes(UInt32(value.utf8.count)) + Array(value.utf8) + } + return message("D", body) + } + + private static func errorResponse(_ failure: Failure) -> [UInt8] { + let fields: [(UInt8, String)] = [ + (UInt8(ascii: "S"), "ERROR"), + (UInt8(ascii: "V"), "ERROR"), + (UInt8(ascii: "C"), failure.sqlState), + (UInt8(ascii: "M"), failure.message), + ] + var body: [UInt8] = [] + for (code, value) in fields { + body += [code] + Array(value.utf8) + [0] + } + return message("E", body + [0]) + } + + // Portal and statement names, the format codes, then each parameter as a length (-1 for null) and its bytes. + private static func parameters(ofBind body: [UInt8]) -> [String?] { + let afterPortal = cString(body, from: 0).next + var offset = cString(body, from: afterPortal).next + guard let formats = int16(body, at: offset) else { return [] } + offset += 2 + 2 * Int(formats) + guard let count = int16(body, at: offset) else { return [] } + offset += 2 + var values: [String?] = [] + for _ in 0..= 0, offset + Int(length) <= body.count else { + values.append(nil) + continue + } + values.append(String(bytes: body[offset.. (text: String, next: Int) { + guard start < body.count else { return ("", body.count) } + let end = body[start...].firstIndex(of: 0) ?? body.count + return (String(bytes: body[start.. Int16? { + guard offset + 2 <= body.count else { return nil } + return Int16(bitPattern: UInt16(body[offset]) << 8 | UInt16(body[offset + 1])) + } + + private static func message(_ type: Character, _ body: [UInt8]) -> [UInt8] { + [type.asciiValue ?? 0] + bytes(UInt32(body.count + 4)) + body + } + + + private static func bytes(_ value: UInt32) -> [UInt8] { + [UInt8((value >> 24) & 0xFF), UInt8((value >> 16) & 0xFF), UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + private static func bytes(_ value: UInt16) -> [UInt8] { + [UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + private static func uint32(_ bytes: C) -> UInt32 where C.Element == UInt8 { + bytes.reduce(0) { $0 << 8 | UInt32($1) } + } + + // A length that counts its own four bytes, then that many minus four. + private func receiveBody(_ client: Int32) -> [UInt8]? { + guard let header = receive(client, 4) else { return nil } + let length = Int(Self.uint32(header)) + guard length >= 4 else { return nil } + return length == 4 ? [] : receive(client, length - 4) + } + + private func acceptClient() -> Int32? { + var pollFD = pollfd(fd: listener, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, Self.pollMilliseconds) > 0 else { return nil } + let client = accept(listener, nil, nil) + guard client >= 0 else { return nil } + var noSignal: Int32 = 1 + setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSignal, socklen_t(MemoryLayout.size)) + return client + } + + private func receive(_ client: Int32, _ count: Int) -> [UInt8]? { + var bytes: [UInt8] = [] + while bytes.count < count { + var pollFD = pollfd(fd: client, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, Self.pollMilliseconds) > 0 else { return nil } + var chunk = [UInt8](repeating: 0, count: count - bytes.count) + let read = recv(client, &chunk, chunk.count, 0) + guard read > 0 else { return nil } + bytes += chunk.prefix(read) + } + return bytes + } + + private func send(_ client: Int32, _ bytes: [UInt8]) -> Bool { + var offset = 0 + while offset < bytes.count { + let sent = bytes[offset...].withUnsafeBytes { Darwin.send(client, $0.baseAddress, $0.count, 0) } + guard sent > 0 else { return false } + offset += sent + } + return true + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQStopBeforeStatementTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQStopBeforeStatementTests.swift new file mode 100644 index 0000000000..8124485cb3 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQStopBeforeStatementTests.swift @@ -0,0 +1,122 @@ +// +// LibPQStopBeforeStatementTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// A Stop that arrives while the driver runs a round trip of its own (the savepoint probe, the cursor read's savepoint) +// is spent on that round trip, so the user's statement must not go out after it. +@Suite(.timeLimit(.minutes(1))) +struct LibPQStopBeforeStatementTests { + private final class ConnectionBox: @unchecked Sendable { + private let lock = NSLock() + private var held: LibPQPluginConnection? + + var connection: LibPQPluginConnection? { + get { lock.withLock { held } } + set { lock.withLock { held = newValue } } + } + + func stop() { + _ = connection?.cancellationGate.cancel() + } + } + + private static func connected(to server: LibPQScriptedServer, box: ConnectionBox) async throws -> LibPQPluginConnection { + let connection = LibPQPluginConnection( + host: "127.0.0.1", port: server.port, user: "postgres", password: nil, database: "postgres" + ) + box.connection = connection + try await connection.connect() + try await connection.finishConnecting() + return connection + } + + @Test("A Stop during the savepoint probe sends none of the user's statement and reports cancelled") + func stopDuringProbeSendsNothing() async throws { + let box = ConnectionBox() + let server = try LibPQScriptedServer(onQuery: { query in + if query.contains("tablepro_probe") { box.stop() } + }) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, box: box) + defer { connection.disconnect() } + + var failure: Error? + do { + _ = try await connection.boundedQuery("DELETE FROM big WHERE flagged", rowCap: 1_000) + } catch { + failure = error + } + + let sent = server.queries + let probed = sent.contains { $0.contains("tablepro_probe") } + let deleteSent = sent.contains { $0.contains("DELETE FROM big") } + #expect(failure is CancellationError) + #expect(probed) + #expect(!deleteSent) + } + + @Test("A Stop during the cursor read's savepoint sends no cursor and reports cancelled") + func stopDuringCursorSavepointSendsNothing() async throws { + let box = ConnectionBox() + let server = try LibPQScriptedServer( + status: { query, earlier in + (earlier + [query]).contains("BEGIN") ? LibPQScriptedServer.inTransaction : LibPQScriptedServer.idle + }, + onQuery: { query in + if query.hasPrefix("SAVEPOINT tablepro_capped_read") { box.stop() } + } + ) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, box: box) + defer { connection.disconnect() } + + _ = try await connection.boundedQuery("SELECT 1", rowCap: 10) + _ = try await connection.executeQuery("BEGIN") + var failure: Error? + do { + _ = try await connection.boundedQuery("SELECT * FROM events", rowCap: 1_000) + } catch { + failure = error + } + + let sent = server.queries + let savepointSent = sent.contains { $0.hasPrefix("SAVEPOINT tablepro_capped_read") } + let readSent = sent.contains { $0.contains("DECLARE") || $0.contains("FROM events") } + #expect(failure is CancellationError) + #expect(savepointSent) + #expect(!readSent) + } + + @Test("A Stop during a stream's savepoint probe sends none of the user's statement and ends cancelled") + func stopDuringStreamProbeSendsNothing() async throws { + let box = ConnectionBox() + let server = try LibPQScriptedServer(onQuery: { query in + if query.contains("tablepro_probe") { box.stop() } + }) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server, box: box) + defer { connection.disconnect() } + + var failure: Error? + do { + for try await _ in connection.streamQuery("SELECT * FROM events") {} + } catch { + failure = error + } + + let sent = server.queries + let probed = sent.contains { $0.contains("tablepro_probe") } + let readSent = sent.contains { $0.contains("FROM events") } + #expect(failure is CancellationError) + #expect(probed) + #expect(!readSent) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLConnectRoundTripTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLConnectRoundTripTests.swift new file mode 100644 index 0000000000..a0a4981176 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLConnectRoundTripTests.swift @@ -0,0 +1,69 @@ +// +// PostgreSQLConnectRoundTripTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct PostgreSQLConnectRoundTripTests { + private static let probes = [ + PostgreSQLSessionFacts.probeQuery, + PostgreSQLCatalogPresence.probeQuery, + PostGISTypes.probeQuery, + PostgreSQLSchemaQueries.enumTypeOidQuery, + PgvectorTypes.probeQuery + ] + + private static func connectedDriver(to server: LibPQScriptedServer) async throws -> PostgreSQLPluginDriver { + let driver = PostgreSQLPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: server.port, username: "postgres", password: "", database: "postgres" + )) + try await driver.connect() + return driver + } + + private static func probeCount(in query: String) -> Int { + probes.filter { query.contains($0) }.count + } + + @Test("Connect sends the five catalog probes in one Query and no string setting the server reports on") + func probesShareOneQuery() async throws { + let server = try LibPQScriptedServer() + defer { server.close() } + server.start() + let driver = try await Self.connectedDriver(to: server) + defer { driver.disconnect() } + + let sent = server.queries + let carriers = sent.filter { Self.probeCount(in: $0) > 0 } + let batch = try #require(carriers.first) + let settingSent = sent.contains { $0.contains(LibPQStringConformance.parameterName) } + #expect(carriers.count == 1) + #expect(Self.probeCount(in: batch) == Self.probes.count) + #expect(batch.hasPrefix(PostgreSQLCatalogSearchPath.pinned)) + #expect(!settingSent) + } + + @Test("A probe the server refuses costs no other: each probe after it goes again in its own Query") + func failedProbeLeavesTheRestSent() async throws { + let server = try LibPQScriptedServer(failingStatement: PostGISTypes.probeQuery) + defer { server.close() } + server.start() + let driver = try await Self.connectedDriver(to: server) + defer { driver.disconnect() } + + let sent = server.queries + let found = sent.firstIndex { Self.probeCount(in: $0) == Self.probes.count } + let batchIndex = try #require(found) + let resent = sent[(batchIndex + 1)...].filter { Self.probeCount(in: $0) > 0 } + let eachAlone = resent.allSatisfy { Self.probeCount(in: $0) == 1 } + #expect(resent.count == 2) + #expect(eachAlone) + #expect(resent.first?.contains(PostgreSQLSchemaQueries.enumTypeOidQuery) == true) + #expect(resent.last?.contains(PgvectorTypes.probeQuery) == true) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLExtensionQueryTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLExtensionQueryTests.swift new file mode 100644 index 0000000000..35fd9ff50e --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLExtensionQueryTests.swift @@ -0,0 +1,123 @@ +// +// PostgreSQLExtensionQueryTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore +import Testing + +struct PostgreSQLExtensionQueryTests { + private func row(_ cells: String?...) -> [PluginCellValue] { + cells.map(PluginCellValue.fromOptional) + } + + private func attribute(_ object: PluginServerObjectInfo?, _ label: String) -> String? { + object?.attributes.first { $0.label == label }?.value + } + + private static let constructsMissingBefore91 = [ + "to_regclass", "LATERAL", "WITH ORDINALITY", "json", "array_remove", "array_position", "FILTER (", "unnest(" + ] + + @Test("The extension reads stay inside what PostgreSQL 9.1 parses") + func readsArePortable() { + let queries = [ + PostgreSQLExtensionQueries.list, + PostgreSQLExtensionQueries.one(oid: 16_384), + PostgreSQLExtensionQueries.defaultVersion(name: "pgcrypto") + ] + for sql in queries { + #expect(Self.constructsMissingBefore91.filter { sql.contains($0) }.isEmpty, "\(sql)") + } + } + + @Test("The listing reads every extension once, by name, without the control files") + func listingIsOneCatalogRead() { + let sql = PostgreSQLExtensionQueries.list + #expect(sql.contains("FROM pg_catalog.pg_extension e")) + #expect(sql.contains("JOIN pg_catalog.pg_namespace n ON n.oid = e.extnamespace")) + #expect(sql.hasSuffix("ORDER BY e.extname")) + #expect(!sql.contains("pg_available_extensions")) + #expect(!sql.contains("WHERE")) + } + + @Test("One extension is read again by its oid alone") + func rereadIsByOid() { + let sql = PostgreSQLExtensionQueries.one(oid: 16_384) + #expect(sql.hasSuffix("WHERE e.oid = 16384")) + #expect(!sql.contains("extname =")) + } + + @Test("The default version read quotes its name as a literal") + func defaultVersionQuotesTheName() { + #expect(PostgreSQLExtensionQueries.defaultVersion(name: "o'brien").contains("WHERE name = 'o''brien'")) + } + + @Test("A plain extension's definition is pg_dump's, with no version") + func plainDefinition() { + #expect(PostgreSQLExtensionQueries.definition(name: "pgcrypto", schema: "public", comment: nil) + == #"CREATE EXTENSION IF NOT EXISTS "pgcrypto" WITH SCHEMA "public";"#) + } + + @Test("A name or schema that needs quoting is quoted as an identifier") + func quotedDefinition() { + #expect(PostgreSQLExtensionQueries.definition(name: "uuid-ossp", schema: "My Schema", comment: "") + == #"CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA "My Schema";"#) + } + + @Test("A comment follows as COMMENT ON EXTENSION, its quote doubled") + func commentWithAQuote() { + let text = PostgreSQLExtensionQueries.definition(name: "hstore", schema: "public", comment: "data type for key/value's") + #expect(text.hasPrefix(#"CREATE EXTENSION IF NOT EXISTS "hstore" WITH SCHEMA "public";"#)) + #expect(text.hasSuffix(#"COMMENT ON EXTENSION "hstore" IS 'data type for key/value''s';"#)) + } + + @Test("A listed row carries its oid, version and definition") + func rowMapping() { + let object = PostgreSQLExtensionQueries.object( + from: row("16384", "pgcrypto", "public", "1.3", "true", "postgres", "cryptographic functions") + ) + + #expect(object?.kindId == "extension") + #expect(object?.identity == "16384") + #expect(object?.name == "pgcrypto") + #expect(object?.schema == "public") + #expect(object?.detail == "1.3") + #expect(object?.isEnabled == nil) + #expect(attribute(object, String(localized: "Version")) == "1.3") + #expect(attribute(object, String(localized: "Owner")) == "postgres") + #expect(attribute(object, String(localized: "Relocatable")) == String(localized: "Yes")) + #expect(attribute(object, String(localized: "Default Version")) == nil) + #expect(object?.definition?.contains("COMMENT ON EXTENSION \"pgcrypto\"") == true) + } + + @Test("The re-read adds the default version, and a non-relocatable extension says No") + func rereadRowMapping() { + let object = PostgreSQLExtensionQueries.object( + from: row("13000", "plpgsql", "pg_catalog", "1.0", "f", "postgres", nil), + defaultVersion: "1.0" + ) + + #expect(attribute(object, String(localized: "Default Version")) == "1.0") + #expect(attribute(object, String(localized: "Relocatable")) == String(localized: "No")) + #expect(attribute(object, String(localized: "Comment")) == nil) + #expect(object?.definition == #"CREATE EXTENSION IF NOT EXISTS "plpgsql" WITH SCHEMA "pg_catalog";"#) + } + + @Test("A row without an oid is not an extension this driver can read again") + func rowWithoutOidIsDropped() { + #expect(PostgreSQLExtensionQueries.object(from: row(nil, "pgcrypto", "public", "1.3", "t", nil, nil)) == nil) + #expect(PostgreSQLExtensionQueries.object(from: row("pgcrypto", "pgcrypto", "public", "1.3", "t", nil, nil)) == nil) + } + + @Test("Extensions are declared from 9.1, at database scope") + func declaredFromNinePointOne() { + #expect(!PostgreSQLExtensions.isDeclared(capabilities: PostgreSQLCapabilities(serverVersion: 90_024))) + #expect(PostgreSQLExtensions.isDeclared(capabilities: PostgreSQLCapabilities(serverVersion: 90_100))) + #expect(PostgreSQLExtensions.isDeclared(capabilities: .assumingModernWhenUnknown(0))) + #expect(PostgreSQLExtensions.kind.scope == .database) + #expect(PostgreSQLExtensions.kind.id == "extension") + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift index 2664e5b315..9556e9c2b3 100644 --- a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift @@ -125,7 +125,15 @@ struct PostgreSQLTableDDLTests { let hostile = "o'rders" let statements = [ PostgreSQLTableDDL.shapeQuery(schema: hostile, table: hostile, capabilities: Self.modern), - PostgreSQLTableDDL.partitionTreeQuery(schema: hostile, table: hostile) + PostgreSQLTableDDL.partitionTreeQuery(schema: hostile, table: hostile), + PostgreSQLTableDDL.partitionElementsQuery( + schema: hostile, table: hostile, scope: .below, capabilities: Self.modern + ), + PostgreSQLTableDDL.partitionElementsQuery( + schema: hostile, table: hostile, scope: .itself, capabilities: Self.modern + ), + PostgreSQLTableDDL.partitionIndexesQuery(schema: hostile, table: hostile), + PostgreSQLTableDDL.partitionCommentsQuery(schema: hostile, table: hostile) ] for sql in statements { #expect(PostgreSQLCatalogQualificationTests.unqualifiedCalls(in: sql).isEmpty, "\(sql)") @@ -134,12 +142,114 @@ struct PostgreSQLTableDDLTests { } } - @Test("The partition tree lists a parent before its own partitions") + @Test("The partition tree lists a parent before its own partitions, foreign ones included") func partitionTreeOrder() { let sql = PostgreSQLTableDDL.partitionTreeQuery(schema: "public", table: "events") #expect(sql.contains("WITH RECURSIVE")) #expect(sql.contains("ORDER BY parts.depth")) - #expect(sql.contains("c.relkind IN ('r', 'p')")) + #expect(sql.contains("c.relkind IN ('r', 'p', 'f')")) + } + + @Test("A foreign table partition's comments are left out with it") + func foreignPartitionCommentsAreLeftOut() { + let sql = PostgreSQLTableDDL.partitionCommentsQuery(schema: "public", table: "events") + #expect(sql.components(separatedBy: "WHERE c.relkind <> 'f'").count == 3) + } + + // Each statement below was replayed on PostgreSQL 18 into an empty database and recreated the partition as read. + @Test("A partition's own defaults, NOT NULLs and constraints go in its element list") + func partitionElements() throws { + var partition = try #require(PostgreSQLTableDDL.partition( + from: ["public.events_2025", "public.events", "FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')", "HASH (id)", "p"] + )) + partition.elements = PostgreSQLTableDDL.elementsByPartition([ + ["public.events_2025", "user_id WITH OPTIONS NOT NULL"], + ["public.events_2025", "note WITH OPTIONS DEFAULT 'y'::text"], + ["public.events_2024", "CONSTRAINT ck_2024 CHECK (user_id > 0)"] + ])["public.events_2025"] ?? [] + #expect(PostgreSQLTableDDL.createPartition(partition) == """ + CREATE TABLE public.events_2025 PARTITION OF public.events ( + user_id WITH OPTIONS NOT NULL, + note WITH OPTIONS DEFAULT 'y'::text + ) FOR VALUES FROM ('2025-01-01') TO ('2026-01-01') PARTITION BY HASH (id); + """) + } + + @Test("A partition asked for by name keeps its own element list") + func partitionByNameKeepsElements() { + var shape = PostgreSQLTableDDL.shape(from: ["p", nil, "FOR VALUES FROM (1) TO (100)", "public.events"]) + shape.partitionElements = ["CONSTRAINT ck CHECK (id > 0)"] + #expect(PostgreSQLTableDDL.createTable(name: "public.events_1", definitions: Self.columns, shape: shape) + == "CREATE TABLE public.events_1 PARTITION OF public.events (\n CONSTRAINT ck CHECK (id > 0)\n) FOR VALUES FROM (1) TO (100);") + } + + // The export reads a foreign partition's rows through the parent; replayed into it, they would go to the server. + @Test("A foreign table partition is written as a comment that names its server and options") + func foreignPartition() throws { + var partition = try #require(PostgreSQLTableDDL.partition(from: [ + "public.logs_old", "public.logs", "FOR VALUES FROM (MINVALUE) TO ('2025-01-01')", nil, "p", + "remote_srv", "schema_name 'public', table_name 'old''logs'" + ])) + partition.elements = ["at WITH OPTIONS DEFAULT '2000-01-01'::date"] + let ddl = PostgreSQLTableDDL.createPartition(partition) + #expect(ddl == """ + -- public.logs_old is a foreign table partition on server remote_srv, left out so a restore writes nothing + -- to that server. Rows exported for it have no partition until it is created: + -- CREATE FOREIGN TABLE public.logs_old PARTITION OF public.logs ( + -- at WITH OPTIONS DEFAULT '2000-01-01'::date + -- ) FOR VALUES FROM (MINVALUE) TO ('2025-01-01') SERVER remote_srv OPTIONS (schema_name 'public', table_name 'old''logs'); + """) + #expect(ddl.split(separator: "\n").allSatisfy { $0.hasPrefix("-- ") }) + + let bare = try #require(PostgreSQLTableDDL.partition( + from: ["public.l", "public.logs", "DEFAULT", nil, "p", "\"Srv\"", nil] + )) + #expect(PostgreSQLTableDDL.createPartition(bare).hasSuffix( + "-- CREATE FOREIGN TABLE public.l PARTITION OF public.logs DEFAULT SERVER \"Srv\";" + )) + + let hostile = try #require(PostgreSQLTableDDL.partition( + from: ["public.\"a\rDROP TABLE t; --\"", "public.logs", "DEFAULT", nil, "p", "s", nil] + )) + let lines = PostgreSQLTableDDL.createPartition(hostile).components(separatedBy: .newlines) + #expect(lines.allSatisfy { $0.hasPrefix("-- ") }) + } + + @Test("A partition's own indexes and comments follow every partition") + func partitionIndexesAndComments() { + let shape = PostgreSQLTableDDL.shape(from: ["p", "RANGE (created_at)", nil, nil]) + let partitions = [ + ["public.events_2024", "public.events", "FOR VALUES FROM ('2024-01-01') TO ('2025-01-01')", nil, "p"] + ].compactMap { PostgreSQLTableDDL.partition(from: $0) } + let comments = PostgreSQLTableDDL.partitionComments([ + ["public.events_2024", "r", nil, "year 2024"], + ["public.events_2024", "r", "note", "it's a note"], + [nil, "r", nil, "dropped"] + ]) + + let ddl = PostgreSQLTableDDL.script( + name: "\"public\".\"events\"", + definitions: Self.columns, + shape: shape, + partitions: partitions, + partitionIndexes: [ + "CREATE INDEX events_2024_user_id_idx ON public.events_2024 USING btree (user_id)", + "CREATE INDEX events_2025_user_id_idx ON ONLY public.events_2025 USING btree (user_id)" + ], + partitionComments: comments + ) + + #expect(ddl.hasSuffix(""" + CREATE TABLE public.events_2024 PARTITION OF public.events FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); + + CREATE INDEX events_2024_user_id_idx ON public.events_2024 USING btree (user_id); + + CREATE INDEX events_2025_user_id_idx ON public.events_2025 USING btree (user_id); + + COMMENT ON TABLE public.events_2024 IS 'year 2024'; + + COMMENT ON COLUMN public.events_2024."note" IS 'it''s a note'; + """)) } @Test("An index over a partitioned table is written so the server builds it on every partition") diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift index 7b6a90473d..a273e1d873 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPlugin.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProQdrantClient import TableProQdrantCore final class QdrantPlugin: NSObject, TableProPlugin, DriverPlugin, PluginDefaultSortProvider { diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift index 4bf1ae92c0..acb8479308 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Execution.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProQdrantClient import TableProQdrantCore extension QdrantPluginDriver { @@ -150,7 +151,7 @@ extension QdrantPluginDriver { // The protocol's default `streamRows` runs `execute` once and buffers the whole collection. func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { if let name = QdrantOperations.decodeExport(query) { @@ -169,7 +170,7 @@ extension QdrantPluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -192,7 +193,7 @@ extension QdrantPluginDriver { estimatedRowCount: collection.pointsCount ))) for try await points in client.pointPages(collection: name, pageSize: Self.exportPageSize) { - continuation.yield(.rows(points.map(cells(columns: columns, typeNames: typeNames)))) + _ = await PluginRowStream.deliver(.rows(points.map(cells(columns: columns, typeNames: typeNames))), to: continuation) } } } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index ca822ad6e3..f5a5767f0d 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProQdrantClient import TableProQdrantCore extension QdrantPluginDriver { @@ -184,11 +185,16 @@ extension QdrantPluginDriver { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex) else { return nil } + // A cell left to the default has no text, so the point leaves it out and a defaulted id gets a new one. let cells: [QdrantCell] if let row = insertedRowData[change.rowIndex] { - cells = zip(columns, row).map { QdrantCell(column: $0, text: $1.asText) } + cells = zip(columns, row).enumerated().map { index, cell in + QdrantCell(column: cell.0, text: change.isColumnDefault(index) ? nil : cell.1.asText) + } } else { - cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } + cells = change.cellChanges.map { + QdrantCell(column: $0.columnName, text: change.isColumnDefault($0.columnIndex) ? nil : $0.newValue.asText) + } } return QdrantRowChange(kind: .insert, rowIndex: change.rowIndex, id: nil, cells: cells) case .update: diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift index fe11918a98..5fa6ecd3ce 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -1,6 +1,7 @@ import Foundation import os import TableProPluginKit +import TableProQdrantClient import TableProQdrantCore internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendable { @@ -21,7 +22,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift index 1494bb383a..0f54ba2843 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantURLSessionTransport.swift @@ -1,6 +1,7 @@ import Foundation import Security import TableProPluginKit +import TableProQdrantClient import TableProQdrantCore import TableProTLSClientIdentity diff --git a/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift b/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift index db79b92cfa..dae6c7f971 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift @@ -228,7 +228,7 @@ enum RedisScanReply { let cursor: String switch parts[0] { case .string(let value), .status(let value): cursor = value - case .data(let value): cursor = String(data: value, encoding: .utf8) ?? "0" + case .data(let value): cursor = RedisWireText.decode(value) ?? "0" case .integer(let value): cursor = String(value) default: cursor = "0" } @@ -236,7 +236,7 @@ enum RedisScanReply { let keys = keyReplies.compactMap { item -> String? in switch item { case .string(let key), .status(let key): return key - case .data(let value): return String(data: value, encoding: .utf8) + case .data(let value): return RedisWireText.decode(value) default: return nil } } diff --git a/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift b/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift index 4890870559..fe99890600 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift @@ -30,6 +30,7 @@ struct RedisKeyContents { let kind: RedisKeyKind let length: Int? let preview: RedisReply? + var stringByteLimit: Int? var lengthCell: PluginCellValue { .fromOptional(length.map(String.init)) @@ -48,11 +49,12 @@ extension RedisCommandChannel { } } - /// A key whose type is unknown gets no probe at all, because the length and preview commands - /// are chosen by type. + // A key whose type is unknown gets no probe at all, because the length and preview commands + // are chosen by type. func readContents( of keys: [String], - describedAs descriptions: [RedisKeyDescription] + describedAs descriptions: [RedisKeyDescription], + stringByteLimit: Int? ) async throws -> [RedisKeyContents?] { let kinds = zip(keys, descriptions).map { (key: $0, kind: $1.kind) } var commands: [[String]] = [] @@ -60,7 +62,7 @@ extension RedisCommandChannel { for entry in kinds { guard let kind = entry.kind else { continue } commands.append(RedisKeySummary.lengthCommand(for: kind, key: entry.key)) - commands.append(RedisKeySummary.previewCommand(for: kind, key: entry.key)) + commands.append(RedisKeySummary.previewCommand(for: kind, key: entry.key, stringByteLimit: stringByteLimit)) } let answers = try await runMetadataReads(commands) @@ -72,7 +74,12 @@ extension RedisCommandChannel { contents.append(nil) continue } - contents.append(RedisKeyContents(kind: kind, length: answers[next]?.intValue, preview: answers[next + 1])) + contents.append(RedisKeyContents( + kind: kind, + length: answers[next]?.intValue, + preview: answers[next + 1], + stringByteLimit: stringByteLimit + )) next += 2 } return contents diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift index ac7b9cbaa8..41964cadbb 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift @@ -942,7 +942,7 @@ enum HiredisReply { switch reply.pointee.type { case REDIS_REPLY_STRING, REDIS_REPLY_BIGNUM, REDIS_REPLY_VERB: guard let data = bytes(of: reply) else { return .null } - return String(data: data, encoding: .utf8).map(RedisReply.string) ?? .data(data) + return RedisWireText.decode(data).map(RedisReply.string) ?? .data(data) case REDIS_REPLY_INTEGER, REDIS_REPLY_BOOL: return .integer(reply.pointee.integer) @@ -966,10 +966,10 @@ enum HiredisReply { return .null case REDIS_REPLY_STATUS: - return .status(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? "") + return .status(bytes(of: reply).flatMap(RedisWireText.decode) ?? "") case REDIS_REPLY_ERROR: - return .error(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? "Unknown error") + return .error(bytes(of: reply).flatMap(RedisWireText.decode) ?? "Unknown error") case REDIS_REPLY_DOUBLE: return .string(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? String(reply.pointee.dval)) diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift index c2f93088de..e06de0f3a8 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift @@ -141,8 +141,9 @@ extension RedisPluginDriver { if opts.xx { args.append("XX".redisArgument) } if opts.keepTTL { args.append("KEEPTTL".redisArgument) } } - try await conn.run(args) - return buildStatusResult("OK", startTime: startTime) + // NX or XX that stops the write answers nil, not an error. + let reply = try await conn.run(args) + return buildStatusResult(reply.displayText, startTime: startTime) case .del(let keys): let args = ["DEL"] + keys @@ -165,7 +166,11 @@ extension RedisPluginDriver { let capped = Array(keys.prefix(PluginRowLimits.emergencyMax)) let keysTruncated = keys.count > PluginRowLimits.emergencyMax return try await buildKeyBrowseResult( - keys: capped, connection: conn, startTime: startTime, isTruncated: keysTruncated + keys: capped, + connection: conn, + startTime: startTime, + isTruncated: keysTruncated, + batchSize: Self.keySummaryBatchSize ) case .scan(let cursor, let pattern, let count, let type): diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift index e705b0bb9c..167701e122 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift @@ -34,18 +34,24 @@ extension RedisPluginDriver { ) } + static let keySummaryBatchSize = 200 + + // Nil sends every key in one pipeline: fine for a grid page the user sized, not for KEYS or SCAN. func buildKeyBrowseResult( keys: [String], connection conn: any RedisCommandChannel, startTime: Date, isTruncated: Bool = false, - statusMessage: String? = nil + statusMessage: String? = nil, + batchSize: Int? = nil ) async throws -> PluginQueryResult { guard !keys.isEmpty else { return buildEmptyKeyResult(startTime: startTime, statusMessage: statusMessage) } - let rows = try await buildKeySummaryRows(keys: keys, connection: conn) + let rows = try await buildKeySummaryRows( + keys: keys, connection: conn, stringByteLimit: RedisStringPreview.byteLimit, batchSize: batchSize + ) return PluginQueryResult( columns: Self.keyBrowseColumns, columnTypeNames: Self.keyBrowseColumnTypeNames, @@ -57,14 +63,35 @@ extension RedisPluginDriver { ) } + // A nil byte limit reads every string whole, which only an export wants. func buildKeySummaryRows( keys: [String], - connection conn: any RedisCommandChannel + connection conn: any RedisCommandChannel, + stringByteLimit: Int?, + batchSize: Int? = nil + ) async throws -> [PluginRow] { + guard let batchSize, batchSize > 0, keys.count > batchSize else { + return try await summarizeKeys(keys, connection: conn, stringByteLimit: stringByteLimit) + } + var rows: [PluginRow] = [] + rows.reserveCapacity(keys.count) + for start in stride(from: 0, to: keys.count, by: batchSize) { + try Task.checkCancellation() + let batch = Array(keys[start ..< min(start + batchSize, keys.count)]) + rows += try await summarizeKeys(batch, connection: conn, stringByteLimit: stringByteLimit) + } + return rows + } + + private func summarizeKeys( + _ keys: [String], + connection conn: any RedisCommandChannel, + stringByteLimit: Int? ) async throws -> [PluginRow] { guard !keys.isEmpty else { return [] } let descriptions = try await conn.describeKeys(keys) - let contents = try await conn.readContents(of: keys, describedAs: descriptions) + let contents = try await conn.readContents(of: keys, describedAs: descriptions, stringByteLimit: stringByteLimit) return zip(keys, zip(descriptions, contents)).map { key, summary in let (description, content) = summary return [ @@ -78,7 +105,24 @@ extension RedisPluginDriver { } private func valueCell(for content: RedisKeyContents) -> PluginCellValue? { - content.preview.map { previewCell($0, kind: content.kind) } + guard let preview = content.preview else { return nil } + guard content.kind == .string, let byteLimit = content.stringByteLimit else { + return previewCell(preview, kind: content.kind) + } + return shownStringCell(preview, length: content.length, byteLimit: byteLimit) + } + + private func shownStringCell(_ reply: RedisReply, length: Int?, byteLimit: Int) -> PluginCellValue { + let bytes: Data + switch reply { + case .string(let text): bytes = Data(text.utf8) + case .data(let data): bytes = data + default: return stringCell(from: reply) + } + switch RedisStringPreview.shown(bytes, length: length, byteLimit: byteLimit) { + case .text(let text): return .text(text) + case .bytes(let data): return .bytes(data) + } } func previewCell(_ reply: RedisReply, kind: RedisKeyKind) -> PluginCellValue { diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift index 519e6b4dd1..7a740f7949 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift @@ -81,12 +81,14 @@ extension RedisPluginDriver { startTime: Date ) async throws -> PluginQueryResult { let outcome = RedisScanPageOutcome(page: page, rowLimit: PluginRowLimits.emergencyMax) + // SCAN COUNT is the user's to set, so a page can hold any number of keys. return try await buildKeyBrowseResult( keys: outcome.keys, connection: conn, startTime: startTime, isTruncated: outcome.isTruncated, - statusMessage: outcome.statusMessage + statusMessage: outcome.statusMessage, + batchSize: Self.keySummaryBatchSize ) } } diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 43d70ee2b9..ff639ea0f0 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -63,7 +63,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - var supported: PluginCapabilities = [.truncateTable, .cancelQuery] + var supported: PluginCapabilities = [.truncateTable, .cancelQuery, .columnDefaultMarks] if redisConnection?.supportsTransactions ?? true { supported.insert(.transactions) } return supported } @@ -307,6 +307,8 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ] } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let tables = try await fetchTables(schema: schema) let columns = try await fetchColumns(table: "", schema: schema) @@ -519,7 +521,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { do { try await self.performStreamRows(query: query, continuation: continuation) @@ -527,9 +529,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in - streamTask.cancel() - } + abort.onAbort { streamTask.cancel() } } } @@ -558,14 +558,18 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation: continuation ) } + case .keys(let pattern): + try await streamKeyRows(connection: conn, pattern: pattern, continuation: continuation) default: let startTime = Date() let result = try await executeOperation(operation, connection: conn, startTime: startTime) - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } @@ -587,7 +591,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ))) var cursor = RedisClusterCursor.start - let batchSize = 200 + let batchSize = Self.keySummaryBatchSize repeat { try Task.checkCancellation() @@ -602,9 +606,9 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try Task.checkCancellation() let batchEnd = min(batchStart + batchSize, page.keys.count) let batchKeys = Array(page.keys[batchStart ..< batchEnd]) - let rowBatch = try await buildKeySummaryRows(keys: batchKeys, connection: conn) + let rowBatch = try await buildKeySummaryRows(keys: batchKeys, connection: conn, stringByteLimit: nil) if !rowBatch.isEmpty { - continuation.yield(.rows(rowBatch)) + _ = await PluginRowStream.deliver(.rows(rowBatch), to: continuation) } batchStart = batchEnd } @@ -613,6 +617,29 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish() } + // KEYS names every match at once; their values are read a batch at a time, each string whole. + private func streamKeyRows( + connection conn: any RedisCommandChannel, + pattern: String, + continuation: AsyncThrowingStream.Continuation + ) async throws { + let keys = (try await conn.run(["KEYS", pattern]).arrayValue ?? []).map(\.displayText) + continuation.yield(.header(PluginStreamHeader( + columns: Self.keyBrowseColumns, + columnTypeNames: Self.keyBrowseColumnTypeNames, + estimatedRowCount: keys.count + ))) + for start in stride(from: 0, to: keys.count, by: Self.keySummaryBatchSize) { + try Task.checkCancellation() + let batch = Array(keys[start ..< min(start + Self.keySummaryBatchSize, keys.count)]) + let rows = try await buildKeySummaryRows(keys: batch, connection: conn, stringByteLimit: nil) + if !rows.isEmpty { + _ = await PluginRowStream.deliver(.rows(rows), to: continuation) + } + } + continuation.finish() + } + // MARK: - Query Building func buildBrowseQuery( @@ -655,6 +682,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let partitioned = redisConnection?.partitionsKeyspace == true let generator = RedisStatementGenerator( namespaceName: table, diff --git a/apple/Plugins/RedisDriverPlugin/RedisReply.swift b/apple/Plugins/RedisDriverPlugin/RedisReply.swift index a4284b2381..0aaeb72278 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisReply.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisReply.swift @@ -9,6 +9,7 @@ import Foundation import TableProPluginKit +import TableProRedisCore // MARK: - Reply Type @@ -24,7 +25,7 @@ enum RedisReply { var stringValue: String? { switch self { case .string(let s), .status(let s): return s - case .data(let d): return String(data: d, encoding: .utf8) + case .data(let d): return RedisWireText.decode(d) default: return nil } } @@ -55,7 +56,7 @@ enum RedisReply { case .string(let text), .status(let text): return text case .error(let message): return "(error) \(message)" case .integer(let value): return String(value) - case .data(let bytes): return String(data: bytes, encoding: .utf8) ?? bytes.base64EncodedString() + case .data(let bytes): return RedisWireText.decode(bytes) ?? bytes.base64EncodedString() case .array(let items): return "[\(items.map(\.displayText).joined(separator: ", "))]" case .null: return "(nil)" } diff --git a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift index 1bb4883226..58c1bbd7b1 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift @@ -54,6 +54,10 @@ struct RedisStatementGenerator { columns.firstIndex(of: "TTL") } + private var lengthColumnIndex: Int? { + columns.firstIndex(of: "Length") + } + private static let insertableTypes: Set = ["string", "hash", "list", "set", "zset"] // MARK: - Public API @@ -114,8 +118,10 @@ struct RedisStatementGenerator { ) throws -> [String] { var key: String? var value: String? + var valueCell: PluginCellValue? var type: String? var ttlText: String? + var lengthText: String? if let values = insertedRowData[change.rowIndex] { if let ki = keyColumnIndex, ki < values.count { @@ -126,17 +132,24 @@ struct RedisStatementGenerator { } if let vi = valueColumnIndex, vi < values.count { value = Self.encodedArgument(values[vi]) + valueCell = values[vi] } if let ttli = ttlColumnIndex, ttli < values.count { ttlText = values[ttli].asText } + if let li = lengthColumnIndex, li < values.count { + lengthText = values[li].asText + } } else { for cellChange in change.cellChanges { switch cellChange.columnName { case "Key": key = cellChange.newValue.asText case "Type": type = cellChange.newValue.asText - case "Value": value = Self.encodedArgument(cellChange.newValue) + case "Value": + value = Self.encodedArgument(cellChange.newValue) + valueCell = cellChange.newValue case "TTL": ttlText = cellChange.newValue.asText + case "Length": lengthText = cellChange.newValue.asText default: break } } @@ -150,6 +163,15 @@ struct RedisStatementGenerator { guard typeName.isEmpty || Self.insertableTypes.contains(typeName) else { throw Self.notAddable(typeName, rowIndex: change.rowIndex) } + if let length = lengthText.flatMap({ Int($0) }), let valueCell, + Self.holdsPart(of: length, value: valueCell, typeName: typeName) { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String( + localized: "This row holds only the part of a value the grid shows, so adding it would store that part. Add the key with a command in the query editor." + ) + ) + } var ttl: Int? if let ttlText { @@ -187,6 +209,23 @@ struct RedisStatementGenerator { private static let collectionTypes: Set = ["hash", "list", "set", "zset"] + // A duplicated row carries the Length the read reported beside the preview it showed. + private static func holdsPart(of length: Int, value: PluginCellValue, typeName: String) -> Bool { + switch (typeName, value) { + case ("string", .text(let text)), ("", .text(let text)): + return length > RedisStringPreview.byteLimit && text.hasSuffix(RedisStringPreview.marker) + && text.utf8.count - RedisStringPreview.marker.utf8.count <= RedisStringPreview.byteLimit + case ("string", .bytes(let bytes)), ("", .bytes(let bytes)): + return length > RedisStringPreview.byteLimit && bytes.count <= RedisStringPreview.byteLimit + case (_, .text(let text)) where collectionTypes.contains(typeName): + guard let parsed = try? JSONSerialization.jsonObject(with: Data(text.utf8)) else { return false } + let count = (parsed as? [String: Any])?.count ?? (parsed as? [Any])?.count + return count.map { $0 < length } ?? false + default: + return false + } + } + private static func notAddable(_ typeName: String, rowIndex: Int) -> PluginRowWriteRefusal { PluginRowWriteRefusal( rowIndex: rowIndex, @@ -289,8 +328,8 @@ struct RedisStatementGenerator { return valueCommands + ttlCommands + (rename.map { [$0] } ?? []) } - /// Only a string's value is the whole of what the grid shows. A collection shows a preview, - /// and a `SET` over it would replace the structure with that text. + // A collection shows a preview and a long string only its start, and a `SET` over either + // would replace the value with that text. private func valueCommands( setting newValue: PluginCellValue, of change: PluginRowChange, @@ -318,12 +357,35 @@ struct RedisStatementGenerator { ) ) } + guard !isShownInPart(change) else { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String( + localized: "The grid shows only the start of this value, so it cannot be edited here. Change it with a command in the query editor." + ) + ) + } let set = "SET \(RedisArgumentCodec.quote(key)) \(encodedValue)" guard !keepsTTLOnSet else { return ["\(set) KEEPTTL"] } guard restoringTTL, let seconds = originalTTL(of: change), seconds > 0 else { return [set] } return [set, "EXPIRE \(RedisArgumentCodec.quote(key)) \(seconds)"] } + // Only the key browse has a Length column, and it reads each string with GETRANGE. + private func isShownInPart(_ change: PluginRowChange) -> Bool { + guard let lengthIndex = lengthColumnIndex, let originalRow = change.originalRow, lengthIndex < originalRow.count else { + return false + } + let length = originalRow[lengthIndex].asText.flatMap { Int($0) } + let shownByteCount: Int + switch valueColumnIndex.flatMap({ $0 < originalRow.count ? originalRow[$0] : nil }) { + case .text(let text)?: shownByteCount = text.utf8.count + case .bytes(let bytes)?: shownByteCount = bytes.count + case .null?, nil: shownByteCount = 0 + } + return RedisStringPreview.isShortened(length: length, shownByteCount: shownByteCount) + } + private func originalTTL(of change: PluginRowChange) -> Int? { guard let ttlIndex = ttlColumnIndex, let originalRow = change.originalRow, ttlIndex < originalRow.count else { return nil diff --git a/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift b/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift index a1e5a4aa40..5d4cf11db7 100644 --- a/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift +++ b/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift @@ -61,6 +61,16 @@ struct HiredisReplyTests { #expect(try Self.convert(wire).displayText == expected) } + @Test("A bulk string that starts with a byte order mark reads with it") + func bulkStringKeepsByteOrderMark() throws { + let reply = try Self.convert("$9\r\n\u{FEFF}user:1\r\n") + guard case .string(let text) = reply else { + Issue.record("expected text, got \(reply.displayText)") + return + } + #expect(Data(text.utf8) == Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8))) + } + private struct Outcome: Sendable { let status: Int32 let error: Int32 diff --git a/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift b/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift new file mode 100644 index 0000000000..853eff5f75 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift @@ -0,0 +1,50 @@ +// +// RedisColumnDefaultTests.swift +// RedisDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct RedisColumnDefaultTests { + private let driver = RedisPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: 6_379, username: "", password: "", database: "0" + )) + + private func valueEdit(to value: PluginCellValue, isDefault: Bool) -> PluginRowChange { + var change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 1, columnName: "Value", oldValue: "old", newValue: value)], + originalRow: ["session:abc", "old", "-1"] + ) + if isDefault { change.defaultColumns = [1] } + return change + } + + private func writes(_ change: PluginRowChange) throws -> [String] { + try driver.generateRowWrites( + table: "", schema: nil, columns: ["Key", "Value", "TTL"], primaryKeyColumns: ["Key"], + changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + )?.map(\.statement) ?? [] + } + + @Test("The driver declares that it reads the default mark") + func declaresTheMark() { + #expect(driver.capabilities.contains(.columnDefaultMarks)) + } + + @Test("The text __DEFAULT__ is written as the value") + func defaultLookingTextIsAValue() throws { + let statements = try writes(valueEdit(to: "__DEFAULT__", isDefault: false)) + #expect(statements.contains { $0.contains("__DEFAULT__") }) + } + + @Test("A cell left to a default is refused, since Redis has none") + func markedDefaultIsRefused() { + #expect(throws: PluginRowWriteRefusal.self) { + _ = try writes(valueEdit(to: "__DEFAULT__", isDefault: true)) + } + } +} diff --git a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift index 72e4aa3bd3..4402cf7e01 100644 --- a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift @@ -180,6 +180,25 @@ struct RedisConditionalWriteTests { #expect(result.rows.first?.first?.asText == "OK") } + @Test("A SET that NX or XX kept from writing shows nil, as the server answered") + func skippedSetShowsNil() async throws { + let channel = StubRedisChannel([.null]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SET lock:job 1 NX"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "(nil)") + #expect(channel.sentCommands == [["SET", "lock:job", "1", "NX"]]) + } + + @Test("A SET that wrote the key shows OK") + func appliedSetShowsOK() async throws { + let channel = StubRedisChannel([.status("OK")]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SET lock:job 1 XX EX 60"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "OK") + } + @Test("A conditional write the server refused outright keeps the server's message") func execKeepsServerErrorForConditionalWrite() { let reply = RedisReply.array([.error("ERR no such key")]) diff --git a/apple/Plugins/RedisDriverTests/RedisKeyBrowseValueTests.swift b/apple/Plugins/RedisDriverTests/RedisKeyBrowseValueTests.swift new file mode 100644 index 0000000000..a3e3c50776 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/RedisKeyBrowseValueTests.swift @@ -0,0 +1,117 @@ +// +// RedisKeyBrowseValueTests.swift +// RedisDriverTests +// + +import Foundation +import TableProPluginKit +import TableProRedisCore +import Testing + +struct RedisKeyBrowseValueTests { + private func driver() -> RedisPluginDriver { + RedisPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: 6_379, username: "", password: "", database: "0" + )) + } + + private static func scanReply(_ keys: [String]) -> RedisReply { + .array([.string("0"), .array(keys.map(RedisReply.string))]) + } + + @Test("The grid reads a long string's start and marks it shortened, with its whole length") + func gridShowsTheStartOfALongString() async throws { + let start = String(repeating: "a", count: RedisStringPreview.byteLimit) + let channel = StubRedisChannel([ + Self.scanReply(["page:1"]), + .status("string"), .integer(-1), + .integer(900_000), .string(start), + ]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYBROWSE LIMIT 100 OFFSET 0"), connection: channel, startTime: Date() + ) + #expect(channel.sentCommands.last == ["GETRANGE", "page:1", "0", String(RedisStringPreview.byteLimit - 1)]) + let row = try #require(result.rows.first) + #expect(row[3] == .text("900000")) + #expect(row[4] == .text(start + RedisStringPreview.marker)) + } + + @Test("A short string in the grid shows whole and unmarked") + func gridShowsAShortStringWhole() async throws { + let channel = StubRedisChannel([ + Self.scanReply(["k"]), + .status("string"), .integer(-1), + .integer(5), .string("hello"), + ]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYBROWSE LIMIT 100 OFFSET 0"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?[4] == .text("hello")) + } + + @Test("A long binary string in the grid stays bytes") + func gridKeepsBinaryAsBytes() async throws { + let blob = Data([0x1F, 0x8B, 0x08, 0x00, 0xFF, 0xFE]) + let channel = StubRedisChannel([ + Self.scanReply(["bin"]), + .status("string"), .integer(-1), + .integer(900_000), .data(blob), + ]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYBROWSE LIMIT 100 OFFSET 0"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?[4] == .bytes(blob)) + } + + @Test("The export summary reads every string whole") + func exportSummaryReadsWholeValues() async throws { + let whole = String(repeating: "b", count: RedisStringPreview.byteLimit + 10) + let channel = StubRedisChannel([ + .status("string"), .integer(-1), + .integer(Int64(whole.utf8.count)), .string(whole), + ]) + let rows = try await driver().buildKeySummaryRows(keys: ["doc"], connection: channel, stringByteLimit: nil) + #expect(channel.sentCommands.last == ["GET", "doc"]) + #expect(rows.first?[4] == .text(whole)) + } + + @Test("KEYS summarizes its keys in batches, never one pipeline for every key") + func keysSummarizedInBatches() async throws { + let keys = (0 ..< 250).map { "k\($0)" } + let channel = StubRedisChannel([.array(keys.map(RedisReply.string))]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYS *"), connection: channel, startTime: Date() + ) + #expect(result.rows.count == 250) + #expect(result.rows.map { $0[0] } == keys.map(PluginCellValue.text)) + #expect(channel.pipelineSizes == [RedisPluginDriver.keySummaryBatchSize * 2, 100]) + } + + @Test("A SCAN page summarizes its keys in batches") + func scanPageSummarizedInBatches() async throws { + let keys = (0 ..< 250).map { "k\($0)" } + let channel = StubRedisChannel([Self.scanReply(keys)]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SCAN 0 COUNT 300"), connection: channel, startTime: Date() + ) + #expect(result.rows.count == 250) + #expect(channel.pipelineSizes == [RedisPluginDriver.keySummaryBatchSize * 2, 100]) + } + + @Test("A key that starts with a byte order mark is deleted by its own name") + func byteOrderMarkKeyDeletesItself() throws { + let markedKey = "\u{FEFF}user:1" + let generator = RedisStatementGenerator(namespaceName: "", columns: ["Key", "Type", "TTL", "Length", "Value"]) + let change = PluginRowChange( + rowIndex: 0, type: .delete, cellChanges: [], originalRow: [.text(markedKey), "string", "-1", "1", "v"] + ) + let writes = try generator.generateRowWrites( + from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: [] + ) + guard case .del(let deleted)? = try? RedisCommandParser.parse(writes[0].statement) else { + Issue.record("expected a DEL") + return + } + #expect(deleted.map { Data($0.utf8) } == [Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8))]) + } +} diff --git a/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift b/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift index f0e9226b82..9289f25820 100644 --- a/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift @@ -165,7 +165,8 @@ struct RedisKeyContentsReadTests { let channel = StubRedisChannel([]) let contents = try await channel.readContents( of: ["other:1"], - describedAs: [RedisKeyDescription(typeName: nil, ttlSeconds: nil)] + describedAs: [RedisKeyDescription(typeName: nil, ttlSeconds: nil)], + stringByteLimit: RedisStringPreview.byteLimit ) #expect(contents.count == 1) #expect(contents[0] == nil) @@ -180,7 +181,8 @@ struct RedisKeyContentsReadTests { let channel = StubRedisChannel([.integer(1), keyDenied]) let contents = try await channel.readContents( of: ["app:h"], - describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)] + describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)], + stringByteLimit: RedisStringPreview.byteLimit ) let hash = try #require(contents[0]) #expect(hash.kind == .hash) @@ -190,12 +192,13 @@ struct RedisKeyContentsReadTests { #expect(channel.sentCommands == [["HLEN", "app:h"], ["HSCAN", "app:h", "0", "COUNT", "100"]]) } - @Test("A refused GET leaves the string length the server gave") + @Test("A refused string read leaves the string length the server gave") func refusedGetKeepsLength() async throws { let channel = StubRedisChannel([.integer(1), keyDenied]) let contents = try await channel.readContents( of: ["app:1"], - describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: nil)] + describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: nil)], + stringByteLimit: RedisStringPreview.byteLimit ) let string = try #require(contents[0]) #expect(string.length == 1) @@ -211,7 +214,8 @@ struct RedisKeyContentsReadTests { RedisKeyDescription(typeName: "string", ttlSeconds: -1), RedisKeyDescription(typeName: nil, ttlSeconds: nil), RedisKeyDescription(typeName: "list", ttlSeconds: -1), - ] + ], + stringByteLimit: nil ) #expect(contents.count == 3) #expect(contents[0]?.length == 3) @@ -223,6 +227,26 @@ struct RedisKeyContentsReadTests { #expect(channel.sentCommands.map(\.first) == ["STRLEN", "GET", "LLEN", "LRANGE"]) } + @Test("With a byte limit a string is read with GETRANGE over that range, without one with GET") + func stringReadFollowsTheLimit() async throws { + let limited = StubRedisChannel([.integer(3), .string("abc")]) + let contents = try await limited.readContents( + of: ["a"], + describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: -1)], + stringByteLimit: 65_536 + ) + #expect(limited.sentCommands == [["STRLEN", "a"], ["GETRANGE", "a", "0", "65535"]]) + #expect(contents[0]?.stringByteLimit == 65_536) + + let whole = StubRedisChannel([.integer(3), .string("abc")]) + _ = try await whole.readContents( + of: ["a"], + describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: -1)], + stringByteLimit: nil + ) + #expect(whole.sentCommands == [["STRLEN", "a"], ["GET", "a"]]) + } + @Test("A key whose type changed under the probe fails the page rather than showing wrong contents") func wrongTypeThrows() async throws { let channel = StubRedisChannel([ @@ -232,7 +256,8 @@ struct RedisKeyContentsReadTests { await #expect(throws: RedisPluginError.self) { try await channel.readContents( of: ["app:h"], - describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)] + describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)], + stringByteLimit: RedisStringPreview.byteLimit ) } } diff --git a/apple/Plugins/RedisDriverTests/RedisReplyTests.swift b/apple/Plugins/RedisDriverTests/RedisReplyTests.swift index 8269d0744c..8ecb837754 100644 --- a/apple/Plugins/RedisDriverTests/RedisReplyTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisReplyTests.swift @@ -58,6 +58,35 @@ struct RedisReplyStringValueTests { let reply = RedisReply.array([.string("a")]) #expect(reply.stringValue == nil) } + + @Test("data that starts with a byte order mark keeps it in stringValue and displayText") + func dataKeepsByteOrderMark() { + let reply = RedisReply.data(Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8))) + #expect(reply.stringValue?.unicodeScalars.first == "\u{FEFF}") + #expect(reply.displayText.unicodeScalars.first == "\u{FEFF}") + } +} + +// MARK: - SCAN reply + +struct RedisScanReplyTests { + private static let markedKey = Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8)) + + @Test("A key that starts with a byte order mark is listed with it") + func keyKeepsByteOrderMark() throws { + let page = RedisScanReply.parse(.array([.string("0"), .array([.data(Self.markedKey), .string("user:1")])])) + #expect(page.keys.count == 2) + let first = try #require(page.keys.first) + #expect(Data(first.utf8) == Self.markedKey) + #expect(page.keys.last == "user:1") + } + + @Test("A key that is not UTF-8 is still left out") + func binaryKeyLeftOut() { + let page = RedisScanReply.parse(.array([.string("7"), .array([.data(Data([0xFF, 0xFE])), .string("a")])])) + #expect(page.cursor == "7") + #expect(page.keys == ["a"]) + } } // MARK: - intValue diff --git a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift index 8710b7d525..1739b3dccb 100644 --- a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift @@ -1219,4 +1219,105 @@ struct RedisStatementGeneratorRefusalTests { #expect(written.map(\.statement) == ["EXPIRE mykey 60", "DEL {u}a {u}b", "DEL x"]) #expect(written.map(\.rowIndices) == [[0], [1, 3], [2]]) } + + private static let shownInPart = PluginRowWriteRefusal( + rowIndex: 0, + reason: "The grid shows only the start of this value, so it cannot be edited here. " + + "Change it with a command in the query editor." + ) + + @Test("A Value edit of a string longer than the grid reads is refused") + func valueShownInPartRefused() { + let change = edit( + [(columnIndex: 4, columnName: "Value", oldValue: "start...", newValue: "start")], + original: ["page:1", "string", "-1", .text(String(RedisStringPreview.byteLimit + 1)), "start..."] + ) + #expect(throws: Self.shownInPart) { try writes(for: [change]) } + } + + @Test("A string exactly as long as the grid reads is written") + func valueAtTheLimitWritten() throws { + let change = edit( + [(columnIndex: 4, columnName: "Value", oldValue: "old", newValue: "new")], + original: ["page:1", "string", "-1", .text(String(RedisStringPreview.byteLimit)), "old"] + ) + #expect(try writes(for: [change]).map(\.statement) == ["SET page:1 new KEEPTTL"]) + } + + @Test("A TTL edit and a rename of a string shown in part still write") + func otherEditsOfAShortenedRowWrite() throws { + let change = edit( + [ + (columnIndex: 2, columnName: "TTL", oldValue: "-1", newValue: "60"), + (columnIndex: 0, columnName: "Key", oldValue: "page:1", newValue: "page:2"), + ], + original: ["page:1", "string", "-1", "900000", "start..."] + ) + #expect(try writes(for: [change]).map(\.statement) == ["EXPIRE page:1 60", "MUSTAPPLY RENAMENX page:1 page:2"]) + } + + @Test("Without a length, a Value cell that filled the read range is refused") + func unknownLengthFilledRangeRefused() throws { + let filled = String(repeating: "a", count: RedisStringPreview.byteLimit - 3) + RedisStringPreview.marker + let refused = edit( + [(columnIndex: 4, columnName: "Value", oldValue: .text(filled), newValue: "b")], + original: ["page:1", "string", "-1", .null, .text(filled)] + ) + #expect(throws: Self.shownInPart) { try writes(for: [refused]) } + + let short = edit( + [(columnIndex: 4, columnName: "Value", oldValue: "old", newValue: "new")], + original: ["page:1", "string", "-1", .null, "old"] + ) + #expect(try writes(for: [short]).map(\.statement) == ["SET page:1 new KEEPTTL"]) + } +} + +// Duplicate copies the Value cell the grid showed, which for a long string or a large collection is only its start. +struct RedisCopiedPreviewInsertTests { + private static let browseColumns = ["Key", "Type", "TTL", "Length", "Value"] + + private static let partRefusal = PluginRowWriteRefusal( + rowIndex: 0, + reason: "This row holds only the part of a value the grid shows, so adding it would store that part. " + + "Add the key with a command in the query editor." + ) + + private func statements(type: String, length: PluginCellValue, value: PluginCellValue) throws -> [String] { + var gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns) + gen.stagingToken = { "t" } + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + return try gen.generateRowWrites( + from: [change], + insertedRowData: [0: ["copy", .text(type), nil, length, value]], + deletedRowIndices: [], + insertedRowIndices: [0] + ).map(\.statement) + } + + @Test("A copied string the grid showed in part is refused") + func copiedShortenedStringRefused() { + let shown = String(repeating: "a", count: RedisStringPreview.byteLimit) + RedisStringPreview.marker + let length = PluginCellValue.text(String(RedisStringPreview.byteLimit + 10)) + #expect(throws: Self.partRefusal) { try statements(type: "string", length: length, value: .text(shown)) } + #expect(throws: Self.partRefusal) { + try statements(type: "string", length: length, value: .bytes(Data(repeating: 0xFF, count: RedisStringPreview.byteLimit))) + } + } + + @Test("A copied string shown whole is added") + func copiedWholeStringAdded() throws { + #expect(try statements(type: "string", length: "5", value: "hello") == ["MUSTAPPLY SET copy hello NX"]) + } + + @Test("A copied collection holding fewer elements than its length is refused") + func copiedCollectionPreviewRefused() { + #expect(throws: Self.partRefusal) { try statements(type: "list", length: "250", value: #"["a","b"]"#) } + #expect(throws: Self.partRefusal) { try statements(type: "hash", length: "3", value: #"{"f":"v"}"#) } + } + + @Test("A copied collection holding every element is added") + func copiedWholeCollectionAdded() throws { + #expect(try statements(type: "set", length: "2", value: #"["a","b"]"#).count == 3) + } } diff --git a/apple/Plugins/RedisDriverTests/StubRedisChannel.swift b/apple/Plugins/RedisDriverTests/StubRedisChannel.swift index c8d8c7a578..edf21d90e9 100644 --- a/apple/Plugins/RedisDriverTests/StubRedisChannel.swift +++ b/apple/Plugins/RedisDriverTests/StubRedisChannel.swift @@ -9,6 +9,7 @@ import Foundation import TableProPluginKit +import TableProRedisCore /// Driven by one task at a time, so the outcomes are handed out in order with no synchronisation. /// It also stands in for one node of a cluster, which hands it the cluster's database. @@ -16,6 +17,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { private var outcomes: [Result] private(set) var sentCommands: [[String]] = [] private(set) var sentScopes: [RedisCommandScope] = [] + private(set) var pipelineSizes: [Int] = [] private(set) var footprint = RedisSessionFootprint() let supportsDatabaseSelection: Bool private(set) var sessionDatabase: RedisSessionDatabase @@ -61,6 +63,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { func forgetSentCommands() { sentCommands = [] sentScopes = [] + pipelineSizes = [] } /// Mirrors the hiredis connection: a SELECT queued in an open block moves nothing yet and @@ -105,6 +108,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { /// connection does, because every command is on the wire before the first reply is read. func executePipeline(_ commands: [[Data]], scope: RedisCommandScope) async throws -> [RedisReply] { let pipeline = commands.map(decoded) + pipelineSizes.append(pipeline.count) try admit(scope, command: pipeline.first ?? []) try moveToCommandDatabase(command: pipeline.first?.first) let replies = try pipeline.map { try send($0, scope: scope) } @@ -136,7 +140,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { } private func decoded(_ args: [Data]) -> [String] { - args.map { String(data: $0, encoding: .utf8) ?? "" } + args.map { RedisWireText.decode($0) ?? "" } } private func admit(_ scope: RedisCommandScope, command: [String]) throws { diff --git a/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift b/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift new file mode 100644 index 0000000000..1b6034c571 --- /dev/null +++ b/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift @@ -0,0 +1,53 @@ +// +// SQLImportCopyRows.swift +// SQLImportPlugin +// + +import Foundation +import TableProPluginKit + +// The driver pulls a COPY's rows on its own thread, so the check for a Stop and the progress note ride on each pull. +internal final class SQLImportCopyRows: @unchecked Sendable { + private static let lineFeed = UInt8(ascii: "\n") + private static let linesPerNote = 10_000 + + private let rows: PluginCopyData + private let line: Int + private let progress: PluginImportProgress + private let lock = NSLock() + private var linesSent = 0 + private var linesNoted: Int? + + private init(_ rows: PluginCopyData, line: Int, progress: PluginImportProgress) { + self.rows = rows + self.line = line + self.progress = progress + } + + internal static func watching(_ rows: PluginCopyData, line: Int, progress: PluginImportProgress) -> PluginCopyData { + let watched = SQLImportCopyRows(rows, line: line, progress: progress) + return PluginCopyData { try watched.nextPiece() } + } + + private func nextPiece() throws -> Data? { + try progress.checkCancellation() + guard let piece = try rows.nextPiece() else { return nil } + let lines = piece.reduce(into: 0) { count, byte in + if byte == Self.lineFeed { count += 1 } + } + let note = lock.withLock { () -> Int? in + linesSent += lines + guard linesNoted.map({ linesSent - $0 >= Self.linesPerNote }) ?? true else { return nil } + linesNoted = linesSent + return linesSent + } + if let note { + progress.setStatus(String( + format: String(localized: "COPY at line %@: %@ lines sent"), + line.formatted(), + note.formatted() + )) + } + return piece + } +} diff --git a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift index 91b6a9c205..bdff60a382 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -51,15 +51,17 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi var errors: [PluginImportResult.ImportStatementError] = [] let maxErrors = 1_000 - let errorMode = settings.errorHandling - let useTransaction = settings.wrapInTransaction && errorMode != .skipAndContinue + // One plugin instance serves every window: a checkbox unticked elsewhere mid-import left the checks off. + let options = settings + let errorMode = options.errorHandling + let useTransaction = options.wrapInTransaction && errorMode != .skipAndContinue let fileSizeBytes = source.fileSizeBytes() let estimatedTotal = max(1, Int(fileSizeBytes / 500)) progress.setEstimatedTotal(estimatedTotal) do { - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { try await sink.disableForeignKeyChecks() } @@ -67,13 +69,15 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi try await sink.beginTransaction() } - let stream = try await source.statements() + let stream = try await source.elements() - for try await (statement, lineNumber) in stream { + for try await element in stream { try progress.checkCancellation() + let statement = element.statementText + let lineNumber = element.lineNumber do { - try await sink.execute(statement: statement, line: lineNumber) + try await Self.run(element, on: sink, progress: progress) executedCount += 1 progress.incrementStatement() } catch is PluginImportCancellationError { @@ -99,7 +103,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi Self.logger.warning("Failed to commit partial import") } } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -134,7 +138,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi didCommit = true } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -156,7 +160,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -176,4 +180,23 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi errors: errors ) } + + private static func run( + _ element: PluginImportElement, + on sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws { + switch element { + case .statement(let statement, let lineNumber): + try await sink.execute(statement: statement, line: lineNumber) + case .copyFromStandardInput(let statement, let lineNumber, let rows): + let watched = SQLImportCopyRows.watching(rows, line: lineNumber, progress: progress) + try await sink.copyFromStandardInput(statement: statement, line: lineNumber, rows: watched) + @unknown default: + throw PluginImportError.importFailed(String( + format: String(localized: "This version of SQL Import cannot run line %@."), + element.lineNumber.formatted() + )) + } + } } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift index 5ff155c7d4..156fe655af 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentBackend.swift @@ -24,6 +24,10 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { private let helloBudget: SQLiteAgentHelloBudget private var busyTimeoutMilliseconds: Int32 = 0 + // Every reply goes to the connection's one pending operation, and an await lets the next call in, so statements queue. + private var statementRunning = false + private var waitingStatements: [CheckedContinuation] = [] + nonisolated let canceller: SQLiteCanceller nonisolated var resolvedServerVersion: String? { connection.cachedServerVersion } @@ -32,9 +36,10 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { port: Int, path: String, token: String, - helloBudget: SQLiteAgentHelloBudget + helloBudget: SQLiteAgentHelloBudget, + heartbeatInterval: TimeInterval = SQLiteAgentConnection.defaultHeartbeatInterval ) { - let connection = SQLiteAgentConnection(host: host, port: port) + let connection = SQLiteAgentConnection(host: host, port: port, heartbeatInterval: heartbeatInterval) self.connection = connection self.path = path self.token = token @@ -77,15 +82,29 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { connection.setBusyTimeout(UInt32(max(0, milliseconds))) } + // The agent stops at the bound's row cap. Its byte limit is not sent: the agent has no such field. func executeQuery( _ query: String, parameters: [PluginCellValue]?, - scope: SQLiteStatementScope + rules: SQLiteStatementRules, + bound: SQLiteReadBound? ) async throws -> SQLiteRawResult { - try await run(query, parameters: (parameters ?? []).map(Self.encode), scope: scope) + try await run( + query, + parameters: (parameters ?? []).map(Self.encode), + rules: rules, + rowCap: bound?.rowCap ?? PluginRowLimits.emergencyMax + ) } - private func run(_ query: String, parameters: [SQLiteAgentValue], scope: SQLiteStatementScope) async throws -> SQLiteRawResult { + private func run( + _ query: String, + parameters: [SQLiteAgentValue], + rules: SQLiteStatementRules, + rowCap: Int + ) async throws -> SQLiteRawResult { + await takeTurn() + defer { endTurn() } let startTime = Date() let accumulator = SQLiteAgentResultAccumulator() @@ -93,10 +112,10 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { try await connection.execute( sql: query, parameters: parameters, - scope: Self.agentScope(scope), + scope: Self.agentScope(rules), // The buffered path holds the whole result in memory, so it takes the same emergency // ceiling the local backend applies; the streaming path below stays uncapped. - rowCap: UInt32(clamping: PluginRowLimits.emergencyMax), + rowCap: UInt32(clamping: rowCap), onHeader: { header in accumulator.setHeader(header) }, onRows: { columnCount, values in guard columnCount > 0 else { return } @@ -124,23 +143,24 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { ) } - func streamQuery( - _ query: String, - scope: SQLiteStatementScope, - continuation: AsyncThrowingStream.Continuation - ) async throws { + func streamQuery(_ query: String, rules: SQLiteStatementRules, into spill: PluginRowSpill) async throws { + await takeTurn() + defer { endTurn() } + try Task.checkCancellation() _ = try await withTaskCancellationHandler { - try await connection.execute( + // A cancel frame sent before the statement starts does nothing, so a reader that left by now ends it here. + try Task.checkCancellation() + return try await connection.execute( sql: query, parameters: [], - scope: Self.agentScope(scope), + scope: Self.agentScope(rules), rowCap: 0, onHeader: { header in - continuation.yield(.header(PluginStreamHeader( + spill.yield(PluginStreamHeader( columns: header.map(\.name), columnTypeNames: header.map { $0.declaredType ?? "" }, estimatedRowCount: nil - ))) + )) }, onRows: { columnCount, values in guard columnCount > 0 else { return } @@ -150,21 +170,40 @@ actor SQLiteAgentBackend: SQLiteExecutionBackend { batch.append(values[index..<(index + columnCount)].map(Self.decode)) index += columnCount } - continuation.yield(.rows(batch)) + // The spill takes rows the stream cannot, so the agent's statement ends as fast as the session carries them. + spill.deliver(batch) } ) } onCancel: { connection.sendCancel() } - continuation.finish() + spill.finish() + } + + private func takeTurn() async { + guard statementRunning else { + statementRunning = true + return + } + await withCheckedContinuation { waitingStatements.append($0) } + } + + private func endTurn() { + guard !waitingStatements.isEmpty else { + statementRunning = false + return + } + waitingStatements.removeFirst().resume() } - // Nil runs the plain execute, so an unrestricted statement sends the frame it always sent. - private static func agentScope(_ scope: SQLiteStatementScope) -> SQLiteAgentStatementScope? { - switch scope { - case .unrestricted: nil - case .noServerCode: .noServerCode - case .readOnly: .readOnly + // Nil runs the plain execute, so an unrestricted statement sends the frame it always sent. The agent loads no + // extension, so a Safe Mode read loses nothing under its read-only scope. + static func agentScope(_ rules: SQLiteStatementRules) -> SQLiteAgentStatementScope? { + if rules.readsOnly { return .readOnly } + switch rules.scope { + case .unrestricted: return nil + case .noServerCode: return .noServerCode + case .readOnly: return .readOnly } } @@ -249,16 +288,18 @@ struct SQLiteAgentExecuteOutcome: Sendable { /// the reader thread's cancel and heartbeat frames never interleave with a partial request. final class SQLiteAgentConnection: @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteAgentConnection") - private static let heartbeatInterval: TimeInterval = 15 + static let defaultHeartbeatInterval: TimeInterval = 15 private let host: String private let port: Int + private let heartbeatInterval: TimeInterval private let stateLock = NSLock() private var socketFD: Int32 = -1 private var pending: SQLiteAgentOperation? private var terminalFailure: Error? private var closed = false + private var readerOwnsDescriptor = false private let writeLock = NSLock() private var reader = SQLiteAgentFrameReader() @@ -272,14 +313,18 @@ final class SQLiteAgentConnection: @unchecked Sendable { set { versionLock.lock(); _cachedServerVersion = newValue; versionLock.unlock() } } - init(host: String, port: Int) { + init(host: String, port: Int, heartbeatInterval: TimeInterval) { self.host = host self.port = port + self.heartbeatInterval = heartbeatInterval } func connect(token: String) throws { let fd = socket(AF_INET, SOCK_STREAM, 0) guard fd >= 0 else { throw SQLitePluginError.connectionFailed("could not create a local socket") } + // A heartbeat after the session went away gets EPIPE; the default SIGPIPE would end the app. + var noSignal: Int32 = 1 + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &noSignal, socklen_t(MemoryLayout.size)) var addr = sockaddr_in() addr.sin_family = sa_family_t(AF_INET) @@ -299,11 +344,31 @@ final class SQLiteAgentConnection: @unchecked Sendable { stateLock.lock() socketFD = fd closed = false + readerOwnsDescriptor = false stateLock.unlock() - try writeAll(SQLiteAgentProtocol.admissionPreamble(token: token)) + do { + try writeAll(SQLiteAgentProtocol.admissionPreamble(token: token)) + } catch { + close() + throw error + } + + stateLock.lock() + guard !closed else { + stateLock.unlock() + throw SQLitePluginError.notConnected + } + readerOwnsDescriptor = true + stateLock.unlock() - let thread = Thread { [weak self] in self?.readLoop() } + let thread = Thread { [weak self] in + guard let self else { + Darwin.close(fd) + return + } + self.readLoop(fd: fd) + } thread.name = "com.TablePro.sqlite.agent-reader" thread.stackSize = 512 * 1_024 readerThread = thread @@ -360,19 +425,29 @@ final class SQLiteAgentConnection: @unchecked Sendable { closed = true let fd = socketFD socketFD = -1 + let readerOwnsDescriptor = self.readerOwnsDescriptor + // Under the lock, so a reader on its way out cannot close the number before this reaches it. + if fd >= 0 { shutdown(fd, SHUT_RDWR) } let inflight = pending pending = nil + let timer = heartbeatTimer + heartbeatTimer = nil stateLock.unlock() - heartbeatTimer?.cancel() - heartbeatTimer = nil - if fd >= 0 { - shutdown(fd, SHUT_RDWR) - Darwin.close(fd) + timer?.cancel() + if fd >= 0, !readerOwnsDescriptor { + closeDescriptor(fd) } inflight?.fail(SQLitePluginError.notConnected) } + // After any writer that read the number before it was cleared, so none writes to a reused descriptor. + private func closeDescriptor(_ fd: Int32) { + writeLock.lock() + Darwin.close(fd) + writeLock.unlock() + } + // MARK: - Operation lifecycle private func withOperation( @@ -394,21 +469,28 @@ final class SQLiteAgentConnection: @unchecked Sendable { } stateLock.lock() - if closed { - stateLock.unlock() - continuation.resume(throwing: SQLitePluginError.notConnected) - return - } /// A terminal reply that arrived before this operation registered, such as the launcher's /// "no python3" notice landing before the hello op was set, is delivered to this /// operation rather than lost. The reader thread stores it under the same lock, so the /// two orderings, store-then-register and register-then-route, both reach here correctly. + /// It comes before the closed check, since the reader closes the connection once it stops. if let stored = terminalFailure { terminalFailure = nil stateLock.unlock() operation.fail(stored) return } + if closed { + stateLock.unlock() + continuation.resume(throwing: SQLitePluginError.notConnected) + return + } + // The backend runs one statement at a time; replacing the pending one would hand it another's replies. + guard pending == nil else { + stateLock.unlock() + operation.fail(SQLitePluginError.queryFailed("another statement is still running on the remote SQLite session")) + return + } pending = operation stateLock.unlock() @@ -460,14 +542,14 @@ final class SQLiteAgentConnection: @unchecked Sendable { // MARK: - Reader - private func readLoop() { + // The reader closes its own descriptor on the way out, so it never reads a number something else reused. + private func readLoop(fd: Int32) { var buffer = [UInt8](repeating: 0, count: 32 * 1_024) while true { stateLock.lock() - let fd = socketFD let isClosed = closed stateLock.unlock() - guard !isClosed, fd >= 0 else { break } + guard !isClosed else { break } let count = read(fd, &buffer, buffer.count) if count <= 0 { break } @@ -483,6 +565,9 @@ final class SQLiteAgentConnection: @unchecked Sendable { } } completeOrStoreFailure(SQLitePluginError.notConnected) + // Nothing reads the socket from here on, so a later request would wait for a reply forever. + close() + closeDescriptor(fd) } private func route(_ reply: SQLiteAgentReply) { @@ -516,8 +601,11 @@ final class SQLiteAgentConnection: @unchecked Sendable { // MARK: - Heartbeat private func startHeartbeat() { + stateLock.lock() + defer { stateLock.unlock() } + guard !closed, heartbeatTimer == nil else { return } let timer = DispatchSource.makeTimerSource(queue: DispatchQueue.global()) - timer.schedule(deadline: .now() + Self.heartbeatInterval, repeating: Self.heartbeatInterval) + timer.schedule(deadline: .now() + heartbeatInterval, repeating: heartbeatInterval) timer.setEventHandler { @Sendable [weak self] in self?.sendHeartbeat() } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift index d7195c7d59..f19c7f8cb8 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteAgentProtocol.swift @@ -212,9 +212,10 @@ struct SQLiteAgentFrameReader { private var readIndex = 0 private var expectsLauncherNotice = true + // A read rarely ends on a frame boundary, so waiting for an empty buffer kept every byte of a long result. mutating func append(_ data: Data) { - if readIndex > 0, readIndex == buffer.count { - buffer.removeAll(keepingCapacity: true) + if readIndex > 0, readIndex >= buffer.count - readIndex { + buffer.removeSubrange(0.. SQLiteRawResult - func streamQuery( - _ query: String, - scope: SQLiteStatementScope, - continuation: AsyncThrowingStream.Continuation - ) async throws + func streamQuery(_ query: String, rules: SQLiteStatementRules, into spill: PluginRowSpill) async throws +} + +// How one statement runs: the authorizer scope, and whether read-only, with `PRAGMA query_only` around it and +// `sqlite3_stmt_readonly` checked before it. Safe Mode's Read-Only keeps the session's authorizer scope, so an in-app +// read keeps the extension functions it uses; an external read takes the `.readOnly` scope, which refuses them. +struct SQLiteStatementRules: Sendable, Equatable { + let scope: SQLiteStatementScope + let readsOnly: Bool + + static let unrestricted = SQLiteStatementRules(scope: .unrestricted, readsOnly: false) + + init(scope: SQLiteStatementScope, readsOnly: Bool) { + self.scope = scope + self.readsOnly = readsOnly || scope == .readOnly + } + + init(_ context: PluginStatementContext) { + if context.readOnly, context.origin == .external { + self.init(scope: .readOnly, readsOnly: true) + } else { + self.init(scope: context.allowsServerCode ? .unrestricted : .noServerCode, readsOnly: context.enforcesReadOnly) + } + } + + // What a statement SQLite does not read as read-only is refused with, in the words of whoever asked. + var notReadRefusal: SQLitePluginError { + scope == .readOnly ? .readOnlyStatement : .safeModeReadOnlyStatement + } +} + +// Where a bounded read stops: once `rowCap` rows are read and another is there, or at the first row that takes the +// decoded rows past `byteLimit`. That row is kept, so the host sees its own limit was reached. +struct SQLiteReadBound: Sendable, Equatable { + let rowCap: Int + let byteLimit: Int? + + init(rowCap: Int, byteLimit: Int?) { + self.rowCap = min(max(rowCap, 1), PluginRowLimits.emergencyMax) + self.byteLimit = byteLimit.map { max($0, 0) } + } + + // Nil when neither bound is set, and the statement is read whole as it always was. + static func ifSet(rowCap: Int?, byteLimit: Int?) -> SQLiteReadBound? { + let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } + guard cap != nil || byteLimit != nil else { return nil } + return SQLiteReadBound(rowCap: cap ?? PluginRowLimits.emergencyMax, byteLimit: byteLimit) + } + + // Text as UTF-8, bytes as they are, null as nothing: never more than the host counts for the same row, so the read + // never stops before the host's own budget is full. + static func byteCount(of row: [PluginCellValue]) -> Int { + row.reduce(0) { total, cell in + switch cell { + case .null: total + case .text(let text): total + text.utf8.count + case .bytes(let data): total + data.count + } + } + } } /// The one thing about a running query that must be reachable while the query holds the backend @@ -65,18 +121,11 @@ extension SQLiteExecutionBackend { func sessionTransactionState() async -> PluginSessionTransactionState { .unknown } func executeQuery(_ query: String) async throws -> SQLiteRawResult { - try await executeQuery(query, parameters: nil, scope: .unrestricted) + try await executeQuery(query, parameters: nil, rules: .unrestricted, bound: nil) } func executeParameterizedQuery(_ query: String, parameters: [PluginCellValue]) async throws -> SQLiteRawResult { - try await executeQuery(query, parameters: parameters, scope: .unrestricted) - } - - func streamQuery( - _ query: String, - continuation: AsyncThrowingStream.Continuation - ) async throws { - try await streamQuery(query, scope: .unrestricted, continuation: continuation) + try await executeQuery(query, parameters: parameters, rules: .unrestricted, bound: nil) } } @@ -282,36 +331,45 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { func executeQuery( _ query: String, parameters: [PluginCellValue]?, - scope: SQLiteStatementScope + rules: SQLiteStatementRules, + bound: SQLiteReadBound? ) throws -> SQLiteRawResult { - try withStatementHandle(scope) { handle, mustRead in - try runStatement(query, parameters: parameters, on: handle, mustRead: mustRead) + try withStatementHandle(rules) { handle, notReadRefusal in + try runStatement(query, parameters: parameters, on: handle, notReadRefusal: notReadRefusal, bound: bound) } } - // `mustRead` is true when the statement has to be one SQLite reads as read-only. A read-only statement runs on the - // session's handle, so temporary objects, attachments, `last_insert_rowid()` and `changes()` read as they do there. + // `notReadRefusal` is set when the statement has to be one SQLite reads as read-only. A read-only statement runs on + // the session's handle, so temporary objects, attachments, `last_insert_rowid()` and `changes()` read as they do there. private func withStatementHandle( - _ scope: SQLiteStatementScope, - _ body: (OpaquePointer, Bool) throws -> T + _ rules: SQLiteStatementRules, + _ body: (OpaquePointer, SQLitePluginError?) throws -> T ) throws -> T { guard let db, let policy else { throw SQLitePluginError.notConnected } busyState.beginOperation() - switch scope { - case .unrestricted: - return try body(db, false) - case .noServerCode: - policy.scope = .noServerCode - defer { policy.scope = .unrestricted } - return try body(db, false) - case .readOnly: - return try withQueryOnly(db, policy: policy) { try body(db, true) } + guard rules.readsOnly else { + return try Self.within(rules.scope, of: policy) { try body(db, nil) } + } + // The pragmas run before the scope applies: the `.readOnly` scope refuses a pragma that sets anything. + return try withQueryOnly(db) { + try Self.within(rules.scope, of: policy) { try body(db, rules.notReadRefusal) } } } - // query_only stops writes to the database files. It does not stop an extension's function or virtual table from - // reaching other files, and the policy refuses those. - private func withQueryOnly(_ db: OpaquePointer, policy: SQLiteAuthorizerPolicy, _ body: () throws -> T) throws -> T { + private static func within( + _ scope: SQLiteStatementScope, + of policy: SQLiteAuthorizerPolicy, + _ body: () throws -> T + ) rethrows -> T { + guard scope != .unrestricted else { return try body() } + policy.scope = scope + defer { policy.scope = .unrestricted } + return try body() + } + + // query_only stops writes to the database files, a write an application function makes among them. It does not stop + // an extension's function or virtual table from reaching other files, and the `.readOnly` scope refuses those. + private func withQueryOnly(_ db: OpaquePointer, _ body: () throws -> T) throws -> T { let wasQueryOnly = try Self.integerPragma("PRAGMA query_only", on: db) == 1 if !wasQueryOnly { try Self.exec("PRAGMA query_only = 1", on: db) @@ -321,8 +379,6 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { try? Self.exec("PRAGMA query_only = 0", on: db) } } - policy.scope = .readOnly - defer { policy.scope = .unrestricted } return try body() } @@ -345,14 +401,15 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { _ query: String, parameters: [PluginCellValue]?, on db: OpaquePointer, - mustRead: Bool + notReadRefusal: SQLitePluginError?, + bound: SQLiteReadBound? ) throws -> SQLiteRawResult { let startTime = Date() let statement = try Self.prepareOnlyStatement(query, db: db) defer { sqlite3_finalize(statement) } // EXPLAIN never runs its statement, but stmt_readonly reports that statement's writes. - if mustRead, sqlite3_stmt_readonly(statement) == 0, sqlite3_stmt_isexplain(statement) == 0 { - throw SQLitePluginError.readOnlyStatement + if let notReadRefusal, sqlite3_stmt_readonly(statement) == 0, sqlite3_stmt_isexplain(statement) == 0 { + throw notReadRefusal } if let parameters { @@ -368,13 +425,23 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { var rows: [[PluginCellValue]] = [] var truncated = false + let rowLimit = bound?.rowCap ?? PluginRowLimits.emergencyMax + var bytes = 0 var stepResult = firstStep.result while stepResult == SQLITE_ROW { - if rows.count >= PluginRowLimits.emergencyMax { + if rows.count >= rowLimit { truncated = true break } - rows.append(Self.readRow(statement, columnCount: columnCount)) + let row = Self.readRow(statement, columnCount: columnCount) + rows.append(row) + if let byteLimit = bound?.byteLimit { + bytes += SQLiteReadBound.byteCount(of: row) + if bytes > byteLimit { + truncated = true + break + } + } stepResult = sqlite3_step(statement) } @@ -429,27 +496,23 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { return Int(sqlite3_changes64(db)) } - func streamQuery( - _ query: String, - scope: SQLiteStatementScope, - continuation: AsyncThrowingStream.Continuation - ) throws { - try withStatementHandle(scope) { handle, mustRead in - try stream(query, on: handle, mustRead: mustRead, continuation: continuation) + func streamQuery(_ query: String, rules: SQLiteStatementRules, into spill: PluginRowSpill) throws { + try withStatementHandle(rules) { handle, notReadRefusal in + try stream(query, on: handle, notReadRefusal: notReadRefusal, into: spill) } } private func stream( _ query: String, on db: OpaquePointer, - mustRead: Bool, - continuation: AsyncThrowingStream.Continuation + notReadRefusal: SQLitePluginError?, + into spill: PluginRowSpill ) throws { let statement = try Self.prepareOnlyStatement(query, db: db) // EXPLAIN never runs its statement, but stmt_readonly reports that statement's writes. - if mustRead, sqlite3_stmt_readonly(statement) == 0, sqlite3_stmt_isexplain(statement) == 0 { + if let notReadRefusal, sqlite3_stmt_readonly(statement) == 0, sqlite3_stmt_isexplain(statement) == 0 { sqlite3_finalize(statement) - throw SQLitePluginError.readOnlyStatement + throw notReadRefusal } let firstStep = SQLiteResultColumns.stepFirst(statement) @@ -458,7 +521,7 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { columns: firstStep.names, columnTypeNames: firstStep.typeNames, estimatedRowCount: nil ) header.columnOrigins = Self.columnOrigins(after: firstStep, of: statement, on: db, text: query) - continuation.yield(.header(header)) + spill.yield(header) let batchSize = 5_000 var batch: [PluginRow] = [] @@ -467,29 +530,33 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { var stepResult = firstStep.result while stepResult == SQLITE_ROW { if Task.isCancelled { - if !batch.isEmpty { continuation.yield(.rows(batch)) } + if !batch.isEmpty { spill.deliver(batch) } sqlite3_finalize(statement) - continuation.finish(throwing: CancellationError()) + spill.finish(throwing: CancellationError()) return } batch.append(Self.readRow(statement, columnCount: columnCount)) if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // Synchronous on the actor's thread: an await here would let another call reach the handle mid-statement. + guard spill.deliver(batch) else { + sqlite3_finalize(statement) + return + } batch.removeAll(keepingCapacity: true) } stepResult = sqlite3_step(statement) } - if !batch.isEmpty { continuation.yield(.rows(batch)) } + if !batch.isEmpty { spill.deliver(batch) } guard stepResult == SQLITE_DONE else { let message = String(cString: sqlite3_errmsg(db)) sqlite3_finalize(statement) - continuation.finish(throwing: SQLitePluginError.queryFailed(message)) + spill.finish(throwing: SQLitePluginError.queryFailed(message)) return } sqlite3_finalize(statement) - continuation.finish() + spill.finish() } private func bind(_ parameters: [PluginCellValue], to statement: OpaquePointer?, db: OpaquePointer) throws { diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift index 439acba77e..c4175d87dc 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift @@ -152,6 +152,7 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .dataCompare, .temporaryTables, .readOnlyStatements, + .enforcesReadOnlyMode, ] } @@ -159,9 +160,13 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { sqliteQuoteIdentifier(name) } - init(config: DriverConnectionConfig) { + convenience init(config: DriverConnectionConfig) { + self.init(config: config, backend: Self.makeBackend(config: config)) + } + + init(config: DriverConnectionConfig, backend: any SQLiteExecutionBackend) { self.config = config - self.backend = Self.makeBackend(config: config) + self.backend = backend } /// A file-backed connection runs on the app's own SQLite; one whose transport marks it a remote @@ -226,9 +231,10 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { - let scope = SQLiteStatementScope(context) - guard scope != .unrestricted else { return try await execute(query: query) } - return Self.queryResult(try await backend.executeQuery(query, parameters: nil, scope: scope)) + let rules = SQLiteStatementRules(context) + guard rules != .unrestricted else { return try await execute(query: query) } + let bound = Self.bound(rowCap: nil, context: context, rules: rules) + return Self.queryResult(try await backend.executeQuery(query, parameters: nil, rules: rules, bound: bound)) } func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { @@ -240,9 +246,19 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - let scope = SQLiteStatementScope(context) - guard scope != .unrestricted else { return try await executeParameterized(query: query, parameters: parameters) } - return Self.queryResult(try await backend.executeQuery(query, parameters: parameters, scope: scope)) + let rules = SQLiteStatementRules(context) + guard rules != .unrestricted else { return try await executeParameterized(query: query, parameters: parameters) } + let bound = Self.bound(rowCap: nil, context: context, rules: rules) + return Self.queryResult(try await backend.executeQuery(query, parameters: parameters, rules: rules, bound: bound)) + } + + // Only a read-only run stops at the byte limit: stopping a write would cancel what the caller approved. + private static func bound( + rowCap: Int?, + context: PluginStatementContext, + rules: SQLiteStatementRules + ) -> SQLiteReadBound? { + SQLiteReadBound.ifSet(rowCap: rowCap, byteLimit: rules.readsOnly ? context.resultByteLimit : nil) } private static func queryResult(_ rawResult: SQLiteRawResult) -> PluginQueryResult { @@ -341,29 +357,39 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return Self.capped(raw, to: rowCap) } + // A capped statement that returns rows stops in the step loop one row past the cap, a read-only one also at the byte + // limit. Any other runs to its end. func executeUserQuery( query: String, rowCap: Int?, parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - guard SQLiteStatementScope(context) != .unrestricted else { + let rules = SQLiteStatementRules(context) + guard rules != .unrestricted else { return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } - if parameters == nil, let cap = rowCap, cap > 0, Self.returnsRows(query) { - return try await PluginBoundedStream.collect( - streamRows(query: query, context: context), - rowCap: cap, - startedAt: Date() - ) - } - let raw: PluginQueryResult - if let parameters { - raw = try await executeParameterized(query: query, parameters: parameters, context: context) - } else { - raw = try await execute(query: query, context: context) + let bound = Self.bound(rowCap: Self.returnsRows(query) ? rowCap : nil, context: context, rules: rules) + let raw = try await backend.executeQuery(query, parameters: parameters, rules: rules, bound: bound) + return Self.capped(Self.queryResult(raw), to: rowCap) + } + + // A context the read-only mode does not cover goes back to the host as PluginKit's default sends it. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + let rules = SQLiteStatementRules(context) + guard rules.readsOnly else { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await executeBoundedQuery(query: query, rowCap: rowCap) } - return Self.capped(raw, to: rowCap) + guard Self.returnsRows(query) else { return nil } + let bound = SQLiteReadBound(rowCap: rowCap, byteLimit: context.resultByteLimit) + return Self.queryResult(try await backend.executeQuery(query, parameters: nil, rules: rules, bound: bound)) } private static func capped(_ raw: PluginQueryResult, to rowCap: Int?) -> PluginQueryResult { @@ -683,24 +709,26 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Streaming func streamRows(query: String) -> AsyncThrowingStream { - stream(query, scope: .unrestricted) + stream(query, rules: .unrestricted) } func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { - stream(query, scope: SQLiteStatementScope(context)) + stream(query, rules: SQLiteStatementRules(context)) } - private func stream(_ query: String, scope: SQLiteStatementScope) -> AsyncThrowingStream { + private func stream(_ query: String, rules: SQLiteStatementRules) -> AsyncThrowingStream { let queryToRun = String(query) - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in + let spill = PluginRowSpill(continuation: continuation) let streamTask = Task { do { - try await self.backend.streamQuery(queryToRun, scope: scope, continuation: continuation) + try await self.backend.streamQuery(queryToRun, rules: rules, into: spill) } catch { - continuation.finish(throwing: error) + spill.finish(throwing: error) } } continuation.onTermination = { @Sendable _ in + spill.abandon() streamTask.cancel() } } @@ -802,6 +830,11 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } + // The schema cookie in the main database header, which SQLite bumps on every schema change. + func schemaChangeToken() async throws -> String? { + try await execute(query: "PRAGMA main.schema_version").rows.first?.first?.asText + } + func columnReorderSchemaFingerprint(table: String, schema: String?) async throws -> String? { try Self.requireMainDatabase(schema) return try await SQLiteColumnReorderPlanner.schemaFingerprint( @@ -843,18 +876,6 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } -extension SQLiteStatementScope { - init(_ context: PluginStatementContext) { - if context.readOnly, context.origin == .external { - self = .readOnly - } else if !context.allowsServerCode { - self = .noServerCode - } else { - self = .unrestricted - } - } -} - // MARK: - Errors enum SQLitePluginError: Error { @@ -864,6 +885,7 @@ enum SQLitePluginError: Error { case unsupportedOperation case attachedDatabaseStructure case readOnlyStatement + case safeModeReadOnlyStatement } extension SQLitePluginError: PluginDriverError { @@ -877,6 +899,8 @@ extension SQLitePluginError: PluginDriverError { return String(localized: "The structure of a table in an attached database cannot be changed here.") case .readOnlyStatement: return String(localized: "This request is read only, and the statement is not a read.") + case .safeModeReadOnlyStatement: + return String(localized: "Safe Mode is set to Read-Only for this connection, and the database refused the statement.") } } } diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift b/apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift new file mode 100644 index 0000000000..557c9addf7 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift @@ -0,0 +1,197 @@ +// +// SQLiteAgentLoopback.swift +// SQLiteDriverTests +// + +import Darwin +import Foundation +import os + +// The SSH session's stand-in: a loopback listener that admits one client by its token line, runs the agent under +// the local python3, and relays each direction on its own thread, so the agent's output can wait on a reader that +// stopped while the client's frames keep reaching it, as over the session pump. When the agent exits, the client's +// socket closes, as the relay's does when its channel closes. +final class SQLiteAgentLoopback: @unchecked Sendable { + static var python3Path: String? { + ["/usr/bin/python3", "/opt/homebrew/bin/python3", "/usr/local/bin/python3"].first { + FileManager.default.isExecutableFile(atPath: $0) + } + } + + // How the launcher runs the shipped program. + static var shippedArguments: [String] { + ["-I", "-S", "-c", RemoteSQLiteAgentSource.interpreterBootstrap, RemoteSQLiteAgentSource.encodedProgram] + } + + let port: Int + let token = UUID().uuidString + + private struct Counters { + var agentBytes = 0 + var heartbeats = 0 + var requests = 0 + var clientClosed = false + } + + private let listenFD: Int32 + private let executable: String + private let arguments: [String] + private let showsErrors: Bool + private let counters = OSAllocatedUnfairLock(initialState: Counters()) + private let process = OSAllocatedUnfairLock(initialState: nil) + + // The local python3 by default; a probe passes ssh and the launcher command to run the agent on a real server, + // and keeps ssh's own errors on standard error. + init( + arguments: [String] = SQLiteAgentLoopback.shippedArguments, + executable: String? = nil, + showsErrors: Bool = false + ) throws { + guard let executable = executable ?? Self.python3Path else { throw LoopbackError.noPython } + self.executable = executable + self.arguments = arguments + self.showsErrors = showsErrors + + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw LoopbackError.socket } + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_addr.s_addr = inet_addr("127.0.0.1") + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(fd, $0, socklen_t(MemoryLayout.size)) + } + } + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(fd, $0, &length) } + } + guard bound == 0, named == 0, listen(fd, 1) == 0 else { + Darwin.close(fd) + throw LoopbackError.socket + } + listenFD = fd + port = Int(UInt16(bigEndian: address.sin_port)) + + Thread.detachNewThread { [self] in acceptOne() } + } + + // Bytes the agent wrote toward the client. + var agentBytes: Int { counters.withLock { $0.agentBytes } } + var heartbeats: Int { counters.withLock { $0.heartbeats } } + // Execute frames the client sent. + var requests: Int { counters.withLock { $0.requests } } + var agentIsRunning: Bool { process.withLock { $0?.isRunning ?? false } } + var clientClosed: Bool { counters.withLock { $0.clientClosed } } + + func killAgent() { + process.withLock { $0?.terminate() } + } + + func close() { + killAgent() + shutdown(listenFD, SHUT_RDWR) + Darwin.close(listenFD) + } + + enum LoopbackError: Error { case noPython, socket } + + private func acceptOne() { + let client = accept(listenFD, nil, nil) + guard client >= 0 else { return } + var noSignal: Int32 = 1 + setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSignal, socklen_t(MemoryLayout.size)) + guard readLine(client) == "TPRSQL1 \(token)" else { + Darwin.close(client) + return + } + + let toAgent = Pipe() + let fromAgent = Pipe() + let agent = Process() + agent.executableURL = URL(fileURLWithPath: executable) + agent.arguments = arguments + agent.standardInput = toAgent + agent.standardOutput = fromAgent + agent.standardError = showsErrors ? FileHandle.standardError : FileHandle.nullDevice + do { + try agent.run() + } catch { + Darwin.close(client) + return + } + process.withLock { $0 = agent } + let agentInput = toAgent.fileHandleForWriting.fileDescriptor + _ = fcntl(agentInput, F_SETNOSIGPIPE, 1) + let agentOutput = fromAgent.fileHandleForReading.fileDescriptor + + Thread.detachNewThread { [self] in + relayToAgent(from: client, to: agentInput) + toAgent.fileHandleForWriting.closeFile() + Darwin.close(client) + counters.withLock { $0.clientClosed = true } + } + Thread.detachNewThread { [self] in + relayToClient(from: agentOutput, to: client) + shutdown(client, SHUT_RDWR) + } + } + + private func readLine(_ fd: Int32) -> String? { + var line = Data() + var byte: UInt8 = 0 + while line.count < 256 { + guard recv(fd, &byte, 1, 0) == 1 else { return nil } + if byte == 0x0A { return String(data: line, encoding: .utf8) } + line.append(byte) + } + return nil + } + + // Counts each frame by its opcode on the way: 0x04 is a heartbeat, 0x02 and 0x06 an execute. + private func relayToAgent(from client: Int32, to agentInput: Int32) { + var buffer = [UInt8](repeating: 0, count: 32_768) + var frames: [UInt8] = [] + while true { + let count = read(client, &buffer, buffer.count) + guard count > 0 else { return } + frames.append(contentsOf: buffer[0..= 5 { + let length = frames[offset..<(offset + 4)].reduce(0) { ($0 << 8) | Int($1) } + guard frames.count - offset >= 4 + length else { break } + let opcode = frames[offset + 4] + counters.withLock { + if opcode == 0x04 { $0.heartbeats += 1 } + if opcode == 0x02 || opcode == 0x06 { $0.requests += 1 } + } + offset += 4 + length + } + frames.removeFirst(offset) + guard writeAll(buffer[0.. 0 else { return } + counters.withLock { $0.agentBytes += count } + guard writeAll(buffer[0.., to fd: Int32) -> Bool { + bytes.withUnsafeBytes { raw in + guard let base = raw.baseAddress else { return true } + var offset = 0 + while offset < raw.count { + let written = write(fd, base.advanced(by: offset), raw.count - offset) + guard written > 0 else { return false } + offset += written + } + return true + } + } +} diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift index 626e1cfc00..de9d2dd4c0 100644 --- a/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift +++ b/apple/Plugins/SQLiteDriverTests/SQLiteAgentProtocolTests.swift @@ -87,6 +87,28 @@ struct SQLiteAgentProtocolTests { #expect(try reader.nextReply() == .done(changes: 3, truncated: false)) } + // The reader drops what it already decoded between reads, so a long stream must not lose or repeat a byte. + @Test func longStreamSplitAcrossReadsDecodesEveryFrame() throws { + let replies = (0..<300).map { index in + SQLiteAgentReply.rows(columnCount: 2, values: [.text(Data("\(index)".utf8)), .blob(Data(repeating: UInt8(index % 256), count: index * 37))]) + } + var stream = Data() + for reply in replies { stream.append(SQLiteAgentFrameEncoder.encode(reply)) } + + var reader = SQLiteAgentFrameReader() + var decoded: [SQLiteAgentReply] = [] + var offset = 0 + var chunk = 1 + while offset < stream.count { + let end = min(offset + chunk, stream.count) + reader.append(stream.subdata(in: offset.. [String] { + let deadline = "IDLE_DEADLINE_SECONDS = 60.0" + let poll = "time.sleep(5.0)" + let source = RemoteSQLiteAgentSource.python + try #require(source.contains(deadline) && source.contains(poll), "the agent's watchdog constants moved") + let scaled = source + .replacingOccurrences(of: deadline, with: "IDLE_DEADLINE_SECONDS = \(watchdogSeconds)") + .replacingOccurrences(of: poll, with: "time.sleep(0.1)") + return ["-I", "-S", "-c", scaled] + } + + private struct Fixture { + let loopback: SQLiteAgentLoopback + let driver: SQLitePluginDriver + let backend: SQLiteAgentBackend + let path: String + } + + private static func makeDatabase() async throws -> String { + let path = FileManager.default.temporaryDirectory.appendingPathComponent("agent-pacing-\(UUID().uuidString).db").path + let local = SQLitePluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: path + )) + try await local.connect() + _ = try await local.execute(query: """ + CREATE TABLE t AS WITH RECURSIVE g(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM g WHERE x < \(rowCount)) + SELECT x AS id, hex(randomblob(100)) AS payload FROM g + """) + local.disconnect() + return path + } + + // Closes the session, the agent and the file on every way out, so a failed case leaves no python3 running. + private static func withAgent( + arguments: [String] = SQLiteAgentLoopback.shippedArguments, + heartbeat: TimeInterval = heartbeatSeconds, + _ body: (Fixture) async throws -> Void + ) async throws { + let path = try await makeDatabase() + defer { try? FileManager.default.removeItem(atPath: path) } + let loopback = try SQLiteAgentLoopback(arguments: arguments) + defer { loopback.close() } + let fields = [SQLiteAgentProtocol.backendFieldKey: SQLiteAgentProtocol.agentBackendValue] + let backend = SQLiteAgentBackend( + host: "127.0.0.1", + port: loopback.port, + path: path, + token: loopback.token, + helloBudget: try #require(SQLiteAgentHelloBudget.forAgent(additionalFields: fields)), + heartbeatInterval: heartbeat + ) + let driver = SQLitePluginDriver( + config: DriverConnectionConfig( + host: "127.0.0.1", port: loopback.port, username: "", password: "", database: path, + additionalFields: fields + ), + backend: backend + ) + do { + try await driver.connect() + try await body(Fixture(loopback: loopback, driver: driver, backend: backend, path: path)) + } catch { + await backend.close() + throw error + } + await backend.close() + } + + private struct NoAnswer: Error {} + + // A statement queued behind a stream that never ends ignores the time limit's cancellation, so it gets its own. + private static func answer(_ statement: @escaping @Sendable () async throws -> PluginQueryResult) async throws -> String? { + let result = await BoundedCall.result(within: .seconds(30)) { + do { + return Result.success(try await statement()) + } catch { + return .failure(error) + } + } + guard let result else { throw NoAnswer() } + return try result.get().rows.first?.first?.asText + } + + private static func ids(_ batch: [PluginRow]) -> [Int] { + batch.compactMap { $0.first?.asText.flatMap(Int.init) } + } + + // Counts the rows taken and the ids that are not the next one expected. + private struct OrderedRead { + var expected = 1 + var mismatches = 0 + + mutating func take(_ element: PluginStreamElement) { + guard case .rows(let batch) = element else { return } + for id in SQLiteAgentStreamPacingTests.ids(batch) { + if id != expected { mismatches += 1 } + expected += 1 + } + } + } + + @Test("A reader held past the agent's watchdog gets every row, in order, and the agent's read ends without it") + func heldReaderOutlastsTheWatchdog() async throws { + try await Self.withAgent(arguments: try Self.scaledProgram()) { fixture in + var iterator = fixture.driver.streamRows(query: Self.scan).makeAsyncIterator() + var read = OrderedRead() + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + + // The agent read the whole table into the spill, so the connection takes the next statement at once. + let driver = fixture.driver + #expect(try await Self.answer { try await driver.execute(query: "SELECT count(*) FROM t") } == String(Self.rowCount)) + + // Long enough for the watchdog to have ended the session twice over without heartbeats. + let heartbeatsBeforeHold = fixture.loopback.heartbeats + let holdHeartbeats = Int(Self.watchdogSeconds * 2.5 / Self.heartbeatSeconds) + try await waitUntil(timeout: .seconds(60)) { + fixture.loopback.heartbeats - heartbeatsBeforeHold >= holdHeartbeats + } + #expect(fixture.loopback.agentIsRunning) + + while let element = try await iterator.next() { read.take(element) } + #expect(read.expected - 1 == Self.rowCount) + #expect(read.mismatches == 0) + #expect(try await Self.answer { try await driver.execute(query: "SELECT 42") } == "42") + } + } + + @Test("Without heartbeats an idle session outlasts the watchdog and its next statement fails") + func idleWithoutHeartbeatsLosesTheSession() async throws { + try await Self.withAgent(arguments: try Self.scaledProgram(), heartbeat: 3_600) { fixture in + _ = try await fixture.driver.execute(query: "SELECT 1") + try await waitUntil(timeout: .seconds(30)) { !fixture.loopback.agentIsRunning } + + let driver = fixture.driver + let failed = await BoundedCall.result(within: .seconds(5)) { + do { + _ = try await driver.execute(query: "SELECT 1") + return false + } catch { + return true + } + } + #expect(failed == true) + } + } + + @Test("A reader that leaves stops the agent's read, and the next statement gets its own answer") + func abandonedReadStopsTheAgent() async throws { + try await Self.withAgent { fixture in + var iterator: AsyncThrowingStream.Iterator? = + fixture.driver.streamRows(query: Self.endless).makeAsyncIterator() + var rows = 0 + while rows == 0, let element = try await iterator?.next() { + if case .rows(let batch) = element { rows += batch.count } + } + iterator = nil + + let driver = fixture.driver + #expect(try await Self.answer { try await driver.execute(query: "SELECT 42") } == "42") + #expect(fixture.loopback.agentIsRunning) + } + } + + @Test("A statement sent while the agent is still reading a stream waits for it instead of taking its rows") + func statementWaitsForTheOpenStream() async throws { + try await Self.withAgent { fixture in + var iterator: AsyncThrowingStream.Iterator? = + fixture.driver.streamRows(query: Self.endless).makeAsyncIterator() + var rows = 0 + while rows == 0, let element = try await iterator?.next() { + if case .rows(let batch) = element { rows += batch.count } + } + + let driver = fixture.driver + let statement = Task { try await driver.execute(query: "SELECT 42") } + // The reader holds, and the agent keeps reading into the spill while the statement stays queued. + let bytesWhenSent = fixture.loopback.agentBytes + try await waitUntil(timeout: .seconds(30)) { fixture.loopback.agentBytes > bytesWhenSent + 20_000_000 } + #expect(fixture.loopback.requests == 1, "the statement went out while the stream was open") + + iterator = nil + #expect(try await Self.answer { try await statement.value } == "42") + } + } + + @Test("An agent that goes away fails the next statement at once instead of ending the app or hanging") + func agentGoneFailsTheNextStatement() async throws { + try await Self.withAgent(heartbeat: 0.05) { fixture in + _ = try await fixture.driver.execute(query: "SELECT 1") + + fixture.loopback.killAgent() + try await waitUntil { fixture.loopback.clientClosed } + + // The second write to a closed socket raised SIGPIPE, and a request nothing would read waited forever. + let driver = fixture.driver + for _ in 0..<3 { + let failed = await BoundedCall.result(within: .seconds(5)) { + do { + _ = try await driver.execute(query: "SELECT 1") + return false + } catch { + return true + } + } + #expect(failed == true) + } + } + } +} diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteReadOnlyStatementTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteReadOnlyStatementTests.swift index 69efdd6ee4..0a3340d1e4 100644 --- a/apple/Plugins/SQLiteDriverTests/SQLiteReadOnlyStatementTests.swift +++ b/apple/Plugins/SQLiteDriverTests/SQLiteReadOnlyStatementTests.swift @@ -36,6 +36,10 @@ struct SQLiteReadOnlyStatementTests { return context } + private static var safeMode: PluginStatementContext { + PluginStatementContext(enforcesReadOnly: true) + } + private func connect(_ database: String) async throws -> SQLitePluginDriver { let driver = SQLitePluginDriver(config: DriverConnectionConfig( host: "", port: 0, username: "", password: "", database: database @@ -54,12 +58,181 @@ struct SQLiteReadOnlyStatementTests { try FileManager.default.contentsOfDirectory(atPath: directory.path).sorted() } - @Test("The driver says it runs a read-only statement in SQLite's own read-only mode") + @Test("The driver says it runs a read-only statement in SQLite's own read-only mode, from outside and at Read-Only") func declaresReadOnlyStatements() { let driver = SQLitePluginDriver(config: DriverConnectionConfig( host: "", port: 0, username: "", password: "", database: path("app.db") )) #expect(driver.capabilities.contains(.readOnlyStatements)) + #expect(driver.capabilities.contains(.enforcesReadOnlyMode)) + } + + @Test("Read-Only reads read-only under the session's own scope, an external read under the read-only scope") + func rulesFollowTheContext() { + var externalSafeModeWrite = Self.external + externalSafeModeWrite.enforcesReadOnly = true + var catalogRead = PluginStatementContext() + catalogRead.readOnly = true + catalogRead.readsCatalog = true + + #expect(SQLiteStatementRules(PluginStatementContext()) == .unrestricted) + #expect(SQLiteStatementRules(catalogRead) == .unrestricted) + #expect(SQLiteStatementRules(Self.safeMode) == SQLiteStatementRules(scope: .unrestricted, readsOnly: true)) + #expect(SQLiteStatementRules(Self.externalRead) == SQLiteStatementRules(scope: .readOnly, readsOnly: true)) + #expect(SQLiteStatementRules(Self.external) == SQLiteStatementRules(scope: .noServerCode, readsOnly: false)) + #expect(SQLiteStatementRules(externalSafeModeWrite) == SQLiteStatementRules(scope: .noServerCode, readsOnly: true)) + #expect(SQLiteAgentBackend.agentScope(SQLiteStatementRules(Self.safeMode)) == .readOnly) + #expect(SQLiteAgentBackend.agentScope(.unrestricted) == nil) + #expect(SQLiteAgentBackend.agentScope(SQLiteStatementRules(Self.external)) == .noServerCode) + } + + // An application function that deletes through its own connection: the shape of a SQL function that writes. + private static let purgeKept: @convention(c) (OpaquePointer?, Int32, UnsafeMutablePointer?) -> Void = { context, _, _ in + let db = sqlite3_context_db_handle(context) + guard sqlite3_exec(db, "DELETE FROM kept", nil, nil, nil) == SQLITE_OK else { + sqlite3_result_error(context, sqlite3_errmsg(db), -1) + return + } + sqlite3_result_int(context, 1) + } + + private typealias ExtensionEntryPoint = @convention(c) ( + OpaquePointer?, UnsafeMutablePointer?>?, UnsafeRawPointer? + ) -> Int32 + + private static let registerPurgeKept: ExtensionEntryPoint = { db, _, _ in + sqlite3_create_function(db, "purge_kept", 0, SQLITE_UTF8, nil, SQLiteReadOnlyStatementTests.purgeKept, nil, nil) + } + + // The driver opens its own handle, so the function goes onto every handle opened while the driver connects. + private func connectWithPurgeKept() async throws -> SQLitePluginDriver { + let entry = unsafeBitCast(Self.registerPurgeKept, to: (@convention(c) () -> Void).self) + #expect(sqlite3_auto_extension(entry) == SQLITE_OK) + defer { sqlite3_cancel_auto_extension(entry) } + return try await connect(path("app.db")) + } + + @Test("At Read-Only a function that writes is refused on every entry point, and the default context lets it write") + func safeModeRefusesAWritingFunction() async throws { + let driver = try await connectWithPurgeKept() + defer { driver.disconnect() } + let call = "SELECT purge_kept()" + + await #expect(throws: (any Error).self) { + _ = try await driver.execute(query: call, context: Self.safeMode) + } + await #expect(throws: (any Error).self) { + _ = try await driver.executeUserQuery(query: call, rowCap: 100, parameters: nil, context: Self.safeMode) + } + await #expect(throws: (any Error).self) { + _ = try await driver.executeParameterized( + query: "SELECT purge_kept(), ?", parameters: [.null], context: Self.safeMode + ) + } + await #expect(throws: (any Error).self) { + _ = try await driver.executeBoundedQuery(query: call, rowCap: 100, context: Self.safeMode) + } + await #expect(throws: (any Error).self) { + for try await _ in driver.streamRows(query: call, context: Self.safeMode) {} + } + await #expect(throws: (any Error).self) { + _ = try await driver.execute(query: call, context: Self.externalRead) + } + #expect(try await count(driver) == "1") + #expect(try await driver.execute(query: "PRAGMA query_only").rows.first?.first?.asText == "0") + + _ = try await driver.execute(query: call, context: PluginStatementContext()) + #expect(try await count(driver) == "0") + } + + @Test("At Read-Only a write is refused, a read runs, and the default context still writes") + func safeModeRefusesWrites() async throws { + let driver = try await connect(path("app.db")) + defer { driver.disconnect() } + + for sql in ["INSERT INTO kept VALUES (2)", "UPDATE kept SET a = 5", "CREATE TABLE added (a INTEGER)"] { + await #expect(throws: (any Error).self, "\(sql)") { + _ = try await driver.execute(query: sql, context: Self.safeMode) + } + } + let safeModeRefusal = await #expect(throws: SQLitePluginError.self) { + _ = try await driver.executeUserQuery( + query: "DELETE FROM kept", rowCap: 10, parameters: nil, context: Self.safeMode + ) + } + #expect(safeModeRefusal?.pluginErrorMessage == SQLitePluginError.safeModeReadOnlyStatement.pluginErrorMessage) + let externalRefusal = await #expect(throws: SQLitePluginError.self) { + _ = try await driver.execute(query: "DELETE FROM kept", context: Self.externalRead) + } + #expect(externalRefusal?.pluginErrorMessage == SQLitePluginError.readOnlyStatement.pluginErrorMessage) + let read = try await driver.executeBoundedQuery(query: "SELECT a FROM kept", rowCap: 10, context: Self.safeMode) + #expect(read?.rows.compactMap { $0.first?.asText } == ["1"]) + #expect(read?.isTruncated == false) + #expect(try await count(driver) == "1") + + _ = try await driver.execute(query: "INSERT INTO kept VALUES (2)", context: PluginStatementContext()) + #expect(try await count(driver) == "2") + } + + @Test("A bounded read the read-only mode does not cover goes back to the host") + func boundedReadGoesBackWhenNotCovered() async throws { + let driver = try await connect(path("app.db")) + defer { driver.disconnect() } + var catalogRead = PluginStatementContext() + catalogRead.readOnly = true + catalogRead.readsCatalog = true + + #expect(try await driver.executeBoundedQuery(query: "SELECT a FROM kept", rowCap: 10, context: catalogRead) == nil) + #expect(try await driver.executeBoundedQuery(query: "SELECT a FROM kept", rowCap: 10, context: Self.external) == nil) + let plain = try await driver.executeBoundedQuery( + query: "SELECT a FROM kept", rowCap: 10, context: PluginStatementContext() + ) + #expect(plain?.rows.count == 1) + } + + @Test("A read-only read stops at the first row past the byte limit and keeps that row") + func readOnlyReadStopsAtTheByteLimit() async throws { + let driver = try await connect(path("app.db")) + defer { driver.disconnect() } + _ = try await driver.execute(query: "DELETE FROM kept") + // Each value reads as six bytes of text. + _ = try await driver.execute(query: "INSERT INTO kept VALUES (100000), (200000), (300000), (400000)") + let sql = "SELECT a FROM kept ORDER BY a" + var limited = Self.externalRead + limited.resultByteLimit = 10 + var safeModeLimited = Self.safeMode + safeModeLimited.resultByteLimit = 10 + + let capped = try await driver.executeUserQuery(query: sql, rowCap: 100, parameters: nil, context: limited) + #expect(capped.rows.compactMap { $0.first?.asText } == ["100000", "200000"]) + #expect(capped.isTruncated) + + let whole = try await driver.execute(query: sql, context: limited) + #expect(whole.rows.count == 2) + #expect(whole.isTruncated) + + let bounded = try await driver.executeBoundedQuery(query: sql, rowCap: 100, context: safeModeLimited) + #expect(bounded?.rows.count == 2) + #expect(bounded?.isTruncated == true) + + let capFirst = try await driver.executeUserQuery(query: sql, rowCap: 1, parameters: nil, context: limited) + #expect(capFirst.rows.compactMap { $0.first?.asText } == ["100000"]) + #expect(capFirst.isTruncated) + + var roomy = Self.externalRead + roomy.resultByteLimit = 1_000 + let all = try await driver.executeUserQuery(query: sql, rowCap: 100, parameters: nil, context: roomy) + #expect(all.rows.count == 4) + #expect(!all.isTruncated) + } + + @Test("A read bound exists only when a cap or a byte limit is set, and a row's size is its decoded bytes") + func readBoundShape() { + #expect(SQLiteReadBound.ifSet(rowCap: nil, byteLimit: nil) == nil) + #expect(SQLiteReadBound.ifSet(rowCap: 0, byteLimit: nil) == nil) + #expect(SQLiteReadBound.ifSet(rowCap: nil, byteLimit: 5)?.rowCap == PluginRowLimits.emergencyMax) + #expect(SQLiteReadBound.ifSet(rowCap: 7, byteLimit: nil) == SQLiteReadBound(rowCap: 7, byteLimit: nil)) + #expect(SQLiteReadBound.byteCount(of: [.null, .text("héllo"), .bytes(Data(count: 4))]) == 10) } @Test("A read-only statement that writes, attaches or copies the database is refused and changes nothing", arguments: [ diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteRowSpillTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteRowSpillTests.swift new file mode 100644 index 0000000000..f97823f276 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteRowSpillTests.swift @@ -0,0 +1,348 @@ +// +// SQLiteRowSpillTests.swift +// SQLiteDriverTests +// + +import Foundation +import os +import TableProPluginKit +import TableProTestSupport +import TableProWireIO +import Testing + +@Suite(.timeLimit(.minutes(2))) +struct SQLiteRowSpillTests { + // Forty batches of 5,000, well past the 16 the stream buffers, so a held reader leaves most of the table unread. + private static let rowCount = 200_000 + private static let scan = "SELECT id, payload FROM t ORDER BY rowid" + + private static func withFile( + wal: Bool = false, + _ body: (_ reader: SQLitePluginDriver, _ writer: SQLitePluginDriver) async throws -> Void + ) async throws { + let path = FileManager.default.temporaryDirectory.appendingPathComponent("row-spill-\(UUID().uuidString).db").path + defer { + for suffix in ["", "-wal", "-shm", "-journal"] { try? FileManager.default.removeItem(atPath: path + suffix) } + } + let config = DriverConnectionConfig(host: "", port: 0, username: "", password: "", database: path) + let reader = SQLitePluginDriver(config: config) + let writer = SQLitePluginDriver(config: config) + try await reader.connect() + defer { reader.disconnect() } + if wal { _ = try await reader.execute(query: "PRAGMA journal_mode=WAL") } + _ = try await reader.execute(query: """ + CREATE TABLE t AS WITH RECURSIVE g(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM g WHERE x < \(rowCount)) + SELECT x AS id, hex(randomblob(50)) AS payload FROM g + """) + try await writer.connect() + defer { writer.disconnect() } + // A lock wait gives up after a second, so a check that the file is free answers quickly either way. + try await writer.applyQueryTimeout(1) + try await body(reader, writer) + } + + // Counts the rows taken and the ids that are not the next one expected. + private struct OrderedRead { + var expected = 1 + var mismatches = 0 + + mutating func take(_ element: PluginStreamElement) { + guard case .rows(let batch) = element else { return } + for id in batch.compactMap({ $0.first?.asText.flatMap(Int.init) }) { + if id != expected { mismatches += 1 } + expected += 1 + } + } + } + + private static func insert(_ batch: [PluginRow]) -> String { + let values = batch.map { row in "(\(row[0].asText ?? "NULL"), '\(row[1].asText ?? "")')" } + return "INSERT INTO copied (id, payload) VALUES " + values.joined(separator: ", ") + } + + @Test("A reader that holds its stream no longer holds the file: another connection can take it for a write") + func heldReaderLeavesTheFileFree() async throws { + try await Self.withFile { reader, writer in + var iterator = reader.streamRows(query: Self.scan).makeAsyncIterator() + var read = OrderedRead() + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + + try await waitUntil(timeout: .seconds(30)) { + (try? await writer.execute(query: "BEGIN EXCLUSIVE")) != nil + } + _ = try await writer.execute(query: "COMMIT") + + while let element = try await iterator.next() { read.take(element) } + #expect(read.expected - 1 == Self.rowCount) + #expect(read.mismatches == 0) + } + } + + @Test("A copy into a second connection to the same file finishes instead of waiting on its own read") + func copyIntoASecondConnectionFinishes() async throws { + try await Self.withFile { reader, writer in + // Long enough for the read to end on a busy machine; main waits on itself past it. + try await writer.applyQueryTimeout(15) + _ = try await writer.execute(query: "CREATE TABLE copied (id INTEGER, payload TEXT)") + for try await element in reader.streamRows(query: Self.scan) { + guard case .rows(let batch) = element else { continue } + _ = try await writer.execute(query: Self.insert(batch)) + } + let copied = try await writer.execute(query: "SELECT count(*), min(id), max(id) FROM copied") + let summary = copied.rows.first?.compactMap { $0.asText } + #expect(summary == [String(Self.rowCount), "1", String(Self.rowCount)]) + } + } + + @Test("In WAL mode a reader that holds its stream no longer keeps a checkpoint from resetting the log") + func heldReaderLetsTheCheckpointRun() async throws { + try await Self.withFile(wal: true) { reader, writer in + var iterator = reader.streamRows(query: Self.scan).makeAsyncIterator() + var read = OrderedRead() + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + + try await waitUntil(timeout: .seconds(30)) { + _ = try await writer.execute(query: "INSERT INTO t (id, payload) VALUES (0, 'written during the read')") + let checkpoint = try await writer.execute(query: "PRAGMA wal_checkpoint(TRUNCATE)") + return checkpoint.rows.first?.first?.asText == "0" + } + + while let element = try await iterator.next() { read.take(element) } + #expect(read.expected - 1 == Self.rowCount) + #expect(read.mismatches == 0) + } + } + + @Test("A read that fails after rows were spilled hands over every earlier row, then the error") + func errorFollowsTheSpilledRows() async throws { + let failAt = 150_000 + try await Self.withFile { reader, writer in + let query = """ + SELECT CASE WHEN id = \(failAt) THEN abs(-9223372036854775808) ELSE id END AS id, payload + FROM t ORDER BY rowid + """ + var iterator = reader.streamRows(query: query).makeAsyncIterator() + var read = OrderedRead() + while read.expected == 1, let element = try await iterator.next() { read.take(element) } + try await waitUntil(timeout: .seconds(30)) { + (try? await writer.execute(query: "BEGIN EXCLUSIVE")) != nil + } + _ = try await writer.execute(query: "COMMIT") + + await #expect(throws: SQLitePluginError.self) { + while let element = try await iterator.next() { read.take(element) } + } + #expect(read.expected == failAt) + #expect(read.mismatches == 0) + } + } + + @Test("A reader that leaves an endless read stops it, and the connection runs the next statement") + func leavingAnEndlessReadStopsIt() async throws { + try await Self.withFile { reader, _ in + let endless = "WITH RECURSIVE g(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM g) SELECT x, hex(randomblob(100)) FROM g" + var iterator: AsyncThrowingStream.Iterator? = + reader.streamRows(query: endless).makeAsyncIterator() + var rows = 0 + while rows == 0, let element = try await iterator?.next() { + if case .rows(let batch) = element { rows += batch.count } + } + iterator = nil + + let answer = await BoundedCall.result(within: .seconds(30)) { + try? await reader.execute(query: "SELECT 42").rows.first?.first?.asText + } + #expect(answer == "42") + } + } + + // MARK: - The spill on its own + + private static let cells: PluginRow = [.null, .text(""), .text("naïve 🧪 \u{0}"), .bytes(Data()), .bytes(Data([0, 255, 7]))] + + private static func batch(_ index: Int) -> [PluginRow] { + (0..<100).map { [.text("\(index)-\($0)")] + cells } + } + + private static func makeStream( + budget: SpoolBudget = SpoolBudget(limit: 1 << 30), + makeSpool: @escaping @Sendable () throws -> SpoolFile = { try SpoolFile() } + ) -> (AsyncThrowingStream, PluginRowSpill) { + let (stream, continuation) = AsyncThrowingStream.makeStream( + of: PluginStreamElement.self, bufferingPolicy: .bufferingOldest(2) + ) + let spill = PluginRowSpill(continuation: continuation, budget: budget, makeSpool: makeSpool) + continuation.onTermination = { @Sendable _ in spill.abandon() } + return (stream, spill) + } + + private static func batches(of stream: AsyncThrowingStream) async throws -> [[PluginRow]] { + var batches: [[PluginRow]] = [] + for try await element in stream { + if case .rows(let rows) = element { batches.append(rows) } + } + return batches + } + + @Test("Batches the buffer cannot take come back from the spool in order, every cell as it went in") + func spilledBatchesKeepOrderAndCells() async throws { + let (stream, spill) = Self.makeStream() + // On its own thread, so a producer that waited for the reader would hit the deadline instead of hanging. + let delivered = await BoundedCall.resultOnItsOwnThread(within: .seconds(30)) { + let accepted = (0..<40).allSatisfy { spill.deliver(Self.batch($0)) } + spill.finish() + return accepted + } + #expect(delivered == true) + let expected = (0..<40).map { Self.batch($0) } + #expect(try await Self.batches(of: stream) == expected) + } + + @Test("An error after spilled batches reaches the reader after them") + func errorAfterSpilledBatches() async throws { + struct Failed: Error {} + let (stream, spill) = Self.makeStream() + for index in 0..<10 { spill.deliver(Self.batch(index)) } + spill.finish(throwing: Failed()) + + var received: [[PluginRow]] = [] + await #expect(throws: Failed.self) { + for try await element in stream { + if case .rows(let rows) = element { received.append(rows) } + } + } + let expected = (0..<10).map { Self.batch($0) } + #expect(received == expected) + } + + @Test("Without a spool the producer waits for the reader, and every batch still arrives in order") + func failedSpoolFallsBackToPacing() async throws { + let attempts = OSAllocatedUnfairLock(initialState: 0) + let (stream, spill) = Self.makeStream { + attempts.withLock { $0 += 1 } + throw SpoolFileError.cannotCreate(errno: ENOSPC) + } + let delivered = Self.produce(0..<20, into: spill) + // Batch 2 meets a full buffer, so it asks for the spool. + try await waitUntil { delivered.withLock { $0 } == 2 } + let expected = (0..<20).map { Self.batch($0) } + #expect(try await Self.batches(of: stream) == expected) + #expect(attempts.withLock { $0 } == 1) + } + + // Delivers on its own thread and counts each deliver that returned. + private static func produce(_ indices: Range, into spill: PluginRowSpill) -> OSAllocatedUnfairLock { + let delivered = OSAllocatedUnfairLock(initialState: 0) + Thread { + for index in indices { + spill.deliver(Self.batch(index)) + delivered.withLock { $0 += 1 } + } + spill.finish() + }.start() + return delivered + } + + private static func rows(_ element: PluginStreamElement?) -> [[PluginRow]] { + guard case .rows(let rows)? = element else { return [] } + return [rows] + } + + // Batches 0 and 1 fill the buffer of two. Batch 2 spills, and the drain takes it out of the file and holds it until + // the buffer has room, so the file is empty again before batches 3 and 4 go in. + private static func fillWithHeldReader(_ spill: PluginRowSpill, spool: () -> SpoolFile?) async throws { + for index in 0..<3 { spill.deliver(Self.batch(index)) } + try await waitUntil { spool()?.byteCount == 0 } + for index in 3..<5 { spill.deliver(Self.batch(index)) } + } + + @Test("A spill charges its spool to the budget, refills the spool once the drain empties it, and gives it all back") + func spillChargesTheBudget() async throws { + let sizes = (0..<10).map { PluginRowSpill.encode(Self.batch($0)).count + 8 } + let budget = SpoolBudget(limit: sizes[3] + sizes[4] + sizes[5] - 1) + let created = OSAllocatedUnfairLock<[SpoolFile]>(initialState: []) + let (stream, spill) = Self.makeStream(budget: budget) { + let spool = try SpoolFile() + created.withLock { $0.append(spool) } + return spool + } + try await Self.fillWithHeldReader(spill) { created.withLock { $0.first } } + let spool = try #require(created.withLock { $0.first }) + #expect(spool.byteCount == sizes[3] + sizes[4]) + #expect(budget.bytesInUse == sizes[3] + sizes[4]) + #expect(!budget.admits(sizes[5])) + + _ = Self.produce(5..<10, into: spill) + var received: [[PluginRow]] = [] + for try await element in stream { received += Self.rows(element) } + let expected = (0..<10).map { Self.batch($0) } + #expect(received == expected) + #expect(created.withLock { $0.count } == 1) + #expect(budget.bytesInUse == 0) + } + + @Test("With no budget left the producer waits for the reader, makes no spool, and every batch arrives in order") + func emptyBudgetPacesEveryBatch() async throws { + let created = OSAllocatedUnfairLock(initialState: 0) + let (stream, spill) = Self.makeStream(budget: SpoolBudget(limit: 0)) { + created.withLock { $0 += 1 } + return try SpoolFile() + } + let delivered = Self.produce(0..<10, into: spill) + // Batch 2 meets a full buffer, so it asks the budget. + try await waitUntil { delivered.withLock { $0 } == 2 } + let expected = (0..<10).map { Self.batch($0) } + #expect(try await Self.batches(of: stream) == expected) + #expect(created.withLock { $0 } == 0) + } + + @Test("Spills charge one budget, and each gives back only what its own spool held") + func spillsShareTheBudget() async throws { + let sizes = (0..<5).map { PluginRowSpill.encode(Self.batch($0)).count + 8 } + let held = sizes[3] + sizes[4] + let budget = SpoolBudget(limit: held * 2) + let spools = OSAllocatedUnfairLock<[SpoolFile]>(initialState: []) + let makeSpool: @Sendable () throws -> SpoolFile = { + let spool = try SpoolFile() + spools.withLock { $0.append(spool) } + return spool + } + let (first, firstSpill) = Self.makeStream(budget: budget, makeSpool: makeSpool) + try await Self.fillWithHeldReader(firstSpill) { spools.withLock { $0.first } } + let (second, secondSpill) = Self.makeStream(budget: budget, makeSpool: makeSpool) + try await Self.fillWithHeldReader(secondSpill) { spools.withLock { $0.count > 1 ? $0[1] : nil } } + #expect(budget.bytesInUse == held * 2) + #expect(!budget.admits(1)) + + firstSpill.finish() + #expect(try await Self.batches(of: first).count == 5) + #expect(budget.bytesInUse == held) + + secondSpill.finish() + #expect(try await Self.batches(of: second).count == 5) + #expect(budget.bytesInUse == 0) + } + + @Test("Once the reader has gone the producer is told so") + func readerGoneStopsDelivery() async throws { + let spill: PluginRowSpill + var iterator: AsyncThrowingStream.Iterator? + do { + let (stream, made) = Self.makeStream() + spill = made + for index in 0..<5 { spill.deliver(Self.batch(index)) } + iterator = stream.makeAsyncIterator() + } + _ = try await iterator?.next() + iterator = nil + try await waitUntil { !spill.deliver(Self.batch(99)) } + } + + @Test("A batch survives the record format unchanged") + func recordRoundTrip() throws { + let original = Self.batch(3) + [[], [.text(String(repeating: "x", count: 100_000))]] + #expect(try PluginRowSpill.decode(PluginRowSpill.encode(original)) == original) + #expect(try PluginRowSpill.decode(PluginRowSpill.encode([])) == []) + #expect(throws: (any Error).self) { try PluginRowSpill.decode([0, 0, 0, 1, 0, 0, 0, 1, 9]) } + } +} diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteSchemaChangeTokenTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteSchemaChangeTokenTests.swift new file mode 100644 index 0000000000..73cb0107f9 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteSchemaChangeTokenTests.swift @@ -0,0 +1,43 @@ +import Foundation +import TableProPluginKit +import Testing + +struct SQLiteSchemaChangeTokenTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("sqlite-schema-token-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + private func connect() async throws -> SQLitePluginDriver { + let driver = SQLitePluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", + database: directory.appendingPathComponent("app.db").path + )) + try await driver.connect() + return driver + } + + @Test("Writing rows leaves the token alone, and another connection's schema change moves it") + func tokenFollowsSchemaNotData() async throws { + let watcher = try await connect() + let writer = try await connect() + defer { + watcher.disconnect() + writer.disconnect() + try? FileManager.default.removeItem(at: directory) + } + _ = try await writer.execute(query: "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") + + let before = try #require(try await watcher.schemaChangeToken()) + _ = try await writer.execute(query: "INSERT INTO users (name) VALUES ('a')") + _ = try await writer.execute(query: "UPDATE users SET name = 'b'") + #expect(try await watcher.schemaChangeToken() == before) + + _ = try await writer.execute(query: "CREATE INDEX users_name ON users (name)") + let after = try #require(try await watcher.schemaChangeToken()) + #expect(after != before) + } +} diff --git a/apple/Plugins/SharedSpill/PluginRowSpill.swift b/apple/Plugins/SharedSpill/PluginRowSpill.swift new file mode 100644 index 0000000000..fad2534f29 --- /dev/null +++ b/apple/Plugins/SharedSpill/PluginRowSpill.swift @@ -0,0 +1,367 @@ +// +// PluginRowSpill.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit +import TableProWireIO + +// The disk every spill in this plugin draws from: half of what the volume has for important data, counting what the +// spools already hold, so concurrent runs leave room for the file an export writes. Read again at most every 10 s. +final class SpoolBudget: @unchecked Sendable { + static let shared = SpoolBudget(limit: nil) + + private let fixedLimit: Int? + private let lock = NSLock() + private var inUse = 0 + private var limit = 0 + private var checkedAt: ContinuousClock.Instant? + + init(limit: Int?) { + fixedLimit = limit + } + + var bytesInUse: Int { + lock.withLock { inUse } + } + + func admits(_ bytes: Int) -> Bool { + if let fixedLimit { + return lock.withLock { inUse + bytes <= fixedLimit } + } + let now = ContinuousClock.now + let (held, cached, stale) = lock.withLock { (inUse, limit, checkedAt.map { now - $0 > .seconds(10) } ?? true) } + var current = cached + if stale { + let values = try? FileManager.default.temporaryDirectory + .resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + current = (Int(clamping: values?.volumeAvailableCapacityForImportantUsage ?? 0) + held) / 2 + lock.withLock { + limit = current + checkedAt = now + } + } + return held + bytes <= current + } + + func add(_ delta: Int) { + lock.withLock { inUse += delta } + } +} + +// Takes a statement's rows without waiting for the reader, so the read, and its lock on a SQLite file, ends at the +// speed SQLite reads. What the stream's buffer refuses goes to a spool file and follows in order. Past the budget, or +// without a spool, the producer waits for the reader. One producer per spill. +final class PluginRowSpill: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "PluginRowSpill") + + private let continuation: AsyncThrowingStream.Continuation + private let budget: SpoolBudget + private let makeSpool: @Sendable () throws -> SpoolFile + + private let lock = NSLock() + private var spool: SpoolFile? + // The spool's size as last counted against the budget. + private var counted = 0 + private var spoolFailed = false + // Batches in the spool that have not reached the stream yet. + private var spilled = 0 + private var draining = false + // The producer's end, held back while spilled batches are still on their way. + private var outcome: Error?? + private var ended = false + + init( + continuation: AsyncThrowingStream.Continuation, + budget: SpoolBudget = .shared, + makeSpool: @escaping @Sendable () throws -> SpoolFile = { try SpoolFile() } + ) { + self.continuation = continuation + self.budget = budget + self.makeSpool = makeSpool + } + + // The header comes first, so the buffer is empty and takes it. + func yield(_ header: PluginStreamHeader) { + continuation.yield(.header(header)) + } + + // False once the reader has gone. + @discardableResult + func deliver(_ batch: [PluginRow]) -> Bool { + lock.lock() + if ended { + lock.unlock() + return false + } + if spilled == 0 { + switch continuation.yield(.rows(batch)) { + case .enqueued: + lock.unlock() + return true + case .terminated: + end() + lock.unlock() + return false + case .dropped: + break + @unknown default: + lock.unlock() + return false + } + } + let failed = spoolFailed + let existing = spool + lock.unlock() + + if !failed, spill(batch, into: existing) { + return true + } + return deliverAtTheReadersPace(batch) + } + + // False when the batch did not go to the spool: past the budget, or the spool failed. + private func spill(_ batch: [PluginRow], into existing: SpoolFile?) -> Bool { + let record = Self.encode(batch) + guard budget.admits(record.count + 8) else { return false } + do { + let target = try existing ?? makeSpool() + try target.append(record) + return didSpill(into: target) + } catch { + Self.logger.error( + "Rows no longer spill to a temporary file: \(String(describing: type(of: error)), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + lock.withLock { + spoolFailed = true + // With no drain left to notice, the space would stay taken until the stream ends. + if !draining { releaseSpool() } + } + return false + } + } + + func finish(throwing error: Error? = nil) { + lock.lock() + guard !ended else { + lock.unlock() + return + } + guard spilled == 0, !draining else { + outcome = .some(error) + lock.unlock() + return + } + end() + lock.unlock() + continuation.finish(throwing: error) + } + + // The stream ended: nothing more goes in. + func abandon() { + lock.withLock { end() } + } + + // Under the lock. The spool's space returns once the drain lets go of it too. + private func end() { + ended = true + releaseSpool() + } + + // Under the lock. + private func releaseSpool() { + spool = nil + budget.add(-counted) + counted = 0 + } + + // Under the lock. + private func count(_ target: SpoolFile) { + guard spool === target else { return } + let size = target.byteCount + budget.add(size - counted) + counted = size + } + + private func didSpill(into target: SpoolFile) -> Bool { + lock.lock() + guard !ended else { + lock.unlock() + return false + } + spool = target + spilled += 1 + count(target) + let startsDrain = !draining + draining = true + lock.unlock() + if startsDrain { + Task { await self.drain(target) } + } + return true + } + + private enum DrainStep { + case stop + case finish(Error?) + case next + } + + private func nextDrainStep() -> DrainStep { + lock.withLock { + if ended { + draining = false + return .stop + } + guard spilled == 0 else { return .next } + draining = false + // A failed spool takes no more batches, so its space goes back now rather than at the end. + if spoolFailed { releaseSpool() } + guard let outcome else { return .stop } + end() + return .finish(outcome) + } + } + + private func drain(_ target: SpoolFile) async { + while true { + switch nextDrainStep() { + case .stop: + return + case .finish(let error): + continuation.finish(throwing: error) + return + case .next: + break + } + + let batch: [PluginRow] + do { + guard let record = try target.next() else { throw SpoolFileError.readFailed(errno: EIO) } + lock.withLock { count(target) } + batch = try Self.decode(record) + } catch { + Self.logger.error( + "Spilled rows could not be read back: \(String(describing: type(of: error)), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + lock.withLock { + end() + draining = false + } + continuation.finish(throwing: error is SpoolFileError ? error : SpoolFileError.readFailed(errno: EIO)) + return + } + guard await PluginRowStream.deliver(.rows(batch), to: continuation) else { + lock.withLock { + end() + draining = false + } + return + } + lock.withLock { spilled -= 1 } + } + } + + // The batches already spilled reach the stream first; the spool is empty again by then. + private func deliverAtTheReadersPace(_ batch: [PluginRow]) -> Bool { + while true { + let (waiting, gone) = lock.withLock { (spilled > 0, ended) } + if gone { return false } + if !waiting { break } + usleep(2_000) + } + guard PluginRowStream.deliverBlocking(.rows(batch), to: continuation) else { + lock.withLock { end() } + return false + } + return true + } + + // MARK: - Record + + private enum Tag: UInt8 { + case null = 0 + case text = 1 + case bytes = 2 + } + + static func encode(_ batch: [PluginRow]) -> [UInt8] { + var size = 4 + for row in batch { + size += 4 + for cell in row { + switch cell { + case .null: size += 1 + case .text(let text): size += 5 + text.utf8.count + case .bytes(let data): size += 5 + data.count + } + } + } + var record: [UInt8] = [] + record.reserveCapacity(size) + appendLength(batch.count, to: &record) + for row in batch { + appendLength(row.count, to: &record) + for cell in row { + switch cell { + case .null: + record.append(Tag.null.rawValue) + case .text(var text): + record.append(Tag.text.rawValue) + text.withUTF8 { utf8 in + appendLength(utf8.count, to: &record) + record.append(contentsOf: utf8) + } + case .bytes(let data): + record.append(Tag.bytes.rawValue) + appendLength(data.count, to: &record) + record.append(contentsOf: data) + } + } + } + return record + } + + static func decode(_ record: [UInt8]) throws -> [PluginRow] { + var reader = ByteReader(record) + let declaredRows = try reader.u32() + let rowCount = try reader.elementCount(declaredRows, minimumElementSize: 4) + var batch: [PluginRow] = [] + batch.reserveCapacity(rowCount) + for _ in 0.. Int { + let length = try reader.u32() + let start = reader.offset + try reader.skip(length) + return start + } + + private static func appendLength(_ length: Int, to record: inout [UInt8]) { + withUnsafeBytes(of: UInt32(length).bigEndian) { record.append(contentsOf: $0) } + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeBrowserAuthServer.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeBrowserAuthServer.swift index 27380d3583..3e272c9bde 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeBrowserAuthServer.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeBrowserAuthServer.swift @@ -7,213 +7,40 @@ // import Foundation -import Network -import os +import TableProHTTP +import TableProSnowflakeCore -final class SnowflakeBrowserAuthServer: @unchecked Sendable { - private var listener: NWListener? - private var connection: NWConnection? - private var readyContinuation: CheckedContinuation? - private var continuation: CheckedContinuation? - private let lock = NSLock() - private var timeoutTask: Task? - private static let logger = Logger(subsystem: "com.TablePro", category: "SnowflakeBrowserAuthServer") +final class SnowflakeBrowserAuthServer: Sendable { + private let listener = LoopbackRedirectListener(port: 0) { head in + switch SnowflakeBrowserCallback.parse(requestHead: head) { + case .token(let token): + return .settle(SnowflakeBrowserAuthServer.successPage, .success(token)) + case .ignore: + return .ignore(LoopbackRedirectPage.badRequest) + } + } func start(timeout: TimeInterval = 120) async throws -> UInt16 { - try await withCheckedThrowingContinuation { cont in - let task = Task { - try? await Task.sleep(for: .seconds(timeout)) - let continuation = self.lock.withLock { () -> CheckedContinuation? in - let continuation = self.readyContinuation - self.readyContinuation = nil - return continuation - } - continuation?.resume(throwing: SnowflakeError.timeout("Browser authentication listener timed out")) - if continuation != nil { self.stop() } - } - lock.withLock { - readyContinuation = cont - timeoutTask = task - } - do { - try startListener() - } catch { - let continuation = lock.withLock { () -> CheckedContinuation? in - let continuation = readyContinuation - readyContinuation = nil - timeoutTask = nil - return continuation - } - task.cancel() - continuation?.resume(throwing: error) - } + do { + return try await listener.start(readyTimeout: timeout) + } catch LoopbackRedirectListenerError.timedOut { + throw SnowflakeError.timeout("Browser authentication listener timed out") + } catch is LoopbackRedirectListenerError { + throw SnowflakeError.authFailed("Browser auth server failed") } } func waitForToken(timeout: TimeInterval = 120) async throws -> String { - try await withCheckedThrowingContinuation { cont in - lock.withLock { continuation = cont } - let task = Task { - try? await Task.sleep(for: .seconds(timeout)) - self.lock.withLock { - if let cont = self.continuation { - self.continuation = nil - cont.resume(throwing: SnowflakeError.timeout("Browser authentication timed out")) - } - } - self.stop() - } - lock.withLock { timeoutTask = task } - } + try await listener.outcome(timeout: timeout, timeoutError: SnowflakeError.timeout("Browser authentication timed out")) } func stop() { - let (task, conn, lst): (Task?, NWConnection?, NWListener?) = lock.withLock { - let values = (timeoutTask, connection, listener) - timeoutTask = nil - connection = nil - listener = nil - return values - } - task?.cancel() - conn?.cancel() - lst?.cancel() - } - - private func startListener() throws { - let params = NWParameters.tcp - params.requiredLocalEndpoint = NWEndpoint.hostPort(host: .ipv4(.loopback), port: .any) - - let listener = try NWListener(using: params) - lock.withLock { self.listener = listener } - - listener.stateUpdateHandler = { [weak self] state in - switch state { - case .ready: - let port = listener.port?.rawValue ?? 0 - let pending = self?.lock.withLock { - let pending = (self?.readyContinuation, self?.timeoutTask) - self?.readyContinuation = nil - self?.timeoutTask = nil - return pending - } - pending?.1?.cancel() - pending?.0?.resume(returning: port) - case .failed(let error): - Self.logger.error("Browser auth server failed: \(error.localizedDescription)") - let authError = SnowflakeError.authFailed("Browser auth server failed: \(error.localizedDescription)") - self?.lock.withLock { - if let cont = self?.readyContinuation { - self?.readyContinuation = nil - cont.resume(throwing: authError) - } - if let cont = self?.continuation { - self?.continuation = nil - cont.resume(throwing: authError) - } - } - default: - break - } - } - - listener.newConnectionHandler = { [weak self] newConnection in - self?.handleConnection(newConnection) - } - - listener.start(queue: .global(qos: .userInitiated)) - } - - private func handleConnection(_ newConnection: NWConnection) { - lock.withLock { connection = newConnection } - newConnection.stateUpdateHandler = { [weak self] state in - if case .ready = state { - self?.readRequest(from: newConnection) - } - } - newConnection.start(queue: .global(qos: .userInitiated)) - } - - private func readRequest(from connection: NWConnection) { - connection.receive(minimumIncompleteLength: 1, maximumLength: 65_536) { [weak self] content, _, _, error in - guard let self, let data = content, error == nil, - let request = String(data: data, encoding: .utf8) else { - self?.resumeWithError(SnowflakeError.authFailed("Failed to read browser callback")) - return - } - - if let token = self.extractToken(from: request) { - self.sendSuccessResponse(to: connection) - self.resumeWithToken(token) - } else if request.contains("error=") { - let desc = self.extractParam(named: "error", from: request) ?? "unknown" - self.sendErrorResponse(to: connection, error: desc) - self.resumeWithError(SnowflakeError.authFailed("SSO authorization failed: \(desc)")) - } else { - self.readRequest(from: connection) - } - } - } - - private func extractToken(from request: String) -> String? { - extractParam(named: "token", from: request) - } - - private func extractParam(named name: String, from request: String) -> String? { - guard let firstLine = request.components(separatedBy: "\r\n").first, - let pathPart = firstLine.components(separatedBy: " ").dropFirst().first, - let components = URLComponents(string: "http://localhost\(pathPart)") - else { return nil } - return components.queryItems?.first(where: { $0.name == name })?.value - } - - private func sendSuccessResponse(to connection: NWConnection) { - let html = """ - -

Authentication Successful

-

You can close this tab and return to TablePro.

- - """ - let response = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n\(html)" - connection.send(content: Data(response.utf8), completion: .contentProcessed { _ in - connection.cancel() - }) - } - - private func sendErrorResponse(to connection: NWConnection, error: String) { - let escaped = error - .replacingOccurrences(of: "&", with: "&") - .replacingOccurrences(of: "<", with: "<") - .replacingOccurrences(of: ">", with: ">") - let html = """ - -

Authentication Failed

-

\(escaped)

- - """ - let response = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nConnection: close\r\n\r\n\(html)" - connection.send(content: Data(response.utf8), completion: .contentProcessed { _ in - connection.cancel() - }) - } - - private func resumeWithToken(_ token: String) { - lock.withLock { - if let cont = continuation { - continuation = nil - cont.resume(returning: token) - } - } - stop() + listener.stop() } - private func resumeWithError(_ error: Error) { - lock.withLock { - if let cont = continuation { - continuation = nil - cont.resume(throwing: error) - } - } - stop() - } + private static let successPage = LoopbackRedirectPage.response( + status: "200 OK", + title: "Authentication Successful", + message: "You can close this tab and return to TablePro." + ) } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift index 6ad5f5bbf2..ba4a63384b 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift @@ -34,8 +34,9 @@ final class SnowflakeConnection: @unchecked Sendable { let params: ResolvedParameters private let connectionIdentifier: String - private let session: URLSession - private let connectSession: URLSession + private let sessions: SnowflakeURLSessions + private var session: URLSession { sessions.query } + private var connectSession: URLSession { sessions.connect } private let lock = NSLock() private let heartbeat = SnowflakeHeartbeat() private var sessionToken: String? @@ -46,6 +47,14 @@ final class SnowflakeConnection: @unchecked Sendable { private var renewalTask: Task? var sessionFingerprint: String { + Self.sessionFingerprint(params: params, account: account, connectionIdentifier: connectionIdentifier) + } + + static func sessionFingerprint( + params: ResolvedParameters, + account: SnowflakeAccount?, + connectionIdentifier: String + ) -> String { SnowflakeSessionKey.fingerprint( connectionId: connectionIdentifier, host: account?.host ?? params.account, @@ -55,6 +64,18 @@ final class SnowflakeConnection: @unchecked Sendable { ) } + static func connectionIdentifier(of config: DriverConnectionConfig) -> String { + config.additionalFields["connectionId"] ?? "" + } + + private var _serverVersion: String? + + // Read once per shared session: every pooled driver reports the same server. + var serverVersion: String? { + get { lock.withLock { _serverVersion } } + set { lock.withLock { _serverVersion = newValue } } + } + /// Statements the connection issues for itself, such as the connect-time `USE` calls, belong to /// no driver and are never the target of a Stop. static let sessionOwner = "session" @@ -76,22 +97,28 @@ final class SnowflakeConnection: @unchecked Sendable { var currentWarehouse: String? { lock.withLock { _currentWarehouse } } var currentRole: String? { lock.withLock { _currentRole } } - init(config: DriverConnectionConfig) { - self.params = Self.resolveParameters(from: config) + convenience init(config: DriverConnectionConfig) { + self.init(config: config, params: Self.resolveParameters(from: config)) + } + + init(config: DriverConnectionConfig, params: ResolvedParameters) { + self.params = params self.account = SnowflakeAccount(params.account) - self.connectionIdentifier = config.additionalFields["connectionId"] ?? "" + self.connectionIdentifier = Self.connectionIdentifier(of: config) let configuration = URLSessionConfiguration.ephemeral configuration.timeoutIntervalForRequest = Self.requestTimeout configuration.timeoutIntervalForResource = 600 - self.session = URLSession( - configuration: configuration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil - ) let connectConfiguration = URLSessionConfiguration.ephemeral connectConfiguration.timeoutIntervalForRequest = TimeInterval(PluginConnectTimeout.maximumMilliseconds) / 1_000 connectConfiguration.timeoutIntervalForResource = TimeInterval(PluginConnectTimeout.maximumMilliseconds) / 1_000 - self.connectSession = URLSession( - configuration: connectConfiguration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + self.sessions = SnowflakeURLSessions( + query: URLSession( + configuration: configuration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + ), + connect: URLSession( + configuration: connectConfiguration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + ) ) self._currentDatabase = params.database.isEmpty ? nil : params.database @@ -102,7 +129,7 @@ final class SnowflakeConnection: @unchecked Sendable { // MARK: - Parameter Resolution - private static func resolveParameters(from config: DriverConnectionConfig) -> ResolvedParameters { + static func resolveParameters(from config: DriverConnectionConfig) -> ResolvedParameters { let fields = config.additionalFields func field(_ key: String) -> String { fields[key]?.trimmingCharacters(in: .whitespaces) ?? "" @@ -714,7 +741,8 @@ final class SnowflakeConnection: @unchecked Sendable { let columns: [SnowflakeColumnMeta] let inlineRows: [[PluginCellValueBox]] let estimatedRowCount: Int - let batches: AsyncThrowingStream<[[PluginCellValueBox]], Error> + let chunkURLs: [URL] + let chunkHeaders: [String: String] } func queryStreamed( @@ -731,58 +759,56 @@ final class SnowflakeConnection: @unchecked Sendable { let headers = Self.chunkRequestHeaders(from: data) let urls = try Self.chunkURLs(chunks) - let stream = AsyncThrowingStream<[[PluginCellValueBox]], Error> { continuation in - guard !urls.isEmpty else { - continuation.finish() - return + return StreamedResult( + columns: columns, + inlineRows: inlineRows, + estimatedRowCount: inlineRows.count + chunkRowCount, + chunkURLs: urls, + chunkHeaders: headers + ) + } + + // Chunks reach `body` in order. A download starts only as a chunk is handed on, so the chunks downloading or + // waiting for one that lags never number more than `chunkDownloadWorkers`. + func forEachChunk( + of result: StreamedResult, + _ body: ([[PluginCellValueBox]]) async throws -> Void + ) async throws { + let urls = result.chunkURLs + guard !urls.isEmpty else { return } + let headers = result.chunkHeaders + let columns = result.columns + let session = self.session + try await withThrowingTaskGroup(of: (Int, [[PluginCellValueBox]]).self) { group in + var nextIndex = 0 + while nextIndex < min(Self.chunkDownloadWorkers, urls.count) { + let index = nextIndex + group.addTask { + (index, try await Self.downloadChunk( + urls[index], headers: headers, session: session, columns: columns + )) + } + nextIndex += 1 } - let task = Task { [session] in - do { - var buffer: [Int: [[PluginCellValueBox]]] = [:] - var nextToYield = 0 - try await withThrowingTaskGroup(of: (Int, [[PluginCellValueBox]]).self) { group in - var nextIndex = 0 - while nextIndex < min(Self.chunkDownloadWorkers, urls.count) { - let index = nextIndex - group.addTask { - (index, try await Self.downloadChunk( - urls[index], headers: headers, session: session, columns: columns - )) - } - nextIndex += 1 - } - while let (index, rows) = try await group.next() { - buffer[index] = rows - while let ready = buffer[nextToYield] { - buffer[nextToYield] = nil - continuation.yield(ready) - nextToYield += 1 - } - if nextIndex < urls.count { - let next = nextIndex - group.addTask { - (next, try await Self.downloadChunk( - urls[next], headers: headers, session: session, columns: columns - )) - } - nextIndex += 1 - } + var held: [Int: [[PluginCellValueBox]]] = [:] + var nextToHandOn = 0 + while let (index, rows) = try await group.next() { + held[index] = rows + while let ready = held.removeValue(forKey: nextToHandOn) { + try await body(ready) + nextToHandOn += 1 + if nextIndex < urls.count { + let next = nextIndex + group.addTask { + (next, try await Self.downloadChunk( + urls[next], headers: headers, session: session, columns: columns + )) } + nextIndex += 1 } - continuation.finish() - } catch { - continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } } - - return StreamedResult( - columns: columns, - inlineRows: inlineRows, - estimatedRowCount: inlineRows.count + chunkRowCount, - batches: stream - ) } private static let chunkDownloadWorkers = 4 @@ -800,9 +826,9 @@ final class SnowflakeConnection: @unchecked Sendable { var nextIndex = 0 while nextIndex < min(Self.chunkDownloadWorkers, urls.count) { let index = nextIndex - group.addTask { [session] in + group.addTask { [sessions] in (index, try await Self.downloadChunk( - urls[index], headers: headers, session: session, columns: columns + urls[index], headers: headers, session: sessions.query, columns: columns )) } nextIndex += 1 @@ -811,9 +837,9 @@ final class SnowflakeConnection: @unchecked Sendable { rowsByChunk[index] = rows if nextIndex < urls.count { let next = nextIndex - group.addTask { [session] in + group.addTask { [sessions] in (next, try await Self.downloadChunk( - urls[next], headers: headers, session: session, columns: columns + urls[next], headers: headers, session: sessions.query, columns: columns )) } nextIndex += 1 @@ -1073,3 +1099,21 @@ private extension SnowflakeConnection.ResolvedParameters { } } } + +// A URLSession keeps its delegate until it is invalidated, so each connection's two sessions are +// invalidated when the last request path holding them lets go. Invalidating any earlier would make a +// late request raise instead of fail. +final class SnowflakeURLSessions: Sendable { + let query: URLSession + let connect: URLSession + + init(query: URLSession, connect: URLSession) { + self.query = query + self.connect = connect + } + + deinit { + query.finishTasksAndInvalidate() + connect.finishTasksAndInvalidate() + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift index 74f97edf4a..ff486dc8ba 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift @@ -10,6 +10,7 @@ import Foundation import TableProPluginKit +import TableProSnowflakeCore final class SnowflakeConnectionRegistry: @unchecked Sendable { static let shared = SnowflakeConnectionRegistry() @@ -17,16 +18,22 @@ final class SnowflakeConnectionRegistry: @unchecked Sendable { private let lock = NSLock() private var entries: [String: (connection: SnowflakeConnection, refCount: Int)] = [:] + // Keyed before anything is built: a connection owns two URLSessions, which a discarded copy would leak. func acquire(config: DriverConnectionConfig) -> SnowflakeConnection { - lock.withLock { - let candidate = SnowflakeConnection(config: config) - let key = candidate.sessionFingerprint + let params = SnowflakeConnection.resolveParameters(from: config) + let key = SnowflakeConnection.sessionFingerprint( + params: params, + account: SnowflakeAccount(params.account), + connectionIdentifier: SnowflakeConnection.connectionIdentifier(of: config) + ) + return lock.withLock { if let entry = entries[key] { entries[key] = (entry.connection, entry.refCount + 1) return entry.connection } - entries[key] = (candidate, 1) - return candidate + let connection = SnowflakeConnection(config: config, params: params) + entries[key] = (connection, 1) + return connection } } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift index d4d5ca8870..b47c91f2b6 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift @@ -43,6 +43,7 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .dataCompare, .rowWrites, .cellExpressions, + .columnDefaultMarks, ] } @@ -89,9 +90,15 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } lock.withLock { _connection = conn } + if let known = conn.serverVersion { + lock.withLock { _serverVersion = known } + return + } if let result = try? await conn.query("SELECT CURRENT_VERSION()", owner: queryOwner), let first = result.rows.first?.first, case .text(let version) = first { - lock.withLock { _serverVersion = "Snowflake \(version)" } + let serverVersion = "Snowflake \(version)" + conn.serverVersion = serverVersion + lock.withLock { _serverVersion = serverVersion } } else { lock.withLock { _serverVersion = "Snowflake" } } @@ -311,6 +318,8 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { }.sorted { $0.name < $1.name } } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let database = connection?.currentDatabase let targetSchema = schema ?? connection?.currentSchema @@ -618,29 +627,29 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { guard let conn = self.connection else { throw SnowflakeError.notConnected } let trimmed = query.replacingOccurrences(of: ";\\s*\\z", with: "", options: .regularExpression) - let streamed = try await conn.queryStreamed(trimmed, owner: queryOwner) + let streamed = try await conn.queryStreamed(trimmed, owner: self.queryOwner) continuation.yield(.header(PluginStreamHeader( columns: streamed.columns.map(\.name), columnTypeNames: streamed.columns.map(SnowflakeTypeMapper.displayType), estimatedRowCount: streamed.estimatedRowCount ))) if !streamed.inlineRows.isEmpty { - continuation.yield(.rows(streamed.inlineRows.map { row in row.map(Self.cellValue) })) + _ = await PluginRowStream.deliver(.rows(streamed.inlineRows.map { row in row.map(Self.cellValue) }), to: continuation) } - for try await batch in streamed.batches { - continuation.yield(.rows(batch.map { row in row.map(Self.cellValue) })) + try await conn.forEachChunk(of: streamed) { batch in + _ = await PluginRowStream.deliver(.rows(batch.map { row in row.map(Self.cellValue) }), to: continuation) } continuation.finish() } catch { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift index 5eac2b6580..8f1c5a4ea7 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift @@ -18,7 +18,7 @@ struct SnowflakeStatementGenerator { let primaryKeyColumns: [String] // Snowflake does not enforce a primary key, so a key match can reach several rows too: every update and delete - // first has to find exactly one. + // first has to find exactly one, and is then held to it by the rows-updated or rows-deleted count it reports. func rowWrites( from changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], @@ -51,7 +51,8 @@ struct SnowflakeStatementGenerator { statement: "UPDATE \(qualifiedTable) SET \(setClauses.joined(separator: ", ")) WHERE \(condition.sql)", parameters: parameters + condition.parameters, rowIndices: [change.rowIndex], - lookup: lookup(condition) + lookup: lookup(condition), + expectedRowCount: 1 )) case .delete: guard deletedRowIndices.contains(change.rowIndex) else { continue } @@ -60,7 +61,8 @@ struct SnowflakeStatementGenerator { statement: "DELETE FROM \(qualifiedTable) WHERE \(condition.sql)", parameters: condition.parameters, rowIndices: [change.rowIndex], - lookup: lookup(condition) + lookup: lookup(condition), + expectedRowCount: 1 )) @unknown default: continue @@ -87,9 +89,6 @@ struct SnowflakeStatementGenerator { ) } - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func isSemiStructured(_ typeName: String) -> Bool { let base = typeName.uppercased().components(separatedBy: "(")[0].trimmingCharacters(in: .whitespaces) return ["VARIANT", "OBJECT", "ARRAY"].contains(base) @@ -100,18 +99,22 @@ struct SnowflakeStatementGenerator { insertedRowData: [Int: [PluginCellValue]] ) -> (statement: String, parameters: [PluginCellValue])? { var values: [(column: String, value: PluginCellValue)] = [] + var hasCells = false if let rowData = insertedRowData[change.rowIndex] { for (index, column) in columns.enumerated() where index < rowData.count { + hasCells = true + guard !change.isColumnDefault(index) else { continue } values.append((column, rowData[index])) } } else { for cellChange in change.cellChanges { + hasCells = true + guard !change.isColumnDefault(cellChange.columnIndex) else { continue } values.append((cellChange.columnName, cellChange.newValue)) } } - guard !values.isEmpty else { return nil } + guard hasCells else { return nil } let expressions = Set((change.expressionColumns ?? []).compactMap { columns.indices.contains($0) ? columns[$0] : nil }) - values.removeAll { $0.value == Self.defaultMarker } guard !values.isEmpty else { guard let firstColumn = columns.first else { return nil } return ("INSERT INTO \(qualifiedTable) (\(quoteIdentifier(firstColumn))) VALUES (DEFAULT)", []) diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift index dc205a2a6a..5352ec4480 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift @@ -188,12 +188,13 @@ extension SpannerPluginDriver { SpannerRowChange.CellChange(column: $0.columnName, value: spannerCell($0.newValue)) }, originalRow: change.originalRow?.map(spannerCell), - expressionColumns: expressionColumnNames(of: change, columns: columns) + expressionColumns: columnNames(change.expressionColumns, of: change, columns: columns), + defaultColumns: columnNames(change.defaultColumns, of: change, columns: columns) ) } - private static func expressionColumnNames(of change: PluginRowChange, columns: [String]) -> Set { - guard let marked = change.expressionColumns, !marked.isEmpty else { return [] } + private static func columnNames(_ marked: Set?, of change: PluginRowChange, columns: [String]) -> Set { + guard let marked, !marked.isEmpty else { return [] } switch change.type { case .update: return Set(change.cellChanges.filter { marked.contains($0.columnIndex) }.map(\.columnName)) diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift index a017a93fa2..c8a212d16b 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Execution.swift @@ -40,8 +40,10 @@ extension SpannerPluginDriver { } } let id = self.registerUserOperation { task.cancel() } - abort.onAbort { - task.cancel() + abort.onAbort { task.cancel() } + // A stream its producer finishes never aborts, so the entry goes when the task ends. + Task { + await task.value self.unregisterUserOperation(id) } } diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift index cd645015f0..60f81f66fd 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift @@ -37,7 +37,10 @@ internal final class SpannerPluginDriver: PluginDatabaseDriver, @unchecked Senda } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites] + [ + .multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, + ] } var serverVersion: String? { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift index 46d05263f4..9db4a8b98c 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift @@ -121,10 +121,7 @@ public enum SurrealCBOR { return .bytes(Data(try cursor.next(count))) case 3: let count = try checkedCount(try readArgument(&cursor, info: info)) - guard let text = String(bytes: try cursor.next(count), encoding: .utf8) else { - throw SurrealCBORError.invalidUTF8 - } - return .string(text) + return .string(try exactText(try cursor.next(count))) case 4: let count = try checkedCount(try readArgument(&cursor, info: info)) var items: [SurrealValue] = [] @@ -154,6 +151,13 @@ public enum SurrealCBOR { } } + // String(bytes:encoding:) drops a leading byte order mark, which names a different record or field. + private static func exactText(_ bytes: [UInt8]) throws -> String { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + guard text.utf8.elementsEqual(bytes) else { throw SurrealCBORError.invalidUTF8 } + return text + } + private static func fallbackKey(_ value: SurrealValue) -> String { switch value { case let .int(number): diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index 1edab23077..4aafdff170 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -38,7 +38,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [.parameterizedQueries, .cancelQuery, .truncateTable, .multiSchema] + [.parameterizedQueries, .cancelQuery, .truncateTable, .multiSchema, .columnDefaultMarks] } var serverVersion: String? { @@ -345,13 +345,15 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let flattened = SurrealRowFlattener.flatten(last.value, length: length) let affected = rowsAffected(last.value) - return PluginQueryResult( + var result = PluginQueryResult( columns: flattened.columns, columnTypeNames: flattened.columnTypeNames, rows: flattened.rows, rowsAffected: affected, executionTime: elapsed ) + result.rowLocators = SurrealRowFlattener.rowLocators(last.value) + return result } private static func rowsAffected(_ value: SurrealValue) -> Int { @@ -457,6 +459,24 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { try SurrealStatementGenerator.rowWrites( table: table, @@ -469,7 +489,8 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { changes: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, - insertedRowIndices: insertedRowIndices + insertedRowIndices: insertedRowIndices, + rowLocators: context.rowLocators ) } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift index bf54b3a6a0..97bb1803c3 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift @@ -46,6 +46,21 @@ public enum SurrealQL { } } + // `recordIdPart` backticks a uuid, an array and an object id like a string that needs quoting, so a backticked + // string is that string only when it cannot read as one of them. + public static func isRecoverableFromText(_ id: SurrealValue) -> Bool { + switch id { + case .int: + return true + case let .string(text): + if isSimpleIdentifier(text) { return true } + guard UUID(uuidString: text) == nil, let first = text.unicodeScalars.first else { return false } + return first != "[" && first != "{" + default: + return false + } + } + public static func recordLiteral(_ record: SurrealRecordID) -> String { quoteIdentifier(record.table) + ":" + recordIdPart(record.id) } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift index a5d692e2ee..b681cf4449 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift @@ -49,6 +49,18 @@ public enum SurrealRowFlattener { ) } + // One per row `flatten` returns: the record's id for a row that is a record, nil for any other row. Nil when the + // rows are not records, which `flatten` shows in a single column with no id to edit by. + public static func rowLocators(_ value: SurrealValue) -> [String?]? { + let records = normalize(value) + guard !records.isEmpty, records.allSatisfy({ $0.objectPairs != nil }) else { return nil } + let locators = records.map { record -> String? in + guard case let .recordId(recordId)? = record[SurrealInfoParser.recordIdColumn] else { return nil } + return SurrealRowLocator.locator(for: recordId) + } + return locators.contains(where: { $0 != nil }) ? locators : nil + } + public static func cell(_ value: SurrealValue, length: SurrealTextLength) -> PluginCellValue { switch value { case .null, .none: @@ -114,3 +126,17 @@ public enum SurrealRowFlattener { } } } + +// The id cell shows a uuid, an array, an object and a string id that needs quoting all as backticked text, so a row +// is found again by its id as CBOR: the same record always encodes to the same bytes. +public enum SurrealRowLocator { + public static func locator(for record: SurrealRecordID) -> String { + SurrealCBOR.encode(.recordId(record)).base64EncodedString() + } + + public static func record(from locator: String) -> SurrealRecordID? { + guard let data = Data(base64Encoded: locator), + case let .recordId(record)? = try? SurrealCBOR.decode(data) else { return nil } + return record + } +} diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index c98626d2bd..627c0bff72 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -8,13 +8,6 @@ import TableProNumberFormatting import TableProPluginKit public enum SurrealStatementGenerator { - static let autoIdMarker = "__DEFAULT__" - - static func isAutoDefault(_ value: PluginCellValue) -> Bool { - guard case let .text(text) = value else { return false } - return text.trimmingCharacters(in: .whitespaces) == autoIdMarker - } - public static func rowWrites( table: String, scope: SurrealScope, @@ -26,28 +19,40 @@ public enum SurrealStatementGenerator { changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, - insertedRowIndices: Set + insertedRowIndices: Set, + rowLocators: [Int: String] = [:] ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] + var insertedWritten: Set = [] let typing = FieldTyping(kinds: kinds, held: heldKinds, records: recordKinds, declared: declaredKinds) - for change in changes where change.type == .update && !insertedRowIndices.contains(change.rowIndex) { - let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) - guard let write else { continue } - writes.append(write) - } - - for index in insertedRowIndices.sorted() { - guard let values = insertedRowData[index] else { continue } - writes.append(try insert( - table: table, scope: scope, columns: columns, typing: typing, values: values, rowIndex: index - )) + var defaults: [Int: Set] = [:] + for change in changes where change.type == .insert { + defaults[change.rowIndex, default: []].formUnion(change.defaultColumns ?? []) } - for change in changes where change.type == .delete || deletedRowIndices.contains(change.rowIndex) { - guard !insertedRowIndices.contains(change.rowIndex) else { continue } - guard let write = delete(table: table, scope: scope, columns: columns, change: change) else { continue } - writes.append(write) + // In the order received, which is edit order: a record deleted to free a unique value goes before the one taking it. + for change in changes { + let index = change.rowIndex + if insertedRowIndices.contains(index) { + guard insertedWritten.insert(index).inserted, let values = insertedRowData[index] else { continue } + writes.append(try insert( + table: table, scope: scope, columns: columns, typing: typing, values: values, + defaults: defaults[index] ?? [], rowIndex: index + )) + continue + } + let locator = rowLocators[index] + if change.type == .update, + let write = try update( + table: table, scope: scope, columns: columns, typing: typing, change: change, locator: locator + ) { + writes.append(write) + } + if change.type == .delete || deletedRowIndices.contains(index), + let write = try delete(table: table, scope: scope, columns: columns, change: change, locator: locator) { + writes.append(write) + } } return writes @@ -89,15 +94,18 @@ public enum SurrealStatementGenerator { scope: SurrealScope, columns: [String], typing: FieldTyping, - change: PluginRowChange + change: PluginRowChange, + locator: String? ) throws(PluginRowWriteRefusal) -> PluginRowWrite? { if let reserved = change.cellChanges.first(where: { SurrealInfoParser.isReservedColumn($0.columnName) }) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: reservedColumnReason(reserved.columnName)) } - let recordKey = Self.recordText(columns: columns, originalRow: change.originalRow) - guard let record = recordKey.flatMap({ SurrealQL.parseRecordId($0, fallbackTable: table) }) else { return nil } - let editable = change.cellChanges.filter { !Self.isAutoDefault($0.newValue) } - guard !editable.isEmpty else { return nil } + let editable = change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) } + guard !editable.isEmpty, + let record = try targetRecord(table: table, columns: columns, change: change, locator: locator) + else { return nil } + // The record kinds are keyed by the literal the read showed. + let recordKey = record.literal var parameters: [PluginCellValue] = [SurrealCellCoder.parameter(.recordId(record))] var assignments: [String] = [] @@ -126,6 +134,7 @@ public enum SurrealStatementGenerator { columns: [String], typing: FieldTyping, values: [PluginCellValue], + defaults: Set, rowIndex: Int ) throws(PluginRowWriteRefusal) -> PluginRowWrite { var parameters: [PluginCellValue] = [] @@ -133,13 +142,13 @@ public enum SurrealStatementGenerator { var target = SurrealQL.quoteIdentifier(table) for (index, column) in columns.enumerated() { - guard index < values.count else { continue } + guard index < values.count, !defaults.contains(index) else { continue } let cell = values[index] if column == SurrealInfoParser.recordIdColumn { guard case let .text(text) = cell else { continue } let trimmed = text.trimmingCharacters(in: .whitespaces) - guard !trimmed.isEmpty, trimmed != Self.autoIdMarker else { continue } + guard !trimmed.isEmpty else { continue } guard let record = SurrealQL.parseRecordId(text, fallbackTable: table) else { continue } parameters.append(SurrealCellCoder.parameter(.recordId(record))) target = "$p\(parameters.count - 1)" @@ -147,7 +156,6 @@ public enum SurrealStatementGenerator { } if case .null = cell { continue } - if Self.isAutoDefault(cell) { continue } try refuseShortened(cell, in: column, rowIndex: rowIndex) let value = typing.value(of: cell, in: column) parameters.append(SurrealCellCoder.parameter(value)) @@ -168,9 +176,12 @@ public enum SurrealStatementGenerator { table: String, scope: SurrealScope, columns: [String], - change: PluginRowChange - ) -> PluginRowWrite? { - guard let record = recordId(table: table, columns: columns, originalRow: change.originalRow) else { return nil } + change: PluginRowChange, + locator: String? + ) throws(PluginRowWriteRefusal) -> PluginRowWrite? { + guard let record = try targetRecord(table: table, columns: columns, change: change, locator: locator) else { + return nil + } return PluginRowWrite( statement: SurrealQueryBuilder.compose(scope: scope, statement: "DELETE $p0;"), parameters: [SurrealCellCoder.parameter(.recordId(record))], @@ -205,14 +216,35 @@ public enum SurrealStatementGenerator { return String(format: format, column) } - private static func recordId( + private static var unreadableRecordIdReason: String { + String( + localized: "This record's id reads like a UUID, array or object id, so its text does not say which record it is. Open the table to edit it, or use a query." + ) + } + + private static var unknownRecordReason: String { + String(localized: "This record cannot be found again from what was read. Refresh and try again.") + } + + // The locator is the id the row was read with. A row without one, from a query tab, is named by its id cell. + private static func targetRecord( table: String, columns: [String], - originalRow: [PluginCellValue]? - ) -> SurrealRecordID? { - recordText(columns: columns, originalRow: originalRow).flatMap { - SurrealQL.parseRecordId($0, fallbackTable: table) + change: PluginRowChange, + locator: String? + ) throws(PluginRowWriteRefusal) -> SurrealRecordID? { + if let locator { + guard let record = SurrealRowLocator.record(from: locator) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: unknownRecordReason) + } + return record + } + guard let text = recordText(columns: columns, originalRow: change.originalRow), + let record = SurrealQL.parseRecordId(text, fallbackTable: table) else { return nil } + guard SurrealQL.isRecoverableFromText(record.id) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: unreadableRecordIdReason) } + return record } private static func recordText(columns: [String], originalRow: [PluginCellValue]?) -> String? { diff --git a/apple/Plugins/TableProPluginKit/AWS/AWSAuthError.swift b/apple/Plugins/TableProPluginKit/AWS/AWSAuthError.swift index 8d6cef9abd..18e61e205a 100644 --- a/apple/Plugins/TableProPluginKit/AWS/AWSAuthError.swift +++ b/apple/Plugins/TableProPluginKit/AWS/AWSAuthError.swift @@ -12,6 +12,8 @@ public enum AWSAuthError: Error, LocalizedError, Equatable { case regionUnknown(host: String) case rdsEndpointUnresolved(host: String) case rdsEndpointInvalid(String) + case rdsEndpointNotDialed(endpoint: String, host: String) + case endpointLookupFailed(host: String) case credentialProcessInvalid(String) case credentialProcessLaunchFailed(profile: String, underlying: String) case credentialProcessFailed(profile: String, status: Int, message: String) @@ -48,6 +50,17 @@ public enum AWSAuthError: Error, LocalizedError, Equatable { format: String(localized: "TablePro cannot sign an RDS token for \"%@\". Enter the RDS Endpoint (for example mydb.abc123.us-east-1.rds.amazonaws.com:5432) when you connect through a port forward or bastion."), host ) + case .rdsEndpointNotDialed(let endpoint, let host): + return String( + // swiftlint:disable:next line_length + format: String(localized: "The RDS Endpoint \"%1$@\" is not the database at \"%2$@\", where the connection goes. Clear the RDS Endpoint unless you connect through a port forward."), + endpoint, host + ) + case .endpointLookupFailed(let host): + return String( + format: String(localized: "TablePro could not look up \"%@\" in DNS to find the AWS endpoint it points to. Check the network and try again."), + host + ) case .rdsEndpointInvalid(let value): return String( format: String(localized: "\"%@\" is not a valid RDS endpoint. Use the form host or host:port."), diff --git a/apple/Plugins/TableProPluginKit/AWS/AWSEndpointHost.swift b/apple/Plugins/TableProPluginKit/AWS/AWSEndpointHost.swift new file mode 100644 index 0000000000..2a365a3f57 --- /dev/null +++ b/apple/Plugins/TableProPluginKit/AWS/AWSEndpointHost.swift @@ -0,0 +1,146 @@ +import Darwin +import Foundation + +// An IAM credential is good for whatever endpoint it names, so these answers decide where one may go. A name counts +// as an RDS endpoint only in the shapes RDS publishes, under a partition's own DNS suffix and with a real region. +public enum AWSEndpointHost { + private static let loopbackNames: Set = ["localhost"] + + public static func normalized(_ host: String) -> String { + var name = host.trimmingCharacters(in: .whitespaces).lowercased() + if name.hasSuffix(".") { + name.removeLast() + } + return name + } + + // Decided by the same getaddrinfo call the driver's dial makes, so `127.1` and `::ffff:127.0.0.1` are loopback, + // while `127.0.0.1.example.com`, `127.0.0.1.`, `127.0.0.1 ` and `[::1]` are names it asks DNS about, which can + // answer anything. The host is read exactly as the driver gets it. + public static func isLoopback(_ host: String) -> Bool { + let name = host.lowercased() + if loopbackNames.contains(name) { + return true + } + switch numericAddress(name) { + case .ipv4(let address): + return address >> 24 == 127 || address == 0 + case .ipv6(let bytes): + let isLoopbackAddress = bytes[0 ..< 15].allSatisfy { $0 == 0 } && bytes[15] == 1 + let mapped = bytes[0 ..< 10].allSatisfy { $0 == 0 } && bytes[10] == 0xFF && bytes[11] == 0xFF + return isLoopbackAddress || (mapped && bytes[12] == 127) + case nil: + return false + } + } + + public static func isRDSEndpoint(_ host: String) -> Bool { + rdsName(normalized(host)) != nil + } + + // Nil for a name AWS did not issue, and for a global database endpoint, which names no region. + public static func rdsRegion(forHost host: String) -> String? { + rdsName(normalized(host))?.region + } + + // The RDS endpoint a name of the user's own stands for, found by following its CNAME records. `notFound` when + // `host` already is one, cannot have one, or leads to none. + public static func rdsEndpoint( + behind host: String, + resolver: any CNAMEResolving, + within timeout: Duration? + ) async -> CNAMELookup { + let name = normalized(host) + guard isAliasCandidate(name), !isRDSEndpoint(name) else { return .notFound } + return await resolver.firstName(behind: name, within: timeout, where: isRDSEndpoint) + } + + // A literal address has no CNAME, a single label is not looked up in DNS, and asking about a `.local` name sends + // multicast that iOS gates behind the Local Network permission. + public static func isAliasCandidate(_ host: String) -> Bool { + let name = normalized(host) + guard name.contains("."), !name.contains(":"), !name.contains("/"), !name.contains(","), + !name.hasSuffix(".local") + else { return false } + return !isLoopback(name) && numericAddress(name) == nil + } + + private struct RDSName { + let region: String? + } + + private static let dnsSuffixes: [String] = AWSPartition.all.reduce(into: []) { suffixes, partition in + if !suffixes.contains(partition.dnsSuffix) { + suffixes.append(partition.dnsSuffix) + } + } + + // `...rds.` everywhere, `..rds..` in China, GovCloud and the + // ISO partitions, and `.global-.global.rds.` for a global database. A Proxy's own endpoints add + // labels in front. + private static func rdsName(_ name: String) -> RDSName? { + for suffix in dnsSuffixes where name.hasSuffix("." + suffix) { + let labels = name.dropLast(suffix.count + 1).split(separator: ".", omittingEmptySubsequences: false) + guard labels.count >= 4, !labels.contains(where: \.isEmpty) else { continue } + let last = String(labels[labels.count - 1]) + let previous = String(labels[labels.count - 2]) + if last == "rds" { + if previous == "global" { + return RDSName(region: nil) + } + if isRegion(previous, under: suffix) { + return RDSName(region: previous) + } + } else if previous == "rds", isRegion(last, under: suffix) { + return RDSName(region: last) + } + } + return nil + } + + // Other services let customers pick labels under the same suffix (`.s3.amazonaws.com`), so the label has + // to read as a region, such as `us-east-1`, `cn-northwest-1` or `us-isob-east-1`. + private static func isRegion(_ label: String, under suffix: String) -> Bool { + let parts = label.split(separator: "-", omittingEmptySubsequences: false) + guard parts.count >= 3, + let number = parts.last, !number.isEmpty, number.allSatisfy({ $0.isASCII && $0.isNumber }), + parts.dropLast().allSatisfy({ !$0.isEmpty && $0.allSatisfy { $0 >= "a" && $0 <= "z" } }) + else { return false } + return AWSPartition.resolve(region: label).dnsSuffix == suffix + } + + private enum NumericAddress: Equatable { + case ipv4(UInt32) + case ipv6([UInt8]) + } + + // AI_NUMERICHOST parses without a lookup. Measured on macOS 27: `127.1`, `0x7f.1` and `2130706433` are + // 127.0.0.1, and `0177.0.0.1` is 177.0.0.1, not the octal 127.0.0.1 inet_aton reads. It also takes + // `::1%x,db.example.com` as ::1, while libpq dials each comma-separated host in turn, so only the characters of a + // bare address are let through. + private static func numericAddress(_ host: String) -> NumericAddress? { + guard !host.isEmpty, host.allSatisfy({ $0.isASCII && ($0.isHexDigit || $0 == "." || $0 == ":" || $0 == "x") }) else { + return nil + } + var hints = addrinfo() + hints.ai_flags = AI_NUMERICHOST + hints.ai_family = AF_UNSPEC + hints.ai_socktype = SOCK_STREAM + var list: UnsafeMutablePointer? + guard getaddrinfo(host, nil, &hints, &list) == 0, let list else { return nil } + defer { freeaddrinfo(list) } + guard let address = list.pointee.ai_addr else { return nil } + switch Int32(address.pointee.sa_family) { + case AF_INET: + return address.withMemoryRebound(to: sockaddr_in.self, capacity: 1) { + .ipv4(UInt32(bigEndian: $0.pointee.sin_addr.s_addr)) + } + case AF_INET6: + return address.withMemoryRebound(to: sockaddr_in6.self, capacity: 1) { pointer in + .ipv6(withUnsafeBytes(of: pointer.pointee.sin6_addr) { Array($0) }) + } + default: + return nil + } + } +} diff --git a/apple/Plugins/TableProPluginKit/AWS/AWSIAMTokenTarget.swift b/apple/Plugins/TableProPluginKit/AWS/AWSIAMTokenTarget.swift index 12e6444385..3fc6dfd570 100644 --- a/apple/Plugins/TableProPluginKit/AWS/AWSIAMTokenTarget.swift +++ b/apple/Plugins/TableProPluginKit/AWS/AWSIAMTokenTarget.swift @@ -47,7 +47,8 @@ public enum AWSIAMTokenTarget: Sendable, Equatable { override: fields["awsRDSEndpoint"], defaultPort: defaultPort ) - guard let region = Self.nonEmpty(fields["awsRegion"]) ?? RDSEndpoint.region(forHost: endpoint.host) else { + // The endpoint's own name decides: a token scoped to another region is refused, whatever the field says. + guard let region = AWSEndpointHost.rdsRegion(forHost: endpoint.host) ?? Self.nonEmpty(fields["awsRegion"]) else { throw AWSAuthError.regionUnknown(host: endpoint.host) } return RDSAuthTokenGenerator.generateToken( diff --git a/apple/Plugins/TableProPluginKit/AWS/AWSPartition.swift b/apple/Plugins/TableProPluginKit/AWS/AWSPartition.swift index 466c6721a7..a5e21e28ca 100644 --- a/apple/Plugins/TableProPluginKit/AWS/AWSPartition.swift +++ b/apple/Plugins/TableProPluginKit/AWS/AWSPartition.swift @@ -36,6 +36,8 @@ public struct AWSPartition: Sendable, Equatable { id: "aws-iso-f", dnsSuffix: "csp.hci.ic.gov", defaultRegion: "us-isof-east-1" ) + static let all: [AWSPartition] = [standard, china, govCloud, europeanSovereign, iso, isoB, isoE, isoF] + private static let byRegionPrefix: [(prefix: String, partition: AWSPartition)] = [ ("cn-", china), ("us-gov-", govCloud), diff --git a/apple/Plugins/TableProPluginKit/AWS/CNAMEResolver.swift b/apple/Plugins/TableProPluginKit/AWS/CNAMEResolver.swift new file mode 100644 index 0000000000..36be624dea --- /dev/null +++ b/apple/Plugins/TableProPluginKit/AWS/CNAMEResolver.swift @@ -0,0 +1,222 @@ +import dnssd +import Foundation + +// A lookup that failed is kept apart from a name with no CNAME: only the second is an answer. The three cases are +// every outcome a lookup has, so the enum is frozen. +@frozen +public enum CNAMELookup: Sendable, Equatable { + case found(String) + case notFound + case failed +} + +public protocol CNAMEResolving: Sendable { + // The target of `name`'s CNAME record: `notFound` when DNS says it has none, `failed` when the lookup errs or + // runs out of `timeout`. + func cnameTarget(of name: String, within timeout: Duration?) async -> CNAMELookup +} + +public extension CNAMEResolving { + // Follows the chain one record at a time, so it can stop at a name in the middle: the end of an RDS chain is an + // EC2, VPC endpoint or replica name that no token is signed for. A loop or an over-long chain finds nothing. + func firstName( + behind host: String, + within timeout: Duration?, + where matches: (String) -> Bool + ) async -> CNAMELookup { + let clock = ContinuousClock() + let deadline = timeout.map { clock.now.advanced(by: $0) } + var visited: Set = [DNSWireName.normalized(host)] + var current = host + for _ in 0 ..< DNSWireName.maximumChainLength { + let remaining = deadline.map { clock.now.duration(to: $0) } + if let remaining, remaining <= .zero { + return .failed + } + switch await cnameTarget(of: current, within: remaining) { + case .found(let target): + if matches(target) { + return .found(target) + } + guard visited.insert(target).inserted else { return .notFound } + current = target + case .notFound: + return .notFound + case .failed: + return .failed + } + } + return .notFound + } +} + +// The system resolver, which is what the driver's own lookup uses: it follows VPN split DNS and scoped resolvers, +// and libresolv does not. +public struct SystemCNAMEResolver: CNAMEResolving { + public init() {} + + public func cnameTarget(of name: String, within timeout: Duration?) async -> CNAMELookup { + let query = CNAMEQuery(name: name) + return await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + query.start(timeout: timeout, continuation: continuation) + } + } onCancel: { + query.cancel() + } + } +} + +enum DNSWireName { + static let maximumChainLength = 8 + + static func normalized(_ name: String) -> String { + var text = name.trimmingCharacters(in: .whitespaces).lowercased() + if text.hasSuffix(".") { + text.removeLast() + } + return text + } + + // dnssd hands over the record's rdata uncompressed, so a pointer here means a malformed answer. + static func decode(_ bytes: UnsafeRawBufferPointer) -> String? { + var labels: [String] = [] + var index = 0 + while index < bytes.count { + let length = Int(bytes[index]) + index += 1 + if length == 0 { + guard !labels.isEmpty, index == bytes.count else { return nil } + let name = labels.joined(separator: ".") + return name.utf8.count <= 253 ? name : nil + } + guard length <= 63, index + length <= bytes.count else { return nil } + let label = bytes[index ..< index + length] + // The name can become the host a driver dials, so it holds only what a host name may. + guard label.allSatisfy(isHostNameByte), let text = String(bytes: label, encoding: .ascii) else { return nil } + labels.append(text.lowercased()) + index += length + } + return nil + } + + private static func isHostNameByte(_ byte: UInt8) -> Bool { + switch byte { + case UInt8(ascii: "a") ... UInt8(ascii: "z"), UInt8(ascii: "A") ... UInt8(ascii: "Z"), + UInt8(ascii: "0") ... UInt8(ascii: "9"), UInt8(ascii: "-"), UInt8(ascii: "_"): + true + default: + false + } + } +} + +// dnssd delivers on the queue a ref was given and wants the ref deallocated there, so all state lives on that queue. +private final class CNAMEQuery: @unchecked Sendable { + private static let queue = DispatchQueue(label: "com.TablePro.CNAMEQuery") + + private let name: String + private var ref: DNSServiceRef? + private var retained: Unmanaged? + private var continuation: CheckedContinuation? + private var isFinished = false + + init(name: String) { + self.name = DNSWireName.normalized(name) + } + + func start(timeout: Duration?, continuation: CheckedContinuation) { + Self.queue.async { + guard !self.isFinished else { + continuation.resume(returning: .failed) + return + } + self.continuation = continuation + if let timeout, timeout <= .zero { + self.finish(.failed) + return + } + let retained = Unmanaged.passRetained(self) + self.retained = retained + var ref: DNSServiceRef? + // Without ReturnIntermediates a name with no CNAME never answers, and the query waits out the system + // timeout of about 30 seconds (measured). + let status = DNSServiceQueryRecord( + &ref, + DNSServiceFlags(kDNSServiceFlagsReturnIntermediates | kDNSServiceFlagsTimeout), + UInt32(kDNSServiceInterfaceIndexAny), + self.name, + UInt16(kDNSServiceType_CNAME), + UInt16(kDNSServiceClass_IN), + { _, flags, _, error, fullname, rrtype, _, length, rdata, _, context in + guard let context else { return } + let query = Unmanaged.fromOpaque(context).takeUnretainedValue() + let owner = fullname.map { String(cString: $0) } + let bytes = rdata.map { UnsafeRawBufferPointer(start: $0, count: Int(length)) } + query.receive(flags: flags, error: error, owner: owner, rrtype: rrtype, rdata: bytes) + }, + retained.toOpaque() + ) + guard status == kDNSServiceErr_NoError, let ref else { + self.finish(.failed) + return + } + self.ref = ref + DNSServiceSetDispatchQueue(ref, Self.queue) + if let timeout { + Self.queue.asyncAfter(deadline: .now() + Self.seconds(timeout)) { + self.finish(.failed) + } + } + } + } + + func cancel() { + Self.queue.async { + self.finish(.failed) + } + } + + private func receive( + flags: DNSServiceFlags, + error: DNSServiceErrorType, + owner: String?, + rrtype: UInt16, + rdata: UnsafeRawBufferPointer? + ) { + guard !isFinished else { return } + guard error == kDNSServiceErr_NoError else { + // With ReturnIntermediates, NoSuchRecord is DNS saying the name has no CNAME, or does not exist. + finish(error == kDNSServiceErr_NoSuchRecord ? .notFound : .failed) + return + } + guard flags & DNSServiceFlags(kDNSServiceFlagsAdd) != 0, + Int(rrtype) == kDNSServiceType_CNAME, + owner.map(DNSWireName.normalized) == name, + let rdata + else { return } + finish(DNSWireName.decode(rdata).map(CNAMELookup.found) ?? .failed) + } + + // A cancel can land before the start block runs: it marks the query finished, and the start block then resumes + // the continuation it was handed. + private func finish(_ outcome: CNAMELookup) { + guard !isFinished else { return } + isFinished = true + if let ref { + DNSServiceRefDeallocate(ref) + self.ref = nil + } + if let continuation { + continuation.resume(returning: outcome) + self.continuation = nil + } + retained?.release() + retained = nil + } + + private static func seconds(_ duration: Duration) -> Double { + let components = duration.components + return Double(components.seconds) + Double(components.attoseconds) / 1e18 + } +} diff --git a/apple/Plugins/TableProPluginKit/AWS/RDSEndpoint.swift b/apple/Plugins/TableProPluginKit/AWS/RDSEndpoint.swift deleted file mode 100644 index ef574bbe44..0000000000 --- a/apple/Plugins/TableProPluginKit/AWS/RDSEndpoint.swift +++ /dev/null @@ -1,15 +0,0 @@ -import Foundation - -enum RDSEndpoint { - static func region(forHost host: String) -> String? { - let parts = host.lowercased().split(separator: ".").map(String.init) - guard let rdsIndex = parts.firstIndex(of: "rds"), - rdsIndex > 0, - parts.count > rdsIndex + 1, - parts[rdsIndex + 1] == "amazonaws" else { - return nil - } - let region = parts[rdsIndex - 1] - return region.isEmpty ? nil : region - } -} diff --git a/apple/Plugins/TableProPluginKit/AWS/RDSSigningEndpoint.swift b/apple/Plugins/TableProPluginKit/AWS/RDSSigningEndpoint.swift index 172b6cdd4c..f6100cc7a7 100644 --- a/apple/Plugins/TableProPluginKit/AWS/RDSSigningEndpoint.swift +++ b/apple/Plugins/TableProPluginKit/AWS/RDSSigningEndpoint.swift @@ -8,8 +8,6 @@ struct RDSSigningEndpoint: Equatable { // RDS validates the token against its own endpoint, while a tunnel or a port forward run outside // TablePro makes the driver dial loopback. This picks the endpoint the token is signed for. enum RDSSigningEndpointResolver { - private static let loopbackHosts: Set = ["localhost", "localhost.localdomain", "::1", "[::1]", "0.0.0.0"] - static func resolve( configuredHost: String, configuredPort: Int, @@ -18,22 +16,23 @@ enum RDSSigningEndpointResolver { override: String?, defaultPort: Int ) throws -> RDSSigningEndpoint { - let dialsLoopback = isLoopback(configuredHost) + let dialledHost = AWSEndpointHost.normalized(configuredHost) + let dialsLoopback = AWSEndpointHost.isLoopback(configuredHost) if let override, !override.trimmingCharacters(in: .whitespaces).isEmpty { let endpoint = try parse(override, defaultPort: defaultPort) - guard dialsLoopback || endpoint.host.caseInsensitiveCompare(configuredHost) == .orderedSame else { - throw AWSAuthError.rdsEndpointUnresolved(host: configuredHost) + guard dialsLoopback || endpoint.host == dialledHost else { + throw AWSAuthError.rdsEndpointNotDialed(endpoint: endpoint.host, host: dialledHost) } return endpoint } - let host = dialsLoopback ? (preTunnelHost ?? configuredHost) : configuredHost + let host = dialsLoopback ? (preTunnelHost.map(AWSEndpointHost.normalized) ?? dialledHost) : dialledHost let port = dialsLoopback ? (preTunnelPort ?? configuredPort) : configuredPort - guard !isLoopback(host) else { + guard !AWSEndpointHost.isLoopback(host) else { throw AWSAuthError.rdsEndpointUnresolved(host: host) } - return RDSSigningEndpoint(host: host.lowercased(), port: port) + return RDSSigningEndpoint(host: host, port: port) } static func parse(_ value: String, defaultPort: Int) throws -> RDSSigningEndpoint { @@ -58,18 +57,14 @@ enum RDSSigningEndpointResolver { return try makeEndpoint(host: String(text[.. Bool { - let normalized = host.trimmingCharacters(in: .whitespaces).lowercased() - return loopbackHosts.contains(normalized) || normalized.hasPrefix("127.") - } - private static func makeEndpoint(host: String, port: Int, rawValue: String) throws -> RDSSigningEndpoint { - let isHostname = !host.isEmpty - && !host.contains(":") - && host.rangeOfCharacter(from: .whitespacesAndNewlines) == nil - guard isHostname, !isLoopback(host) else { + let name = AWSEndpointHost.normalized(host) + let isHostname = !name.isEmpty + && !name.contains(":") + && name.rangeOfCharacter(from: .whitespacesAndNewlines) == nil + guard isHostname, !AWSEndpointHost.isLoopback(name) else { throw AWSAuthError.rdsEndpointInvalid(rawValue) } - return RDSSigningEndpoint(host: host.lowercased(), port: port) + return RDSSigningEndpoint(host: name, port: port) } } diff --git a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift index 702eaa1e56..1153452bb7 100644 --- a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift +++ b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift @@ -31,6 +31,7 @@ public enum PluginBoundedStream { var rows: [PluginRow] = [] rows.reserveCapacity(min(cap, 10_000)) var truncated = false + var declaredLimit: Int? /// The first batch to arrive is the first row the server produced, which is the only part /// of a streamed read that separates the query's own cost from the cost of moving its rows. var firstRowTime: TimeInterval? @@ -41,6 +42,7 @@ public enum PluginBoundedStream { columns = header.columns columnTypeNames = header.columnTypeNames columnOrigins = header.columnOrigins + declaredLimit = header.rowLimit case .rows(let batch): if firstRowTime == nil, !batch.isEmpty { firstRowTime = Date().timeIntervalSince(startedAt) @@ -55,6 +57,10 @@ public enum PluginBoundedStream { } if truncated { break } } + // A stream that ended at the limit its header declared was cut there, below the cap. + if let declaredLimit, rows.count >= declaredLimit { + truncated = true + } var result = PluginQueryResult( columns: columns, diff --git a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift index 59872dfaef..0eb929ecbc 100644 --- a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift @@ -41,4 +41,13 @@ public struct PluginCapabilities: OptionSet, Sendable { // The driver refuses what `PluginStatementContext.allowsDestructive` and `allowsServerCode` forbid, for a language // the host cannot read in full. public static let statementPolicy = PluginCapabilities(rawValue: 1 << 22) + // The driver writes the column default only where `PluginRowChange.defaultColumns` marks one, so the text + // `__DEFAULT__` in any other cell is saved as text. + public static let columnDefaultMarks = PluginCapabilities(rawValue: 1 << 23) + // A `PluginStatementContext.enforcesReadOnly` statement runs in the engine's read-only mode or throws, from every + // entry point that takes a context, `executeBoundedQuery(query:rowCap:context:)` included. + public static let enforcesReadOnlyMode = PluginCapabilities(rawValue: 1 << 24) + // The driver runs a `COPY ... FROM STDIN` and sends its rows through + // `PluginDatabaseDriver.copyFromStandardInput(statement:rows:context:)`. + public static let copyFromStandardInput = PluginCapabilities(rawValue: 1 << 25) } diff --git a/apple/Plugins/TableProPluginKit/PluginCopyData.swift b/apple/Plugins/TableProPluginKit/PluginCopyData.swift new file mode 100644 index 0000000000..7a49c89e39 --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginCopyData.swift @@ -0,0 +1,24 @@ +// +// PluginCopyData.swift +// TableProPluginKit +// + +import Foundation + +/// The rows a `COPY ... FROM STDIN` loads, handed over a piece at a time so a large block is never held whole. +/// +/// The driver pulls each piece on the thread that holds its connection for the whole copy, so a read may block on a +/// file. A piece can end anywhere, inside a row too, and holds row data only: never psql's `\.` line. +public struct PluginCopyData: Sendable { + private let read: @Sendable () throws -> Data? + + public init(read: @escaping @Sendable () throws -> Data?) { + self.read = read + } + + /// The next piece, encoded for the session (UTF-8 here), or nil once every row was read. A throw ends the copy + /// with a failure, so the server keeps none of its rows, and the driver rethrows it. + public func nextPiece() throws -> Data? { + try read() + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index c173fc6838..991d7721a6 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -55,6 +55,7 @@ public struct PluginRowChange: Sendable { case delete } + // One row can arrive as several changes, each with its own key and the row as it stands before that change. public let rowIndex: Int public let type: ChangeType public let cellChanges: [(columnIndex: Int, columnName: String, oldValue: PluginCellValue, newValue: PluginCellValue)] @@ -70,6 +71,11 @@ public struct PluginRowChange: Sendable { /// that declares `PluginCapabilities.cellExpressions` is handed any. public var expressionColumns: Set? + // The columns whose new value is the column's default rather than a value: an update's `cellChanges`, or a new + // row's `insertedRowData` cells. Such a cell still holds the text `__DEFAULT__`, which a driver built before this + // field reads as the default. Every other cell is a value, that text included. + public var defaultColumns: Set? + public init( rowIndex: Int, type: ChangeType, @@ -81,6 +87,10 @@ public struct PluginRowChange: Sendable { self.cellChanges = cellChanges self.originalRow = originalRow } + + public func isColumnDefault(_ columnIndex: Int) -> Bool { + defaultColumns?.contains(columnIndex) ?? false + } } public protocol PluginDatabaseDriver: AnyObject, Sendable { @@ -117,6 +127,15 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// statement on the server. func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? + // The same bounded read with what the host decided about it, under the same contract. The default runs the + // requirement above for a context that asks nothing that call cannot honor, and returns nil otherwise, so the host + // falls back to a call that takes the context. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? + /// Sends `query` to the server as one batch and reads it to the end, returning every result set it produced and /// every error it raised. `rowCap` bounds each result set on its own. A driver that implements this declares /// ``PluginCapabilities/resultSetBatches``; the default returns nil, and the host runs the text statement by @@ -126,6 +145,15 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// statements after it may still have run and returned results. func executeBatch(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginBatchResult? + /// Runs `statement`, a `COPY ... FROM STDIN`, and sends it `rows` a piece at a time through the engine's copy + /// protocol, holding the session until the copy ends. A driver declares `PluginCapabilities.copyFromStandardInput` + /// for it; the default throws `PluginDriverUnsupportedOperation.copyFromStandardInput`. + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult + func fetchTables(schema: String?) async throws -> [PluginTableInfo] /// What `fetchTables(schema:)` lists for every schema `fetchSchemas()` lists, in one call, each @@ -200,6 +228,14 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func fetchEvents(schema: String?) async throws -> [PluginEventInfo] func fetchEventDDL(_ event: PluginEventInfo) async throws -> String + // Asked of the live driver, so one plugin can answer per server flavor and version. + var serverObjectKinds: [PluginServerObjectKind] { get } + // `schema` is the container for a `.container` kind and nil for a `.database` kind. A server + // without the catalog answers empty rather than throwing. + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [PluginServerObjectInfo] + // Reads one listed object again with its definition; `identity` is the driver's own key. + func fetchServerObject(_ object: PluginServerObjectInfo) async throws -> PluginServerObjectInfo + /// The settings the server recorded for an object when it was created and applies whenever it runs, such as a /// MySQL routine's `sql_mode`, as statements a script writes around the object's `CREATE`. Nil when there are none. func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? @@ -306,6 +342,7 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] var providesBulkColumnFetch: Bool { get } + var fetchesAllColumnsInOneRequest: Bool { get } func sampleFieldPaths(table: String, schema: String?, limit: Int) async throws -> [PluginFieldPath] func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] var providesBulkForeignKeyFetch: Bool { get } @@ -495,6 +532,10 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// answer, which stands the check down for an engine TablePro never runs a rebuild on anyway. func columnReorderSchemaFingerprint(table: String, schema: String?) async throws -> String? + // A cheap value that changes whenever the catalog does, so a write to a database file that left the schema + // alone needs no catalog reload. Nil where the engine keeps none; the host then reloads on every change. + func schemaChangeToken() async throws -> String? + var unsupportedStructureColumnFields: Set { get } var unsupportedIndexTypes: Set { get } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? @@ -824,6 +865,17 @@ public extension PluginDatabaseDriver { return definition } + var serverObjectKinds: [PluginServerObjectKind] { [] } + + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [PluginServerObjectInfo] { [] } + + func fetchServerObject(_ object: PluginServerObjectInfo) async throws -> PluginServerObjectInfo { + guard let definition = object.definition, !definition.isEmpty else { + throw PluginObjectSourceError.unsupported(object.name) + } + return object + } + func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { nil } func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { nil } @@ -1021,12 +1073,16 @@ public extension PluginDatabaseDriver { /// difference at all. var providesBulkColumnFetch: Bool { false } + // Whether `fetchAllColumns` reads every table in one request or a fixed few, whatever its completeness. + var fetchesAllColumnsInOneRequest: Bool { providesBulkColumnFetch } + /// Default: fetches columns per-table sequentially (N+1 round-trips). /// SQL drivers should override with a single bulk query (e.g. INFORMATION_SCHEMA.COLUMNS). func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let tables = try await fetchTables(schema: schema) var result: [String: [PluginColumnInfo]] = [:] for table in tables { + try Task.checkCancellation() result[table.name] = try await fetchColumns(table: table.name, schema: schema) } return result @@ -1061,6 +1117,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: [PluginForeignKeyInfo]] = [:] for table in tables { + try Task.checkCancellation() let fks = try await fetchForeignKeys(table: table.name, schema: schema) if !fks.isEmpty { result[table.name] = fks } } @@ -1090,6 +1147,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: [PluginIndexInfo]] = [:] for table in tables { + try Task.checkCancellation() let indexes = try await fetchIndexes(table: table.name, schema: schema) if !indexes.isEmpty { result[table.name] = indexes } } @@ -1108,6 +1166,7 @@ public extension PluginDatabaseDriver { let tables = try await fetchTables(schema: schema) var result: [String: PluginTableMetadata] = [:] for table in tables { + try Task.checkCancellation() guard let metadata = try? await fetchTableMetadata(table: table.name, schema: schema) else { continue } result[table.name] = metadata } @@ -1118,6 +1177,7 @@ public extension PluginDatabaseDriver { let dbs = try await fetchDatabases() var result: [PluginDatabaseMetadata] = [] for db in dbs { + try Task.checkCancellation() do { result.append(try await fetchDatabaseMetadata(db)) } catch { @@ -1315,6 +1375,8 @@ public extension PluginDatabaseDriver { func columnReorderSchemaFingerprint(table: String, schema: String?) async throws -> String? { nil } + func schemaChangeToken() async throws -> String? { nil } + var unsupportedStructureColumnFields: Set { [] } var unsupportedIndexTypes: Set { [] } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? { nil } @@ -1440,10 +1502,31 @@ public extension PluginDatabaseDriver { func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { nil } + // A driver built before the context overload ignores every field, so a read-only mode, a statement policy or a + // catalog pin it was asked for would silently not happen. + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await executeBoundedQuery(query: query, rowCap: rowCap) + } + func executeBatch(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginBatchResult? { nil } + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + /// The bounded read for a driver whose `streamRows` yields rows as they arrive and whose /// producer stops when its consumer does. Opt in by returning this from `executeBoundedQuery`. /// @@ -1471,7 +1554,7 @@ public extension PluginDatabaseDriver { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - try await PluginParameterBinder.$hostFeatures.withValue(context.lexicalFeatures) { + try await PluginParameterBinder.withHostContext(context) { try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } } @@ -1481,7 +1564,7 @@ public extension PluginDatabaseDriver { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await PluginParameterBinder.$hostFeatures.withValue(context.lexicalFeatures) { + try await PluginParameterBinder.withHostContext(context) { try await executeParameterized(query: query, parameters: parameters) } } @@ -1500,13 +1583,15 @@ public extension PluginDatabaseDriver { func streamRows(query: String) -> AsyncThrowingStream { AsyncThrowingStream { continuation in - Task { + let producer = Task { do { + try Task.checkCancellation() let result = try await self.execute(query: query) - let header = PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames ) + if result.isTruncated { header.rowLimit = result.rows.count } continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) @@ -1516,6 +1601,8 @@ public extension PluginDatabaseDriver { continuation.finish(throwing: error) } } + // An unstructured task is no child of the consumer, so its cancel reaches execute only through here. + continuation.onTermination = { @Sendable _ in producer.cancel() } } } @@ -1531,9 +1618,12 @@ public extension PluginDatabaseDriver { guard !parameters.isEmpty else { return try await execute(query: query) } + let kinds = PluginParameterBinder.hostParameterKinds let sql = try PluginParameterBinder.bind( query, - literals: parameters.map(sqlLiteral(for:)), + literals: parameters.enumerated().map { index, value in + sqlLiteral(for: value, kind: kinds.flatMap { $0.indices.contains(index) ? $0[index] : nil }) + }, style: parameterStyle, features: literalFeatures ) @@ -1541,11 +1631,17 @@ public extension PluginDatabaseDriver { } func sqlLiteral(for value: PluginCellValue) -> String { + sqlLiteral(for: value, kind: nil) + } + + // A nil kind is a value nobody typed, such as a grid cell, whose text the host wrote: number-shaped text is a number. + func sqlLiteral(for value: PluginCellValue, kind: PluginColumnKind?) -> String { switch value { case .null: return "NULL" case .text(let s): - return escapedParameterValue(s) + guard let kind else { return escapedParameterValue(s) } + return typedParameterValue(s, kind: kind) case .bytes(let data): var hex = "X'" hex.reserveCapacity(2 + data.count * 2 + 1) @@ -1558,7 +1654,8 @@ public extension PluginDatabaseDriver { } func escapedParameterValue(_ value: String) -> String { - if Self.isNumericLiteral(value) { + // No numeric column reads as `00123` or `-007`, so that text is a code or a zip and stays a string. + if Self.isNumericLiteral(value), !opensWithPaddingZero(value) { return value } return PluginSQLQuoting.stringLiteral(value, escapes: PluginSQLQuoting.Escapes(features: literalFeatures)) @@ -1568,6 +1665,23 @@ public extension PluginDatabaseDriver { PluginNumericLiteral.isValid(value) } + // Text the server cannot read as the declared number or boolean is sent as a string, for the server to refuse. + func typedParameterValue(_ value: String, kind: PluginColumnKind) -> String { + switch kind { + case .integer, .decimal: + if PluginSQLLiteral.isNumericLiteral(value, kind: kind) { return value } + case .boolean: + switch PluginSQLLiteral.booleanSynonym(for: value, kind: kind) { + case .isTrue: return "TRUE" + case .isFalse: return "FALSE" + case nil: break + } + case .text, .other: + break + } + return PluginSQLQuoting.stringLiteral(value, escapes: PluginSQLQuoting.Escapes(features: literalFeatures)) + } + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { let raw: PluginQueryResult if let parameters { @@ -1602,3 +1716,10 @@ extension PluginDatabaseDriver { return backslash ? [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes] : [] } } + +// `007`, `-0042`: an integer part that opens with 0 and goes on with another digit. +private func opensWithPaddingZero(_ value: String) -> Bool { + let unsigned = value.unicodeScalars.drop { $0 == "-" || $0 == "+" } + guard unsigned.first == "0", let next = unsigned.dropFirst().first else { return false } + return ("0"..."9").contains(next) +} diff --git a/apple/Plugins/TableProPluginKit/PluginDriverError.swift b/apple/Plugins/TableProPluginKit/PluginDriverError.swift index 9581028e68..09b1da9da8 100644 --- a/apple/Plugins/TableProPluginKit/PluginDriverError.swift +++ b/apple/Plugins/TableProPluginKit/PluginDriverError.swift @@ -31,3 +31,17 @@ public extension PluginDriverError { return desc } } + +// A statement Stop ended, after which the server rolled back the transaction it ran in. A driver throws it in place of +// the cancellation, so the host can say what the Stop undid. +public struct PluginStopRolledBackTransaction: PluginDriverError { + public let pluginErrorCode: Int? + + public init(code: Int? = nil) { + pluginErrorCode = code + } + + public var pluginErrorMessage: String { + String(localized: "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone.") + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift b/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift index 75d947a168..695612e291 100644 --- a/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift +++ b/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift @@ -22,6 +22,7 @@ public enum PluginDriverUnsupportedOperation: Error, LocalizedError, Sendable { case writeDocument case explainPlan case importExportedDatabase + case copyFromStandardInput public var errorDescription: String? { switch self { @@ -37,6 +38,8 @@ public enum PluginDriverUnsupportedOperation: Error, LocalizedError, Sendable { return String(localized: "This driver cannot explain a statement") case .importExportedDatabase: return String(localized: "This driver cannot restore an exported folder. Update it in Settings > Extensions.") + case .copyFromStandardInput: + return String(localized: "This driver cannot send rows to COPY FROM STDIN.") } } } diff --git a/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift b/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift index 726b45de69..a89463cd38 100644 --- a/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift @@ -109,6 +109,72 @@ public enum PluginExportUtilities { } } +// Shared so XLSX number cells and the CSV decimal comma read a column's declared type the same way. +public enum PluginColumnNumbers: Equatable, Sendable { + case text + case exact + case approximate + case untyped + + public init(typeName: String, bitsAreStrings: Bool = false) { + let base = Self.baseName(typeName) + if base.isEmpty { + self = .untyped + } else if Self.bitTypes.contains(base) { + self = bitsAreStrings ? .text : .exact + } else if Self.approximateTypes.contains(base) { + self = .approximate + } else if Self.exactTypes.contains(base) { + self = .exact + } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { + self = .text + } else { + self = .untyped + } + } + + public var isNumeric: Bool { + self == .exact || self == .approximate + } + + // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the + // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. + private static func baseName(_ typeName: String) -> String { + var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + while name.hasSuffix(")"), let open = name.firstIndex(of: "("), + ["nullable", "lowcardinality"].contains(String(name[.. = [ + "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", + "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", + "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" + ] + + private static let approximateTypes: Set = [ + "float", "double", "real", "binary_float", "binary_double" + ] + + private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] + + private static let bitTypes: Set = ["bit", "varbit"] + + // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. + public static func bitsAreStrings(databaseTypeId: String) -> Bool { + enginesWithBitStrings.contains(databaseTypeId) + } + + private static let enginesWithBitStrings: Set = [ + "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" + ] + + private static let textMarkers = ["char", "text", "string", "clob"] +} + public extension String { func toUTF8Data() throws -> Data { guard let data = self.data(using: .utf8) else { diff --git a/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift b/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift index 21d25f4d95..440057949f 100644 --- a/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift +++ b/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift @@ -15,6 +15,9 @@ public protocol PluginImportDataSink: AnyObject, Sendable { /// its lines from the batch's first. func execute(statement: String, line: Int) async throws + /// Runs a `COPY ... FROM STDIN` read from the file's `line` and sends it `rows`. + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws + func insertRow(_ values: [String: PluginCellValue]) async throws func insertRows(_ rows: [[String: PluginCellValue]]) async throws func deleteAllRowsFromTargetTable() async throws @@ -30,6 +33,11 @@ public extension PluginImportDataSink { func execute(statement: String, line: Int) async throws { try await execute(statement: statement) } + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + throw PluginImportError.importFailed( + String(localized: "This connection cannot load the rows of COPY FROM STDIN.") + ) + } func insertRow(_ values: [String: PluginCellValue]) async throws { throw PluginImportError.importFailed("Row-based import is not supported by this connection") } diff --git a/apple/Plugins/TableProPluginKit/PluginImportElement.swift b/apple/Plugins/TableProPluginKit/PluginImportElement.swift new file mode 100644 index 0000000000..0f26191005 --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginImportElement.swift @@ -0,0 +1,31 @@ +// +// PluginImportElement.swift +// TableProPluginKit +// + +import Foundation + +/// One thing a SQL file asks the import to run, in file order. +public enum PluginImportElement: Sendable { + case statement(String, lineNumber: Int) + + /// A `COPY ... FROM STDIN` and the rows the file holds after it. The rows are read from the file while they are + /// sent; once the next element is asked for, any left unread are passed over. + case copyFromStandardInput(statement: String, lineNumber: Int, rows: PluginCopyData) +} + +public extension PluginImportElement { + var statementText: String { + switch self { + case .statement(let statement, _), .copyFromStandardInput(let statement, _, _): + return statement + } + } + + var lineNumber: Int { + switch self { + case .statement(_, let lineNumber), .copyFromStandardInput(_, let lineNumber, _): + return lineNumber + } + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginImportSource.swift b/apple/Plugins/TableProPluginKit/PluginImportSource.swift index ca602a0ee5..9d857a1781 100644 --- a/apple/Plugins/TableProPluginKit/PluginImportSource.swift +++ b/apple/Plugins/TableProPluginKit/PluginImportSource.swift @@ -7,6 +7,11 @@ import Foundation public protocol PluginImportSource: AnyObject, Sendable { func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> + + /// The file's statements and `COPY ... FROM STDIN` blocks, in order. A source that reads no `COPY` rows answers + /// with its statements. + func elements() async throws -> AsyncThrowingStream + func fileURL() -> URL func fileSizeBytes() -> Int64 func cleanup() @@ -14,4 +19,25 @@ public protocol PluginImportSource: AnyObject, Sendable { public extension PluginImportSource { func cleanup() {} + + func elements() async throws -> AsyncThrowingStream { + let statements = StatementElements(try await statements()) + return AsyncThrowingStream(unfolding: { + try await statements.next() + }) + } +} + +// The stream asks for one element at a time, so its iterator is never used from two places at once. +private final class StatementElements: @unchecked Sendable { + private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator + + init(_ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>) { + self.statements = statements.makeAsyncIterator() + } + + func next() async throws -> PluginImportElement? { + guard let (statement, lineNumber) = try await statements.next() else { return nil } + return .statement(statement, lineNumber: lineNumber) + } } diff --git a/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift b/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift index d2a06245da..1669cfdc11 100644 --- a/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift +++ b/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift @@ -20,6 +20,21 @@ public enum PluginParameterBinder { // The host's reading of the connection, set around a statement it hands to a driver. @TaskLocal static var hostFeatures: SQLLexicalFeatures? + @TaskLocal static var hostParameterKinds: [PluginColumnKind?]? + + // A driver whose context overload forwards to its context-free twin wraps the call in this, so the default binder + // still writes each value the way the host's context says. + public static func withHostContext( + _ context: PluginStatementContext, + operation: () async throws -> T + ) async rethrows -> T { + try await $hostFeatures.withValue(context.lexicalFeatures) { + try await $hostParameterKinds.withValue(context.parameterKinds) { + try await operation() + } + } + } + public static func bind( _ template: String, literals parameters: [String], diff --git a/apple/Plugins/TableProPluginKit/PluginRowWrite.swift b/apple/Plugins/TableProPluginKit/PluginRowWrite.swift index 317bdad935..e31c7b5250 100644 --- a/apple/Plugins/TableProPluginKit/PluginRowWrite.swift +++ b/apple/Plugins/TableProPluginKit/PluginRowWrite.swift @@ -19,17 +19,32 @@ public struct PluginRowWrite: Sendable { public let parameters: [PluginCellValue] public let rowIndices: [Int] public let lookup: PluginRowLookup? + // Set only when the engine's count for this statement is the rows it matched, a row that already held the value + // included. The host then refuses the save when the count is higher, or lower for an UPDATE, and rolls it back. + public let expectedRowCount: Int? @_disfavoredOverload public init(statement: String, parameters: [PluginCellValue] = [], rowIndices: [Int]) { - self.init(statement: statement, parameters: parameters, rowIndices: rowIndices, lookup: nil) + self.init(statement: statement, parameters: parameters, rowIndices: rowIndices, lookup: nil, expectedRowCount: nil) } + @_disfavoredOverload public init(statement: String, parameters: [PluginCellValue] = [], rowIndices: [Int], lookup: PluginRowLookup? = nil) { + self.init(statement: statement, parameters: parameters, rowIndices: rowIndices, lookup: lookup, expectedRowCount: nil) + } + + public init( + statement: String, + parameters: [PluginCellValue] = [], + rowIndices: [Int], + lookup: PluginRowLookup? = nil, + expectedRowCount: Int? = nil + ) { self.statement = statement self.parameters = parameters self.rowIndices = rowIndices self.lookup = lookup + self.expectedRowCount = expectedRowCount } } @@ -76,4 +91,13 @@ public struct PluginRowWriteRefusal: Error, LocalizedError, Sendable, Equatable } public var errorDescription: String? { reason } + + // For a driver whose engine has no column defaults: the first change that leaves a cell to one. + public static func columnDefaultsUnsupported(in changes: [PluginRowChange]) -> PluginRowWriteRefusal? { + guard let change = changes.first(where: { !($0.defaultColumns ?? []).isEmpty }) else { return nil } + return PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "This database has no column defaults, so a cell cannot be left to one. Enter a value.") + ) + } } diff --git a/apple/Plugins/TableProPluginKit/PluginRowWriters.swift b/apple/Plugins/TableProPluginKit/PluginRowWriters.swift index e92617afb2..d3c0be5caa 100644 --- a/apple/Plugins/TableProPluginKit/PluginRowWriters.swift +++ b/apple/Plugins/TableProPluginKit/PluginRowWriters.swift @@ -18,9 +18,7 @@ public struct PluginCsvWriteOptions: Sendable, Equatable { public let lineEnding: String public let nullAsEmpty: Bool - /// Prefixes a value opening with `=`, `+`, `-` or `@` with a quote. A spreadsheet treats a cell - /// starting with one of those as a formula, and the values in an export come from the database - /// rather than from the person opening the file. + // The quote stays in the value for any reader but a spreadsheet, so it suits files meant for one. public let sanitizesFormulas: Bool /// Replaces a line break inside a value with a space. Off means the value keeps its break and is @@ -73,9 +71,7 @@ public enum PluginRowWriters { .replacingOccurrences(of: "\r", with: " ", options: .literal) } var wasNeutralised = false - // Scalars, not Characters: `\r\n` is one Character, so a Character test misses a Windows line break. - if options.sanitizesFormulas, let first = processed.unicodeScalars.first, - isFormulaLead(Character(first)), !isPlainNumber(processed) { + if options.sanitizesFormulas, isFormulaLike(processed) { processed = "'" + processed wasNeutralised = true } @@ -95,12 +91,19 @@ public enum PluginRowWriters { private static let quotedScalars: Set = ["\"", "\n", "\r", "\t"] - /// A leading tab or carriage return is treated as a formula lead too: Excel strips it before - /// parsing the cell, so `\t=1+1` reaches the formula engine exactly as `=1+1` would. - private static func isFormulaLead(_ character: Character) -> Bool { - formulaPrefixes.contains(character) || character == "\t" || character == "\r" + // Public so a format plugin can count the values sanitizing changes. + public static func isFormulaLike(_ value: String) -> Bool { + // Scalars, not Characters: `\r\n` is one Character, so a Character test misses a Windows line break. + guard let first = value.unicodeScalars.first, formulaLeads.contains(first) else { return false } + return !isPlainNumber(value) } + // OWASP's list: the four operators, the whitespace Excel strips before parsing (so `\t=1+1` runs as `=1+1`), + // and the full-width operators a Japanese locale reads as formulas. The same set as the shared core's `FORMULA_LEADS`. + private static let formulaLeads: Set = [ + "=", "+", "-", "@", "\t", "\r", "\n", "\u{FF1D}", "\u{FF0B}", "\u{FF0D}", "\u{FF20}" + ] + // A number is never a formula, and `'-5` is text to every reader, a spreadsheet included. // The same rule as the Linux app's `is_plain_number`. private static func isPlainNumber(_ value: String) -> Bool { diff --git a/apple/Plugins/TableProPluginKit/PluginServerObject.swift b/apple/Plugins/TableProPluginKit/PluginServerObject.swift new file mode 100644 index 0000000000..460117f92f --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginServerObject.swift @@ -0,0 +1,68 @@ +// +// PluginServerObject.swift +// TableProPluginKit +// + +import Foundation + +public enum PluginServerObjectScope: String, Codable, Sendable { + // Listed in every container the engine lists tables in: a MySQL database, a PostgreSQL schema. + case container + // Listed once per database, beside its schemas. + case database +} + +public struct PluginServerObjectKind: Codable, Sendable, Hashable { + public let id: String + public let displayName: String + public let pluralDisplayName: String + public let iconName: String + public let scope: PluginServerObjectScope + + public init( + id: String, + displayName: String, + pluralDisplayName: String, + iconName: String, + scope: PluginServerObjectScope = .container + ) { + self.id = id + self.displayName = displayName + self.pluralDisplayName = pluralDisplayName + self.iconName = iconName + self.scope = scope + } +} + +public struct PluginServerObjectInfo: Codable, Sendable { + public let kindId: String + public let name: String + public let schema: String? + // The driver's own key for reading this object again, opaque to the app. + public let identity: String? + // One short label drawn beside the name, such as "Recurring" or "1.3". + public let detail: String? + public let isEnabled: Bool? + public let definition: String? + public let attributes: [PluginObjectAttribute] + + public init( + kindId: String, + name: String, + schema: String? = nil, + identity: String? = nil, + detail: String? = nil, + isEnabled: Bool? = nil, + definition: String? = nil, + attributes: [PluginObjectAttribute] = [] + ) { + self.kindId = kindId + self.name = name + self.schema = schema + self.identity = identity + self.detail = detail + self.isEnabled = isEnabled + self.definition = definition + self.attributes = attributes + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift index 2af809f03b..09c2b1c98c 100644 --- a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift +++ b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift @@ -9,7 +9,8 @@ import Foundation public struct PluginStatementContext: Sendable, Equatable { // From an external origin, a driver that declares `PluginCapabilities.readOnlyStatements` runs the statement in // the engine's read-only mode and throws when the session state does not allow it. A scripting driver refuses - // every write whatever the origin. + // every write whatever the origin. A SQL driver runs an app statement marked this way as it is, a catalog read + // among them: Safe Mode's Read-Only level is `enforcesReadOnly`. public var readOnly = false public var origin: PluginStatementOrigin = .app @@ -31,7 +32,35 @@ public struct PluginStatementContext: Sendable, Equatable { // that does not say. public var lexicalFeatures: SQLLexicalFeatures? + // The type the user gave each parameter, by position: `.text` and `.other` are written as strings, `.integer` and + // `.decimal` as numbers, `.boolean` as TRUE or FALSE. Nil, or a nil entry, where none was given. + public var parameterKinds: [PluginColumnKind?]? + + // The type of the column each parameter is written to or compared with, by position, as the target's driver names + // it. Nil, or a nil entry, where the host does not know it. + public var parameterTypeNames: [String?]? + + // Safe Mode is Read-Only for the connection. A driver that declares `PluginCapabilities.enforcesReadOnlyMode` runs + // the statement in the engine's read-only mode whatever the origin, or throws. + public var enforcesReadOnly = false + + // False when the host read the statement and found no call off the engine's built-in function list. At + // `enforcesReadOnly`, a driver whose session holds a transaction or a lock the read-only mode would end runs such a + // statement in it as it is, which Safe Mode's function check allows, and refuses any other. + public var callsUnlistedFunction = true + + // Stop reading rows once the decoded rows pass this many bytes, keeping the row that crossed it, so the host sees + // the limit was reached. Nil for no limit. The host sets it on reads only, since stopping a write cancels it. + public var resultByteLimit: Int? + + // Kept at its published signature: a defaulted parameter would replace its mangled symbol. + @_disfavoredOverload public init() {} + + public init(enforcesReadOnly: Bool = false, resultByteLimit: Int? = nil) { + self.enforcesReadOnly = enforcesReadOnly + self.resultByteLimit = resultByteLimit + } } public enum PluginStatementOrigin: Sendable, Equatable { diff --git a/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift b/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift index 7ed43ac87e..f5dec35e28 100644 --- a/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift +++ b/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift @@ -48,16 +48,9 @@ public final class PluginStreamAbort: @unchecked Sendable { } public enum PluginRowStream { - /// Builds a row stream whose abort signal is already wired to its termination. - /// - /// The handler is installed before `build` runs and sets the flag synchronously on whichever - /// thread terminated the stream, so a producer that has not started yet still sees it and can - /// decline to send the query at all. A producer that hops the signal onto its own serial queue - /// instead sees it only after it has finished, which is the defect this exists to make - /// unrepeatable. - /// - /// Termination follows the lifetime of the stream, not the consumer's `break`: a consumer that - /// stops reading but holds the stream in scope has not terminated it. + // The handler is installed before `build` runs and sets the flag on whichever thread ended the stream, so a + // producer that has not started yet still sees it and can decline to send the query. Termination follows the + // stream's lifetime, not the consumer's `break`: a consumer that stops reading but holds the stream has not ended it. public static func make( bufferingPolicy: AsyncThrowingStream.Continuation.BufferingPolicy = .unbounded, _ build: @escaping @Sendable ( @@ -67,8 +60,116 @@ public enum PluginRowStream { ) -> AsyncThrowingStream { let abort = PluginStreamAbort() return AsyncThrowingStream(bufferingPolicy: bufferingPolicy) { continuation in - continuation.onTermination = { @Sendable _ in abort.abort() } + // Only the consumer ends a stream as `.cancelled`, by cancelling its task or letting the stream go. A + // producer's own `finish` is `.finished`, and aborting then ran its cancel actions after every whole read. + continuation.onTermination = { @Sendable termination in + if case .cancelled = termination { abort.abort() } + } build(continuation, abort) } } + + // A batch is a few thousand rows, so this caps what a reader that falls behind leaves in memory. + public static let boundedBufferBatches = 16 + + // Every element the producer sends goes through `deliverBlocking` or `deliver`: a full buffer drops a plain yield. + public static func makeBounded( + _ build: @escaping @Sendable ( + AsyncThrowingStream.Continuation, + PluginStreamAbort + ) -> Void + ) -> AsyncThrowingStream { + make(bufferingPolicy: .bufferingOldest(boundedBufferBatches), build) + } + + // Blocks the calling thread while the buffer is full, so it is only for a producer on its own serial queue or + // thread, never the cooperative pool. False, with the element not delivered, once the reader has gone or `stops` + // answers true. + public static func deliverBlocking( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation, + stops: () -> Bool = { false } + ) -> Bool { + deliverBlocking(element, to: continuation, stops: stops, keepingAliveEvery: .infinity) {} + } + + // For a server that ends a read nobody reads from: while the buffer stays full, `keepAlive` runs on the producer's + // thread every `interval` seconds, so the server holds the read for as long as the reader holds the stream. Its + // error ends the wait and is thrown. + public static func deliverBlocking( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation, + stops: () -> Bool = { false }, + keepingAliveEvery interval: TimeInterval, + _ keepAlive: () throws -> Void + ) rethrows -> Bool { + var pending = element + var lastContact = ContinuousClock.now + while true { + switch continuation.yield(pending) { + case .enqueued: + return true + case .dropped(let refused): + guard !stops() else { return false } + pending = refused + if isDue(since: lastContact, every: interval) { + try keepAlive() + lastContact = .now + } + usleep(pollMicroseconds) + case .terminated: + return false + @unknown default: + return false + } + } + } + + // `deliverBlocking` for a producer running as a task: it suspends while the buffer is full. + public static func deliver( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation + ) async -> Bool { + await deliver(element, to: continuation, keepingAliveEvery: .infinity) {} + } + + // `deliverBlocking` with a keep-alive, for a producer running as a task. + public static func deliver( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation, + keepingAliveEvery interval: TimeInterval, + _ keepAlive: () async throws -> Void + ) async rethrows -> Bool { + var pending = element + var lastContact = ContinuousClock.now + while true { + switch continuation.yield(pending) { + case .enqueued: + return true + case .dropped(let refused): + pending = refused + if isDue(since: lastContact, every: interval) { + try await keepAlive() + lastContact = .now + } + do { + try await Task.sleep(nanoseconds: UInt64(pollMicroseconds) * 1_000) + } catch { + return false + } + case .terminated: + return false + @unknown default: + return false + } + } + } + + // Compared as seconds, so an infinite interval is never due and any interval converts without trapping. + private static func isDue(since lastContact: ContinuousClock.Instant, every interval: TimeInterval) -> Bool { + let elapsed = (ContinuousClock.now - lastContact).components + return Double(elapsed.seconds) + Double(elapsed.attoseconds) / 1e18 >= interval + } + + private static let pollMicroseconds: UInt32 = 2_000 } diff --git a/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift b/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift index 12cec34a01..914e3fc847 100644 --- a/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift +++ b/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift @@ -8,6 +8,8 @@ public struct PluginStreamHeader: Sendable { public let estimatedRowCount: Int? // See `PluginQueryResult.columnOrigins`. public var columnOrigins: [PluginColumnOrigin?]? + // The most rows the driver lets this statement return. A stream that ends at it was cut there; nil when nothing cuts it. + public var rowLimit: Int? public init(columns: [String], columnTypeNames: [String], estimatedRowCount: Int? = nil) { self.columns = columns diff --git a/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift b/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift index b6e034b8e4..bfbd14b67f 100644 --- a/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift +++ b/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift @@ -249,7 +249,7 @@ struct SQLFeatureLexer { } private func prefixedLiteralEnd(at index: Int, unit: UInt16) -> Int? { - guard index == 0 || !isWordUnit(units[index - 1]) else { return nil } + guard index == 0 || !continuesNameBeforePrefix(units[index - 1]) else { return nil } if features.contains(.escapeStringPrefix), unit == 0x45 || unit == 0x65, unitAt(index + 1) == Self.singleQuote { return quotedEnd(from: index + 2, quote: Self.singleQuote, backslashEscapes: true) } @@ -371,8 +371,25 @@ struct SQLFeatureLexer { } } + // A prefix glued to a name is part of it: `$`, `#` where it continues a name, and every non-ASCII unit but the + // spaces DuckDB blanks before it lexes. DuckDB is the engine whose plugin reads `E'...'` through the kit, and this + // is its reading outside the quotes its pre-lexing pass sees; the app's gate also reads the other way. + private func continuesNameBeforePrefix(_ unit: UInt16) -> Bool { + if unit >= 0x80 { + return !Self.duckDBBlanks.contains(unit) + } + return isASCIIIdentifierPart(unit) || unit == Self.dollar + || (unit == Self.hash && features.contains(.dollarAndHashInIdentifiers)) + } + + private static let duckDBBlanks: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) + .union(0x2000...0x200B) + private func continuesIdentifierBeforeDollar(_ unit: UInt16) -> Bool { - isASCIIIdentifierPart(unit) || unit == Self.dollar || unit >= 0x80 + if unit >= 0x80, features.contains(.escapeStringPrefix) { + return !Self.duckDBBlanks.contains(unit) + } + return isASCIIIdentifierPart(unit) || unit == Self.dollar || unit >= 0x80 } private func isTagStart(_ unit: UInt16) -> Bool { diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index ef3166e4c1..e8b6bbd7a0 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -1,9 +1,12 @@ import Foundation +import TableProPluginKit import TableProTeradataCore final class TeradataAsyncConnection: @unchecked Sendable { private let connection: TeradataConnection private let queue = DispatchQueue(label: "com.TablePro.teradata.connection") + private let runningLock = NSLock() + private var running: TeradataAsyncCancellation? init(config: TeradataConnectionConfig) { connection = TeradataConnection(config: config) @@ -16,6 +19,12 @@ final class TeradataAsyncConnection: @unchecked Sendable { // Read outside the queue, so asking never waits behind a running statement. var hasLostConnection: Bool { connection.hasLostConnection } + var transactionState: TeradataTransactionState { connection.transactionState } + + var sessionTransactionMode: TeradataTransactionMode { connection.sessionTransactionMode } + + var sessionModeSource: TeradataSessionModeSource { connection.sessionModeSource } + func connect() async throws { try await run { try $0.connect() } } @@ -28,6 +37,35 @@ final class TeradataAsyncConnection: @unchecked Sendable { try await run { try $0.execute(sql) } } + func execute(_ sql: String, rowCap: Int?) async throws -> TeradataCappedResult { + try await run { try $0.execute(sql, rowCap: rowCap) } + } + + // One queue item for the whole answer: the session holds one request, so no other call may run between its replies. + func stream( + _ sql: String, + onColumns: @escaping @Sendable ([TeradataColumn]) throws -> Void, + onRows: @escaping @Sendable ([[TeradataValue]]) throws -> Bool + ) async throws { + try await run { try $0.stream(sql, onColumns: onColumns, onRows: onRows) } + } + + func beginTransaction() async throws { + try await run { try $0.beginTransaction() } + } + + func commitTransaction() async throws { + try await run { try $0.commitTransaction() } + } + + func rollbackTransaction() async throws { + try await run { try $0.rollbackTransaction() } + } + + func switchDatabase(to database: String) async throws { + try await run { try $0.switchDatabase(to: database) } + } + func disconnect() { queue.sync { connection.disconnect() } } @@ -41,11 +79,21 @@ final class TeradataAsyncConnection: @unchecked Sendable { return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in queue.async { + // Begun before the call is marked running, so a Stop either side of the mark reaches it. + self.connection.beginOperation() + self.runningLock.withLock { self.running = cancellation } + defer { + self.runningLock.withLock { self.running = nil } + self.connection.endOperation() + } do { try cancellation.throwIfCancelled() let value = try body(self.connection) try cancellation.throwIfCancelled() continuation.resume(returning: value) + } catch TeradataWireError.rolledBackByStop { + // Kept past the cancel, so the host can say what the Stop undid. + continuation.resume(throwing: PluginStopRolledBackTransaction(code: TeradataConnection.userAbortCode)) } catch { continuation.resume(throwing: cancellation.isCancelled ? CancellationError() : error) } @@ -53,9 +101,12 @@ final class TeradataAsyncConnection: @unchecked Sendable { } } onCancel: { cancellation.cancel() - // TeradataConnection snapshots its transport under a lock; the transport's cancel - // path uses shutdown rather than disconnecting concurrently with protocol cleanup. - connection.cancel() + // A call still queued never starts; stopping the connection would stop the request of the one running. + // Held across the stop, so the call cannot end and hand the session to the next one meanwhile. + runningLock.withLock { + guard running === cancellation else { return } + connection.cancel() + } } } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift index 6790d03b9d..f3c5c8b7ac 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift @@ -15,15 +15,19 @@ extension TeradataValue { } } +extension TeradataColumn { + var pluginTypeName: String { TeradataColumnType.wireTypeName(typeCode) } +} + extension TeradataResultSet { - func toPluginResult(executionTime: TimeInterval) -> PluginQueryResult { + func toPluginResult(executionTime: TimeInterval, isTruncated: Bool = false) -> PluginQueryResult { PluginQueryResult( columns: columns.map { $0.name }, - columnTypeNames: columns.map { TeradataColumnType.wireTypeName($0.typeCode) }, + columnTypeNames: columns.map { $0.pluginTypeName }, rows: rows.map { row in row.map { $0.asPluginCell } }, rowsAffected: activityCount, executionTime: executionTime, - isTruncated: false + isTruncated: isTruncated ) } } @@ -136,7 +140,10 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { true } var capabilities: PluginCapabilities { - [.cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites] + [ + .cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, + ] } init(config: DriverConnectionConfig) { @@ -190,6 +197,7 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { connection.disconnect() throw error } + logSessionMode(connection) self.connection = connection } @@ -203,34 +211,118 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func execute(query: String) async throws -> PluginQueryResult { - guard let connection else { throw TeradataWireError.connectionFailed("not connected") } - let start = Date() - let result = try await connection.execute(query) - return result.toPluginResult(executionTime: Date().timeIntervalSince(start)) + try await execute(query, rowCap: nil) } func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { - guard let parameters else { return try await execute(query: query) } - return try await executeParameterized(query: query, parameters: parameters) + try await execute(bound(query, parameters ?? []), rowCap: rowCap) + } + + // Not the stream: a Stop cancels the reading task, which ends a stream before a rollback error can cross it. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await execute(query, rowCap: rowCap) } - // Teradata reads quotes, names and comments the ANSI way, which is no lexical feature. func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { - guard !parameters.isEmpty else { return try await execute(query: query) } - let bound = try PluginParameterBinder.bind( + try await execute(bound(query, parameters), rowCap: nil) + } + + func streamRows(query: String) -> AsyncThrowingStream { + let connection = self.connection + return PluginRowStream.makeBounded { continuation, abort in + let task = Task { + guard let connection else { + continuation.finish(throwing: TeradataWireError.connectionFailed("not connected")) + return + } + do { + // Each reply's rows go out as it arrives, and the next one is asked for once the reader has room. + try await connection.stream( + query, + onColumns: { columns in + continuation.yield(.header(PluginStreamHeader( + columns: columns.map { $0.name }, + columnTypeNames: columns.map { $0.pluginTypeName } + ))) + }, + onRows: { rows in + guard !abort.isAborted else { return false } + let delivered = PluginRowStream.deliverBlocking( + .rows(rows.map { row in row.map { $0.asPluginCell } }), + to: continuation + ) + return delivered && !abort.isAborted + } + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { task.cancel() } + } + } + + // Teradata reads quotes, names and comments the ANSI way, which is no lexical feature. + private func bound(_ query: String, _ parameters: [PluginCellValue]) throws -> String { + guard !parameters.isEmpty else { return query } + return try PluginParameterBinder.bind( query, literals: parameters.map(TeradataRowEditSQL.literal), style: .questionMark, features: [] ) - return try await execute(query: bound) + } + + private func execute(_ query: String, rowCap: Int?) async throws -> PluginQueryResult { + guard let connection else { throw TeradataWireError.connectionFailed("not connected") } + let start = Date() + let capped = try await connection.execute(query, rowCap: rowCap) + return capped.resultSet.toPluginResult( + executionTime: Date().timeIntervalSince(start), isTruncated: capped.isTruncated) } func cancelQuery() throws { connection?.cancel() } - func beginTransaction() async throws { _ = try await execute(query: "BEGIN TRANSACTION") } - func commitTransaction() async throws { _ = try await execute(query: "END TRANSACTION") } - func rollbackTransaction() async throws { _ = try await execute(query: "ROLLBACK") } + func beginTransaction() async throws { + try await liveConnection().beginTransaction() + } + + func commitTransaction() async throws { + try await liveConnection().commitTransaction() + } + + func rollbackTransaction() async throws { + try await liveConnection().rollbackTransaction() + } + + func sessionTransactionState() async -> PluginSessionTransactionState { + guard let connection, !connection.hasLostConnection else { return .unknown } + switch connection.transactionState { + case .idle: return .idle + case .open: return .inTransaction + case .unknown: return .unknown + } + } + + func liveConnection() throws -> TeradataAsyncConnection { + guard let connection else { throw TeradataWireError.connectionFailed("not connected") } + return connection + } + + private func logSessionMode(_ connection: TeradataAsyncConnection) { + let mode = connection.sessionTransactionMode.rawValue + switch connection.sessionModeSource { + case .configured: + Self.logger.debug("Transaction mode \(mode, privacy: .public) as configured") + case .helpSession: + Self.logger.info("Transaction mode \(mode, privacy: .public) read from HELP SESSION") + case .sessionInfoView: + Self.logger.info("Transaction mode \(mode, privacy: .public) read from DBC.SessionInfoV") + case .unresolved: + Self.logger.warning("Neither HELP SESSION nor DBC.SessionInfoV gave the transaction mode; using Teradata semantics") + } + } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+DDL.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+DDL.swift index 8adab53cd0..2c577c9976 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+DDL.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+DDL.swift @@ -105,6 +105,11 @@ extension TeradataPluginDriver { ) } + // Teradata has no TRUNCATE; DELETE ... ALL is its documented way to empty a table, and it has no CASCADE. + func truncateTableStatements(table: String, schema: String?, cascade: Bool) -> [String]? { + ["DELETE FROM \(TeradataSchemaQueries.qualifiedName(database: effectiveDatabase(schema), table: table)) ALL"] + } + func createViewTemplate() -> String? { "CREATE VIEW view_name AS\nSELECT column1, column2\nFROM table_name\nWHERE condition;" } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift index e9c842a0f7..a60e747208 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift @@ -36,7 +36,8 @@ extension TeradataPluginDriver { target: target, columns: columns, values: values, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) statements.append((insert, [])) case .update: @@ -45,7 +46,8 @@ extension TeradataPluginDriver { TeradataRowEditSQL.assignment( column: $0.columnName, value: $0.newValue, - isExpression: expressionColumns.contains($0.columnIndex) + isExpression: expressionColumns.contains($0.columnIndex), + isDefault: change.isColumnDefault($0.columnIndex) ) } guard !assignments.isEmpty, diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift index f6c68eb7a3..0a3483f3fb 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift @@ -131,7 +131,7 @@ extension TeradataPluginDriver { } func switchDatabase(to database: String) async throws { - _ = try await execute(query: TeradataSchemaQueries.setDatabase(database)) + try await liveConnection().switchDatabase(to: database) currentDatabaseName = database } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift b/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift index 7c1b7420ad..1ff5996b2d 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift @@ -11,9 +11,6 @@ import TableProTeradataCore /// Teradata literals and the INSERT a new grid row becomes. The driver sends statements as text, so every value is /// written as a literal here rather than bound. enum TeradataRowEditSQL { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func literal(_ value: PluginCellValue) -> String { switch value { case .null: @@ -29,9 +26,10 @@ enum TeradataRowEditSQL { target: String, columns: [String], values: [PluginCellValue], - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) -> String { - let written = zip(columns, values).enumerated().filter { $0.element.1 != defaultMarker } + let written = zip(columns, values).enumerated().filter { !defaultColumns.contains($0.offset) } guard !written.isEmpty else { return "INSERT INTO \(target) DEFAULT VALUES" } let columnList = written.map { TeradataSchemaQueries.quoteIdentifier($0.element.0) }.joined(separator: ", ") let valueList = written.map { cell in @@ -40,9 +38,9 @@ enum TeradataRowEditSQL { return "INSERT INTO \(target) (\(columnList)) VALUES (\(valueList))" } - static func assignment(column: String, value: PluginCellValue, isExpression: Bool) -> String { + static func assignment(column: String, value: PluginCellValue, isExpression: Bool, isDefault: Bool = false) -> String { let name = TeradataSchemaQueries.quoteIdentifier(column) - guard value != defaultMarker else { return "\(name) = DEFAULT" } + guard !isDefault else { return "\(name) = DEFAULT" } return "\(name) = \(Self.value(value, isExpression: isExpression))" } diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift index 1cdc981634..6a12640a9d 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift @@ -10,6 +10,7 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private let lock = NSLock() private var _client: TrinoStatementClient? private var _serverVersion: String? + private var _heartbeat = TrinoHeartbeat.repeatLastPage private var _columnTypes: [String: [String: String]] = [:] static let logger = Logger(subsystem: "com.TablePro", category: "TrinoPluginDriver") @@ -27,7 +28,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .materializedViews, .dataCompare, .rowWrites, .cellExpressions, .uuidCellText] + [ + .multiSchema, .cancelQuery, .materializedViews, .dataCompare, .rowWrites, .cellExpressions, .uuidCellText, + .columnDefaultMarks, + ] } func cacheColumnTypes(_ types: [String: String], key: String) { @@ -84,7 +88,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { : try? await client.execute(flavor.versionStatement) let product = flavor.productName if case .text(let version)? = versionResult?.rows.first?.first { - lock.withLock { _serverVersion = "\(product) \(version)" } + lock.withLock { + _serverVersion = "\(product) \(version)" + _heartbeat = TrinoHeartbeat(flavor: flavor, version: version) + } } else { lock.withLock { _serverVersion = product } } @@ -129,8 +136,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - try await Self.$hostSplit.withValue(context.hostSplit) { - try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + try await PluginParameterBinder.withHostContext(context) { + try await Self.$hostSplit.withValue(context.hostSplit) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } } } @@ -139,8 +148,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await Self.$hostSplit.withValue(context.hostSplit) { - try await executeParameterized(query: query, parameters: parameters) + try await PluginParameterBinder.withHostContext(context) { + try await Self.$hostSplit.withValue(context.hostSplit) { + try await executeParameterized(query: query, parameters: parameters) + } } } @@ -157,7 +168,7 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } private func stream(_ statement: String) -> AsyncThrowingStream { - PluginRowStream.make { continuation, abort in + PluginRowStream.makeBounded { continuation, abort in let driver = self let streamTask = Task { guard let client = driver.client else { @@ -166,20 +177,32 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } do { var headerSent = false - for try await element in client.executeStreamed(statement) { - switch element { - case .columns(let columns): - continuation.yield(.header(PluginStreamHeader( + try await client.executeStreamed( + statement, + heartbeat: driver.heartbeat, + onColumns: { columns in + let header = PluginStreamHeader( columns: columns.map(\.name), columnTypeNames: columns.map(\.typeName) - ))) + ) + guard await PluginRowStream.deliver(.header(header), to: continuation) else { + throw CancellationError() + } headerSent = true - case .rows(let page): - continuation.yield(.rows(page.map { row in row.map(Self.cellValue) })) + }, + onPage: { page, keepAlive in + let rows = PluginStreamElement.rows(page.map { row in row.map(Self.cellValue) }) + let delivered = try await PluginRowStream.deliver( + rows, + to: continuation, + keepingAliveEvery: Self.heartbeatInterval, + keepAlive + ) + guard delivered else { throw CancellationError() } } - } + ) if !headerSent { - continuation.yield(.header(PluginStreamHeader(columns: [], columnTypeNames: []))) + _ = await PluginRowStream.deliver(.header(PluginStreamHeader(columns: [], columnTypeNames: [])), to: continuation) } continuation.finish() } catch { @@ -190,6 +213,13 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + private var heartbeat: TrinoHeartbeat { + lock.withLock { _heartbeat } + } + + // The official client's interval: a coordinator abandons a query after query.client.timeout, 5 minutes by default. + private static let heartbeatInterval: TimeInterval = 30 + private func pluginResult(_ result: TrinoResultSet, executionTime: TimeInterval) -> PluginQueryResult { guard !result.columns.isEmpty else { let message = Self.statusMessage(for: result) diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift b/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift index 0a4da883cf..e2cf65dcc6 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift @@ -9,10 +9,7 @@ import TableProTrinoCore /// The column values a grid change writes, in the form `TrinoRowEditSQL` renders. enum TrinoRowEditValues { - /// What the grid stages for a column the user leaves to the server's default. Trino has no `DEFAULT` keyword in - /// `VALUES`, so the column stays out of the insert and the connector fills it. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - + // Trino has no `DEFAULT` keyword in `VALUES`, so a defaulted column stays out of the insert and the connector fills it. static func insertValues( _ change: PluginRowChange, columns: [String], @@ -21,11 +18,11 @@ enum TrinoRowEditValues { ) -> [TrinoColumnValue] { if let rowData = insertedRowData[change.rowIndex] { return columns.enumerated().compactMap { index, column in - guard index < rowData.count, rowData[index] != defaultMarker else { return nil } + guard index < rowData.count, !change.isColumnDefault(index) else { return nil } return TrinoColumnValue(name: column, value: trinoValue(rowData[index]), typeName: typeName(column)) } } - return change.cellChanges.filter { $0.newValue != defaultMarker }.map { + return change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) }.map { TrinoColumnValue(name: $0.columnName, value: trinoValue($0.newValue), typeName: typeName($0.columnName)) } } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift b/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift index c55ad2c08b..017488be85 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesenseConnection.swift @@ -50,6 +50,9 @@ internal struct TypesenseResponse { internal final class TypesenseConnection: NSObject, @unchecked Sendable { private let lock = NSLock() private var _session: URLSession? + // An export holds one response for as long as its consumer takes, and a paused read still counts toward the + // resource limit, so it runs on a session that keeps the system default. + private var _streamSession: URLSession? private var _currentTask: URLSessionDataTask? private var _serverVersion: String? private let queryTimeout = HttpQueryTimeoutBox() @@ -98,7 +101,13 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { sessionConfig.timeoutIntervalForRequest = connectTimeout sessionConfig.timeoutIntervalForResource = HttpQueryTimeout.sessionResourceTimeout let session = URLSession(configuration: sessionConfig, delegate: self, delegateQueue: nil) - lock.withLock { _session = session } + // The session made its own copy of the configuration above. + sessionConfig.timeoutIntervalForResource = URLSessionConfiguration.default.timeoutIntervalForResource + let streamSession = URLSession(configuration: sessionConfig, delegate: self, delegateQueue: nil) + lock.withLock { + _session = session + _streamSession = streamSession + } let response = try await request( method: "GET", @@ -118,6 +127,8 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { _currentTask = nil _session?.finishTasksAndInvalidate() _session = nil + _streamSession?.finishTasksAndInvalidate() + _streamSession = nil } } @@ -180,10 +191,10 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { /// collection never holds it in memory. func streamExport( collection: String, - onDocument: @escaping ([String: Any]) -> Bool + onDocument: ([String: Any]) async -> Bool ) async throws { let session: URLSession = try lock.withLock { - guard let session = _session else { throw TypesenseError.notConnected } + guard let session = _streamSession else { throw TypesenseError.notConnected } return session } let path = TypesenseOperations.exportPath(collection: collection) @@ -224,7 +235,7 @@ internal final class TypesenseConnection: NSObject, @unchecked Sendable { let data = line.data(using: .utf8), let document = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } - if !onDocument(document) { return } + if await !onDocument(document) { return } } } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift index 0e86a82cdf..b9f4c274a5 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver+Execution.swift @@ -109,11 +109,10 @@ extension TypesensePluginDriver { // MARK: - Streaming Export - /// Export reads the collection's JSONL endpoint a line at a time, so a collection larger than - /// memory still exports and the 250-hit search ceiling never applies. Anything that is not an - /// export tag runs as an ordinary query and is handed over in one batch. + // Export reads the collection's JSONL endpoint a line at a time, so a collection larger than memory still exports + // and the 250-hit search ceiling never applies. Anything else runs as an ordinary query, handed over in one batch. func streamRows(query: String) -> AsyncThrowingStream { - AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.performStreamRows(query: query, continuation: continuation) @@ -121,7 +120,7 @@ extension TypesensePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -154,12 +153,12 @@ extension TypesensePluginDriver { try await connection.streamExport(collection: collection) { document in batch.append(document) guard batch.count >= Self.exportBatchSize else { return true } - continuation.yield(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole))) + _ = await PluginRowStream.deliver(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole)), to: continuation) batch.removeAll(keepingCapacity: true) return true } if !batch.isEmpty { - continuation.yield(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole))) + _ = await PluginRowStream.deliver(.rows(TypesenseSchema.rows(for: batch, columns: columns, length: .whole)), to: continuation) } continuation.finish() } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift index e00f521323..4ed6c0f139 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift @@ -28,7 +28,7 @@ internal final class TypesensePluginDriver: PluginDatabaseDriver, @unchecked Sen var supportsTransactions: Bool { false } /// `.userManagement` is what enables Database > Users & Roles, which lists Typesense API keys. - var capabilities: PluginCapabilities { [.cancelQuery, .userManagement] } + var capabilities: PluginCapabilities { [.cancelQuery, .userManagement, .columnDefaultMarks] } init(config: DriverConnectionConfig) { self.config = config @@ -237,6 +237,7 @@ internal final class TypesensePluginDriver: PluginDatabaseDriver, @unchecked Sen deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let fields = lock.withLock { _schemaCache[table] }?.fieldsByName ?? [:] let generator = TypesenseStatementGenerator(collection: table, columns: columns, fields: fields) return try generator.generateRowWrites( diff --git a/apple/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift b/apple/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift index 1b1a3d2d41..b0779b42ec 100644 --- a/apple/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift +++ b/apple/Plugins/WeaviateDriverPlugin/WeaviatePlugin.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProWeaviateClient import TableProWeaviateCore final class WeaviatePlugin: NSObject, TableProPlugin, DriverPlugin { diff --git a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift index 0142ec23db..47d0a6572c 100644 --- a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift +++ b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Execution.swift @@ -1,5 +1,6 @@ import Foundation import TableProPluginKit +import TableProWeaviateClient import TableProWeaviateCore extension WeaviatePluginDriver { @@ -41,14 +42,13 @@ extension WeaviatePluginDriver { static let exportPageSize = 500 - /// Exports one collection by walking it a page at a time, yielding each page instead of - /// holding the whole collection. The protocol default runs `execute` once and buffers - /// everything, which is fine for a grid page and not for an export. + // An export walks the collection a page at a time instead of holding it whole. The protocol default runs + // `execute` once and buffers everything, fine for a grid page and not for an export. func streamRows(query: String) -> AsyncThrowingStream { guard let collectionName = WeaviateOperations.decodeExport(query) else { return defaultStreamRows(query: query) } - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return PluginRowStream.makeBounded { continuation, abort in let task = Task { do { try await self.streamCollection(collectionName, continuation: continuation) @@ -57,7 +57,7 @@ extension WeaviatePluginDriver { continuation.finish(throwing: error) } } - continuation.onTermination = { @Sendable _ in task.cancel() } + abort.onAbort { task.cancel() } } } @@ -75,11 +75,12 @@ extension WeaviatePluginDriver { ))) for try await objects in client.objectPages(collection: name, pageSize: Self.exportPageSize) { - continuation.yield(.rows(objects.map { object in + let rows: [PluginRow] = objects.map { object in WeaviateObjectCodec.row(for: object, columns: columns).map { value in value.map(PluginCellValue.text) ?? .null } - })) + } + _ = await PluginRowStream.deliver(.rows(rows), to: continuation) } } diff --git a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift index ba8a40fbde..5af4278f1b 100644 --- a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift +++ b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver+Metadata.swift @@ -1,6 +1,7 @@ import Foundation import os import TableProPluginKit +import TableProWeaviateClient import TableProWeaviateCore extension WeaviatePluginDriver { diff --git a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift index f777b70f49..50a08da72e 100644 --- a/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift +++ b/apple/Plugins/WeaviateDriverPlugin/WeaviatePluginDriver.swift @@ -1,6 +1,7 @@ import Foundation import os import TableProPluginKit +import TableProWeaviateClient import TableProWeaviateCore internal final class WeaviatePluginDriver: PluginDatabaseDriver, @unchecked Sendable { diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index 31f57cd3ac..6ed1863eac 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -43,6 +43,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug destination: URL, progress: PluginExportProgress ) async throws -> ExportFormatResult { + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings let writer = XLSXWriter(databaseTypeId: dataSource.databaseTypeId) var didSplitSheets = false @@ -53,10 +55,31 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug var isFirstBatch = true var rowBatch: [[PluginCellValue]] = [] - var currentSheetRowCount = 0 var columns: [String] = [] var columnTypeNames: [String] = [] - let headerRowCount = settings.includeHeaderRow ? 1 : 0 + var sheetRows = XLSXSheetRows(limit: Self.maxRowsPerSheet, headerRowCount: options.includeHeaderRow ? 1 : 0) + + func writeBatch() throws { + for chunk in sheetRows.chunks(of: rowBatch.count) { + if chunk.startsNewSheet { + try writer.continueSheet( + baseName: table.name, + columns: columns, + columnTypeNames: columnTypeNames, + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty + ) + didSplitSheets = true + } + try autoreleasepool { + try writer.addRows(Array(rowBatch[chunk.rows]), convertNullToEmpty: options.convertNullToEmpty) + } + } + for _ in 0..= 5_000 { - let remaining = Self.maxRowsPerSheet - currentSheetRowCount - if rowBatch.count <= remaining { - autoreleasepool { - writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) - } - currentSheetRowCount += rowBatch.count - } else { - let fitting = Array(rowBatch.prefix(remaining)) - let overflow = Array(rowBatch.dropFirst(remaining)) - if !fitting.isEmpty { - autoreleasepool { - writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) - } - currentSheetRowCount += fitting.count - } - writer.continueSheet( - baseName: table.name, - columns: columns, - columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty - ) - didSplitSheets = true - currentSheetRowCount = headerRowCount - if !overflow.isEmpty { - autoreleasepool { - writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) - } - currentSheetRowCount += overflow.count - } - } - let batchCount = rowBatch.count - rowBatch.removeAll(keepingCapacity: true) - for _ in 0.. + let startsNewSheet: Bool + } + + let limit: Int + let headerRowCount: Int + private(set) var rowsInSheet: Int + + init(limit: Int, headerRowCount: Int) { + self.limit = limit + self.headerRowCount = headerRowCount + self.rowsInSheet = headerRowCount + } + + // A new sheet starts only when a row needs it, so a batch that fills a sheet exactly leaves no empty one. + mutating func chunks(of count: Int) -> [Chunk] { + var chunks: [Chunk] = [] + var start = 0 + while start < count { + var startsNewSheet = false + if rowsInSheet >= limit { + rowsInSheet = headerRowCount + startsNewSheet = true + } + let end = min(count, start + max(1, limit - rowsInSheet)) + chunks.append(Chunk(rows: start...stride + var name: [Int32] = [CTL_KERN, KERN_PROC, KERN_PROC_PID, getpid()] + guard sysctl(&name, u_int(name.count), &info, &size, nil, 0) == 0 else { return Date() } + let start = info.kp_proc.p_un.__p_starttime + return Date(timeIntervalSince1970: TimeInterval(start.tv_sec) + TimeInterval(start.tv_usec) / 1_000_000) + }() + + private var sheets: [XLSXSheetPart] = [] + private var sheetBytesInArchive = 0 + private var partsDirectory: URL? + private var partsLock: Int32? + private let flushThreshold: Int + private let maximumArchiveSize: Int private static let maximumSheetNameLength = 31 private var takenSheetNames: Set = [] - /// Pre-cached column letter lookups private var columnLetterCache: [String] = [] - /// Tracks the current row number for the active sheet being built private var currentRowNumber: Int = 0 - /// Whether the current sheet has a header row (used for bold styling) - private var currentSheetHasHeader: Bool = false - enum CellValue: Equatable { case string(String) case number(String) case empty } - enum ColumnNumbers: Equatable { - case text - case exact - case approximate - case untyped - - init(typeName: String, bitsAreStrings: Bool = false) { - let base = Self.baseName(typeName) - if base.isEmpty { - self = .untyped - } else if Self.bitTypes.contains(base) { - self = bitsAreStrings ? .text : .exact - } else if Self.approximateTypes.contains(base) { - self = .approximate - } else if Self.exactTypes.contains(base) { - self = .exact - } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { - self = .text - } else { - self = .untyped - } - } - - // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the - // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. - private static func baseName(_ typeName: String) -> String { - var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - while name.hasSuffix(")"), let open = name.firstIndex(of: "("), - ["nullable", "lowcardinality"].contains(String(name[.. = [ - "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", - "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", - "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" - ] - - private static let approximateTypes: Set = [ - "float", "double", "real", "binary_float", "binary_double" - ] - - private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] - - private static let bitTypes: Set = ["bit", "varbit"] - - // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. - static func bitsAreStrings(databaseTypeId: String) -> Bool { - enginesWithBitStrings.contains(databaseTypeId) - } - - private static let enginesWithBitStrings: Set = [ - "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" - ] - - private static let textMarkers = ["char", "text", "string", "clob"] - } + typealias ColumnNumbers = PluginColumnNumbers // Excel keeps 15 significant digits of a number. private static let maximumExactDigits = 15 @@ -113,24 +55,49 @@ final class XLSXWriter { private let bitsAreStrings: Bool - init(databaseTypeId: String = "") { + // A sheet holds at most `flushThreshold` bytes plus one row in memory before they go to its file. + // `maximumArchiveSize` is the ZIP's 4 GB offset limit, checked as the sheets grow. + init( + databaseTypeId: String = "", + partsDirectory: URL? = nil, + flushThreshold: Int = XLSXWriter.defaultFlushThreshold, + maximumArchiveSize: Int = Int(UInt32.max) + ) { bitsAreStrings = ColumnNumbers.bitsAreStrings(databaseTypeId: databaseTypeId) + self.partsDirectory = partsDirectory + self.flushThreshold = max(0, flushThreshold) + self.maximumArchiveSize = min(maximumArchiveSize, Int(UInt32.max)) + } + + deinit { + sheets.forEach { $0.closeWithoutWriting() } + defer { + if let partsLock { close(partsLock) } + } + guard let partsDirectory else { return } + do { + try FileManager.default.removeItem(at: partsDirectory) + } catch { + guard (error as? CocoaError)?.code != .fileNoSuchFile else { return } + Self.logger.warning("Could not remove the workbook's sheet files: \(error.localizedDescription, privacy: .private)") + } } // MARK: - Sheet Building API - /// Begin a new worksheet. Must be followed by `addRows` calls and then `finishSheet`. func beginSheet( name: String, columns: [String], columnTypeNames: [String] = [], includeHeader: Bool, convertNullToEmpty: Bool - ) { + ) throws { + if let previous = sheets.last, previous.isOpen { + try closeSheet(previous) + } let sheetName = unusedSheetName(for: name) currentColumnNumbers = columnTypeNames.map { ColumnNumbers(typeName: $0, bitsAreStrings: bitsAreStrings) } currentRowNumber = 0 - currentSheetHasHeader = includeHeader let maxCols = max(columns.count, columnLetterCache.count) if maxCols > columnLetterCache.count { @@ -139,25 +106,22 @@ final class XLSXWriter { } } - var d = Data() - d.appendUTF8("\n") - d.appendUTF8("") + let part = try XLSXSheetPart(name: sheetName, fileURL: try partURL(forSheet: sheets.count + 1)) + sheets.append(part) + sheetBytesInArchive += ZipBuilder.localHeaderSize(path: Self.sheetPath(sheets.count)) + part.pending.appendUTF8("\n") + part.pending.appendUTF8("") if includeHeader { let headerCells: [CellValue] = columns.map { .string($0) } - appendRow(headerCells, isHeader: true, to: &d) + appendRow(headerCells, isHeader: true, to: &part.pending) } - - sheets.append((name: sheetName, data: d)) + try flush(part) } - /// Add a batch of raw rows to the current (last) sheet. - /// Converts `[PluginCellValue]` to `CellValue` and writes XML immediately, - /// so the caller can release the raw row data after this call returns. - func addRows(_ rows: [[PluginCellValue]], convertNullToEmpty: Bool) { - guard !sheets.isEmpty else { return } - - var sheetData = sheets[sheets.count - 1].data + // Writes each row's XML as it converts it, so the caller can release the rows once this returns. + func addRows(_ rows: [[PluginCellValue]], convertNullToEmpty: Bool) throws { + guard let part = sheets.last, part.isOpen else { return } for row in rows { autoreleasepool { @@ -166,7 +130,7 @@ final class XLSXWriter { case .null: return convertNullToEmpty ? .empty : .string("NULL") case .bytes(let data): - if data.isEmpty { return .empty } + // Zero bytes are `0x`, as in CSV: an empty cell reads back as NULL. let hex = data.map { String(format: "%02X", $0) }.joined() return .string("0x" + hex) case .text(let val): @@ -174,11 +138,10 @@ final class XLSXWriter { return Self.cell(forText: val, numbers: numbers) } } - appendRow(cellRow, isHeader: false, to: &sheetData) + appendRow(cellRow, isHeader: false, to: &part.pending) } + try flush(part) } - - sheets[sheets.count - 1].data = sheetData } // A text column writes text whatever its values look like. A numeric column writes a number @@ -203,6 +166,8 @@ final class XLSXWriter { let isPlain = literal == text && !literal.contains("e") let digitCount = literal.filter { $0.isNumber }.count return isPlain && digitCount <= maximumExactDigits ? .number(literal) : .string(text) + @unknown default: + return .string(text) } } @@ -220,23 +185,22 @@ final class XLSXWriter { return digits[...last].count } - /// Finish the current sheet by closing the XML tags. - func finishSheet() { - guard !sheets.isEmpty else { return } - sheets[sheets.count - 1].data.appendUTF8("") + func finishSheet() throws { + guard let part = sheets.last, part.isOpen else { return } + part.pending.appendUTF8("") + try closeSheet(part) } - /// Finish the current sheet and start a continuation sheet with the same columns. - /// The new sheet is named "BaseName (N)" where N increments. + // Finishes the current sheet and starts "BaseName (N)" with the same columns. func continueSheet( baseName: String, columns: [String], columnTypeNames: [String] = [], includeHeader: Bool, convertNullToEmpty: Bool - ) { - finishSheet() - beginSheet( + ) throws { + try finishSheet() + try beginSheet( name: baseName, columns: columns, columnTypeNames: columnTypeNames, @@ -245,35 +209,14 @@ final class XLSXWriter { ) } - // MARK: - Legacy Convenience API - - /// Add a complete worksheet with all rows at once (legacy compatibility). - /// For better memory usage, prefer `beginSheet` / `addRows` / `finishSheet`. - func addSheet( - name: String, - columns: [String], - columnTypeNames: [String] = [], - rows: [[PluginCellValue]], - includeHeader: Bool, - convertNullToEmpty: Bool - ) { - beginSheet( - name: name, - columns: columns, - columnTypeNames: columnTypeNames, - includeHeader: includeHeader, - convertNullToEmpty: convertNullToEmpty - ) - addRows(rows, convertNullToEmpty: convertNullToEmpty) - finishSheet() - } - - /// Write the XLSX file to the given URL func write(to url: URL) async throws { + if let part = sheets.last, part.isOpen { + try closeSheet(part) + } let entries = archiveEntries() + let maximumSize = maximumArchiveSize try await Task.detached(priority: .userInitiated) { - let zipData = try ZipBuilder.build(entries: entries) - try zipData.write(to: url, options: .atomic) + try ZipBuilder.write(entries, to: url, maximumSize: maximumSize) }.value } @@ -288,19 +231,89 @@ final class XLSXWriter { for (index, sheet) in sheets.enumerated() { entries.append(ZipFileEntry( - path: "xl/worksheets/sheet\(index + 1).xml", - data: sheet.data + path: Self.sheetPath(index + 1), + file: sheet.fileURL, + size: sheet.byteCount, + crc: sheet.crc )) } return entries } + private static func sheetPath(_ number: Int) -> String { + "xl/worksheets/sheet\(number).xml" + } + + // MARK: - Sheet Files + + private func partURL(forSheet number: Int) throws -> URL { + try openPartsDirectory().appendingPathComponent("sheet\(number).xml") + } + + // The folder stays locked while this writer lives, which tells a later export's sweep that it is in use. + private func openPartsDirectory() throws -> URL { + if let partsDirectory, partsLock != nil { return partsDirectory } + let directory: URL + if let partsDirectory { + directory = partsDirectory + } else { + let parent = FileManager.default.temporaryDirectory + Self.removeAbandonedPartsDirectories(in: parent, modifiedBefore: Self.processStart) + directory = parent.appendingPathComponent(Self.partsDirectoryPrefix + UUID().uuidString, isDirectory: true) + } + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + partsDirectory = directory + partsLock = try Self.lock(directory) + return directory + } + + // A crash or a force quit skips the deinit that removes a writer's folder. A folder older than + // this process that no writer holds, in this process or another, was left that way. + static func removeAbandonedPartsDirectories(in parent: URL, modifiedBefore cutoff: Date) { + let keys: Set = [.isDirectoryKey, .contentModificationDateKey] + guard let entries = try? FileManager.default.contentsOfDirectory( + at: parent, includingPropertiesForKeys: Array(keys) + ) else { return } + for entry in entries where entry.lastPathComponent.hasPrefix(partsDirectoryPrefix) { + guard let values = try? entry.resourceValues(forKeys: keys), values.isDirectory == true, + let modified = values.contentModificationDate, modified < cutoff, + let descriptor = try? lock(entry) else { continue } + try? FileManager.default.removeItem(at: entry) + close(descriptor) + } + } + + // `flock` is released when the descriptor closes, also when the process dies. + private static func lock(_ directory: URL) throws -> Int32 { + let descriptor = open(directory.path(percentEncoded: false), O_RDONLY | O_CLOEXEC) + guard descriptor >= 0 else { throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) } + guard flock(descriptor, LOCK_EX | LOCK_NB) == 0 else { + let code = POSIXErrorCode(rawValue: errno) ?? .EIO + close(descriptor) + throw POSIXError(code) + } + return descriptor + } + + private func flush(_ part: XLSXSheetPart) throws { + guard part.pending.count >= flushThreshold else { return } + try account(writtenBytes: part.writePending()) + } + + private func closeSheet(_ part: XLSXSheetPart) throws { + try account(writtenBytes: part.close()) + } + + // The sheets alone already past the limit means `write(to:)` would refuse the archive, so the + // export stops now instead of after reading every remaining row. + private func account(writtenBytes: Int) throws { + sheetBytesInArchive += writtenBytes + guard sheetBytesInArchive <= maximumArchiveSize else { throw ZipBuilder.ZipError.fileTooLarge } + } + // MARK: - Row XML Generation - /// Append a single row of cells to the given Data buffer using inline strings. - /// Inline strings use `t="inlineStr"` with `text` to avoid - /// the shared string table entirely (MEM-15 fix). private func appendRow(_ cells: [CellValue], isHeader: Bool, to data: inout Data) { currentRowNumber += 1 let rowNum = currentRowNumber @@ -316,7 +329,6 @@ final class XLSXWriter { switch cell { case .string(let value): if isHeader { - // Header cells get bold style (s="1") + inline string data.appendUTF8("") } else { data.appendUTF8("") @@ -335,7 +347,7 @@ final class XLSXWriter { data.appendUTF8("
") } - // MARK: - XML Generation (Data-based to avoid O(n^2) String concatenation) + // MARK: - XML Generation private func contentTypesXML() -> Data { var d = Data() @@ -452,7 +464,6 @@ final class XLSXWriter { // MARK: - Data XML Helpers private extension Data { - /// Append a UTF-8 string directly to Data (O(1) amortized, no intermediate String copies) mutating func appendUTF8(_ string: String) { string.utf8.withContiguousStorageIfAvailable { buffer in if let baseAddress = buffer.baseAddress { @@ -461,9 +472,6 @@ private extension Data { } ?? self.append(contentsOf: string.utf8) } - /// Append XML-escaped text directly to Data without creating intermediate Strings. - /// Strips XML 1.0 illegal control characters (0x00–0x08, 0x0B, 0x0C, 0x0E–0x1F) - /// that can appear in binary/hex database columns and would produce malformed XML. mutating func appendXMLEscaped(_ text: String) { for byte in text.utf8 { switch byte { @@ -490,105 +498,211 @@ private extension Data { } } +// MARK: - Sheet Part + +private final class XLSXSheetPart { + let name: String + let fileURL: URL + var pending = Data() + private var handle: FileHandle? + private(set) var byteCount = 0 + private(set) var crc: UInt32 = 0 + + var isOpen: Bool { handle != nil } + + init(name: String, fileURL: URL) throws { + self.name = name + self.fileURL = fileURL + guard FileManager.default.createFile(atPath: fileURL.path(percentEncoded: false), contents: nil) else { + throw CocoaError(.fileWriteUnknown, userInfo: [NSURLErrorKey: fileURL]) + } + handle = try FileHandle(forWritingTo: fileURL) + } + + func writePending() throws -> Int { + guard let handle, !pending.isEmpty else { return 0 } + try handle.write(contentsOf: pending) + crc = ZipBuilder.crc32(of: pending, continuing: crc) + let written = pending.count + byteCount += written + pending.removeAll(keepingCapacity: true) + return written + } + + func close() throws -> Int { + guard let handle else { return 0 } + let written = try writePending() + self.handle = nil + try handle.close() + return written + } + + func closeWithoutWriting() { + try? handle?.close() + handle = nil + } +} + // MARK: - ZIP File Builder -/// Minimal ZIP file builder (store-only, no compression) +// A stored (uncompressed) ZIP, written entry by entry. A sheet's bytes are copied from its file. private struct ZipFileEntry { + enum Contents { + case data(Data) + case file(URL) + } + let path: String - let data: Data + let contents: Contents + let size: Int + let crc: UInt32 + + init(path: String, data: Data) { + self.path = path + contents = .data(data) + size = data.count + crc = ZipBuilder.crc32(of: data) + } + + init(path: String, file: URL, size: Int, crc: UInt32) { + self.path = path + contents = .file(file) + self.size = size + self.crc = crc + } } private enum ZipBuilder { enum ZipError: LocalizedError { case fileTooLarge + case tooManySheets var errorDescription: String? { switch self { case .fileTooLarge: - return "XLSX file exceeds 4 GB ZIP limit" + return String(localized: "XLSX file exceeds 4 GB ZIP limit") + case .tooManySheets: + return String(localized: "The workbook has more sheets than an XLSX file can hold.") } } } - static func build(entries: [ZipFileEntry]) throws -> Data { - var totalSize = 22 - for entry in entries { - let pathLen = entry.path.utf8.count - totalSize += 30 + pathLen + entry.data.count - totalSize += 46 + pathLen - } + private static let copyChunkSize = 1 << 20 - var output = Data(capacity: totalSize) - var centralDirectory = Data() - var offsets: [Int] = [] + static func localHeaderSize(path: String) -> Int { + 30 + path.utf8.count + } + static func write(_ entries: [ZipFileEntry], to destination: URL, maximumSize: Int) throws { + var offsets: [Int] = [] + offsets.reserveCapacity(entries.count) + var offset = 0 for entry in entries { - let currentOffset = output.count - guard currentOffset <= UInt32.max, entry.data.count <= UInt32.max else { + guard offset <= maximumSize, entry.size <= maximumSize else { throw ZipError.fileTooLarge } - offsets.append(currentOffset) - - let pathData = Data(entry.path.utf8) - let crc = zlibCRC32(entry.data) - - output.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) - output.appendUInt16(10) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt32(crc) - output.appendUInt32(UInt32(entry.data.count)) - output.appendUInt32(UInt32(entry.data.count)) - output.appendUInt16(UInt16(pathData.count)) - output.appendUInt16(0) - output.append(pathData) - output.append(entry.data) - - centralDirectory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) - centralDirectory.appendUInt16(20) - centralDirectory.appendUInt16(10) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt32(crc) - centralDirectory.appendUInt32(UInt32(entry.data.count)) - centralDirectory.appendUInt32(UInt32(entry.data.count)) - centralDirectory.appendUInt16(UInt16(pathData.count)) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt16(0) - centralDirectory.appendUInt32(0) - centralDirectory.appendUInt32(UInt32(currentOffset)) - centralDirectory.append(pathData) + offsets.append(offset) + offset += localHeaderSize(path: entry.path) + entry.size } - - let centralDirOffset = output.count - guard centralDirOffset <= UInt32.max else { + guard offset <= maximumSize else { throw ZipError.fileTooLarge } - output.append(centralDirectory) + guard entries.count <= Int(UInt16.max) else { + throw ZipError.tooManySheets + } + + let (handle, temporaryURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) + var isOpen = true + do { + var centralDirectory = Data() + for (entry, entryOffset) in zip(entries, offsets) { + let pathData = Data(entry.path.utf8) + + var header = Data() + header.append(contentsOf: [0x50, 0x4B, 0x03, 0x04]) + header.appendUInt16(10) + header.appendUInt16(0) + header.appendUInt16(0) + header.appendUInt16(0) + header.appendUInt16(0) + header.appendUInt32(entry.crc) + header.appendUInt32(UInt32(entry.size)) + header.appendUInt32(UInt32(entry.size)) + header.appendUInt16(UInt16(pathData.count)) + header.appendUInt16(0) + header.append(pathData) + try handle.write(contentsOf: header) + try writeContents(of: entry, to: handle) + + centralDirectory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) + centralDirectory.appendUInt16(20) + centralDirectory.appendUInt16(10) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt32(entry.crc) + centralDirectory.appendUInt32(UInt32(entry.size)) + centralDirectory.appendUInt32(UInt32(entry.size)) + centralDirectory.appendUInt16(UInt16(pathData.count)) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt16(0) + centralDirectory.appendUInt32(0) + centralDirectory.appendUInt32(UInt32(entryOffset)) + centralDirectory.append(pathData) + } - output.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) - output.appendUInt16(0) - output.appendUInt16(0) - output.appendUInt16(UInt16(entries.count)) - output.appendUInt16(UInt16(entries.count)) - output.appendUInt32(UInt32(centralDirectory.count)) - output.appendUInt32(UInt32(centralDirOffset)) - output.appendUInt16(0) + var end = Data() + end.append(contentsOf: [0x50, 0x4B, 0x05, 0x06]) + end.appendUInt16(0) + end.appendUInt16(0) + end.appendUInt16(UInt16(entries.count)) + end.appendUInt16(UInt16(entries.count)) + end.appendUInt32(UInt32(centralDirectory.count)) + end.appendUInt32(UInt32(offset)) + end.appendUInt16(0) + + try handle.write(contentsOf: centralDirectory) + try handle.write(contentsOf: end) + isOpen = false + try handle.close() + try PluginExportUtilities.commitAtomicWrite(from: temporaryURL, to: destination) + } catch { + if isOpen { + try? handle.close() + } + PluginExportUtilities.rollbackAtomicWrite(at: temporaryURL) + throw error + } + } - return output + private static func writeContents(of entry: ZipFileEntry, to handle: FileHandle) throws { + switch entry.contents { + case .data(let data): + try handle.write(contentsOf: data) + case .file(let url): + let reader = try FileHandle(forReadingFrom: url) + defer { try? reader.close() } + var copied = 0 + while true { + let chunk = try autoreleasepool { try reader.read(upToCount: copyChunkSize) } ?? Data() + guard !chunk.isEmpty else { break } + try handle.write(contentsOf: chunk) + copied += chunk.count + } + // The header already promised this size and CRC. + guard copied == entry.size else { + throw CocoaError(.fileReadCorruptFile, userInfo: [NSURLErrorKey: url]) + } + } } - /// CRC-32 using system zlib (hardware-accelerated) - private static func zlibCRC32(_ data: Data) -> UInt32 { + static func crc32(of data: Data, continuing crc: UInt32 = 0) -> UInt32 { data.withUnsafeBytes { buffer in - guard let ptr = buffer.baseAddress else { return 0 } - return UInt32(zlib.crc32(0, ptr.assumingMemoryBound(to: UInt8.self), uInt(buffer.count))) + guard let ptr = buffer.baseAddress, !buffer.isEmpty else { return crc } + return UInt32(zlib.crc32(uLong(crc), ptr.assumingMemoryBound(to: UInt8.self), uInt(buffer.count))) } } } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift index 6f7592ab3a..d74bafbfba 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift @@ -39,13 +39,13 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var fieldDetectionSignature: String { settings.detectionSignature } - var rowImportConfiguration: RowImportRunner.Configuration? { runConfiguration } + var rowImportConfiguration: RowImportRunner.Configuration? { Self.runConfiguration(settings) } - private var runConfiguration: RowImportRunner.Configuration { + private static func runConfiguration(_ options: XLSXImportOptions) -> RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } @@ -78,11 +78,13 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress: PluginImportProgress ) async throws -> PluginImportResult { let started = Date() + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings let sheet: XLSXImportSheet do { sheet = try XLSXImportSheet.read( at: source.fileURL(), - hasHeaderRow: settings.hasHeaderRow, + hasHeaderRow: options.hasHeaderRow, isCancelled: { progress.isCancelled } ) } catch is TabularCancellation { @@ -96,7 +98,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress.setEstimatedTotal(max(1, sheet.rowCount)) var cursor = 0 let outcome = try await RowImportRunner.run( - configuration: runConfiguration, + configuration: Self.runConfiguration(options), sink: sink, progress: progress ) { @@ -108,7 +110,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug if filled.count < rows.count { progress.incrementStatement(by: rows.count - filled.count) } - return filled.map { (line: $0.line, row: self.row($0.values, header: header)) } + return filled.map { (line: $0.line, row: Self.row($0.values, header: header, options: options)) } } return PluginImportResult( @@ -121,7 +123,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Private - private func row(_ values: [String?], header: [String]) -> [String: PluginCellValue] { + private static func row(_ values: [String?], header: [String], options: XLSXImportOptions) -> [String: PluginCellValue] { var mapped: [String: PluginCellValue] = [:] for (index, column) in header.enumerated() { let raw = values[safe: index] ?? nil @@ -129,10 +131,10 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug mapped[column] = .null continue } - let trimmed = settings.trimWhitespace + let trimmed = options.trimWhitespace ? raw.trimmingCharacters(in: .whitespacesAndNewlines) : raw - if settings.emptyAsNull, trimmed.isEmpty { + if options.emptyAsNull, trimmed.isEmpty { mapped[column] = .null } else { mapped[column] = .text(trimmed) diff --git a/apple/RELEASING.md b/apple/RELEASING.md index 413df53c7b..89c9441c67 100644 --- a/apple/RELEASING.md +++ b/apple/RELEASING.md @@ -11,7 +11,7 @@ The macOS app, its registry plugins, the static libraries and the manual iOS flo ## Storage check -Sparkle never downgrades, so a release never writes storage the previous stable cannot read. Before beta.1, before every stable, and on a pick that touches the `.claude/rules/data-sync-storage.md` paths, list the commits on those paths and read each for a changed storage or sync format: +Sparkle never downgrades, so a release never writes storage the previous stable cannot read, apart from the one exception in `.claude/rules/data-sync-storage.md`. Before beta.1, before every stable, and on a pick that touches the `.claude/rules/data-sync-storage.md` paths, list the commits on those paths and read each for a changed storage or sync format: ```bash awk -F'"' '/^ - "/ {print ":(glob)" $2}' .claude/rules/data-sync-storage.md \ diff --git a/apple/TablePro/AppDelegate.swift b/apple/TablePro/AppDelegate.swift index 87a96c7c4c..b15592456e 100644 --- a/apple/TablePro/AppDelegate.swift +++ b/apple/TablePro/AppDelegate.swift @@ -85,9 +85,10 @@ class AppDelegate: NSObject, NSApplicationDelegate { KeyRepeatFilter.shared.install() TableClickFocus.shared.install() RecentTabSwitcherController.installEditorKeyClaim() - let syncSettings = AppSettingsStorage.shared.loadSync() - let passwordSyncExpected = syncSettings.enabled && syncSettings.syncConnections && syncSettings.syncPasswords - AppStorageEnvironment.shared.defaults.set(passwordSyncExpected, forKey: KeychainHelper.passwordSyncEnabledKey) + AppStorageEnvironment.shared.defaults.set( + AppSettingsStorage.shared.loadSync().syncsPasswords, + forKey: KeychainHelper.passwordSyncEnabledKey + ) DatabaseManager.shared.startObservingSystemEvents() DatabaseManager.shared.tabStatePersister = SessionTabStatePersister() @@ -169,6 +170,21 @@ class AppDelegate: NSObject, NSApplicationDelegate { true } + // MARK: - Remote Notifications + + func application(_ application: NSApplication, didRegisterForRemoteNotificationsWithDeviceToken deviceToken: Data) { + SyncCoordinator.shared.pushRegistrationDidFinish(registered: true) + } + + func application(_ application: NSApplication, didFailToRegisterForRemoteNotificationsWithError error: any Error) { + Self.logger.warning("Push registration failed: \(error.localizedDescription, privacy: .private)") + SyncCoordinator.shared.pushRegistrationDidFinish(registered: false) + } + + func application(_ application: NSApplication, didReceiveRemoteNotification userInfo: [String: Any]) { + SyncCoordinator.shared.handleRemoteNotification(userInfo) + } + /// Unhiding is the one way a window comes back without any window notification firing, and the /// app reports every window it owns as invisible while it is hidden. func applicationDidUnhide(_ notification: Notification) { @@ -176,16 +192,25 @@ class AppDelegate: NSObject, NSApplicationDelegate { } func applicationShouldTerminate(_ sender: NSApplication) -> NSApplication.TerminateReply { - guard QuitApproval.approve() else { + guard let plan = QuitApproval.approve() else { AppRelauncher.shared.terminationWasCancelled() return .terminateCancel } Task { - // First, while the drivers are open: a debug stop rolls back the routine's transaction. + // Before anything the app could not take back if it stays open. + guard await QuitApproval.commitBeforeQuitting(plan) else { + NSApp.reply(toApplicationShouldTerminate: false) + AppRelauncher.shared.terminationWasCancelled() + return + } + RunningWorkRegistry.shared.cancel(plan.work) + // While the drivers are open: a debug stop rolls back the routine's transaction. await AppModuleTermination.stopModuleWork() await MCPServerManager.shared.stop() await DatabaseManager.shared.runPostDisconnectScriptsBeforeQuit() + // A Vault lease outlives the app on the server until its TTL, so its database user goes now. + await VaultDatabaseCredentials.shared.revokeAllLeases() NSApp.reply(toApplicationShouldTerminate: true) } return .terminateLater diff --git a/apple/TablePro/Assets.xcassets/claude-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/claude-icon.imageset/Contents.json new file mode 100644 index 0000000000..9eea838495 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/claude-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "claude.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/claude-icon.imageset/claude.svg b/apple/TablePro/Assets.xcassets/claude-icon.imageset/claude.svg new file mode 100644 index 0000000000..1beee86121 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/claude-icon.imageset/claude.svg @@ -0,0 +1 @@ +Claude \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/claudecode-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/claudecode-icon.imageset/Contents.json new file mode 100644 index 0000000000..a339b05512 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/claudecode-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "claudecode.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/claudecode-icon.imageset/claudecode.svg b/apple/TablePro/Assets.xcassets/claudecode-icon.imageset/claudecode.svg new file mode 100644 index 0000000000..9b0061b760 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/claudecode-icon.imageset/claudecode.svg @@ -0,0 +1 @@ +Claude Code \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/cline-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/cline-icon.imageset/Contents.json new file mode 100644 index 0000000000..6ec74c460e --- /dev/null +++ b/apple/TablePro/Assets.xcassets/cline-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "cline.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/cline-icon.imageset/cline.svg b/apple/TablePro/Assets.xcassets/cline-icon.imageset/cline.svg new file mode 100644 index 0000000000..0fe6a51fd8 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/cline-icon.imageset/cline.svg @@ -0,0 +1 @@ +Cline \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/cockroachdb-icon.imageset/cockroachdb.svg b/apple/TablePro/Assets.xcassets/cockroachdb-icon.imageset/cockroachdb.svg index 9d5feb8704..d0f8e29fc1 100644 --- a/apple/TablePro/Assets.xcassets/cockroachdb-icon.imageset/cockroachdb.svg +++ b/apple/TablePro/Assets.xcassets/cockroachdb-icon.imageset/cockroachdb.svg @@ -1 +1 @@ -CL +CL diff --git a/apple/TablePro/Assets.xcassets/cursor-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/cursor-icon.imageset/Contents.json new file mode 100644 index 0000000000..fe93e63c45 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/cursor-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "cursor.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/cursor-icon.imageset/cursor.svg b/apple/TablePro/Assets.xcassets/cursor-icon.imageset/cursor.svg new file mode 100644 index 0000000000..61303fbcc3 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/cursor-icon.imageset/cursor.svg @@ -0,0 +1 @@ +Cursor \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/githubcopilot-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/githubcopilot-icon.imageset/Contents.json new file mode 100644 index 0000000000..c9eed271be --- /dev/null +++ b/apple/TablePro/Assets.xcassets/githubcopilot-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "githubcopilot.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/githubcopilot-icon.imageset/githubcopilot.svg b/apple/TablePro/Assets.xcassets/githubcopilot-icon.imageset/githubcopilot.svg new file mode 100644 index 0000000000..f064947d1d --- /dev/null +++ b/apple/TablePro/Assets.xcassets/githubcopilot-icon.imageset/githubcopilot.svg @@ -0,0 +1 @@ +GitHub Copilot \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/googlegemini-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/googlegemini-icon.imageset/Contents.json new file mode 100644 index 0000000000..2681885282 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/googlegemini-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "googlegemini.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/googlegemini-icon.imageset/googlegemini.svg b/apple/TablePro/Assets.xcassets/googlegemini-icon.imageset/googlegemini.svg new file mode 100644 index 0000000000..765adb0887 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/googlegemini-icon.imageset/googlegemini.svg @@ -0,0 +1 @@ +Google Gemini \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/goose-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/goose-icon.imageset/Contents.json new file mode 100644 index 0000000000..7b8ae75508 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/goose-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "goose.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/goose-icon.imageset/goose.svg b/apple/TablePro/Assets.xcassets/goose-icon.imageset/goose.svg new file mode 100644 index 0000000000..94152e79d3 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/goose-icon.imageset/goose.svg @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/apple/TablePro/Assets.xcassets/jetbrains-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/jetbrains-icon.imageset/Contents.json new file mode 100644 index 0000000000..039658a592 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/jetbrains-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "jetbrains.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/jetbrains-icon.imageset/jetbrains.svg b/apple/TablePro/Assets.xcassets/jetbrains-icon.imageset/jetbrains.svg new file mode 100644 index 0000000000..2f669ff6a4 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/jetbrains-icon.imageset/jetbrains.svg @@ -0,0 +1 @@ +JetBrains \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/kiro-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/kiro-icon.imageset/Contents.json new file mode 100644 index 0000000000..2a0a6a2951 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/kiro-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "kiro.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "original" + } +} diff --git a/apple/TablePro/Assets.xcassets/kiro-icon.imageset/kiro.svg b/apple/TablePro/Assets.xcassets/kiro-icon.imageset/kiro.svg new file mode 100644 index 0000000000..5fe3cf6847 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/kiro-icon.imageset/kiro.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/apple/TablePro/Assets.xcassets/llamacpp-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/llamacpp-icon.imageset/Contents.json new file mode 100644 index 0000000000..95261ea879 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/llamacpp-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "llamacpp.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/llamacpp-icon.imageset/llamacpp.svg b/apple/TablePro/Assets.xcassets/llamacpp-icon.imageset/llamacpp.svg new file mode 100644 index 0000000000..05424790af --- /dev/null +++ b/apple/TablePro/Assets.xcassets/llamacpp-icon.imageset/llamacpp.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/apple/TablePro/Assets.xcassets/lmstudio-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/lmstudio-icon.imageset/Contents.json new file mode 100644 index 0000000000..12c3071e78 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/lmstudio-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "lmstudio.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/lmstudio-icon.imageset/lmstudio.svg b/apple/TablePro/Assets.xcassets/lmstudio-icon.imageset/lmstudio.svg new file mode 100644 index 0000000000..e324916aaf --- /dev/null +++ b/apple/TablePro/Assets.xcassets/lmstudio-icon.imageset/lmstudio.svg @@ -0,0 +1 @@ +LM Studio \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/meta-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/meta-icon.imageset/Contents.json new file mode 100644 index 0000000000..7eaa0ac309 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/meta-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "meta.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/meta-icon.imageset/meta.svg b/apple/TablePro/Assets.xcassets/meta-icon.imageset/meta.svg new file mode 100644 index 0000000000..d9a1631d1a --- /dev/null +++ b/apple/TablePro/Assets.xcassets/meta-icon.imageset/meta.svg @@ -0,0 +1 @@ +Meta \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/mlx-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/mlx-icon.imageset/Contents.json new file mode 100644 index 0000000000..0bf36eb608 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/mlx-icon.imageset/Contents.json @@ -0,0 +1,26 @@ +{ + "images" : [ + { + "filename" : "mlx.pdf", + "idiom" : "universal" + }, + { + "appearances" : [ + { + "appearance" : "luminosity", + "value" : "dark" + } + ], + "filename" : "mlx-dark.pdf", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "original" + } +} diff --git a/apple/TablePro/Assets.xcassets/mlx-icon.imageset/mlx-dark.pdf b/apple/TablePro/Assets.xcassets/mlx-icon.imageset/mlx-dark.pdf new file mode 100644 index 0000000000..3f95f37c12 Binary files /dev/null and b/apple/TablePro/Assets.xcassets/mlx-icon.imageset/mlx-dark.pdf differ diff --git a/apple/TablePro/Assets.xcassets/mlx-icon.imageset/mlx.pdf b/apple/TablePro/Assets.xcassets/mlx-icon.imageset/mlx.pdf new file mode 100644 index 0000000000..c25193d799 Binary files /dev/null and b/apple/TablePro/Assets.xcassets/mlx-icon.imageset/mlx.pdf differ diff --git a/apple/TablePro/Assets.xcassets/ollama-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/ollama-icon.imageset/Contents.json new file mode 100644 index 0000000000..7b3662889b --- /dev/null +++ b/apple/TablePro/Assets.xcassets/ollama-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "ollama.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/ollama-icon.imageset/ollama.svg b/apple/TablePro/Assets.xcassets/ollama-icon.imageset/ollama.svg new file mode 100644 index 0000000000..bc368e99c5 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/ollama-icon.imageset/ollama.svg @@ -0,0 +1 @@ +Ollama \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/openai-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/openai-icon.imageset/Contents.json new file mode 100644 index 0000000000..7f508a44dd --- /dev/null +++ b/apple/TablePro/Assets.xcassets/openai-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "openai.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/openai-icon.imageset/openai.svg b/apple/TablePro/Assets.xcassets/openai-icon.imageset/openai.svg new file mode 100644 index 0000000000..e04db75a5b --- /dev/null +++ b/apple/TablePro/Assets.xcassets/openai-icon.imageset/openai.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/opencode-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/opencode-icon.imageset/Contents.json new file mode 100644 index 0000000000..60dd3911f8 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/opencode-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "opencode.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/opencode-icon.imageset/opencode.svg b/apple/TablePro/Assets.xcassets/opencode-icon.imageset/opencode.svg new file mode 100644 index 0000000000..1e0da06f97 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/opencode-icon.imageset/opencode.svg @@ -0,0 +1 @@ +OpenCode \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/openrouter-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/openrouter-icon.imageset/Contents.json new file mode 100644 index 0000000000..1ba973b6a6 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/openrouter-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "openrouter.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/openrouter-icon.imageset/openrouter.svg b/apple/TablePro/Assets.xcassets/openrouter-icon.imageset/openrouter.svg new file mode 100644 index 0000000000..83ec807bc9 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/openrouter-icon.imageset/openrouter.svg @@ -0,0 +1 @@ +OpenRouter \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/raycast-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/raycast-icon.imageset/Contents.json new file mode 100644 index 0000000000..630a8ef983 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/raycast-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "raycast.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/raycast-icon.imageset/raycast.svg b/apple/TablePro/Assets.xcassets/raycast-icon.imageset/raycast.svg new file mode 100644 index 0000000000..9e69b8d9bf --- /dev/null +++ b/apple/TablePro/Assets.xcassets/raycast-icon.imageset/raycast.svg @@ -0,0 +1 @@ +Raycast \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/requesty-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/requesty-icon.imageset/Contents.json new file mode 100644 index 0000000000..373d859eb1 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/requesty-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "requesty.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "original" + } +} diff --git a/apple/TablePro/Assets.xcassets/requesty-icon.imageset/requesty.svg b/apple/TablePro/Assets.xcassets/requesty-icon.imageset/requesty.svg new file mode 100644 index 0000000000..d6f1a1073a --- /dev/null +++ b/apple/TablePro/Assets.xcassets/requesty-icon.imageset/requesty.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/apple/TablePro/Assets.xcassets/snowflake-icon.imageset/snowflake.svg b/apple/TablePro/Assets.xcassets/snowflake-icon.imageset/snowflake.svg index 551e9d821a..dbaa3e7fab 100644 --- a/apple/TablePro/Assets.xcassets/snowflake-icon.imageset/snowflake.svg +++ b/apple/TablePro/Assets.xcassets/snowflake-icon.imageset/snowflake.svg @@ -1,4 +1,4 @@ - + diff --git a/apple/TablePro/Assets.xcassets/starrocks-icon.imageset/starrocks.svg b/apple/TablePro/Assets.xcassets/starrocks-icon.imageset/starrocks.svg index 1de2232110..2301c95fc1 100644 --- a/apple/TablePro/Assets.xcassets/starrocks-icon.imageset/starrocks.svg +++ b/apple/TablePro/Assets.xcassets/starrocks-icon.imageset/starrocks.svg @@ -1 +1 @@ - + diff --git a/apple/TablePro/Assets.xcassets/vscode-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/vscode-icon.imageset/Contents.json new file mode 100644 index 0000000000..803aa273d9 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/vscode-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "vscode.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "original" + } +} diff --git a/apple/TablePro/Assets.xcassets/vscode-icon.imageset/vscode.svg b/apple/TablePro/Assets.xcassets/vscode-icon.imageset/vscode.svg new file mode 100644 index 0000000000..c453e633f3 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/vscode-icon.imageset/vscode.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apple/TablePro/Assets.xcassets/warp-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/warp-icon.imageset/Contents.json new file mode 100644 index 0000000000..1d4bbf3f3e --- /dev/null +++ b/apple/TablePro/Assets.xcassets/warp-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "warp.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/warp-icon.imageset/warp.svg b/apple/TablePro/Assets.xcassets/warp-icon.imageset/warp.svg new file mode 100644 index 0000000000..b825120839 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/warp-icon.imageset/warp.svg @@ -0,0 +1 @@ +Warp \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/windsurf-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/windsurf-icon.imageset/Contents.json new file mode 100644 index 0000000000..87899b36f6 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/windsurf-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "windsurf.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/windsurf-icon.imageset/windsurf.svg b/apple/TablePro/Assets.xcassets/windsurf-icon.imageset/windsurf.svg new file mode 100644 index 0000000000..bb77fc9348 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/windsurf-icon.imageset/windsurf.svg @@ -0,0 +1 @@ +Windsurf \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/xai-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/xai-icon.imageset/Contents.json new file mode 100644 index 0000000000..a0cd740ec8 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/xai-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "xai.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/xai-icon.imageset/xai.svg b/apple/TablePro/Assets.xcassets/xai-icon.imageset/xai.svg new file mode 100644 index 0000000000..b08ffff656 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/xai-icon.imageset/xai.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/apple/TablePro/Assets.xcassets/zed-icon.imageset/Contents.json b/apple/TablePro/Assets.xcassets/zed-icon.imageset/Contents.json new file mode 100644 index 0000000000..37797c2e01 --- /dev/null +++ b/apple/TablePro/Assets.xcassets/zed-icon.imageset/Contents.json @@ -0,0 +1,16 @@ +{ + "images" : [ + { + "filename" : "zed.svg", + "idiom" : "universal" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + }, + "properties" : { + "preserves-vector-representation" : true, + "template-rendering-intent" : "template" + } +} diff --git a/apple/TablePro/Assets.xcassets/zed-icon.imageset/zed.svg b/apple/TablePro/Assets.xcassets/zed-icon.imageset/zed.svg new file mode 100644 index 0000000000..02327fd13b --- /dev/null +++ b/apple/TablePro/Assets.xcassets/zed-icon.imageset/zed.svg @@ -0,0 +1 @@ +Zed Industries \ No newline at end of file diff --git a/apple/TablePro/CLI/BridgeMain.swift b/apple/TablePro/CLI/BridgeMain.swift index 3c1ac69d89..5fc45ac04a 100644 --- a/apple/TablePro/CLI/BridgeMain.swift +++ b/apple/TablePro/CLI/BridgeMain.swift @@ -1,7 +1,7 @@ import Foundation import os import TableProMCP -import TableProUtilities +import TableProProcess @main struct TableProMcpBridge { @@ -40,7 +40,7 @@ struct TableProMcpBridge { return "The local MCP endpoint did not prove it belongs to TablePro. Restart TablePro and try again." case MCPBridgeStartupError.unsupportedUpstreamVersion: return "This TablePro build does not speak \(MCPProtocolVersion.latest.rawValue). Update TablePro." - case MCPHandshakeAcquisitionError.serverTurnedOff: + case MCPHandshakeAcquisitionError.serverTurnedOff, MCPHandshakeAcquisitionError.serverFailedToStart: return error.localizedDescription default: return "TablePro is not running. Launch the app and enable the MCP server." diff --git a/apple/TablePro/Core/AI/AIProviderDraftRules.swift b/apple/TablePro/Core/AI/AIProviderDraftRules.swift index 63eed9b41b..f54979369a 100644 --- a/apple/TablePro/Core/AI/AIProviderDraftRules.swift +++ b/apple/TablePro/Core/AI/AIProviderDraftRules.swift @@ -24,6 +24,34 @@ internal enum AIProviderDraftRules { return fetched.count == 1 ? fetched.first?.id : nil } + // Without an effort, a model that always reasons runs the server's default, which can be its costliest. + internal static func effort( + afterPicking current: ReasoningEffort?, + reasoning: AIReasoningSupport, + curated: CuratedModel? + ) -> ReasoningEffort? { + let levels = reasoning.effortLevels + guard !levels.isEmpty else { return nil } + if let current, levels.contains(current) { return current } + guard reasoning.isMandatory else { return curated?.defaultEffort } + return current.flatMap(reasoning.clampedEffort) + ?? curated?.defaultEffort + ?? reasoning.defaultEffort + ?? levels.first + } + + // A typed id is incomplete until the last keystroke; the send path fits the effort to the model anyway. + internal static func effort( + keeping current: ReasoningEffort?, + reasoning: AIReasoningSupport + ) -> ReasoningEffort? { + guard reasoning.isMandatory, !reasoning.effortLevels.isEmpty else { return current } + if let current, reasoning.effortLevels.contains(current) { return current } + return current.flatMap(reasoning.clampedEffort) + ?? reasoning.defaultEffort + ?? reasoning.effortLevels.first + } + /// Save waits for a model only while the provider's own list is there to pick one from. With /// no list, because the fetch is blocked or failed or the provider has not been signed in to /// yet, an empty model is saved as before and can be set later. diff --git a/apple/TablePro/Core/AI/AIProviderFactory.swift b/apple/TablePro/Core/AI/AIProviderFactory.swift index 87ac8543c5..3d01d434e2 100644 --- a/apple/TablePro/Core/AI/AIProviderFactory.swift +++ b/apple/TablePro/Core/AI/AIProviderFactory.swift @@ -69,6 +69,25 @@ enum AIProviderFactory { } } + // `onlyWhereRequired` is for inline suggestions: they send an effort only where a model always reasons. + static func reasoningEffort( + _ requested: ReasoningEffort?, + config: AIProviderConfig, + model: String, + onlyWhereRequired: Bool = false, + catalog: AIModelCatalog = .shared + ) -> ReasoningEffort? { + guard let descriptor = AIProviderRegistry.shared.descriptor(for: config.type.rawValue) else { + return onlyWhereRequired ? nil : requested + } + let fetched = catalog.fetchedInfo(providerID: config.id, modelID: model) + if onlyWhereRequired, + !descriptor.reasoning(forModelID: model, fetched: fetched, preset: config.preset).isMandatory { + return nil + } + return descriptor.effort(requested, forModelID: model, fetched: fetched, preset: config.preset) + } + static func resolve( settings: AISettings, overrideProviderId: UUID? = nil, @@ -76,11 +95,14 @@ enum AIProviderFactory { ) -> ResolvedProvider? { guard settings.enabled else { return nil } let config: AIProviderConfig? + // A model picked for a provider that was since removed belongs to that provider, not the active one. + var model = overrideModel if let overrideProviderId, let match = settings.providers.first(where: { $0.id == overrideProviderId }) { config = match } else { config = settings.activeProvider + if overrideProviderId != nil { model = nil } } guard let config else { return nil } let apiKey: String? @@ -91,7 +113,6 @@ enum AIProviderFactory { apiKey = nil } let provider = createProvider(for: config, apiKey: apiKey) - let model = overrideModel ?? config.model - return ResolvedProvider(provider: provider, model: model, config: config) + return ResolvedProvider(provider: provider, model: model ?? config.model, config: config) } } diff --git a/apple/TablePro/Core/AI/AnthropicProvider.swift b/apple/TablePro/Core/AI/AnthropicProvider.swift index 0019e43fda..7106932e2f 100644 --- a/apple/TablePro/Core/AI/AnthropicProvider.swift +++ b/apple/TablePro/Core/AI/AnthropicProvider.swift @@ -405,7 +405,14 @@ final class AnthropicProvider: ChatTransport { let tokens = usage["output_tokens"] as? Int { state.outputTokens = tokens } - return [] + switch (json["delta"] as? [String: Any])?["stop_reason"] as? String { + case "max_tokens": + return [.outputLimitReached(.outputTokens)] + case "model_context_window_exceeded": + return [.outputLimitReached(.contextWindow)] + default: + return [] + } case "error": if let errorObj = json["error"] as? [String: Any], let message = errorObj["message"] as? String { diff --git a/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift b/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift index c465866ced..1a3cc41198 100644 --- a/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift +++ b/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift @@ -40,6 +40,7 @@ struct AgentCLILaunch: Sendable { } } +// Never FileHandle.bytes: every AsyncBytes read shares one serial queue, so a quiet pipe stalls every other read. struct AgentCLIProcess: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "AgentCLIProcess") private static let terminationGracePeriod: Duration = .milliseconds(750) @@ -56,20 +57,33 @@ struct AgentCLIProcess: Sendable { let combined = Pipe() process.standardOutput = combined process.standardError = combined + // Read while the child runs: one that fills the pipe never exits. + let output = OSAllocatedUnfairLock(initialState: Data()) + let reader = PipeReader(combined.fileHandleForReading) return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in process.terminationHandler = { finished in - let data = (try? combined.fileHandleForReading.readToEnd() ?? Data()) ?? Data() + reader.stopAtEndOfFile() + let data = output.withLock { $0 } continuation.resume( returning: (finished.terminationStatus, String(data: data, encoding: .utf8) ?? "") ) } + reader.start { chunk in + output.withLock { $0.append(chunk) } + } do { try process.run() } catch { process.terminationHandler = nil + reader.stop() continuation.resume(throwing: AgentCLIError.launchFailed(error.localizedDescription)) + return + } + // A cancel that landed before launch found nothing running to stop. + if Task.isCancelled { + Self.terminate(process, label: label) } } } onCancel: { @@ -92,37 +106,45 @@ struct AgentCLIProcess: Sendable { process.standardError = stderr let label = self.label - let stderrDrain = Task { - for try await line in stderr.fileHandleForReading.bytes.lines where !line.isEmpty { - Self.logger.error("\(label, privacy: .public) stderr: \(line, privacy: .private)") + let stdoutLines = OSAllocatedUnfairLock(initialState: LineSplitter()) + let stderrLines = OSAllocatedUnfairLock(initialState: LineSplitter()) + let stdoutReader = PipeReader(stdout.fileHandleForReading) + let stderrReader = PipeReader(stderr.fileHandleForReading) + + continuation.onTermination = { _ in + Self.terminate(process, label: label) + stdoutReader.stop() + stderrReader.stop(drainingUpTo: DescriptorRead.pipeCapacity) + Self.logStderr(stderrLines.withLock { $0.finish() }, label: label) + } + + stderrReader.start { chunk in + Self.logStderr(stderrLines.withLock { $0.append(chunk) }, label: label) + } atEnd: { _ in + Self.logStderr(stderrLines.withLock { $0.finish() }, label: label) + } + stdoutReader.start { chunk in + for line in stdoutLines.withLock({ $0.append(chunk) }) { + continuation.yield(line) } + } atEnd: { failure in + for line in stdoutLines.withLock({ $0.finish() }) { + continuation.yield(line) + } + continuation.finish(throwing: failure) } do { try process.run() } catch { - stderrDrain.cancel() continuation.finish(throwing: AgentCLIError.launchFailed(error.localizedDescription)) - return - } - - let reader = Task { - do { - for try await line in stdout.fileHandleForReading.bytes.lines { - if Task.isCancelled { break } - continuation.yield(line) - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } } + } + } - continuation.onTermination = { _ in - reader.cancel() - stderrDrain.cancel() - Self.terminate(process, label: label) - } + private static func logStderr(_ lines: [String], label: String) { + for line in lines { + logger.error("\(label, privacy: .public) stderr: \(line, privacy: .private)") } } @@ -147,3 +169,36 @@ struct AgentCLIProcess: Sendable { } } } + +extension AgentCLIProcess { + // LF alone ends a line: stream-json text can hold a raw U+2028, which FileHandle's `lines` also splits on. + struct LineSplitter: Sendable { + private var partial = Data() + + mutating func append(_ chunk: Data) -> [String] { + var lines: [String] = [] + var lineStart = chunk.startIndex + while let newline = chunk[lineStart...].firstIndex(of: 0x0A) { + partial.append(chunk[lineStart ..< newline]) + if let line = Self.decode(partial) { + lines.append(line) + } + partial.removeAll(keepingCapacity: true) + lineStart = chunk.index(after: newline) + } + partial.append(chunk[lineStart...]) + return lines + } + + mutating func finish() -> [String] { + defer { partial = Data() } + return Self.decode(partial).map { [$0] } ?? [] + } + + private static func decode(_ bytes: Data) -> String? { + let line = bytes.last == 0x0D ? bytes.dropLast() : bytes + guard !line.isEmpty else { return nil } + return String(bytes: line, encoding: .utf8) + } + } +} diff --git a/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift b/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift index 63302e7804..b499238ae4 100644 --- a/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift +++ b/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift @@ -37,6 +37,19 @@ enum ChatToolCallRepair { return repaired } + // A stored card has nothing left to answer it, so storage and a loaded transcript never hold `.pending`. + static func settlingPendingApprovals(_ turns: [ChatTurnWire]) -> [ChatTurnWire] { + turns.map { turn in + var settled = turn + settled.blocks = turn.blocks.map { block in + guard case .toolUse(var call) = block.kind, case .pending = call.approvalState else { return block } + call.approvalState = .cancelled + return ChatContentBlockWire(id: block.id, kind: .toolUse(call)) + } + return settled + } + } + static func interruptedResults(for blocks: [ToolUseBlock]) -> [ToolResultBlock] { blocks.map { block in switch block.approvalState { @@ -59,11 +72,23 @@ enum ChatToolCallRepair { for block in turn.blocks { guard case .toolUse(let call) = block.kind, !answered.contains(call.id) else { continue } answered.insert(call.id) - owed.append(ToolResultBlock(toolUseId: call.id, content: interruptedContent, isError: true)) + owed.append(ToolResultBlock(toolUseId: call.id, content: unansweredContent(for: call), isError: true)) } return owed } + // An approved call with no result may have run; a cancelled or denied one never did. + private static func unansweredContent(for call: ToolUseBlock) -> String { + switch call.approvalState { + case .approved, .pending: + return interruptedContent + case .cancelled: + return stoppedBeforeRunContent + case .denied(let reason): + return reason + } + } + private static func toolResultIds(in turn: ChatTurnWire) -> Set { Set(turn.blocks.compactMap { block -> String? in guard case .toolResult(let result) = block.kind else { return nil } diff --git a/apple/TablePro/Core/AI/Chat/ChatToolContext.swift b/apple/TablePro/Core/AI/Chat/ChatToolContext.swift index 8724ff8fa4..b5ea5bf614 100644 --- a/apple/TablePro/Core/AI/Chat/ChatToolContext.swift +++ b/apple/TablePro/Core/AI/Chat/ChatToolContext.swift @@ -5,15 +5,8 @@ import Foundation -/// Per-call context passed to `ChatTool.execute(input:context:)`. Carries the session's connection, -/// the shared `MCPConnectionBridge` actor that does the underlying database work, and the -/// `MCPAuthPolicy` that authorizes the call and gates writes through the connection's safe-mode -/// dialog. -/// -/// `connectionId` is the session's own connection and is the only connection a tool may act on. A -/// tool never reads `connection_id` out of the model's input: that is what let an approval computed -/// against one connection authorize a statement executed against another. `ChatToolTarget` is the -/// one door, and it is why this type no longer carries a resolver of its own. +// `connectionId` is the only connection a tool may act on: a tool never reads `connection_id` from the model's input, +// which once let an approval for one connection run a statement on another. `ChatToolTarget` is the one door. struct ChatToolContext: Sendable { typealias PolicyLookup = @Sendable (UUID) async -> AIConnectionPolicy? @@ -21,18 +14,14 @@ struct ChatToolContext: Sendable { let bridge: MCPConnectionBridge let authPolicy: MCPAuthPolicy - /// The agent session this call belongs to. Only the audit entry for an outside MCP server reads - /// it, and it reads it because "which conversation sent this" is the first question asked of a - /// call that left the machine. + // Read only by an outside MCP server's audit entry, where the conversation that sent a call is the first question. let sessionId: UUID? - /// Whether the user answered this call's own approval card with Run or Always. - /// - /// Only an explicit answer pre-clears the execution gate's confirmation, because only then has - /// the user been shown this statement and this connection. A grant carried over from - /// `aiAlwaysAllowedTools` is consent for the tool, not for the statement, so the gate still asks - /// and still shows the SQL. Every chat tool used to pass `.confirmationPreCleared` - /// unconditionally, which skipped the only screen naming what was about to run and where. + // The mode the tool was resolved in, which also limits what it may run: Edit runs data statements only. + let chatMode: AIChatMode + + // Only a Run or Always on this call's own card pre-clears the gate's confirmation: only then was the user shown + // this statement on this connection. A grant from `aiAlwaysAllowedTools` is consent to the tool, not the statement. let approvalWasExplicit: Bool private let policyLookup: PolicyLookup @@ -41,6 +30,7 @@ struct ChatToolContext: Sendable { connectionId: UUID?, bridge: MCPConnectionBridge, authPolicy: MCPAuthPolicy, + chatMode: AIChatMode, sessionId: UUID? = nil, approvalWasExplicit: Bool = false, policyLookup: @escaping PolicyLookup = ChatToolContext.savedPolicy @@ -48,27 +38,25 @@ struct ChatToolContext: Sendable { self.connectionId = connectionId self.bridge = bridge self.authPolicy = authPolicy + self.chatMode = chatMode self.sessionId = sessionId self.approvalWasExplicit = approvalWasExplicit self.policyLookup = policyLookup } - /// The same context bound to one block's approval outcome. - /// - /// One context is built per streaming round and shared by every block in it, including blocks - /// that run concurrently, so provenance cannot live on the shared value. + // One context serves a round's blocks, some running at once, so each block's approval is bound to a copy. func carrying(approvalWasExplicit: Bool) -> ChatToolContext { ChatToolContext( connectionId: connectionId, bridge: bridge, authPolicy: authPolicy, + chatMode: chatMode, sessionId: sessionId, approvalWasExplicit: approvalWasExplicit, policyLookup: policyLookup ) } - /// This connection's AI Policy, or nil when there is no such connection. func aiPolicy(for connectionId: UUID) async -> AIConnectionPolicy? { await policyLookup(connectionId) } @@ -84,7 +72,6 @@ struct ChatToolContext: Sendable { } } - /// The write capabilities this call may claim, which is where approval provenance is spent. var writeCapabilities: CallerCapabilities { approvalWasExplicit ? [.mayWrite, .mayRunDestructive, .confirmationPreCleared] diff --git a/apple/TablePro/Core/AI/Chat/ChatTransport.swift b/apple/TablePro/Core/AI/Chat/ChatTransport.swift index 5ef3b2230a..2b0b566c0c 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTransport.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTransport.swift @@ -24,17 +24,14 @@ struct ChatTransportOptions: Sendable { var temperature: Double? var tools: [ChatToolSpec] var reasoningEffort: ReasoningEffort? - /// Which agent session this turn belongs to. - /// - /// A stateful transport keeps conversation state per session rather than per provider - /// configuration. `AIProviderFactory` caches one provider per configuration, so two sessions on - /// one Copilot configuration appended their turns to a single server-side conversation and each - /// was answered with the other's context, across connections included, while their local - /// transcripts stayed correctly separate. + // A stateful transport keys its conversation by session: providers are cached per configuration, and two + // sessions on one Copilot configuration once shared a single server-side conversation. var sessionId: UUID // The connection the conversation is attached to. A transport that hands the model its own // database tools scopes them to this one, and hands out none without it. var connectionId: UUID? + // A transport that grants its own database tools narrows them by mode; a caller that leaves it out gets Ask. + var chatMode: AIChatMode init( model: String, @@ -44,7 +41,8 @@ struct ChatTransportOptions: Sendable { tools: [ChatToolSpec] = [], reasoningEffort: ReasoningEffort? = nil, sessionId: UUID = UUID(), - connectionId: UUID? = nil + connectionId: UUID? = nil, + chatMode: AIChatMode = .ask ) { self.model = model self.systemPrompt = systemPrompt @@ -54,6 +52,7 @@ struct ChatTransportOptions: Sendable { self.reasoningEffort = reasoningEffort self.sessionId = sessionId self.connectionId = connectionId + self.chatMode = chatMode } } @@ -83,6 +82,12 @@ struct ChatToolSpec: Codable, Equatable, Sendable { } } +// Why a provider stopped the response before the model finished it. +enum ChatOutputLimit: String, Codable, Sendable { + case outputTokens + case contextWindow +} + enum ChatStreamEvent: Sendable { case textDelta(String) case toolUseStart(id: String, name: String, providerMetadata: [String: String]? = nil) @@ -93,6 +98,7 @@ enum ChatStreamEvent: Sendable { case reasoningStart(id: String) case reasoningDelta(id: String, text: String) case reasoningEnd(id: String, opaque: ReasoningOpaque?) + case outputLimitReached(ChatOutputLimit) } final class ToolReplyToken: Sendable { diff --git a/apple/TablePro/Core/AI/Chat/ChatTurn.swift b/apple/TablePro/Core/AI/Chat/ChatTurn.swift index b7bb4c1d5f..429a06f5b5 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTurn.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTurn.swift @@ -105,6 +105,7 @@ final class ChatTurn: ObservableObject, Identifiable { @Published var usage: AITokenUsage? @Published var modelId: String? @Published var providerId: String? + @Published var outputLimit: ChatOutputLimit? init( id: UUID = UUID(), @@ -113,7 +114,8 @@ final class ChatTurn: ObservableObject, Identifiable { timestamp: Date = Date(), usage: AITokenUsage? = nil, modelId: String? = nil, - providerId: String? = nil + providerId: String? = nil, + outputLimit: ChatOutputLimit? = nil ) { self.id = id self.role = role @@ -122,6 +124,7 @@ final class ChatTurn: ObservableObject, Identifiable { self.usage = usage self.modelId = modelId self.providerId = providerId + self.outputLimit = outputLimit } init(wire: ChatTurnWire) { @@ -132,6 +135,7 @@ final class ChatTurn: ObservableObject, Identifiable { self.usage = wire.usage self.modelId = wire.modelId self.providerId = wire.providerId + self.outputLimit = wire.outputLimit } var plainText: String { @@ -152,7 +156,8 @@ final class ChatTurn: ObservableObject, Identifiable { timestamp: timestamp, usage: usage, modelId: modelId, - providerId: providerId + providerId: providerId, + outputLimit: outputLimit ) } @@ -337,6 +342,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { var usage: AITokenUsage? var modelId: String? var providerId: String? + var outputLimit: ChatOutputLimit? init( id: UUID = UUID(), @@ -345,7 +351,8 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { timestamp: Date = Date(), usage: AITokenUsage? = nil, modelId: String? = nil, - providerId: String? = nil + providerId: String? = nil, + outputLimit: ChatOutputLimit? = nil ) { self.id = id self.role = role @@ -354,6 +361,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { self.usage = usage self.modelId = modelId self.providerId = providerId + self.outputLimit = outputLimit } init(from decoder: Decoder) throws { @@ -364,6 +372,10 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { usage = try container.decodeIfPresent(AITokenUsage.self, forKey: .usage) modelId = try container.decodeIfPresent(String.self, forKey: .modelId) providerId = try container.decodeIfPresent(String.self, forKey: .providerId) + // A value a later version wrote reads as no note rather than failing the conversation. + outputLimit = (try? container.decodeIfPresent(String.self, forKey: .outputLimit)).flatMap { + $0.flatMap(ChatOutputLimit.init(rawValue:)) + } if let decodedBlocks = try container.decodeIfPresent([LossyContentBlockWire].self, forKey: .blocks) { blocks = decodedBlocks.compactMap(\.block) @@ -386,6 +398,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { try container.encodeIfPresent(usage, forKey: .usage) try container.encodeIfPresent(modelId, forKey: .modelId) try container.encodeIfPresent(providerId, forKey: .providerId) + try container.encodeIfPresent(outputLimit, forKey: .outputLimit) } var plainText: String { @@ -399,7 +412,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { } private enum CodingKeys: String, CodingKey { - case id, role, blocks, timestamp, usage, modelId, providerId + case id, role, blocks, timestamp, usage, modelId, providerId, outputLimit } private enum LegacyKeys: String, CodingKey { diff --git a/apple/TablePro/Core/AI/Chat/ToolApprovalCenter.swift b/apple/TablePro/Core/AI/Chat/ToolApprovalCenter.swift index 6e0677cc21..eee5e72245 100644 --- a/apple/TablePro/Core/AI/Chat/ToolApprovalCenter.swift +++ b/apple/TablePro/Core/AI/Chat/ToolApprovalCenter.swift @@ -124,6 +124,13 @@ final class ToolApprovalCenter { } } + // Not observable: safe for a view to read only because every change here lands in the same + // main-actor turn as the transcript change that draws or settles the card. + func isAwaiting(sessionId: UUID, toolUseId: String) -> Bool { + let key = ApprovalRequestID(sessionId: sessionId, toolUseId: toolUseId) + return expected.contains(key) || pending[key] != nil + } + func hasPending(sessionId: UUID) -> Bool { pending.keys.contains { $0.sessionId == sessionId } || expected.contains { key in diff --git a/apple/TablePro/Core/AI/Chat/Tools/ConfirmDestructiveOperationChatTool.swift b/apple/TablePro/Core/AI/Chat/Tools/ConfirmDestructiveOperationChatTool.swift index 417bb6b8ba..44c10fac30 100644 --- a/apple/TablePro/Core/AI/Chat/Tools/ConfirmDestructiveOperationChatTool.swift +++ b/apple/TablePro/Core/AI/Chat/Tools/ConfirmDestructiveOperationChatTool.swift @@ -98,7 +98,8 @@ struct ConfirmDestructiveOperationChatTool: ChatTool { operation: operation, maxRows: maxRows, timeoutSeconds: MCPLimitResolver.resolveTimeoutSeconds(requested: nil, settings: mcpSettings), - principal: .inAppAssistant + principal: .inAppAssistant, + budget: .assistantAnswer ) return ChatToolResult(content: payload.jsonString(prettyPrinted: true)) } diff --git a/apple/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift b/apple/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift index 66f7fa5da1..6aceb67145 100644 --- a/apple/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift +++ b/apple/TablePro/Core/AI/Chat/Tools/ExecuteQueryChatTool.swift @@ -61,8 +61,8 @@ struct ExecuteQueryChatTool: ChatTool { takingScriptsOn: meta.databaseType ) - // The refusals MCP makes, from the same gate. AI Policy, checked above, governs the assistant, so the - // External Clients level is not asked. + // The refusals MCP makes, from the same gate, plus Edit's. AI Policy, checked above, governs the assistant, so + // the External Clients level is not asked. let classification: QueryClassification do { classification = try ExternalStatementGate.classify( @@ -76,7 +76,8 @@ struct ExecuteQueryChatTool: ChatTool { allowsMultiStatement: capabilities.contains(.mayRunMultiStatement), destructiveAlternative: String( localized: "In Agent mode, use confirm_destructive_operation with the explicit confirmation phrase." - ) + ), + allowsOnlyDataStatements: context.chatMode != .agent ) ) } catch let refusal as ExternalStatementGateError { @@ -98,6 +99,13 @@ struct ExecuteQueryChatTool: ChatTool { database: database, schema: schema ) + // A table named without a schema lands in the scope's, and the engine's own schemas hold its grant, setup and + // settings tables. + if context.chatMode != .agent, classification.tier != .safe, QueryClassifier.writesSystemScope( + database: scope.database, schema: scope.schema, databaseType: meta.databaseType + ) { + return ChatToolResult(content: ExternalStatementGate.dataStatementRefusal, isError: true) + } let planned = try await context.bridge.plannedScript(scope: scope, query: query, maxRows: maxRows) let operation = try await context.authPolicy.checkSafeModeDialog( operation: PendingOperation( @@ -118,7 +126,8 @@ struct ExecuteQueryChatTool: ChatTool { operation: operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - principal: .inAppAssistant + principal: .inAppAssistant, + budget: .assistantAnswer ) return ChatToolResult(content: payload.jsonString(prettyPrinted: true)) } diff --git a/apple/TablePro/Core/AI/Chat/Tools/ListConnectionsChatTool.swift b/apple/TablePro/Core/AI/Chat/Tools/ListConnectionsChatTool.swift index 92c74bda4d..2245a87974 100644 --- a/apple/TablePro/Core/AI/Chat/Tools/ListConnectionsChatTool.swift +++ b/apple/TablePro/Core/AI/Chat/Tools/ListConnectionsChatTool.swift @@ -8,12 +8,13 @@ import TableProMCP struct ListConnectionsChatTool: ChatTool { let name = "list_connections" - let description = String(localized: "List all saved database connections with their current status.") + let description = String(localized: "Show the connection this conversation works on, with its status.") let inputSchema: JsonValue = ChatToolSchemaBuilder.object(properties: [:]) let mode: ChatToolMode = .readOnly func execute(input: JsonValue, context: ChatToolContext) async throws -> ChatToolResult { - let payload = await context.bridge.listConnections() + let connectionId = try await ChatToolTarget.authorized(context: context, input: input, tool: name) + let payload = await context.bridge.listConnection(id: connectionId) return ChatToolResult(content: payload.jsonString(prettyPrinted: true)) } } diff --git a/apple/TablePro/Core/AI/ChatGPTCodex/ChatGPTCodexCallbackServer.swift b/apple/TablePro/Core/AI/ChatGPTCodex/ChatGPTCodexCallbackServer.swift index fcf2eb8d3c..716dced00e 100644 --- a/apple/TablePro/Core/AI/ChatGPTCodex/ChatGPTCodexCallbackServer.swift +++ b/apple/TablePro/Core/AI/ChatGPTCodex/ChatGPTCodexCallbackServer.swift @@ -7,14 +7,65 @@ // import Foundation -import Network import os +import TableProHTTP -final class ChatGPTCodexCallbackServer: @unchecked Sendable { - enum ServerError: Error, LocalizedError { +// Only the redirect carrying this sign-in's `state` settles it (RFC 6749 section 4.1.2, RFC 8252 section 7.3). +enum ChatGPTCodexCallback: Sendable, Equatable { + case code(String) + case denied(error: String, description: String?) + case ignore + + static let callbackPath = URLComponents(string: ChatGPTCodex.redirectURI)?.path ?? "" + + static let unknownError = "unknown_error" + + private static let maximumErrorLength = 300 + + static func parse(requestHead: String, expectedState: String) -> ChatGPTCodexCallback { + guard !expectedState.isEmpty, + !callbackPath.isEmpty, + let target = LoopbackRedirectRequest.getTarget(requestHead), + let components = URLComponents(string: target), + components.host == nil, + components.path == callbackPath + else { + return .ignore + } + let items = components.queryItems ?? [] + guard let state = LoopbackRedirectRequest.singleValue(named: "state", in: items), + LoopbackRedirectRequest.constantTimeEquals(state, expectedState) + else { + return .ignore + } + if items.contains(where: { $0.name == "error" }) { + let error = LoopbackRedirectRequest.singleValue(named: "error", in: items).flatMap(sanitized) ?? unknownError + let formItems = LoopbackRedirectRequest.formDecodedItems(in: components) + let description = LoopbackRedirectRequest.singleValue(named: "error_description", in: formItems).flatMap(sanitized) + return .denied(error: error, description: description) + } + if let code = LoopbackRedirectRequest.singleValue(named: "code", in: items), !code.isEmpty { + return .code(code) + } + return .ignore + } + + // RFC 6749 allows only printable ASCII other than `"` and `\` in `error` and `error_description`. + private static func sanitized(_ value: String) -> String? { + let allowed = value.unicodeScalars.filter { scalar in + (0x20...0x7E).contains(scalar.value) && scalar != "\"" && scalar != "\\" + } + let text = String(String.UnicodeScalarView(allowed.prefix(maximumErrorLength))) + .trimmingCharacters(in: .whitespaces) + return text.isEmpty ? nil : text + } +} + +final class ChatGPTCodexCallbackServer: Sendable { + enum ServerError: Error, LocalizedError, Equatable { case portInUse case timedOut - case stateMismatch + case authorizationDenied(error: String, description: String?) var errorDescription: String? { switch self { @@ -22,8 +73,12 @@ final class ChatGPTCodexCallbackServer: @unchecked Sendable { return String(localized: "Port 1455 is in use. Quit the Codex CLI or another sign-in and try again.") case .timedOut: return String(localized: "Sign-in timed out. Please try again.") - case .stateMismatch: - return String(localized: "Sign-in could not be verified. Please try again.") + case .authorizationDenied(let error, let description): + if error == "access_denied" { + guard let description else { return String(localized: "ChatGPT sign-in was denied.") } + return String(format: String(localized: "ChatGPT sign-in was denied: %@"), description) + } + return String(format: String(localized: "ChatGPT sign-in failed: %@"), description ?? error) } } } @@ -32,182 +87,55 @@ final class ChatGPTCodexCallbackServer: @unchecked Sendable { private static let timeout: TimeInterval = 300 private static let startTimeout: TimeInterval = 10 - private let expectedState: String - private let lock = NSLock() - private var listener: NWListener? - private var connection: NWConnection? - private var readyContinuation: CheckedContinuation? - private var codeContinuation: CheckedContinuation? - private var readyTimeoutTask: Task? - private var timeoutTask: Task? - - init(expectedState: String) { - self.expectedState = expectedState - } - - func start() async throws { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - let timeout = Task { [weak self] in - try? await Task.sleep(nanoseconds: UInt64(Self.startTimeout * 1_000_000_000)) - self?.finishReady(.failure(ServerError.timedOut)) - } - lock.withLock { - readyContinuation = continuation - readyTimeoutTask = timeout - } - do { - try startListener() - } catch { - finishReady(.failure(ServerError.portInUse)) - } - } - } + private let listener: LoopbackRedirectListener - func waitForCode() async throws -> String { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - lock.withLock { codeContinuation = continuation } - let task = Task { [weak self] in - try? await Task.sleep(nanoseconds: UInt64(Self.timeout * 1_000_000_000)) - self?.finishCode(.failure(ServerError.timedOut)) - } - lock.withLock { timeoutTask = task } + init(expectedState: String, port: UInt16 = ChatGPTCodex.redirectPort, headTimeout: TimeInterval = 5) { + listener = LoopbackRedirectListener(port: port, headTimeout: headTimeout) { head in + Self.answer(head, expectedState: expectedState) } } - func stop() { - let (task, readyTask, conn, lst): ( - Task?, Task?, NWConnection?, NWListener? - ) = lock.withLock { - let values = (timeoutTask, readyTimeoutTask, connection, listener) - timeoutTask = nil - readyTimeoutTask = nil - connection = nil - listener = nil - return values - } - task?.cancel() - readyTask?.cancel() - conn?.cancel() - lst?.cancel() - } - - private func startListener() throws { - guard let port = NWEndpoint.Port(rawValue: ChatGPTCodex.redirectPort) else { + @discardableResult + func start() async throws -> UInt16 { + do { + return try await listener.start(readyTimeout: Self.startTimeout) + } catch LoopbackRedirectListenerError.timedOut { + throw ServerError.timedOut + } catch is LoopbackRedirectListenerError { throw ServerError.portInUse } - let parameters = NWParameters.tcp - parameters.requiredLocalEndpoint = NWEndpoint.hostPort(host: .ipv4(.loopback), port: port) - - let listener = try NWListener(using: parameters) - lock.withLock { self.listener = listener } - - listener.stateUpdateHandler = { [weak self] state in - switch state { - case .ready: - self?.finishReady(.success(())) - case .failed(let error): - Self.logger.error("Callback listener failed: \(error.publicLogShape, privacy: .public)") - self?.finishReady(.failure(ServerError.portInUse)) - self?.finishCode(.failure(ServerError.portInUse)) - default: - break - } - } - listener.newConnectionHandler = { [weak self] connection in - self?.handleConnection(connection) - } - listener.start(queue: .global(qos: .userInitiated)) - } - - private func handleConnection(_ newConnection: NWConnection) { - lock.withLock { connection = newConnection } - newConnection.stateUpdateHandler = { [weak self] state in - if case .ready = state { - self?.readRequest(from: newConnection) - } - } - newConnection.start(queue: .global(qos: .userInitiated)) } - private func readRequest(from connection: NWConnection) { - connection.receive(minimumIncompleteLength: 1, maximumLength: 16_384) { [weak self] content, _, isComplete, error in - guard let self else { return } - if error != nil { - connection.cancel() - return - } - guard let data = content, let request = String(data: data, encoding: .utf8), - let query = Self.parseCallback(request) else { - if isComplete { - connection.cancel() - } else { - self.readRequest(from: connection) - } - return - } - let matches = query.state == self.expectedState - Self.send(html: matches ? Self.successPage : Self.failurePage, to: connection) - self.finishCode(matches ? .success(query.code) : .failure(ServerError.stateMismatch)) - } - } - - private func finishReady(_ result: Result) { - let (continuation, timeout): (CheckedContinuation?, Task?) = lock.withLock { - let pendingContinuation = readyContinuation - let pendingTimeout = readyTimeoutTask - readyContinuation = nil - readyTimeoutTask = nil - return (pendingContinuation, pendingTimeout) - } - timeout?.cancel() - continuation?.resume(with: result) + func waitForCode() async throws -> String { + try await listener.outcome(timeout: Self.timeout, timeoutError: ServerError.timedOut) } - private func finishCode(_ result: Result) { - let continuation: CheckedContinuation? = lock.withLock { - let pending = codeContinuation - codeContinuation = nil - return pending - } - guard let continuation else { return } - continuation.resume(with: result) - stop() + func stop() { + listener.stop() } - private static func parseCallback(_ request: String) -> (code: String, state: String)? { - guard let requestLine = request.components(separatedBy: "\r\n").first, - let target = requestLine.components(separatedBy: " ").dropFirst().first, - let components = URLComponents(string: "http://127.0.0.1\(target)"), - let items = components.queryItems, - let code = items.first(where: { $0.name == "code" })?.value, - let state = items.first(where: { $0.name == "state" })?.value else { - return nil + private static func answer(_ head: Data, expectedState: String) -> LoopbackRedirectAnswer { + let text = LoopbackRedirectRequest.text(of: head) + switch ChatGPTCodexCallback.parse(requestHead: text, expectedState: expectedState) { + case .ignore: + return .ignore(LoopbackRedirectPage.badRequest) + case .code(let code): + return .settle(successPage, .success(code)) + case .denied(let error, let description): + logger.info("The ChatGPT sign-in redirect carried an OAuth error: \(error, privacy: .private)") + return .settle(failurePage, .failure(ServerError.authorizationDenied(error: error, description: description))) } - return (code, state) - } - - private static func send(html: String, to connection: NWConnection) { - let body = Array(html.utf8) - let headers = "HTTP/1.1 200 OK\r\n" - + "Content-Type: text/html; charset=utf-8\r\n" - + "Content-Length: \(body.count)\r\n" - + "Connection: close\r\n\r\n" - var response = Data(headers.utf8) - response.append(contentsOf: body) - connection.send(content: response, completion: .contentProcessed { _ in - connection.cancel() - }) } - private static let successPage = """ - TablePro - -

Signed in to ChatGPT

You can return to TablePro.

- """ - - private static let failurePage = """ - TablePro - -

Sign-in failed

Please return to TablePro and try again.

- """ + private static let successPage = LoopbackRedirectPage.response( + status: "200 OK", + title: "Signed in to ChatGPT", + message: "You can return to TablePro." + ) + + private static let failurePage = LoopbackRedirectPage.response( + status: "200 OK", + title: "Sign-in failed", + message: "Please return to TablePro and try again." + ) } diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift index 3450ce5425..f602367ee0 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift @@ -1,8 +1,3 @@ -// -// ClaudeAgentMCPBridge.swift -// TablePro -// - import Foundation import os import TableProMCP @@ -10,22 +5,49 @@ import TableProMCP struct ClaudeAgentToolGrant: Sendable, Equatable { let configPath: String let tokenId: UUID + // Nil leaves every tool the token's scopes allow. + let allowedToolNames: [String]? +} + +// A turn that was asked to use a connection stops with one of these rather than run without its tools. +enum ClaudeAgentToolGrantError: Error, LocalizedError, Equatable { + case tokenNotSaved + case tokenNotCreated + case configNotWritten + + var errorDescription: String? { + switch self { + case .tokenNotSaved: + String(localized: "Claude Agent can't use this connection: TablePro couldn't save its access token to the Keychain. Try again.") + case .tokenNotCreated: + String(localized: "Claude Agent can't use this connection: TablePro couldn't create its access token. Try again.") + case .configNotWritten: + String(localized: "Claude Agent can't use this connection: TablePro couldn't write its MCP settings file. Try again.") + } + } } protocol ClaudeAgentMCPBridging: Sendable { - func makeGrant(connectionId: UUID?) async -> ClaudeAgentToolGrant? + func makeGrant(connectionId: UUID?, mode: AIChatMode) async throws -> ClaudeAgentToolGrant? func release(_ grant: ClaudeAgentToolGrant?) async } struct ClaudeAgentMCPBridge: ClaudeAgentMCPBridging { private static let logger = Logger(subsystem: "com.TablePro", category: "ClaudeAgentMCPBridge") - private static let tokenName = "__claude_agent__" private static let tokenLifetime: TimeInterval = 15 * 60 private let permissions: TokenPermissions + private let endpoint: @Sendable () async -> URL? + private let tokenStore: @Sendable () async -> MCPTokenStore? - init(permissions: TokenPermissions = .readOnly) { + init( + permissions: TokenPermissions = .readOnly, + endpoint: (@Sendable () async -> URL?)? = nil, + tokenStore: (@Sendable () async -> MCPTokenStore?)? = nil + ) { self.permissions = permissions + self.endpoint = endpoint ?? { await ClaudeAgentMCPBridge.resolveEndpoint() } + self.tokenStore = tokenStore ?? { await MCPServerManager.shared.tokenStore } } static func connectionAccess(for connectionId: UUID?) -> ConnectionAccess? { @@ -33,37 +55,62 @@ struct ClaudeAgentMCPBridge: ClaudeAgentMCPBridging { return .limited([connectionId]) } - func makeGrant(connectionId: UUID?) async -> ClaudeAgentToolGrant? { + // Ask promises schema lookups only, so the server holds the token to them; Edit and Agent keep the read-only token. + static func toolProfile(for mode: AIChatMode) -> MCPToolProfile? { + switch mode { + case .ask: .schemaLookup + case .edit, .agent: nil + } + } + + func makeGrant(connectionId: UUID?, mode: AIChatMode) async throws -> ClaudeAgentToolGrant? { guard let connectionAccess = Self.connectionAccess(for: connectionId) else { Self.logger.info("No connection attached; Claude Agent runs without database tools") return nil } - guard let endpoint = await Self.resolveEndpoint() else { return nil } - guard let tokenStore = await MCPServerManager.shared.tokenStore else { return nil } - - guard let generated = try? await tokenStore.generate( - name: Self.tokenName, - permissions: permissions, - connectionAccess: connectionAccess, - expiresAt: Date.now.addingTimeInterval(Self.tokenLifetime), - isBridgeCredential: false, - isClaudeAgentCredential: true, - extraScopes: [] - ) else { return nil } + guard let endpoint = await self.endpoint() else { return nil } + guard let tokenStore = await self.tokenStore() else { return nil } + + let toolProfile = Self.toolProfile(for: mode) + let generated: (token: MCPAuthToken, plaintext: String) + do { + generated = try await tokenStore.generate( + name: MCPTokenStore.claudeAgentTokenName, + permissions: permissions, + connectionAccess: connectionAccess, + expiresAt: Date.now.addingTimeInterval(Self.tokenLifetime), + isBridgeCredential: false, + isClaudeAgentCredential: true, + extraScopes: [], + toolProfile: toolProfile + ) + } catch { + Self.logger.error("Claude Agent token was not issued: \(error.publicLogShape, privacy: .public)") + switch error as? MCPTokenStoreError { + case .notSaved?, .keychainUnavailable?: + throw ClaudeAgentToolGrantError.tokenNotSaved + case .storedTokensUnreadable?, .randomGenerationFailed?, nil: + throw ClaudeAgentToolGrantError.tokenNotCreated + } + } guard let configPath = Self.writeConfig(endpoint: endpoint, token: generated.plaintext) else { await tokenStore.retire(tokenId: generated.token.id) - return nil + throw ClaudeAgentToolGrantError.configNotWritten } Self.logger.debug("Issued Claude Agent MCP grant") - return ClaudeAgentToolGrant(configPath: configPath, tokenId: generated.token.id) + return ClaudeAgentToolGrant( + configPath: configPath, + tokenId: generated.token.id, + allowedToolNames: toolProfile.map { $0.toolNames.sorted() } + ) } func release(_ grant: ClaudeAgentToolGrant?) async { guard let grant else { return } try? FileManager.default.removeItem(atPath: grant.configPath) - guard let tokenStore = await MCPServerManager.shared.tokenStore else { return } + guard let tokenStore = await self.tokenStore() else { return } await tokenStore.retire(tokenId: grant.tokenId) Self.logger.debug("Released Claude Agent MCP grant") } diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift index e21593ca8f..a9afb00acd 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift @@ -1,8 +1,3 @@ -// -// ClaudeAgentProvider.swift -// TablePro -// - import Foundation import os @@ -31,14 +26,15 @@ final class ClaudeAgentProvider: ChatTransport { let task = Task { var grant: ClaudeAgentToolGrant? do { - grant = await toolBridge.makeGrant(connectionId: options.connectionId) + grant = try await toolBridge.makeGrant(connectionId: options.connectionId, mode: options.chatMode) let prompt = AgentPromptRenderer.renderPrompt(turns: turns, includingSystemPrompt: nil) let lines = try cli.streamLines( arguments: Self.inferenceArguments( prompt: prompt, model: model, systemPrompt: options.systemPrompt ?? ClaudeAgent.systemPrompt, - mcpConfigPath: grant?.configPath + mcpConfigPath: grant?.configPath, + allowedToolNames: grant?.allowedToolNames ), currentDirectory: FileManager.default.temporaryDirectory ) @@ -51,6 +47,8 @@ final class ClaudeAgentProvider: ChatTransport { continuation.yield(.textDelta(text)) case .usage(let usage): continuation.yield(.usage(usage)) + case .outputLimit(let limit): + continuation.yield(.outputLimitReached(limit)) case .failure(let message): await toolBridge.release(grant) grant = nil @@ -90,6 +88,7 @@ final class ClaudeAgentProvider: ChatTransport { enum DecodedEvent: Equatable { case text(String) case usage(AITokenUsage) + case outputLimit(ChatOutputLimit) case failure(String) } @@ -97,7 +96,8 @@ final class ClaudeAgentProvider: ChatTransport { prompt: String, model: String, systemPrompt: String?, - mcpConfigPath: String? + mcpConfigPath: String?, + allowedToolNames: [String]? ) -> [String] { var arguments = [ "-p", @@ -111,7 +111,7 @@ final class ClaudeAgentProvider: ChatTransport { ] if let mcpConfigPath { - arguments += ["--mcp-config", mcpConfigPath, "--allowedTools", "mcp__\(ClaudeAgent.mcpServerName)__*"] + arguments += ["--mcp-config", mcpConfigPath, "--allowedTools", allowedToolsArgument(allowedToolNames)] } arguments.append("--strict-mcp-config") @@ -125,10 +125,18 @@ final class ClaudeAgentProvider: ChatTransport { return arguments } + // The server's token profile is the gate; naming the tools here keeps the CLI from offering any other. + static func allowedToolsArgument(_ toolNames: [String]?) -> String { + let prefix = "mcp__\(ClaudeAgent.mcpServerName)__" + guard let toolNames else { return prefix + "*" } + return toolNames.map { prefix + $0 }.joined(separator: ",") + } + static func decodeEvent(_ line: String) -> DecodedEvent? { guard let json = decodeJSON(line), let type = json["type"] as? String else { return nil } switch type { case "stream_event": + if let limit = outputLimit(json) { return .outputLimit(limit) } guard let text = partialText(json) else { return nil } return .text(text) case "result": @@ -149,6 +157,18 @@ final class ClaudeAgentProvider: ChatTransport { return text } + // The CLI passes the Messages API stream through, so a cut response ends with its message_delta stop reason. + static func outputLimit(_ json: [String: Any]) -> ChatOutputLimit? { + guard let event = json["event"] as? [String: Any], + event["type"] as? String == "message_delta", + let delta = event["delta"] as? [String: Any] else { return nil } + switch delta["stop_reason"] as? String { + case "max_tokens": return .outputTokens + case "model_context_window_exceeded": return .contextWindow + default: return nil + } + } + static func resultEvent(_ json: [String: Any]) -> DecodedEvent? { if json["is_error"] as? Bool == true { let message = (json["result"] as? String).flatMap { $0.isEmpty ? nil : $0 } diff --git a/apple/TablePro/Core/AI/Copilot/CopilotDocumentServer.swift b/apple/TablePro/Core/AI/Copilot/CopilotDocumentServer.swift new file mode 100644 index 0000000000..da0fb9b029 --- /dev/null +++ b/apple/TablePro/Core/AI/Copilot/CopilotDocumentServer.swift @@ -0,0 +1,38 @@ +import Foundation + +@MainActor +protocol CopilotDocumentServer { + var generation: Int { get } + var hasClient: Bool { get } + func didOpen(_ document: LSPTextDocumentItem) async + func didChange(_ change: LSPDidChangeParams) async + func didFocus(uri: String) async + func didClose(uri: String) async +} + +@MainActor +struct LiveCopilotDocumentServer: CopilotDocumentServer { + var generation: Int { CopilotService.shared.generation } + + var hasClient: Bool { CopilotService.shared.client != nil } + + func didOpen(_ document: LSPTextDocumentItem) async { + await CopilotService.shared.client?.didOpenDocument(document) + } + + func didChange(_ change: LSPDidChangeParams) async { + await CopilotService.shared.client?.didChangeDocument( + uri: change.textDocument.uri, + version: change.textDocument.version, + changes: change.contentChanges + ) + } + + func didFocus(uri: String) async { + await CopilotService.shared.client?.didFocusDocument(uri: uri) + } + + func didClose(uri: String) async { + await CopilotService.shared.client?.didCloseDocument(uri: uri) + } +} diff --git a/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift b/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift index 20a137cb63..9b2d594d07 100644 --- a/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift +++ b/apple/TablePro/Core/AI/Copilot/CopilotDocumentSync.swift @@ -1,117 +1,384 @@ -// -// CopilotDocumentSync.swift -// TablePro -// - -import Foundation +import AppKit import os +import TableProTextEngine -/// Manages LSP document lifecycle for Copilot. Prepends the schema preamble -/// to all document text sent to the server. @MainActor final class CopilotDocumentSync { + private enum Outgoing { + case open(uri: String, text: String, generation: Int) + case change(uri: String, changes: [LSPTextDocumentContentChangeEvent], generation: Int) + case focus(uri: String, generation: Int) + case close(uri: String, generation: Int) + + var generation: Int { + switch self { + case .open(_, _, let generation), .change(_, _, let generation), + .focus(_, let generation), .close(_, let generation): + generation + } + } + } + + private struct OpenDocument { + let uri: String + let generation: Int + } + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "CopilotDocumentSync") + private static let carriageReturn: unichar = 0x0D + private static let lineFeed: unichar = 0x0A - private let documentManager = LSPDocumentManager() let preambleBuilder = CopilotPreambleBuilder() - private var currentURI: String? - private var serverSyncedURIs: Set = [] - private var pendingText: [String: String] = [:] + private let server: any CopilotDocumentServer + private let lengthLimit: Int + private let documentManager = LSPDocumentManager() + // Every editor talks to the one server, so a document number is never reused by another editor's sync. + private static var nextDocumentNumber = 1 private var uriMap: [UUID: String] = [:] - private var nextID = 1 - private var lastKnownGeneration: Int = 0 + + private var openDocument: OpenDocument? + private var needsFullText = false + private var mayContainCarriageReturn = false + private var linesMayDisagree = false + private var lineCheckTask: Task? + + private var outbox: [Outgoing] = [] + private var drainTask: Task? + + private weak var textView: TextView? + private weak var observedStorage: NSTextStorage? + private var editObservation: CopilotEditObservation? + private var currentTabID: @MainActor () -> UUID? = { nil } + private var isAllowed: @MainActor () -> Bool = { false } + private var schemaSource: @MainActor () -> CopilotPreambleBuilder.Source? = { nil } + private var preambleRebuild: Task? + + init(server: any CopilotDocumentServer, lengthLimit: Int) { + self.server = server + self.lengthLimit = lengthLimit + } func documentURI(for tabID: UUID) -> String { if let existing = uriMap[tabID] { return existing } - let fileURL = CopilotPreambleBuilder.contextDirectory.appendingPathComponent("query-\(nextID).sql") - nextID += 1 + let fileURL = CopilotPreambleBuilder.contextDirectory + .appendingPathComponent("query-\(Self.nextDocumentNumber).sql") + Self.nextDocumentNumber += 1 let uri = fileURL.absoluteString uriMap[tabID] = uri return uri } - func resetServerState() { - serverSyncedURIs.removeAll() - documentManager.resetAll() - pendingText.removeAll() + // MARK: - Editor + + func attach( + to textView: TextView, + tabID: @escaping @MainActor () -> UUID?, + isAllowed: @escaping @MainActor () -> Bool, + schemaSource: @escaping @MainActor () -> CopilotPreambleBuilder.Source? + ) { + self.textView = textView + currentTabID = tabID + self.isAllowed = isAllowed + self.schemaSource = schemaSource + guard let storage = textView.textStorage, let lines = textView.layoutManager?.lineStorage else { return } + observe(storage) + // Edits made before this observer existed can have left the line storage off the server's lines. + if storage.mutableString.range(of: "\r", options: .literal).location != NSNotFound { + linesMayDisagree = !lines.breaksLines(like: storage.mutableString) + } + synchronize(from: storage) + } + + // Only a tab the user types in, or asks a suggestion in, reaches the server. + func textViewDidReplaceDocument() { + guard let storage = textView?.textStorage else { return } + observe(storage) + needsFullText = true + } + + func detach() { + preambleRebuild?.cancel() + preambleRebuild = nil + editObservation = nil + observedStorage = nil + textView = nil + currentTabID = { nil } + isAllowed = { false } + schemaSource = { nil } + guard let open = openDocument else { return } + openDocument = nil + enqueue(.close(uri: open.uri, generation: open.generation)) } - /// Register document locally. Does NOT send to server. - func ensureDocumentOpen(tabID: UUID, text: String, languageId: String = "sql") { + func activate(tabID: UUID, text: String) { + guard canSend(length: text.utf16.count) else { + needsFullText = true + return + } let uri = documentURI(for: tabID) - let fullText = preambleBuilder.prependToText(text) - if !documentManager.isOpen(uri) { - _ = documentManager.openDocument(uri: uri, languageId: languageId, text: fullText) + let generation = server.generation + if let open = openDocument, open.uri != uri { + enqueue(.close(uri: open.uri, generation: open.generation)) + openDocument = nil } - currentURI = uri + sendFullText(text, uri: uri, generation: generation) + enqueue(.focus(uri: uri, generation: generation)) } - /// Open document at the server with preamble-prepended text - func didActivateTab(tabID: UUID, text: String, languageId: String = "sql") async { - let currentGeneration = CopilotService.shared.generation - if currentGeneration != lastKnownGeneration { - resetServerState() - lastKnownGeneration = currentGeneration + // MARK: - Server + + // Column reads can queue behind a running query, so a request never waits on them; the next sync sends the result. + func refreshPreamble(forStatementAt cursorOffset: Int, in text: String) { + guard preambleRebuild == nil, textView != nil, isAllowed(), let source = schemaSource() else { return } + preambleRebuild = Task { [weak self] in + guard let self else { return } + let rebuilt = await self.preambleBuilder.rebuiltPreamble(forStatementAt: cursorOffset, in: text, from: source) + self.preambleRebuild = nil + guard !Task.isCancelled, let rebuilt else { return } + let changesText = rebuilt.preamble != self.preambleBuilder.preamble + guard self.preambleBuilder.adopt(rebuilt), changesText else { return } + // Set with the new preamble, so no edit is placed with its line count against a document that has the old one. + self.needsFullText = true + } + } + + func flush() async { + if lineCheckTask != nil { + checkLines() + } + resynchronizeIfStale() + while let task = drainTask { + await task.value + } + } + + func currentDocumentInfo() -> (uri: String, version: Int)? { + guard let open = openDocument, !needsFullText, outbox.isEmpty, drainTask == nil, + open.generation == server.generation, + let tabID = currentTabID(), uriMap[tabID] == open.uri, + let version = documentManager.version(for: open.uri) else { return nil } + return (open.uri, version) + } + + // MARK: - Edits + + private func observe(_ storage: NSTextStorage) { + observedStorage = storage + linesMayDisagree = false + // Every way the text changes passes through the storage, the typing filters' edits included. + let token = NotificationCenter.default.addObserver( + forName: NSTextStorage.willProcessEditingNotification, + object: storage, + queue: nil + ) { [weak self] _ in + MainActor.assumeIsolated { self?.storageWillProcessEditing() } } + editObservation = CopilotEditObservation(token) + } + private func storageWillProcessEditing() { + guard let storage = observedStorage, storage.editedMask.contains(.editedCharacters) else { return } + guard textView?.textStorage === storage else { + needsFullText = true + return + } + guard let tabID = currentTabID(), canSend(length: storage.length) else { + needsFullText = true + return + } let uri = documentURI(for: tabID) - let fullText = preambleBuilder.prependToText(text) - ensureDocumentOpen(tabID: tabID, text: text, languageId: languageId) - - guard let client = CopilotService.shared.client else { return } - if !serverSyncedURIs.contains(uri) { - let item = LSPTextDocumentItem( - uri: uri, - languageId: languageId, - version: documentManager.version(for: uri) ?? 0, - text: fullText - ) - await client.didOpenDocument(item) - serverSyncedURIs.insert(uri) - - if let pending = pendingText.removeValue(forKey: uri) { - let pendingFull = preambleBuilder.prependToText(pending) - if let (versioned, changes) = documentManager.changeDocument(uri: uri, newText: pendingFull) { - await client.didChangeDocument(uri: versioned.uri, version: versioned.version, changes: changes) - } + guard let open = openDocument, open.uri == uri, open.generation == server.generation else { + activate(tabID: tabID, text: storage.string) + return + } + guard !needsFullText, let change = incrementalChange(in: storage) else { + sendFullText(storage.string, uri: uri, generation: open.generation) + return + } + enqueueChange(change, uri: uri, generation: open.generation) + } + + private func incrementalChange(in storage: NSTextStorage) -> LSPTextDocumentContentChangeEvent? { + guard let lines = textView?.layoutManager?.lineStorage else { return nil } + let editedRange = storage.editedRange + let replacedLength = editedRange.length - storage.changeInLength + // The layout manager updates its lines after this notification, so they still describe the text before the edit. + guard lines.length == storage.length - storage.changeInLength else { + Self.logger.debug("Line storage out of step with the edit, sending the full text") + return nil + } + let text = storage.mutableString + let inserted = text.substring(with: editedRange) + if inserted.utf16.contains(Self.carriageReturn) { + mayContainCarriageReturn = true + } + if mayContainCarriageReturn { + if Self.touchesLineBreakPair(in: text, editedRange: editedRange, replacedLength: replacedLength) { + linesMayDisagree = true } + if linesMayDisagree { return nil } + } + let lineOffset = preambleBuilder.preambleLineCount + guard let start = Self.position(of: editedRange.location, in: lines, lineOffset: lineOffset), + let end = Self.position(of: editedRange.location + replacedLength, in: lines, lineOffset: lineOffset) else { + return nil } - await client.didFocusDocument(uri: uri) + return LSPTextDocumentContentChangeEvent(range: LSPRange(start: start, end: end), text: inserted) } - /// Send text change with preamble prepended - func didChangeText(tabID: UUID, newText: String) async { - let currentGeneration = CopilotService.shared.generation - if currentGeneration != lastKnownGeneration { - resetServerState() - lastKnownGeneration = currentGeneration + // The server clamps a position between CR and LF to before the CR, and the editor's line storage splits or joins a + // CR LF pair an edit crosses, so such an edit sends the full text, as does every later one until no CR is left. + private static func touchesLineBreakPair(in text: NSString, editedRange: NSRange, replacedLength: Int) -> Bool { + if editedRange.location > 0, text.character(at: editedRange.location - 1) == carriageReturn { + return true } + let end = NSMaxRange(editedRange) + guard end < text.length, text.character(at: end) == lineFeed else { return false } + return replacedLength > 0 || (editedRange.length > 0 && text.character(at: end - 1) == carriageReturn) + } + + private static func position(of offset: Int, in lines: TextLineStorage, lineOffset: Int) -> LSPPosition? { + guard let line = offset == lines.length ? lines.last : lines.getLine(atOffset: offset) else { return nil } + return LSPPosition(line: line.index + lineOffset, character: offset - line.range.location) + } + // MARK: - Full Text + + private func canSend(length: Int) -> Bool { + length <= lengthLimit && server.hasClient && isAllowed() + } + + private func synchronize(from storage: NSTextStorage) { + guard let tabID = currentTabID(), canSend(length: storage.length) else { + needsFullText = true + return + } let uri = documentURI(for: tabID) - guard serverSyncedURIs.contains(uri) else { - pendingText[uri] = newText + if let open = openDocument, open.uri == uri, open.generation == server.generation { + sendFullText(storage.string, uri: uri, generation: open.generation) + } else { + activate(tabID: tabID, text: storage.string) + } + } + + private func resynchronizeIfStale() { + guard let storage = textView?.textStorage else { return } + if storage !== observedStorage { + observe(storage) + needsFullText = true + } + guard let tabID = currentTabID() else { return } + if let open = openDocument, open.uri == uriMap[tabID], open.generation == server.generation, !needsFullText { return } - let fullText = preambleBuilder.prependToText(newText) - guard let client = CopilotService.shared.client else { return } - guard let (versioned, changes) = documentManager.changeDocument(uri: uri, newText: fullText) else { return } - await client.didChangeDocument(uri: versioned.uri, version: versioned.version, changes: changes) + synchronize(from: storage) } - func didCloseTab(tabID: UUID) async { - let uri = documentURI(for: tabID) - guard let client = CopilotService.shared.client else { return } - guard let docId = documentManager.closeDocument(uri: uri) else { return } - await client.didCloseDocument(uri: docId.uri) - serverSyncedURIs.remove(uri) - uriMap.removeValue(forKey: tabID) - if currentURI == uri { currentURI = nil } + private func sendFullText(_ text: String, uri: String, generation: Int) { + mayContainCarriageReturn = (text as NSString).range(of: "\r", options: .literal).location != NSNotFound + if !mayContainCarriageReturn { + linesMayDisagree = false + } else if linesMayDisagree { + scheduleLineCheck() + } + needsFullText = false + let fullText = preambleBuilder.prependToText(text) + if let open = openDocument, open.uri == uri, open.generation == generation { + enqueueChange(LSPTextDocumentContentChangeEvent(text: fullText), uri: uri, generation: generation) + } else { + openDocument = OpenDocument(uri: uri, generation: generation) + enqueue(.open(uri: uri, text: fullText, generation: generation)) + } } - func currentDocumentInfo() -> (uri: String, version: Int)? { - guard let uri = currentURI else { return nil } - guard serverSyncedURIs.contains(uri) else { return nil } - guard let version = documentManager.version(for: uri) else { return nil } - return (uri, version) + // The storage notifies before the layout manager takes an edit in, so the check runs once the edit is done. + private func scheduleLineCheck() { + guard lineCheckTask == nil else { return } + lineCheckTask = Task { [weak self] in + guard !Task.isCancelled else { return } + self?.checkLines() + } + } + + // Every edit since the last full text went out as full text, so lines that follow the text now follow the server's. + private func checkLines() { + lineCheckTask?.cancel() + lineCheckTask = nil + guard linesMayDisagree, let textView, let storage = textView.textStorage, storage === observedStorage, + storage.editedMask.isEmpty, let lines = textView.layoutManager?.lineStorage else { return } + linesMayDisagree = !lines.breaksLines(like: storage.mutableString) + } + + // MARK: - Outbox + + private func enqueue(_ message: Outgoing) { + outbox.append(message) + scheduleDrain() + } + + private func enqueueChange(_ change: LSPTextDocumentContentChangeEvent, uri: String, generation: Int) { + if change.range == nil { + outbox.removeAll { message in + if case .change(let queuedURI, _, _) = message { return queuedURI == uri } + return false + } + } + if case .change(let lastURI, let changes, let lastGeneration)? = outbox.last, + lastURI == uri, lastGeneration == generation { + outbox[outbox.count - 1] = .change(uri: uri, changes: changes + [change], generation: generation) + return + } + enqueue(.change(uri: uri, changes: [change], generation: generation)) + } + + // One task sends everything in edit order; a task per notification gives no order between them. + private func scheduleDrain() { + guard drainTask == nil else { return } + drainTask = Task { + await self.drainOutbox() + } + } + + private func drainOutbox() async { + while !outbox.isEmpty { + await send(outbox.removeFirst()) + } + drainTask = nil + } + + private func send(_ message: Outgoing) async { + guard server.hasClient, server.generation == message.generation else { return } + switch message { + case let .close(uri, _): + guard documentManager.closeDocument(uri: uri) != nil else { return } + await server.didClose(uri: uri) + case let .open(uri, _, generation) where !isAllowed(): + if openDocument?.uri == uri, openDocument?.generation == generation { + openDocument = nil + } + case .change where !isAllowed(): + needsFullText = true + case .focus where !isAllowed(): + break + case let .open(uri, text, _): + await server.didOpen(documentManager.openDocument(uri: uri, languageId: "sql", text: text)) + case let .change(uri, changes, _): + guard let params = documentManager.changeDocument(uri: uri, changes: changes) else { return } + await server.didChange(params) + case let .focus(uri, _): + await server.didFocus(uri: uri) + } + } +} + +private final class CopilotEditObservation { + private let token: any NSObjectProtocol + + init(_ token: any NSObjectProtocol) { + self.token = token + } + + deinit { + NotificationCenter.default.removeObserver(token) } } diff --git a/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift b/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift index 6c5f4a1f8e..2733c6abf7 100644 --- a/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift +++ b/apple/TablePro/Core/AI/Copilot/CopilotInlineSource.swift @@ -22,7 +22,10 @@ final class CopilotInlineSource: InlineSuggestionSource { } func requestSuggestion(context: SuggestionContext) async throws -> InlineSuggestion? { - guard let client = CopilotService.shared.client else { return nil } + guard CopilotService.shared.client != nil else { return nil } + documentSync.refreshPreamble(forStatementAt: context.cursorOffset, in: context.fullText) + await documentSync.flush() + guard !Task.isCancelled, let client = CopilotService.shared.client else { return nil } guard let docInfo = documentSync.currentDocumentInfo() else { return nil } let editorSettings = AppSettingsManager.shared.editor diff --git a/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift b/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift index 4ba91198ca..514cc3a350 100644 --- a/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift +++ b/apple/TablePro/Core/AI/Copilot/CopilotPreambleBuilder.swift @@ -6,9 +6,36 @@ import Foundation import os import TableProPluginKit +import TableProSQLGrammar @MainActor final class CopilotPreambleBuilder { + struct Source { + let schemaProvider: SQLSchemaProvider + let databaseName: String + let databaseType: DatabaseType + let grammar: SQLLexicalGrammar + let budget: Int + } + + struct Rebuilt { + let preamble: String + let lineCount: Int + fileprivate let describedTables: [TableKey] + fileprivate let inputs: Source + fileprivate let revision: Int + } + + fileprivate struct TableKey: Hashable { + let name: String + let schema: String? + + init(_ table: TableInfo) { + name = table.name + schema = table.schema + } + } + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "CopilotPreambleBuilder") static let contextDirectory: URL = { @@ -20,33 +47,147 @@ final class CopilotPreambleBuilder { private(set) var preambleLineCount: Int = 0 + private var inputs: Source? + // Every table whose columns were read, with or without any, so a table that has none is not read on every request. + private var describedTables: [TableKey] = [] + private var revision = 0 + func buildPreamble( schemaProvider: SQLSchemaProvider, databaseName: String, - databaseType: DatabaseType + databaseType: DatabaseType, + referencing documentText: String? = nil, + grammar: SQLLexicalGrammar? = nil, + maxTables: Int = AppSettingsManager.shared.ai.maxSchemaTables ) async { try? FileManager.default.createDirectory(at: Self.contextDirectory, withIntermediateDirectories: true) + let inputs = Source( + schemaProvider: schemaProvider, + databaseName: databaseName, + databaseType: databaseType, + grammar: grammar ?? databaseType.lexicalGrammar, + budget: max(maxTables, 0) + ) + self.inputs = inputs let tables = await schemaProvider.getTables() - guard !tables.isEmpty else { - preamble = "" - preambleLineCount = 0 - return + let defaultSchema = await schemaProvider.getDefaultSchema() + var first: [Int] = [] + if tables.count > inputs.budget, let documentText { + first = await Self.referencedTableIndices( + in: documentText, + cursorOffset: nil, + grammar: inputs.grammar, + tables: tables, + defaultSchema: defaultSchema + ) } + let picked = Self.budgeted(tables, first: first, keeping: [], budget: inputs.budget) + guard let rendered = await Self.render( + picked, catalogCount: tables.count, inputs: inputs, defaultSchema: defaultSchema + ) else { return } + apply(rendered) - let defaultSchema = await schemaProvider.getDefaultSchema() + Self.logger.info("Copilot schema preamble: \(tables.count) tables, \(self.preambleLineCount) lines") + } + + // A tab moved to another database scope names that scope's tables, so a new scope starts the preamble over, as + // does an editor that had no schema when Copilot started. + func rebuiltPreamble(forStatementAt cursorOffset: Int, in text: String, from source: Source) async -> Rebuilt? { + let base = revision + // A lower "Max schema tables" has to take effect on the next request, as it does in the chat. + let startsOver = inputs.map { + $0.schemaProvider !== source.schemaProvider || $0.databaseName != source.databaseName + || $0.budget != source.budget || $0.databaseType != source.databaseType || $0.grammar != source.grammar + } ?? true + let target = startsOver ? source : inputs ?? source + let kept = startsOver ? [] : describedTables + let described = Set(kept) + let tables = await target.schemaProvider.getTables() + if !startsOver, tables.count <= target.budget, tables.allSatisfy({ described.contains(TableKey($0)) }) { + return nil + } + let defaultSchema = await target.schemaProvider.getDefaultSchema() + let referenced = await Self.referencedTableIndices( + in: text, + cursorOffset: cursorOffset, + grammar: target.grammar, + tables: tables, + defaultSchema: defaultSchema + ).prefix(target.budget) + guard startsOver || referenced.contains(where: { !described.contains(TableKey(tables[$0])) }) else { return nil } + let picked = Self.budgeted(tables, first: Array(referenced), keeping: kept, budget: target.budget) + guard let rendered = await Self.render( + picked, catalogCount: tables.count, inputs: target, defaultSchema: defaultSchema + ) else { return nil } + return Rebuilt( + preamble: rendered.preamble, + lineCount: rendered.lineCount, + describedTables: rendered.describedTables, + inputs: target, + revision: base + ) + } + + // A rebuild that started before another build landed describes a preamble the sync no longer has. + func adopt(_ rebuilt: Rebuilt) -> Bool { + guard rebuilt.revision == revision else { return false } + apply(rebuilt) + return true + } + func prependToText(_ text: String) -> String { + guard !preamble.isEmpty else { return text } + return preamble + text + } + + private func apply(_ rebuilt: Rebuilt) { + preamble = rebuilt.preamble + preambleLineCount = rebuilt.lineCount + describedTables = rebuilt.describedTables + inputs = rebuilt.inputs + revision += 1 + } + + private static func budgeted(_ tables: [TableInfo], first: [Int], keeping kept: [TableKey], budget: Int) -> [TableInfo] { + var catalog: [TableKey: TableInfo] = [:] + for table in tables where catalog[TableKey(table)] == nil { + catalog[TableKey(table)] = table + } + var picked: [TableInfo] = [] + var seen = Set() + for table in first.map({ tables[$0] }) + kept.compactMap({ catalog[$0] }) + tables { + guard picked.count < budget else { break } + if seen.insert(TableKey(table)).inserted { + picked.append(table) + } + } + return picked + } + + private static func render( + _ picked: [TableInfo], + catalogCount: Int, + inputs: Source, + defaultSchema: String? + ) async -> Rebuilt? { + guard catalogCount > 0 else { + return Rebuilt(preamble: "", lineCount: 0, describedTables: [], inputs: inputs, revision: 0) + } + + let dialect = singleLine(inputs.databaseType.rawValue) var lines: [String] = [] - lines.append("-- Database: \(databaseName)") - lines.append("-- Dialect: \(databaseType.rawValue): use \(databaseType.rawValue) syntax only") + lines.append("-- Database: \(singleLine(inputs.databaseName))") + lines.append("-- Dialect: \(dialect): use \(dialect) syntax only") lines.append("--") - for table in tables { - let columns = await schemaProvider.getColumns(for: table.name, schema: table.schema) + for table in picked { + guard !Task.isCancelled else { return nil } + let columns = await inputs.schemaProvider.getColumns(for: table.name, schema: table.schema) guard !columns.isEmpty else { continue } let colDefs = columns.map { col -> String in - var parts = ["\(col.name) \(col.dataType)"] + var parts = ["\(singleLine(col.name)) \(singleLine(col.dataType))"] if col.isPrimaryKey { parts.append("PK") } if !col.isNullable { parts.append("NOT NULL") } return parts.joined(separator: " ") @@ -57,19 +198,45 @@ final class CopilotPreambleBuilder { implicitSchemaName: defaultSchema, quote: { $0 } ) - lines.append("-- \(tableName)(\(colDefs.joined(separator: ", ")))") + lines.append("-- \(singleLine(tableName))(\(colDefs.joined(separator: ", ")))") } - lines.append("") + if catalogCount > picked.count { + lines.append("-- … and \(catalogCount - picked.count) more tables") + } - preamble = lines.joined(separator: "\n") - preambleLineCount = lines.count - 1 + lines.append("") + return Rebuilt( + preamble: lines.joined(separator: "\n"), + lineCount: lines.count - 1, + describedTables: picked.map(TableKey.init), + inputs: inputs, + revision: 0 + ) + } - Self.logger.info("Copilot schema preamble: \(tables.count) tables, \(self.preambleLineCount) lines") + // The document sync places edits by line number, so a name must not add a line break. + private static func singleLine(_ name: String) -> String { + name.replacingOccurrences(of: "\r", with: " ", options: .literal) + .replacingOccurrences(of: "\n", with: " ", options: .literal) } - func prependToText(_ text: String) -> String { - guard !preamble.isEmpty else { return text } - return preamble + text + // Scanning a large script takes too long for the main actor. + @concurrent + nonisolated private static func referencedTableIndices( + in text: String, + cursorOffset: Int?, + grammar: SQLLexicalGrammar, + tables: [TableInfo], + defaultSchema: String? + ) async -> [Int] { + let scope = cursorOffset.map { + SQLStatementScanner.statementAtCursor(in: text, cursorPosition: $0, grammar: grammar) + } ?? text + return AISchemaContext.indices( + of: AISchemaContext.referencedTables(in: scope, grammar: grammar), + in: tables, + defaultSchema: defaultSchema + ) } } diff --git a/apple/TablePro/Core/AI/GeminiProvider.swift b/apple/TablePro/Core/AI/GeminiProvider.swift index b59e1a7663..b2197bd9cd 100644 --- a/apple/TablePro/Core/AI/GeminiProvider.swift +++ b/apple/TablePro/Core/AI/GeminiProvider.swift @@ -321,6 +321,10 @@ final class GeminiProvider: ChatTransport { } } } + if let candidates = json["candidates"] as? [[String: Any]], + candidates.first?["finishReason"] as? String == "MAX_TOKENS" { + events.append(.outputLimitReached(.outputTokens)) + } if let usageMetadata = json["usageMetadata"] as? [String: Any] { if let prompt = usageMetadata["promptTokenCount"] as? Int { state.inputTokens = prompt diff --git a/apple/TablePro/Core/AI/InlineSuggestion/AIChatInlineSource.swift b/apple/TablePro/Core/AI/InlineSuggestion/AIChatInlineSource.swift index 266516fc4c..2ddff7a1fd 100644 --- a/apple/TablePro/Core/AI/InlineSuggestion/AIChatInlineSource.swift +++ b/apple/TablePro/Core/AI/InlineSuggestion/AIChatInlineSource.swift @@ -72,6 +72,12 @@ final class AIChatInlineSource: InlineSuggestionSource { options: ChatTransportOptions( model: resolved.model, systemPrompt: systemPrompt, + reasoningEffort: AIProviderFactory.reasoningEffort( + resolved.config.reasoningEffort, + config: resolved.config, + model: resolved.model, + onlyWhereRequired: true + ), sessionId: sessionId ) ) diff --git a/apple/TablePro/Core/AI/Models/AIModelCatalog.swift b/apple/TablePro/Core/AI/Models/AIModelCatalog.swift index 3483f78f50..3a4c8788e5 100644 --- a/apple/TablePro/Core/AI/Models/AIModelCatalog.swift +++ b/apple/TablePro/Core/AI/Models/AIModelCatalog.swift @@ -15,6 +15,7 @@ final class AIModelCatalog: @unchecked Sendable { private let lock = NSLock() private var fetched: [UUID: [String: AIModelInfo]] = [:] private var refreshTokens: [UUID: UUID] = [:] + private var revisions: [UUID: Int] = [:] init() {} @@ -24,6 +25,7 @@ final class AIModelCatalog: @unchecked Sendable { defer { lock.unlock() } refreshTokens.removeValue(forKey: providerID) fetched[providerID] = Self.indexed(models) + revisions[providerID, default: 0] += 1 } func remove(providerID: UUID) { @@ -31,6 +33,24 @@ final class AIModelCatalog: @unchecked Sendable { defer { lock.unlock() } refreshTokens.removeValue(forKey: providerID) fetched.removeValue(forKey: providerID) + revisions[providerID, default: 0] += 1 + } + + func revision(providerID: UUID) -> Int { + lock.lock() + defer { lock.unlock() } + return revisions[providerID, default: 0] + } + + // For a reader such as the chat panel: it never overtakes a refresh a Save started, and its + // answer is dropped once anything newer was written or begun since it read `revision`. + func store(providerID: UUID, models: [AIModelInfo], ifUnchangedSince revision: Int) { + guard !models.isEmpty else { return } + lock.lock() + defer { lock.unlock() } + guard revisions[providerID, default: 0] == revision else { return } + fetched[providerID] = Self.indexed(models) + revisions[providerID, default: 0] += 1 } /// Loads a provider's list again after Save dropped the one on record. Open chat windows read @@ -49,6 +69,7 @@ final class AIModelCatalog: @unchecked Sendable { lock.lock() defer { lock.unlock() } refreshTokens[providerID] = token + revisions[providerID, default: 0] += 1 return token } @@ -59,6 +80,7 @@ final class AIModelCatalog: @unchecked Sendable { refreshTokens.removeValue(forKey: providerID) guard !models.isEmpty else { return } fetched[providerID] = Self.indexed(models) + revisions[providerID, default: 0] += 1 } private static func indexed(_ models: [AIModelInfo]) -> [String: AIModelInfo] { diff --git a/apple/TablePro/Core/AI/Models/AIModelOverlay.swift b/apple/TablePro/Core/AI/Models/AIModelOverlay.swift index 45993de477..8770d970c2 100644 --- a/apple/TablePro/Core/AI/Models/AIModelOverlay.swift +++ b/apple/TablePro/Core/AI/Models/AIModelOverlay.swift @@ -47,11 +47,9 @@ enum AIModelOverlay { id: modelID, maxOutputTokens: entry.maxOutputTokens, modalities: [.text, .image], - reasoning: AIReasoningSupport( - mode: .effortOnly, - effortLevels: entry.effortLevels, - defaultEffort: .medium - ) + reasoning: entry.effortLevels.isEmpty + ? .unsupported + : AIReasoningSupport(mode: .effortOnly, effortLevels: entry.effortLevels, defaultEffort: .medium) ) } @@ -78,7 +76,14 @@ enum AIModelOverlay { let maxOutputTokens: Int? } + // First match wins. The models without reasoning come first: OpenAI rejects a reasoning + // effort for them, and `gpt-5-chat-latest` would otherwise match `gpt-5`. private static let openAITable: [OpenAIEntry] = [ + OpenAIEntry( + patterns: ["chat-latest", "chatgpt-4o", "gpt-4", "gpt-3.5"], + effortLevels: [], + maxOutputTokens: nil + ), OpenAIEntry( patterns: ["gpt-5.6", "gpt-5-6"], effortLevels: ReasoningEffort.allCases, diff --git a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift index 87cba934b7..03e7c264ec 100644 --- a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift +++ b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift @@ -22,6 +22,7 @@ final class OpenAICompatibleProvider: ChatTransport { private let model: String private let maxOutputTokens: Int? private let treatsForbiddenAsAuthFailure: Bool + private let modelIDs: AIModelIDFilter private let providerID: UUID? private let catalog: AIModelCatalog private let session: URLSession @@ -36,6 +37,7 @@ final class OpenAICompatibleProvider: ChatTransport { model: String = "", maxOutputTokens: Int? = nil, treatsForbiddenAsAuthFailure: Bool = false, + modelIDs: AIModelIDFilter = .all, providerID: UUID? = nil, catalog: AIModelCatalog = .shared, session: URLSession = URLSession(configuration: .ephemeral) @@ -49,6 +51,7 @@ final class OpenAICompatibleProvider: ChatTransport { self.model = model.trimmingCharacters(in: .whitespacesAndNewlines) self.maxOutputTokens = maxOutputTokens self.treatsForbiddenAsAuthFailure = treatsForbiddenAsAuthFailure + self.modelIDs = modelIDs self.providerID = providerID self.catalog = catalog self.session = session @@ -66,6 +69,7 @@ final class OpenAICompatibleProvider: ChatTransport { model: config.model, maxOutputTokens: config.maxOutputTokens, treatsForbiddenAsAuthFailure: config.preset?.rejectsBadKeyWithForbidden ?? false, + modelIDs: config.preset?.modelIDFilter(forEndpoint: config.endpoint) ?? .all, providerID: config.id, session: session ) @@ -171,6 +175,13 @@ final class OpenAICompatibleProvider: ChatTransport { if finishReason == "tool_calls" { events.append(contentsOf: state.flushToolUseEnds()) } + if finishReason == "length" { + events.append(.outputLimitReached(.outputTokens)) + } + } + // Ollama's own API names the cut in `done_reason`. + if (json["done"] as? Bool) == true, (json["done_reason"] as? String) == "length" { + events.append(.outputLimitReached(.outputTokens)) } if let usage = json["usage"] as? [String: Any], @@ -319,6 +330,11 @@ final class OpenAICompatibleProvider: ChatTransport { return Self.looksLikeAnAPIResponse(data: data, response: httpResponse) } + // The placeholder model is unknown by design; that answer proves the route and the key. + if model.isEmpty, statusCode == 404, Self.isUnknownModelError(data) { + return true + } + let errorBody = String(data: data, encoding: .utf8) ?? "" throw AIProviderError.mapHTTPError( statusCode: statusCode, @@ -329,6 +345,13 @@ final class OpenAICompatibleProvider: ChatTransport { } } + private static func isUnknownModelError(_ data: Data) -> Bool { + guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let error = json["error"] as? [String: Any] + else { return false } + return error["code"] as? String == "model_not_found" + } + /// A wrong Base URL often lands on a reverse proxy's login page or a single-page app's /// fallback route, both of which answer 200 with HTML that the chat stream cannot read. private static func looksLikeAnAPIResponse(data: Data, response: HTTPURLResponse) -> Bool { @@ -600,7 +623,9 @@ final class OpenAICompatibleProvider: ChatTransport { ) } - return modelsArray.compactMap(Self.decodeModel(_:)).sorted { $0.id < $1.id } + return modelsArray.compactMap(Self.decodeModel(_:)) + .filter { modelIDs.lists($0.id) } + .sorted { $0.id < $1.id } } private func fetchOllamaModels() async throws -> [String] { diff --git a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift index e246479c13..2ab633f0f4 100644 --- a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift +++ b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift @@ -66,11 +66,68 @@ final class OpenAIResponsesProvider: ChatTransport { } func fetchAvailableModels() async throws -> [AIModelInfo] { - try await fetchModelIDs().map { AIModelInfo(id: $0) } + if dialect == .xai, let languageModels = await fetchXAILanguageModels() { + return languageModels + } + let host = resolvedEndpoint?.apiBase.host + return try await fetchModelEntries(resource: AIEndpointStyle.responses.modelsResource, key: "data") + .filter { Self.listsChatModel($0, dialect: dialect, host: host) } + .compactMap { $0["id"] as? String } + .sorted() + .map { AIModelInfo(id: $0) } } - private func fetchModelIDs() async throws -> [String] { - let url = try requestURL(AIEndpointStyle.responses.modelsResource) + // `/v1/models` carries no modality; a gateway in xAI's place may not serve `language-models`. + private func fetchXAILanguageModels() async -> [AIModelInfo]? { + guard let entries = try? await fetchModelEntries(resource: "language-models", key: "models") else { + return nil + } + let models = entries.compactMap(Self.decodeXAILanguageModel(_:)).sorted { $0.id < $1.id } + return models.isEmpty ? nil : models + } + + static func decodeXAILanguageModel(_ json: [String: Any]) -> AIModelInfo? { + guard let id = json["id"] as? String, !id.isEmpty, + let outputs = json["output_modalities"] as? [String], outputs.contains("text") + else { return nil } + let inputs = json["input_modalities"] as? [String] ?? [] + return AIModelInfo(id: id, modalities: inputs.contains("image") ? [.text, .image] : [.text]) + } + + static func listsChatModel(_ json: [String: Any], dialect: ResponsesDialect, host: String?) -> Bool { + guard let id = json["id"] as? String, !id.isEmpty else { return false } + switch dialect { + case .xai: + if json["image_price"] != nil { return false } + return !["imagine", "-image", "voice"].contains(where: { id.contains($0) }) + case .openAI: + guard let host, isOpenAIHost(host) else { return true } + return isOpenAIChatModel(id) + } + } + + // Only OpenAI's own hosts are known to list its non-chat models; a gateway lists what it routes. + private static func isOpenAIHost(_ host: String) -> Bool { + let host = host.lowercased() + return host == "api.openai.com" || host.hasSuffix(".api.openai.com") + } + + // Checked against the Responses column of every live model on platform.openai.com/docs/models. + // A fine-tune is `ft::::`; only its base model says what it is. + static func isOpenAIChatModel(_ id: String) -> Bool { + let base = id.hasPrefix("ft:") ? String(id.dropFirst(3).prefix { $0 != ":" }) : id + let families = ["gpt-", "chatgpt-", "chat-latest", "codex-"] + let isOSeries = base.first == "o" && base.dropFirst().first?.isNumber == true + guard isOSeries || families.contains(where: { base.hasPrefix($0) }) else { return false } + let nonChat = [ + "image", "audio", "realtime", "transcribe", "tts", "whisper", "-live", "search", + "embedding", "moderation", "instruct", "deep-research", "computer-use" + ] + return !nonChat.contains(where: { base.contains($0) }) + } + + private func fetchModelEntries(resource: String, key: String) async throws -> [[String: Any]] { + let url = try requestURL(resource) var request = URLRequest(url: url) request.timeoutInterval = AIProvider.modelListTimeout if let apiKey, !apiKey.isEmpty { @@ -87,11 +144,11 @@ final class OpenAIResponsesProvider: ChatTransport { guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 200, let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let modelsArray = json["data"] as? [[String: Any]] + let entries = json[key] as? [[String: Any]] else { throw AIProviderError.networkError("Failed to fetch models") } - return modelsArray.compactMap { $0["id"] as? String }.sorted() + return entries } func testConnection() async throws -> Bool { @@ -371,15 +428,17 @@ final class OpenAIResponsesProvider: ChatTransport { return [.textDelta(delta)] } return [] - case "response.completed": - if let responseObj = json["response"] as? [String: Any], - let usage = responseObj["usage"] as? [String: Any] { + case "response.completed", "response.incomplete": + let responseObj = json["response"] as? [String: Any] + if let usage = responseObj?["usage"] as? [String: Any] { let input = usage["input_tokens"] as? Int ?? 0 let output = usage["output_tokens"] as? Int ?? 0 state.inputTokens = input state.outputTokens = output } - return [] + let details = responseObj?["incomplete_details"] as? [String: Any] + guard type == "response.incomplete", details?["reason"] as? String == "max_output_tokens" else { return [] } + return [.outputLimitReached(.outputTokens)] case "response.failed": if let responseObj = json["response"] as? [String: Any], let errorObj = responseObj["error"] as? [String: Any], diff --git a/apple/TablePro/Core/AI/Registry/AIProviderDescriptor.swift b/apple/TablePro/Core/AI/Registry/AIProviderDescriptor.swift index a85ce002bc..1fba48bf3c 100644 --- a/apple/TablePro/Core/AI/Registry/AIProviderDescriptor.swift +++ b/apple/TablePro/Core/AI/Registry/AIProviderDescriptor.swift @@ -53,24 +53,60 @@ struct AIProviderDescriptor: Sendable { var allowsMaxOutputTokens: Bool { capabilities.contains(.maxOutputTokens) } var fetchesModelList: Bool { capabilities.contains(.modelListFetchable) } - func curatedModel(forID id: String) -> CuratedModel? { - curatedModels.first(where: { $0.id == id }) + func curatedModels(for preset: AIProviderPreset?) -> [CuratedModel] { + guard let preset, !preset.curatedModels.isEmpty else { return curatedModels } + return preset.curatedModels } - /// `fetched` is what the provider's own model list said about the model, when it has been - /// loaded. It wins over the offline tables, and its silence leaves the provider's envelope open. - func supportedEffortLevels(forModelID id: String, fetched: AIModelInfo? = nil) -> [ReasoningEffort] { - guard supportsReasoning else { return [] } - if let reasoning = fetched?.reasoning ?? AIModelOverlay.reasoning(providerTypeID: typeID, modelID: id) { - return reasoning.effortLevels + func curatedModel(forID id: String, preset: AIProviderPreset? = nil) -> CuratedModel? { + curatedModels(for: preset).first(where: { $0.id == id }) + } + + // The provider's own list wins over the offline tables; its silence leaves the envelope open. + func reasoning( + forModelID id: String, + fetched: AIModelInfo? = nil, + preset: AIProviderPreset? = nil + ) -> AIReasoningSupport { + guard supportsReasoning else { return .unsupported } + if let known = fetched?.reasoning + ?? preset?.reasoning(forModelID: id) + ?? AIModelOverlay.reasoning(providerTypeID: typeID, modelID: id) { + return known } if let effortLevelResolver { - return effortLevelResolver(id) + return AIReasoningSupport(mode: .effortOnly, effortLevels: effortLevelResolver(id)) } - if let curated = curatedModel(forID: id), !curated.supportedEffortLevels.isEmpty { - return curated.supportedEffortLevels + if let curated = curatedModel(forID: id, preset: preset), !curated.supportedEffortLevels.isEmpty { + return AIReasoningSupport( + mode: .effortOnly, + effortLevels: curated.supportedEffortLevels, + defaultEffort: curated.defaultEffort + ) + } + return AIReasoningSupport(mode: .effortOnly, effortLevels: [.low, .medium, .high]) + } + + func supportedEffortLevels( + forModelID id: String, + fetched: AIModelInfo? = nil, + preset: AIProviderPreset? = nil + ) -> [ReasoningEffort] { + reasoning(forModelID: id, fetched: fetched, preset: preset).effortLevels + } + + // The saved effort belongs to the saved model, and the chat panel can send another one. + func effort( + _ requested: ReasoningEffort?, + forModelID id: String, + fetched: AIModelInfo? = nil, + preset: AIProviderPreset? = nil + ) -> ReasoningEffort? { + let reasoning = reasoning(forModelID: id, fetched: fetched, preset: preset) + guard let requested else { + return reasoning.isMandatory ? reasoning.defaultEffort ?? reasoning.effortLevels.first : nil } - return [.low, .medium, .high] + return reasoning.clampedEffort(requested) } func supportsImages(fetched: AIModelInfo?) -> Bool { diff --git a/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift b/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift index c58dd2fce1..5101636ad3 100644 --- a/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift +++ b/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift @@ -1,10 +1,3 @@ -// -// SQLSchemaProvider.swift -// TablePro -// -// Cached database schema provider for autocomplete -// - import Foundation import os import TableProAutocomplete @@ -35,7 +28,9 @@ actor SQLSchemaProvider { } private var tables: [TableInfo] = [] - private var columnCache: [ColumnCacheKey: [ColumnInfo]] = [:] + private var columnCache: [ColumnCacheKey: [ColumnInfo]] = [:] { + didSet { cachedTableColumnItems = nil } + } private var columnAccessOrder: [ColumnCacheKey] = [] private var columnTasks: [ColumnCacheKey: Task<[ColumnInfo]?, Never>] = [:] private var loadTask: Task? @@ -61,20 +56,27 @@ actor SQLSchemaProvider { let fetchAllColumns: @Sendable () async throws -> [String: [ColumnInfo]] let fetchSchemaTables: (@Sendable (_ schema: String) async throws -> [TableInfo])? let sampleFieldPaths: (@Sendable (_ table: String, _ limit: Int) async throws -> [PluginFieldPath])? + let fetchForeignKeys: (@Sendable (_ table: String, _ schema: String?) async throws -> [ForeignKeyInfo])? init( fetchColumns: @escaping @Sendable (_ table: String, _ schema: String?) async throws -> [ColumnInfo], fetchAllColumns: @escaping @Sendable () async throws -> [String: [ColumnInfo]], fetchSchemaTables: (@Sendable (_ schema: String) async throws -> [TableInfo])? = nil, - sampleFieldPaths: (@Sendable (_ table: String, _ limit: Int) async throws -> [PluginFieldPath])? = nil + sampleFieldPaths: (@Sendable (_ table: String, _ limit: Int) async throws -> [PluginFieldPath])? = nil, + fetchForeignKeys: (@Sendable (_ table: String, _ schema: String?) async throws -> [ForeignKeyInfo])? = nil ) { self.fetchColumns = fetchColumns self.fetchAllColumns = fetchAllColumns self.fetchSchemaTables = fetchSchemaTables self.sampleFieldPaths = sampleFieldPaths + self.fetchForeignKeys = fetchForeignKeys } } + // Keyed like the column cache and cleared with it, so a table's keys are read once per table list. + private var foreignKeyCache: [ColumnCacheKey: [SQLCompletionForeignKey]] = [:] + private var foreignKeyTasks: [ColumnCacheKey: Task<[SQLCompletionForeignKey]?, Never>] = [:] + private var fieldPathCache: [String: [PluginFieldPath]] = [:] private var fieldPathTasks: [String: Task<[PluginFieldPath], Never>] = [:] @@ -90,7 +92,19 @@ actor SQLSchemaProvider { // Every table-slot request reads all of them, so they are built once per table list, not per keystroke. private var unqualifiedTableItems: (defaultSchema: String?, items: [SQLCompletionItem])? + private struct TableColumnItems { + let defaultSchema: String? + let eagerLoadSchema: String? + let items: [SQLCompletionItem] + } + + // Every column request before a FROM reads all of them, so they are built once per column cache and table list. + private var cachedTableColumnItems: TableColumnItems? + private(set) var tableColumnItemsBuildCount = 0 + private weak var cachedDriver: (any DatabaseDriver)? + // Read from the driver that filled the table list, so learning it opens no connection. + private var fetchesAllColumnsInOneRequest = false private let metadataSource: ColumnMetadataSource? private var connectionInfo: DatabaseConnection? @@ -104,7 +118,6 @@ actor SQLSchemaProvider { // MARK: - Public API - /// Load schema from the database (driver should already be connected). /// Concurrent callers await the same in-flight Task instead of firing duplicate queries. func loadSchema(using driver: DatabaseDriver, connection: DatabaseConnection? = nil) async { if let existing = loadTask { @@ -119,6 +132,7 @@ actor SQLSchemaProvider { Self.logger.info("[schema] loadSchema starting new fetch") let t0 = Date() self.cachedDriver = driver + self.fetchesAllColumnsInOneRequest = driver.fetchesAllColumnsInOneRequest self.eagerLoadSchema = (driver as? SchemaSwitchable)?.currentSchema if let connection { self.connectionInfo = connection } @@ -143,17 +157,14 @@ actor SQLSchemaProvider { noteTableListWritten() } - /// Get the current connection info func getConnectionInfo() -> DatabaseConnection? { connectionInfo } - /// Get all tables func getTables() -> [TableInfo] { tables } - /// Get columns for a specific table (with LRU caching) func getColumns(for tableName: String, schema: String? = nil) async -> [ColumnInfo] { let key = cacheKey(table: tableName, schema: schema) @@ -198,9 +209,62 @@ actor SQLSchemaProvider { while columnAccessOrder.count > Self.maxCachedTables { let evicted = columnAccessOrder.removeFirst() columnCache.removeValue(forKey: evicted) + foreignKeyCache.removeValue(forKey: evicted) } } + // Read after the ON of a join, never per keystroke: concurrent callers await the read in flight, and a failed read + // is kept as no keys until the table list is read again, so a catalog that refuses it is not asked on every key. + func foreignKeys(of tableName: String, schema: String? = nil) async -> [SQLCompletionForeignKey] { + let key = cacheKey(table: tableName, schema: schema) + if let cached = foreignKeyCache[key] { return cached } + if let inFlight = foreignKeyTasks[key] { return await inFlight.value ?? [] } + guard connectionInfo?.type.supportsForeignKeys != false else { return [] } + + let readSchema = schema.map { catalogSpelling(ofSchema: $0, isQuoted: false) ?? $0 } + let read: @Sendable () async throws -> [ForeignKeyInfo] + if let source = metadataSource { + guard let fetch = source.fetchForeignKeys else { return [] } + read = { try await fetch(tableName, readSchema) } + } else if let driver = cachedDriver { + guard driver.connection.type.supportsForeignKeys else { return [] } + read = { try await driver.fetchForeignKeys(table: tableName, schema: readSchema) } + } else { + return [] + } + + let task = Task<[SQLCompletionForeignKey]?, Never> { + do { + let rows = try await read() + return Self.completionForeignKeys(rows) + } catch { + Self.logger.debug( + "[schema] foreign key read failed for completion: \(error.publicLogShape, privacy: .public)" + ) + return nil + } + } + foreignKeyTasks[key] = task + let fetched = await task.value + guard foreignKeyTasks[key] == task else { return fetched ?? [] } + foreignKeyTasks[key] = nil + let keys = fetched ?? [] + foreignKeyCache[key] = keys + return keys + } + + private static func completionForeignKeys(_ rows: [ForeignKeyInfo]) -> [SQLCompletionForeignKey] { + SQLCompletionForeignKey.grouping(rows.map { + SQLCompletionForeignKey.Row( + name: $0.name, + column: $0.column, + referencedTable: $0.referencedTable, + referencedSchema: $0.referencedSchema, + referencedColumn: $0.referencedColumn + ) + }) + } + /// A lookup that names no schema reads the schema an unqualified fetch targets, which is the one /// the column preload fills, so `orders` and `public.orders` on PostgreSQL share one entry. private func cacheKey(table: String, schema: String?) -> ColumnCacheKey { @@ -243,11 +307,14 @@ actor SQLSchemaProvider { self.columnCache.removeAll() self.columnAccessOrder.removeAll() self.columnTasks.removeAll() + self.foreignKeyCache.removeAll() + self.foreignKeyTasks.removeAll() self.fieldPathCache.removeAll() self.fieldPathTasks.removeAll() self.onDemandSchemaTables.removeAll() self.onDemandSchemaTableTasks.removeAll() self.cachedDriver = driver + self.fetchesAllColumnsInOneRequest = driver.fetchesAllColumnsInOneRequest self.eagerLoadSchema = (driver as? SchemaSwitchable)?.currentSchema if let connection { self.connectionInfo = connection } noteTableListWritten() @@ -260,6 +327,8 @@ actor SQLSchemaProvider { columnCache.removeAll() columnAccessOrder.removeAll() columnTasks.removeAll() + foreignKeyCache.removeAll() + foreignKeyTasks.removeAll() fieldPathCache.removeAll() fieldPathTasks.removeAll() } @@ -306,6 +375,13 @@ actor SQLSchemaProvider { columnPreload = .awaitingTables(generation: generation) return } + guard fetchesAllColumnsInOneRequest else { + Self.logger.info( + "[schema] column preload skipped tableCount=\(tableCount) fetchesAllColumnsInOneRequest=false" + ) + columnPreload = .settled(generation: generation, outcome: .skipped) + return + } guard tableCount <= Self.maxCachedTables else { Self.logger.info( "[schema] column preload skipped tableCount=\(tableCount) limit=\(Self.maxCachedTables)" @@ -354,6 +430,7 @@ actor SQLSchemaProvider { private func noteTableListWritten() { tableListGeneration &+= 1 unqualifiedTableItems = nil + cachedTableColumnItems = nil switch columnPreload { case .awaitingTables: startColumnPreloadIfStale(retryingFailure: true) @@ -397,7 +474,6 @@ actor SQLSchemaProvider { await task.value } - /// Find table name from alias func resolveAlias(_ aliasOrName: String, in references: [TableReference]) -> String? { let lowerName = aliasOrName.lowercased() @@ -650,7 +726,6 @@ actor SQLSchemaProvider { return [] } - /// Get completion items for all columns of tables in scope func allColumnsInScope(for references: [TableReference]) async -> [SQLCompletionItem] { // swiftlint:disable:next large_tuple var itemDataBuilder: [( @@ -710,12 +785,29 @@ actor SQLSchemaProvider { } } - /// Only the bulk cache can answer a statement that names no table, so once an editor has asked, a - /// stale cache starts a read here. This call answers from what is cached and never waits. + // Only the bulk cache can answer a statement that names no table, so once an editor has asked, a stale cache + // starts a read here. This call answers from what is cached and never waits. func allColumnsFromCachedTables() async -> [SQLCompletionItem] { if hasEditorDemand { startColumnPreloadIfStale(retryingFailure: false) } guard !columnCache.isEmpty else { return [] } + let defaultSchema = getDefaultSchema() + if let cached = cachedTableColumnItems, + cached.defaultSchema == defaultSchema, + cached.eagerLoadSchema == eagerLoadSchema { + return cached.items + } + let items = buildTableColumnItems() + tableColumnItemsBuildCount += 1 + cachedTableColumnItems = TableColumnItems( + defaultSchema: defaultSchema, + eagerLoadSchema: eagerLoadSchema, + items: items + ) + return items + } + + private func buildTableColumnItems() -> [SQLCompletionItem] { let canonicalNames = Dictionary( tables.map { ($0.name.lowercased(), $0.name) }, uniquingKeysWith: { first, _ in first } diff --git a/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift b/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift index 33c13b438c..5c2e7b65bf 100644 --- a/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/AnyChangeManager.swift @@ -15,6 +15,7 @@ protocol ChangeManaging: AnyObject { var supportsFieldRemoval: Bool { get } var cellWriterSupport: CellWriterSupport { get } func isRowDeleted(_ rowID: RowID) -> Bool + func pendingChange(forRow rowID: RowID) -> RowChange? func isExpression(rowID: RowID, columnIndex: Int) -> Bool func expressionColumns(forRow rowID: RowID) -> Set func recordCellChange( @@ -47,8 +48,7 @@ protocol ChangeManaging: AnyObject { func undoRowDeletion(rowID: RowID) } -/// Only the data grid tracks server-computed columns; the structure and -/// inspector grids edit schema definitions, where the concept does not apply. +// Only the data grid tracks server-computed columns; the structure and inspector grids edit schema definitions. extension ChangeManaging { var generatedColumns: Set { [] } @@ -60,11 +60,15 @@ extension ChangeManaging { var cellWriterSupport: CellWriterSupport { .none } + func pendingChange(forRow rowID: RowID) -> RowChange? { + rowChanges.first { $0.rowID == rowID } + } + func isExpression(rowID: RowID, columnIndex: Int) -> Bool { false } func expressionColumns(forRow rowID: RowID) -> Set { [] } - /// Only the data grid can show a row without a field; every other grid records the value. + // Only the data grid can show a row without a field; every other grid records the value. func recordCellChange( rowID: RowID, columnIndex: Int, @@ -84,7 +88,7 @@ extension ChangeManaging { ) } - /// Only the data grid writes SQL expressions; `cellWriterSupport` keeps every other grid from offering one. + // Only the data grid writes SQL expressions; `cellWriterSupport` keeps every other grid from offering one. func recordCellChange( rowID: RowID, columnIndex: Int, @@ -126,6 +130,10 @@ final class AnyChangeManager: ObservableObject { wrapped.isRowDeleted(rowID) } + func pendingChange(forRow rowID: RowID) -> RowChange? { + wrapped.pendingChange(forRow: rowID) + } + func isExpression(rowID: RowID, columnIndex: Int) -> Bool { wrapped.isExpression(rowID: rowID, columnIndex: columnIndex) } diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift index 55cb981c82..51793a5616 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift @@ -109,6 +109,34 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private var loadedValues: [RowID: [Int: PluginCellValue]] = [:] private var coalescedOrder: [CoalescedCellKey] = [] + // Process-wide, so a run restored from another tab never shares an id with a new command. + private static var lastActionID = 0 + private var commandAction: Int? + private var typingAction: Int? + + private static func newActionID() -> Int { + lastActionID += 1 + return lastActionID + } + + // One undo step over many cells, so a row it writes more than once stays one write in the save. + func recordAsOneAction(_ body: () -> Void) { + guard commandAction == nil else { + body() + return + } + commandAction = Self.newActionID() + defer { commandAction = nil } + body() + } + + private func typingRunAction() -> Int { + if let typingAction { return typingAction } + let action = Self.newActionID() + typingAction = action + return action + } + /// Whether a typed edit is waiting to become an undo step. The Edit menu reads it, because the /// undo manager has nothing registered yet while the run is open. var hasCoalescedUndoRun: Bool { !coalescedOrder.isEmpty } @@ -323,6 +351,14 @@ final class DataChangeManager: ObservableObject, ChangeManaging { wasExpression: pending.isExpression(rowID: rowID, columnIndex: columnIndex), isExpression: isExpression ) + // The grid's row shows the row's other pending cells; an undo that has to stage the row again needs it as read. + var absence = absence + let originalRow = originalRow.map { shown in + let image = pending.imageAsRead(ofRow: rowID, current: shown, absentColumns: absence.originalRow) + absence.originalRow = image.absentColumns + return image.row + } + let action = coalescesWithPrevious ? typingRunAction() : commandAction ?? Self.newActionID() objectWillChange.send() let recorded = pending.recordCellChange( rowID: rowID, @@ -332,7 +368,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { newValue: newValue, originalRow: originalRow, absence: absence, - isExpression: isExpression + isExpression: isExpression, + action: action ) guard recorded else { hasChanges = !pending.isEmpty @@ -346,11 +383,14 @@ final class DataChangeManager: ObservableObject, ChangeManaging { ) } else { registerUndo(actionName: String(localized: "Edit Cell")) { target in - target.applyDataUndo(.cellEdit( - rowID: rowID, columnIndex: columnIndex, columnName: columnName, - previousValue: oldValue, newValue: newValue, originalRow: originalRow, - absence: absence, expression: expression - )) + target.applyDataUndo( + .cellEdit( + rowID: rowID, columnIndex: columnIndex, columnName: columnName, + previousValue: oldValue, newValue: newValue, originalRow: originalRow, + absence: absence, expression: expression + ), + actionID: action + ) } } hasChanges = !pending.isEmpty @@ -382,25 +422,28 @@ final class DataChangeManager: ObservableObject, ChangeManaging { coalescedOrder.append(key) } - /// Registers the run's cells as one undo step, in the order they were first written. - /// - /// A cell typed back to where it started contributes nothing: registering it would leave a step - /// that restores a value the cell already holds, and undoing it would record a pending change - /// over a value that already matches the server. + // A cell typed back to where it started is left out: undoing it would stage a value the server already holds. func endCoalescedUndoRun() { - guard !coalescedOrder.isEmpty else { return } + guard !coalescedOrder.isEmpty else { + typingAction = nil + return + } let edits = coalescedOrder.compactMap { coalescedEdits[$0] }.filter { !$0.isNoOp } + let action = typingAction discardCoalescedUndoRun() guard !edits.isEmpty, let undoManager = undoManagerProvider?() else { return } undoManager.beginUndoGrouping() for edit in edits { undoManager.registerUndo(withTarget: self) { target in - target.applyDataUndo(.cellEdit( - rowID: edit.rowID, columnIndex: edit.columnIndex, columnName: edit.columnName, - previousValue: edit.previousValue, newValue: edit.newValue, originalRow: edit.originalRow, - absence: edit.absence, expression: edit.expression - )) + target.applyDataUndo( + .cellEdit( + rowID: edit.rowID, columnIndex: edit.columnIndex, columnName: edit.columnName, + previousValue: edit.previousValue, newValue: edit.newValue, originalRow: edit.originalRow, + absence: edit.absence, expression: edit.expression + ), + actionID: action + ) } } undoManager.setActionName(String(localized: "Edit Cell")) @@ -410,6 +453,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private func discardCoalescedUndoRun() { coalescedEdits.removeAll() coalescedOrder.removeAll() + typingAction = nil } private func discardUndoHistory() { @@ -534,12 +578,49 @@ final class DataChangeManager: ObservableObject, ChangeManaging { // MARK: - Undo Operations func undoRowDeletion(rowID: RowID) { - guard pending.isRowDeleted(rowID) else { return } + restoreDeletedRows([rowID]) + } + + // A step of its own, so Undo deletes the rows again rather than replaying the delete under it the wrong way round. + func restoreDeletedRows(_ rowIDs: [RowID]) { + let restored = rowIDs.compactMap { rowID -> (rowID: RowID, originalRow: [PluginCellValue], absent: Set)? in + guard let delete = pending.change(forRow: rowID, type: .delete) else { return nil } + return (rowID, completedImage(delete.originalRow ?? [], of: rowID), delete.absentColumns) + } + guard !restored.isEmpty else { return } objectWillChange.send() - guard pending.undoRowDeletion(rowID: rowID) else { return } + for row in restored { + _ = pending.undoRowDeletion(rowID: row.rowID) + } + if restored.count == 1, let row = restored.first { + registerUndo(actionName: Self.restoreRowActionName) { target in + target.applyDataUndo(.rowDeletion(rowID: row.rowID, originalRow: row.originalRow, absentColumns: row.absent)) + } + } else { + let rows = restored.map { (rowID: $0.rowID, originalRow: $0.originalRow) } + let absentColumns = Dictionary( + restored.filter { !$0.absent.isEmpty }.map { ($0.rowID, $0.absent) }, + uniquingKeysWith: { first, _ in first } + ) + registerUndo(actionName: Self.restoreRowsActionName) { target in + target.applyDataUndo(.batchRowDeletion(rows: rows, absentColumns: absentColumns)) + } + } hasChanges = !pending.isEmpty } + private static var restoreRowActionName: String { String(localized: "Restore Row") } + private static var restoreRowsActionName: String { String(localized: "Restore Rows") } + + // A delete and a restore replay through one step, which keeps the name of the command that made it. + private func rowDeletionStepName(rowIsDeleted: Bool, batch: Bool) -> String { + let isUndoing = undoManagerProvider?()?.isUndoing ?? false + if isUndoing == rowIsDeleted { + return batch ? String(localized: "Delete Rows") : String(localized: "Delete Row") + } + return batch ? Self.restoreRowsActionName : Self.restoreRowActionName + } + func undoBatchRowInsertion(rows: [InsertedRowLocation]) { let validRows = rows.filter { pending.isRowInserted($0.rowID) } guard !validRows.isEmpty else { return } @@ -560,7 +641,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { // MARK: - Core Undo Application - private func applyDataUndo(_ action: UndoAction) { + private func applyDataUndo(_ action: UndoAction, actionID: Int? = nil) { objectWillChange.send() switch action { case .cellEdit(let rowID, let columnIndex, let columnName, let previousValue, let newValue, let originalRow, @@ -568,7 +649,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { applyCellEditUndo( rowID: rowID, columnIndex: columnIndex, columnName: columnName, previousValue: previousValue, newValue: newValue, originalRow: originalRow, - absence: absence, expression: expression, action: action + absence: absence, expression: expression, action: action, actionID: actionID ) case .rowInsertion(let rowID, let image): @@ -597,20 +678,25 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private func applyCellEditUndo( rowID: RowID, columnIndex: Int, columnName: String, previousValue: PluginCellValue, newValue: PluginCellValue, originalRow: [PluginCellValue]?, - absence: FieldAbsence, expression: ExpressionTransition, action: UndoAction + absence: FieldAbsence, expression: ExpressionTransition, action: UndoAction, actionID: Int? ) { + let isUndoing = undoManagerProvider?()?.isUndoing ?? false registerUndo(actionName: String(localized: "Edit Cell")) { target in - target.applyDataUndo(.cellEdit( - rowID: rowID, columnIndex: columnIndex, columnName: columnName, - previousValue: newValue, newValue: previousValue, originalRow: originalRow, - absence: absence.reversed, expression: expression.reversed - )) + target.applyDataUndo( + .cellEdit( + rowID: rowID, columnIndex: columnIndex, columnName: columnName, + previousValue: newValue, newValue: previousValue, originalRow: originalRow, + absence: absence.reversed, expression: expression.reversed + ), + actionID: actionID + ) } + // A redo is a new write, so it goes back after the other rows' changes still pending, where it was. let updateHoldsCell = pending.change(forRow: rowID, type: .update)? .cellChanges.contains { $0.columnIndex == columnIndex } ?? false - if updateHoldsCell { - pending.revertUpdateCell( + if updateHoldsCell, isUndoing { + pending.revertCellWrite( rowID: rowID, columnIndex: columnIndex, columnName: columnName, previousValue: previousValue, previousIsAbsent: absence.wasAbsent, @@ -626,8 +712,10 @@ final class DataChangeManager: ObservableObject, ChangeManaging { pending.reapplyCellChange( rowID: rowID, columnIndex: columnIndex, columnName: columnName, - originalDBValue: newValue, newValue: previousValue, originalRow: originalRow, - absence: absence.reversed, expression: expression.reversed + originalDBValue: newValue, newValue: previousValue, + originalRow: originalRow.map { completedImage($0, of: rowID) }, + absence: absence.reversed, expression: expression.reversed, + action: actionID ) } lastUndoResult = UndoResult( @@ -673,7 +761,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { ) { // The pending deletion's image, which a value loaded since the delete has completed. let image = completedImage(pending.change(forRow: rowID, type: .delete)?.originalRow ?? originalRow, of: rowID) - registerUndo(actionName: String(localized: "Delete Row")) { target in + let stepName = rowDeletionStepName(rowIsDeleted: pending.isRowDeleted(rowID), batch: false) + registerUndo(actionName: stepName) { target in target.applyDataUndo(.rowDeletion(rowID: rowID, originalRow: image, absentColumns: absentColumns)) } @@ -701,7 +790,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { let image = pending.change(forRow: row.rowID, type: .delete)?.originalRow ?? row.originalRow return (rowID: row.rowID, originalRow: completedImage(image, of: row.rowID)) } - registerUndo(actionName: String(localized: "Delete Rows")) { target in + let stepName = rowDeletionStepName(rowIsDeleted: rows.contains { pending.isRowDeleted($0.rowID) }, batch: true) + registerUndo(actionName: stepName) { target in target.applyDataUndo(.batchRowDeletion(rows: images, absentColumns: absentColumns)) } @@ -928,6 +1018,14 @@ final class DataChangeManager: ObservableObject, ChangeManaging { pending.isRowInserted(rowID) } + func pendingChange(forRow rowID: RowID) -> RowChange? { + pending.firstChange(forRow: rowID) + } + + func pendingUpdate(forRow rowID: RowID) -> RowChange? { + pending.change(forRow: rowID, type: .update) + } + func isCellModified(rowID: RowID, columnIndex: Int) -> Bool { pending.isCellModified(rowID: rowID, columnIndex: columnIndex) } diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift index 7a795ff87d..a3910c6c8f 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift @@ -49,8 +49,8 @@ struct CellChange: Identifiable, Equatable { let oldIsAbsent: Bool let newIsAbsent: Bool let oldIsExpression: Bool - /// The new value is `.text` holding SQL from `CellExpressionVocabulary`, written unquoted. Never set with - /// `newIsAbsent`. + /// The new value is `.text` holding SQL from `CellExpressionVocabulary`, or `PluginCellValue.columnDefault`, + /// written unquoted. Never set with `newIsAbsent`. let newIsExpression: Bool init( @@ -79,6 +79,47 @@ struct CellChange: Identifiable, Equatable { var restoresOriginal: Bool { oldValue == newValue && oldIsAbsent == newIsAbsent && oldIsExpression == newIsExpression } + + func followed(by later: CellChange) -> CellChange { + CellChange( + columnIndex: columnIndex, + columnName: later.columnName, + oldValue: oldValue, + newValue: later.newValue, + oldIsAbsent: oldIsAbsent, + newIsAbsent: later.newIsAbsent, + oldIsExpression: oldIsExpression, + newIsExpression: later.newIsExpression + ) + } +} + +// A cell's old value is the cell as this run found it, not as the row was read. +struct UpdateRun: Equatable { + let sequence: Int + let action: Int + var cellChanges: [CellChange] + + func holds(_ columnIndex: Int) -> Bool { + cellChanges.contains { $0.columnIndex == columnIndex } + } + + func writesExpression(into columns: Set) -> Bool { + cellChanges.contains { $0.newIsExpression && columns.contains($0.columnIndex) } + } + + mutating func write(_ cellChange: CellChange) { + guard let index = cellChanges.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) else { + cellChanges.append(cellChange) + return + } + let written = cellChanges[index].followed(by: cellChange) + if written.restoresOriginal { + cellChanges.remove(at: index) + } else { + cellChanges[index] = written + } + } } struct RowChange: Identifiable, Equatable { @@ -92,8 +133,8 @@ struct RowChange: Identifiable, Equatable { /// stands for an insert. Empty on every engine that cannot tell a missing field from NULL. var absentColumns: Set - /// An insert's cells that hold a SQL expression, as the row stands. An update keeps its own on each - /// `CellChange`. + /// An insert's cells that hold a SQL expression or the column default, as the row stands. An update keeps its own + /// on each `CellChange`. var expressionColumns: Set /// The order the user made this change in. @@ -104,6 +145,9 @@ struct RowChange: Identifiable, Equatable { /// and reusing its unique value in a new one only works in that order. var sequence: Int + // An update's edits in the order they were made. Empty reads as one run at `sequence`. + var runs: [UpdateRun] + init( rowID: RowID, type: ChangeType, @@ -111,7 +155,8 @@ struct RowChange: Identifiable, Equatable { originalRow: [PluginCellValue]? = nil, absentColumns: Set = [], expressionColumns: Set = [], - sequence: Int = 0 + sequence: Int = 0, + runs: [UpdateRun] = [] ) { self.id = UUID() self.rowID = rowID @@ -121,6 +166,76 @@ struct RowChange: Identifiable, Equatable { self.absentColumns = absentColumns self.expressionColumns = expressionColumns self.sequence = sequence + self.runs = runs + } + + // A row keeps one UPDATE at its first run when a later step could not find it: with no key, a keyless match fails + // once the server rewrites a column on the first UPDATE, and an earlier run's SQL in a key is not the key it wrote. + static func saveSteps(_ changes: [RowChange], keyColumns: Set) -> [RowChange] { + var steps: [(step: RowChange, source: Int)] = [] + for (source, change) in changes.enumerated() { + guard change.type == .update, let first = change.runs.first else { + steps.append((change, source)) + continue + } + guard !keyColumns.isEmpty, change.runs.count > 1, var row = change.originalRow, + !change.runs.dropLast().contains(where: { $0.writesExpression(into: keyColumns) }) + else { + var placed = change + placed.sequence = first.sequence + steps.append((placed, source)) + continue + } + var absent = change.absentColumns + for run in change.runs { + steps.append(( + RowChange( + rowID: change.rowID, type: .update, cellChanges: run.cellChanges, originalRow: row, + absentColumns: absent, expressionColumns: change.expressionColumns, sequence: run.sequence + ), + source + )) + for cellChange in run.cellChanges { + if row.indices.contains(cellChange.columnIndex) { + row[cellChange.columnIndex] = cellChange.newValue + } + if cellChange.newIsAbsent { + absent.insert(cellChange.columnIndex) + } else { + absent.remove(cellChange.columnIndex) + } + } + } + } + + let ordered = steps.enumerated() + .sorted { ($0.element.step.sequence, $0.offset) < ($1.element.step.sequence, $1.offset) } + .map(\.element) + // Two runs with nothing left between them, once a change recorded there was cancelled, are one UPDATE. + var merged: [(step: RowChange, source: Int)] = [] + for entry in ordered { + if let last = merged.last, last.source == entry.source, entry.step.type == .update { + merged[merged.count - 1].step = last.step.followed(by: entry.step) + } else { + merged.append(entry) + } + } + return merged.map(\.step) + } + + private func followed(by later: RowChange) -> RowChange { + var cells = cellChanges + for cellChange in later.cellChanges { + if let index = cells.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) { + cells[index] = cells[index].followed(by: cellChange) + } else { + cells.append(cellChange) + } + } + return RowChange( + rowID: rowID, type: type, cellChanges: cells, originalRow: originalRow, + absentColumns: absentColumns, expressionColumns: expressionColumns, sequence: sequence + ) } // A value loaded after the change was recorded replaces the placeholder its pre-image captured. diff --git a/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift b/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift new file mode 100644 index 0000000000..bdbb48fc6b --- /dev/null +++ b/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift @@ -0,0 +1,38 @@ +// +// DefaultedCells.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// The cells a pending insert or edit set to the column's default, by row and data index. Only those rows can hold one. +enum DefaultedCells { + static func marked( + in rows: TableRows, + rowIDs: Set, + expressionColumns: (RowID) -> Set + ) -> [RowID: Set] { + var cells: [RowID: Set] = [:] + for rowID in rowIDs { + guard let row = rows.row(withID: rowID) else { continue } + let defaulted = expressionColumns(rowID).filter { + row.values.indices.contains($0) && row.values[$0].isColumnDefault(isExpression: true) + } + if !defaulted.isEmpty { cells[rowID] = defaulted } + } + return cells + } +} + +extension ChangeManaging { + func defaultedCells(in rows: TableRows) -> [RowID: Set] { + DefaultedCells.marked(in: rows, rowIDs: insertedRowIDs.union(modifiedCells.keys)) { expressionColumns(forRow: $0) } + } +} + +extension AnyChangeManager { + func defaultedCells(in rows: TableRows) -> [RowID: Set] { + DefaultedCells.marked(in: rows, rowIDs: insertedRowIDs.union(modifiedCells.keys)) { expressionColumns(forRow: $0) } + } +} diff --git a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift index aeb641a715..e9d21fe2bc 100644 --- a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift +++ b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift @@ -2,12 +2,6 @@ // PendingChanges.swift // TablePro // -// Value type holding all uncommitted edits to a result set. -// Owns the consistency invariants between `changes`, `changeIndex`, -// `deletedRowIDs`, `insertedRowIDs`, `modifiedCells`, and -// `insertedRowData`. Callers mutate through methods that maintain -// the cross-collection state. -// import Foundation import TableProPluginKit @@ -16,9 +10,7 @@ struct PendingChanges: Equatable { private(set) var changes: [RowChange] = [] private(set) var deletedRowIDs: Set = [] - /// Stamped onto every change so statement generation can recover the order the user worked in. - /// `changes` cannot carry that order itself: a cancelled change is removed by swapping the last - /// element into its slot. + // `changes` cannot carry the edit order: a cancelled change is swap-removed from it. private var nextSequence = 0 private(set) var insertedRowIDs: Set = [] private(set) var modifiedCells: [RowID: Set] = [:] @@ -79,6 +71,16 @@ struct PendingChanges: Equatable { return changes[idx] } + // The row's change that comes first in `changes`, which is what a scan for its id would find. + func firstChange(forRow rowID: RowID) -> RowChange? { + var first: Int? + for type in [ChangeType.insert, .update, .delete] { + guard let idx = changeIndex[RowChangeKey(rowID: rowID, type: type)] else { continue } + first = min(first ?? idx, idx) + } + return first.map { changes[$0] } + } + /// The row's cells that hold a SQL expression now: a new row's from its insert, a saved row's from its edits. func expressionColumns(forRow rowID: RowID) -> Set { if let insert = change(forRow: rowID, type: .insert) { @@ -98,11 +100,7 @@ struct PendingChanges: Equatable { // MARK: - Mutate (recording user edits) - /// Whether the recorded edit is a no-op (oldValue == newValue with no prior modification). - /// Returns the result so the caller can decide whether to register undo. - /// - /// A field going missing, or coming back, is a change even when the value reads `.null` on - /// both sides. So is an expression put in place of text that reads the same, or text in place of one. + // False for a no-op, so no undo is registered. A nil `action` is an undo step of its own. @discardableResult mutating func recordCellChange( rowID: RowID, @@ -112,7 +110,8 @@ struct PendingChanges: Equatable { newValue: PluginCellValue, originalRow: [PluginCellValue]? = nil, absence: FieldAbsence = FieldAbsence(), - isExpression: Bool = false + isExpression: Bool = false, + action: Int? = nil ) -> Bool { let wasExpression = self.isExpression(rowID: rowID, columnIndex: columnIndex) if oldValue == newValue && absence.wasAbsent == absence.isAbsent && wasExpression == isExpression { @@ -142,14 +141,12 @@ struct PendingChanges: Equatable { } if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { - mergeUpdateCell(at: updateIdx, cellChange: cellChange) + mergeUpdateCell(at: updateIdx, cellChange: cellChange, action: action) } else { - appendChange(RowChange( - rowID: rowID, type: .update, - cellChanges: [cellChange], originalRow: originalRow, - absentColumns: absence.originalRow - )) - modifiedCells[rowID, default: []].insert(columnIndex) + appendUpdate( + rowID: rowID, cellChange: cellChange, originalRow: originalRow, + absentColumns: absence.originalRow, action: action + ) } return true } @@ -236,9 +233,7 @@ struct PendingChanges: Equatable { recordRowDeletion(rowID: rowID, originalRow: originalRow, absentColumns: absentColumns) } - /// Re-apply a cell edit during undo replay (skips undo registration). - /// `originalDBValue` is the cell's value in the unmodified database row. - /// It must be preserved so that a later collapse compares correctly. + // `originalDBValue` is the cell as it stands before this write. mutating func reapplyCellChange( rowID: RowID, columnIndex: Int, @@ -247,7 +242,8 @@ struct PendingChanges: Equatable { newValue: PluginCellValue, originalRow: [PluginCellValue]?, absence: FieldAbsence = FieldAbsence(), - expression: ExpressionTransition = ExpressionTransition() + expression: ExpressionTransition = ExpressionTransition(), + action: Int? = nil ) { let cellChange = CellChange( columnIndex: columnIndex, @@ -269,14 +265,12 @@ struct PendingChanges: Equatable { } if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { - mergeUpdateCell(at: updateIdx, cellChange: cellChange) + mergeUpdateCell(at: updateIdx, cellChange: cellChange, action: action) } else { - appendChange(RowChange( - rowID: rowID, type: .update, - cellChanges: [cellChange], originalRow: originalRow, - absentColumns: absence.originalRow - )) - modifiedCells[rowID, default: []].insert(columnIndex) + appendUpdate( + rowID: rowID, cellChange: cellChange, originalRow: originalRow, + absentColumns: absence.originalRow, action: action + ) } } @@ -296,8 +290,8 @@ struct PendingChanges: Equatable { ) } - /// Restore a cell's value during undo replay when an existing change matches. - mutating func revertUpdateCell( + // Undo takes back the cell's latest write, which the newest run holding the cell made. + mutating func revertCellWrite( rowID: RowID, columnIndex: Int, columnName: String, @@ -306,29 +300,24 @@ struct PendingChanges: Equatable { previousIsExpression: Bool = false ) { guard let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)], - let cellIdx = changes[updateIdx].cellChanges.firstIndex(where: { $0.columnIndex == columnIndex }) + let current = changes[updateIdx].cellChanges.first(where: { $0.columnIndex == columnIndex }) else { return } - let original = changes[updateIdx].cellChanges[cellIdx] - if previousValue == original.oldValue && previousIsAbsent == original.oldIsAbsent - && previousIsExpression == original.oldIsExpression { - changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: columnIndex) - if changes[updateIdx].cellChanges.isEmpty { - removeChangeAt(updateIdx) - } - } else { - changes[updateIdx].cellChanges[cellIdx] = CellChange( - columnIndex: columnIndex, - columnName: columnName, - oldValue: original.oldValue, - newValue: previousValue, - oldIsAbsent: original.oldIsAbsent, - newIsAbsent: previousIsAbsent, - oldIsExpression: original.oldIsExpression, - newIsExpression: previousIsExpression - ) - } + materializeRuns(at: updateIdx) + let reverted = CellChange( + columnIndex: columnIndex, + columnName: columnName, + oldValue: current.newValue, + newValue: previousValue, + oldIsAbsent: current.newIsAbsent, + newIsAbsent: previousIsAbsent, + oldIsExpression: current.newIsExpression, + newIsExpression: previousIsExpression + ) + guard writeNetCell(reverted, at: updateIdx), + let runIdx = changes[updateIdx].runs.lastIndex(where: { $0.holds(columnIndex) }) + else { return } + write(reverted, intoRun: runIdx, at: updateIdx) } /// Insert a synthetic .insert RowChange for undo replay (e.g., after redoing a deletion's undo). @@ -420,7 +409,7 @@ struct PendingChanges: Equatable { modifiedCells = snapshot.modifiedCells insertedRowData = snapshot.insertedRowData supersededUpdates = snapshot.supersededUpdates - let sequences = changes.map(\.sequence) + supersededUpdates.values.map(\.sequence) + let sequences = (changes + supersededUpdates.values).flatMap { [$0.sequence] + $0.runs.map(\.sequence) } nextSequence = (sequences.max() ?? -1) + 1 rebuildChangeIndex() } @@ -448,6 +437,29 @@ struct PendingChanges: Equatable { changeIndex[RowChangeKey(rowID: stamped.rowID, type: stamped.type)] = changes.count - 1 } + private mutating func appendUpdate( + rowID: RowID, + cellChange: CellChange, + originalRow: [PluginCellValue]?, + absentColumns: Set, + action: Int? + ) { + let run = UpdateRun( + sequence: nextSequence, action: action ?? Self.ownAction(at: nextSequence), cellChanges: [cellChange] + ) + appendChange(RowChange( + rowID: rowID, type: .update, + cellChanges: [cellChange], originalRow: originalRow, + absentColumns: absentColumns, runs: [run] + )) + modifiedCells[rowID, default: []].insert(cellChange.columnIndex) + } + + // Below every id DataChangeManager hands out, so no other write ever joins it. + private static func ownAction(at sequence: Int) -> Int { + -(sequence + 1) + } + @discardableResult private mutating func removeChange(rowID: RowID, type: ChangeType) -> Bool { let key = RowChangeKey(rowID: rowID, type: type) @@ -518,32 +530,72 @@ struct PendingChanges: Equatable { } } - private mutating func mergeUpdateCell(at updateIdx: Int, cellChange: CellChange) { + private mutating func mergeUpdateCell(at updateIdx: Int, cellChange: CellChange, action: Int?) { + materializeRuns(at: updateIdx) + guard writeNetCell(cellChange, at: updateIdx) else { return } + + // The newest run takes the write when the same step opened it or nothing was recorded since. + if let newest = changes[updateIdx].runs.indices.last { + let run = changes[updateIdx].runs[newest] + if (action != nil && run.action == action) || run.sequence == nextSequence - 1 { + write(cellChange, intoRun: newest, at: updateIdx) + return + } + } + changes[updateIdx].runs.append(UpdateRun( + sequence: nextSequence, action: action ?? Self.ownAction(at: nextSequence), cellChanges: [cellChange] + )) + nextSequence += 1 + } + + // An update built outside this type reads as one run at its own place. + private mutating func materializeRuns(at updateIdx: Int) { + let update = changes[updateIdx] + guard update.runs.isEmpty, !update.cellChanges.isEmpty else { return } + changes[updateIdx].runs = [UpdateRun( + sequence: update.sequence, action: Self.ownAction(at: update.sequence), cellChanges: update.cellChanges + )] + } + + // False when the write left the cell as it was read, so no run writes it. + private mutating func writeNetCell(_ cellChange: CellChange, at updateIdx: Int) -> Bool { let rowID = changes[updateIdx].rowID guard let cellIdx = changes[updateIdx].cellChanges.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) else { changes[updateIdx].cellChanges.append(cellChange) modifiedCells[rowID, default: []].insert(cellChange.columnIndex) - return + return true } - let original = changes[updateIdx].cellChanges[cellIdx] - let merged = CellChange( - columnIndex: cellChange.columnIndex, - columnName: cellChange.columnName, - oldValue: original.oldValue, - newValue: cellChange.newValue, - oldIsAbsent: original.oldIsAbsent, - newIsAbsent: cellChange.newIsAbsent, - oldIsExpression: original.oldIsExpression, - newIsExpression: cellChange.newIsExpression - ) - changes[updateIdx].cellChanges[cellIdx] = merged + let merged = changes[updateIdx].cellChanges[cellIdx].followed(by: cellChange) + guard merged.restoresOriginal else { + changes[updateIdx].cellChanges[cellIdx] = merged + return true + } + removeUpdateCell(at: updateIdx, cellIdx: cellIdx) + return false + } - guard merged.restoresOriginal else { return } + private mutating func write(_ cellChange: CellChange, intoRun runIdx: Int, at updateIdx: Int) { + changes[updateIdx].runs[runIdx].write(cellChange) + if changes[updateIdx].runs[runIdx].cellChanges.isEmpty { + changes[updateIdx].runs.remove(at: runIdx) + } + } + + // A cell the grid shows unchanged leaves every run too, so no trigger fires for a value nobody changed. + private mutating func removeUpdateCell(at updateIdx: Int, cellIdx: Int) { + let rowID = changes[updateIdx].rowID + let columnIndex = changes[updateIdx].cellChanges[cellIdx].columnIndex changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: cellChange.columnIndex) + removeModifiedCell(rowID: rowID, columnIndex: columnIndex) + for runIdx in changes[updateIdx].runs.indices.reversed() { + changes[updateIdx].runs[runIdx].cellChanges.removeAll { $0.columnIndex == columnIndex } + if changes[updateIdx].runs[runIdx].cellChanges.isEmpty { + changes[updateIdx].runs.remove(at: runIdx) + } + } if changes[updateIdx].cellChanges.isEmpty { removeChangeAt(updateIdx) } @@ -565,11 +617,7 @@ struct PendingChanges: Equatable { changes[updateIdx].cellChanges[cellIdx].oldIsExpression == restoredIsExpression else { return false } - changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: columnIndex) - if changes[updateIdx].cellChanges.isEmpty { - removeChangeAt(updateIdx) - } + removeUpdateCell(at: updateIdx, cellIdx: cellIdx) return true } } diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 17f1871fa9..6cec2f9225 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -11,18 +11,26 @@ import os import TableProPluginKit import TableProSQLGrammar -/// A parameterized SQL statement with placeholders and bound values struct ParameterizedStatement: @unchecked Sendable { let sql: String let parameters: [Any?] + // By position, the type of the column each parameter is written to or compared with. Nil when none is known. + var parameterTypeNames: [String?]? + + // What a driver needs to write each value for its column; everything else stays the host's default. + var context: PluginStatementContext { + var context = PluginStatementContext() + context.parameterTypeNames = parameterTypeNames + return context + } } -/// A statement plus the rows it is meant to touch, so a caller can hold the server to that count -/// and tell a change that became a statement from one that did not. +// The rows it should touch hold the server to a count; the changes it writes tell a written change from a dropped one. struct AttributedStatement: @unchecked Sendable { let statement: ParameterizedStatement let kind: RowWriteKind let rowIDs: [RowID] + let changeIDs: [UUID] var refusesSharedKey = false var rowCount: Int { rowIDs.count } @@ -49,6 +57,7 @@ struct SQLStatementGenerator { /// Set only by a restore that writes a row back with the identity it had: PostgreSQL refuses an explicit value /// for a `GENERATED ALWAYS` column unless the INSERT says so. let insertOverridesSystemValue: Bool + private let columnTypeNames: [String: String] // BIT columns of a family that reads them as a number. Text bound to one is stored and compared as its bytes. private let numberBitColumns: Set private let quoteIdentifierFn: (String) -> String @@ -82,6 +91,7 @@ struct SQLStatementGenerator { self.databaseType = databaseType self.parameterStyle = parameterStyle ?? Self.defaultParameterStyle(for: databaseType) self.insertOverridesSystemValue = insertOverridesSystemValue + self.columnTypeNames = columnTypeNames self.numberBitColumns = Self.numberBitColumns(in: columnTypeNames, databaseType: databaseType) if let quoteIdentifier { self.quoteIdentifierFn = quoteIdentifier @@ -125,16 +135,8 @@ struct SQLStatementGenerator { ).map(\.statement) } - /// The same statements, each carrying how many rows it is meant to touch. - /// - /// Emitted in the order the user made the changes, because that order can be load-bearing: a - /// row deleted to free a unique value, and a new row taking that value, only work if the - /// DELETE runs first. Grouping every INSERT ahead of every DELETE, which is what this used to - /// do, turned that save into a constraint violation and rolled the whole thing back. - /// - /// Deletes still batch, but only across a consecutive run of them. That keeps the case that - /// matters, selecting many rows and pressing Delete, on one statement, while a delete - /// separated from another delete by an insert stays on its own side of it. + // In edit order, since a row deleted to free a unique value must go before the row that takes it. Only a + // consecutive run of deletes batches. func generateAttributedStatements( from changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], @@ -156,7 +158,8 @@ struct SQLStatementGenerator { flushDeleteRun() if let stmt = generateUpdateSQL(for: change) { statements.append(AttributedStatement( - statement: stmt, kind: .update, rowIDs: [change.rowID], refusesSharedKey: matchesEitherWay(change) + statement: stmt, kind: .update, rowIDs: [change.rowID], changeIDs: [change.id], + refusesSharedKey: matchesEitherWay(change) )) } case .insert: @@ -164,7 +167,9 @@ struct SQLStatementGenerator { guard insertedRowIDs.contains(change.rowID) else { continue } flushDeleteRun() if let stmt = generateInsertSQL(for: change, insertedRowData: insertedRowData) { - statements.append(AttributedStatement(statement: stmt, kind: .insert, rowIDs: [change.rowID])) + statements.append(AttributedStatement( + statement: stmt, kind: .insert, rowIDs: [change.rowID], changeIDs: [change.id] + )) } case .delete: // SAFETY: Verify the row is still marked as deleted @@ -196,18 +201,43 @@ struct SQLStatementGenerator { return Set(bitColumns.keys) } + private struct Bindings { + var values: [Any?] = [] + var typeNames: [String?] = [] + + var count: Int { values.count } + + mutating func append(_ value: Any?, typeName: String?) { + values.append(value) + typeNames.append(typeName) + } + } + + private func statement(_ sql: String, _ bindings: Bindings) -> ParameterizedStatement { + ParameterizedStatement( + sql: sql, + parameters: bindings.values, + parameterTypeNames: bindings.typeNames.contains { $0 != nil } ? bindings.typeNames : nil + ) + } + + private func typeName(of column: String?) -> String? { + column.flatMap { columnTypeNames[$0] } + } + // Blank text goes as '', which the server stores as 0: cast, strict mode refuses it as a truncated integer, and a // space bound as it is would be its byte, 32. - private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout [Any?]) -> String { + private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout Bindings) -> String { guard let column, numberBitColumns.contains(column), case .text(let text) = value else { - parameters.append(value.asAny) + parameters.append(value.asAny, typeName: typeName(of: column)) return placeholder(at: parameters.count - 1) } guard !text.trimmingCharacters(in: .whitespaces).isEmpty else { - parameters.append("") + parameters.append("", typeName: typeName(of: column)) return placeholder(at: parameters.count - 1) } - parameters.append(Self.decimalBitText(text)) + // The value is the CAST's operand, not the column's. + parameters.append(Self.decimalBitText(text), typeName: nil) return "CAST(\(placeholder(at: parameters.count - 1)) AS UNSIGNED)" } @@ -254,10 +284,11 @@ struct SQLStatementGenerator { { var nonDefaultColumns: [String] = [] var placeholderParts: [String] = [] - var bindParameters: [Any?] = [] + var bindParameters = Bindings() for (index, value) in values.enumerated() { - if case .text(let s) = value, s == "__DEFAULT__" { continue } + // Left out rather than written as DEFAULT, which SQLite does not take in VALUES. + if value.isColumnDefault(isExpression: expressionColumns.contains(index)) { continue } guard index < columns.count else { continue } let columnName = columns[index] @@ -282,13 +313,17 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnList))\(override) VALUES (\(placeholders))" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } private func allDefaultsInsertStatement() -> ParameterizedStatement? { - let firstWritable = columns.first { !generatedColumns.contains($0) }.map(quoteIdentifierFn) - return AllDefaultsInsert.sql( - into: qualifiedTableName, databaseType: databaseType, firstWritableColumn: firstWritable + allDefaultsInsertStatement(firstColumn: columns.first { !generatedColumns.contains($0) }) + } + + // For a generator built without the table's columns, the caller names the one Oracle needs. + func allDefaultsInsertStatement(firstColumn: String?) -> ParameterizedStatement? { + AllDefaultsInsert.sql( + into: qualifiedTableName, databaseType: databaseType, firstWritableColumn: firstColumn.map(quoteIdentifierFn) ).map { ParameterizedStatement(sql: $0, parameters: []) } } @@ -297,7 +332,7 @@ struct SQLStatementGenerator { { guard !insertColumns.isEmpty, insertColumns.count == values.count else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") let placeholders = zip(insertColumns, values).map { column, value in bind(value, toColumn: column, into: &bindParameters) @@ -306,7 +341,7 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES (\(placeholders))" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } func insertStatement(columns insertColumns: [String], rows: [[PluginCellValue]]) @@ -315,7 +350,7 @@ struct SQLStatementGenerator { guard !insertColumns.isEmpty, !rows.isEmpty, rows.allSatisfy({ $0.count == insertColumns.count }) else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") let rowTuples = rows.map { values -> String in let placeholders = zip(insertColumns, values).map { column, value in @@ -327,14 +362,14 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES \(rowTuples)" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } // The server compares, so its casting and collation decide a match, as they do for the UPDATE that follows. func keyMatchCountStatement(keyColumns: [String], keys: [[PluginCellValue]]) -> ParameterizedStatement? { guard !keyColumns.isEmpty, !keys.isEmpty, keys.allSatisfy({ $0.count == keyColumns.count }) else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() let selects = keys.enumerated().map { position, values -> String in let conditions = zip(keyColumns, values).map { column, value -> String in "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" @@ -342,7 +377,7 @@ struct SQLStatementGenerator { return "SELECT \(position), COUNT(*) FROM \(qualifiedTableName) WHERE \(conditions)" } - return ParameterizedStatement(sql: selects.joined(separator: " UNION ALL "), parameters: bindParameters) + return statement(selects.joined(separator: " UNION ALL "), bindParameters) } func keyedUpdateStatement( @@ -354,7 +389,7 @@ struct SQLStatementGenerator { guard !setColumns.isEmpty, setColumns.count == values.count, !keyColumns.isEmpty, keyColumns.count == keyValues.count else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() func bound(_ column: String, _ value: PluginCellValue) -> String { "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" } @@ -363,13 +398,14 @@ struct SQLStatementGenerator { let sql = rowUpdateFn?(qualifiedTableName, assignments, conditions) ?? "UPDATE \(qualifiedTableName) SET \(assignments) WHERE \(conditions)" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } var maxBindParameters: Int { switch databaseType { - case .sqlite: 32_766 + case .sqlite, .libsql, .turso: 32_766 case .mssql: 2_100 + case .cloudflareD1: 100 default: 65_535 } } @@ -383,7 +419,7 @@ struct SQLStatementGenerator { guard !change.cellChanges.isEmpty else { return nil } let nonDefaultChanges = change.cellChanges.filter { - $0.newValue != .text("__DEFAULT__") && !generatedColumns.contains($0.columnName) + !$0.newValue.isColumnDefault(isExpression: $0.newIsExpression) && !generatedColumns.contains($0.columnName) } guard !nonDefaultChanges.isEmpty else { return nil } @@ -392,7 +428,7 @@ struct SQLStatementGenerator { quoteIdentifierFn($0.columnName) }.joined(separator: ", ") - var parameters: [Any?] = [] + var parameters = Bindings() let placeholders = nonDefaultChanges.map { cellChange -> String in switch cellChange.newValue { case .text(let expression) where cellChange.newIsExpression: @@ -405,7 +441,7 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnNames)) VALUES (\(placeholders))" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } // MARK: - UPDATE Generation @@ -416,11 +452,9 @@ struct SQLStatementGenerator { let writableChanges = change.cellChanges.filter { !generatedColumns.contains($0.columnName) } guard !writableChanges.isEmpty else { return nil } - var parameters: [Any?] = [] + var parameters = Bindings() let setClauses = writableChanges.map { cellChange -> String in switch cellChange.newValue { - case .text(let s) where s == "__DEFAULT__": - return "\(quoteIdentifierFn(cellChange.columnName)) = DEFAULT" case .text(let expression) where cellChange.newIsExpression: return "\(quoteIdentifierFn(cellChange.columnName)) = \(expression)" default: @@ -457,7 +491,7 @@ struct SQLStatementGenerator { let whereClause = rowCondition(terms, parameters: ¶meters) let sql = "UPDATE \(qualifiedTableName) SET \(setClauses) WHERE \(whereClause)" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } else { guard let originalRow = change.originalRow else { Self.logger.warning( @@ -488,7 +522,7 @@ struct SQLStatementGenerator { let sql = "UPDATE \(qualifiedTableName) SET \(setClauses) WHERE \(whereClause)" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } } @@ -507,22 +541,23 @@ struct SQLStatementGenerator { /// affinity, and the grid read either one as `5`, so a number the grid read matches both, held by CAST to the /// text SQLite reads it back as so a REAL 5.0 is not matched too; a typed column converts either literal to its /// own type, so the form is exact there too. When both rows exist the statement itself refuses them. - private func rowCondition(_ terms: [RowMatchTerm], parameters: inout [Any?]) -> String { + private func rowCondition(_ terms: [RowMatchTerm], parameters: inout Bindings) -> String { let clause = conditions(terms, parameters: ¶meters) guard terms.contains(where: { matchesEitherWay($0.value) }) else { return clause } let again = conditions(terms, parameters: ¶meters) return "\(clause) AND (SELECT COUNT(*) FROM \(qualifiedTableName) WHERE \(again)) = 1" } - private func conditions(_ terms: [RowMatchTerm], parameters: inout [Any?]) -> String { + private func conditions(_ terms: [RowMatchTerm], parameters: inout Bindings) -> String { terms.map { term in guard let value = term.value else { return "\(term.matched) IS NULL" } guard matchesEitherWay(value), case .text(let text) = value else { return "\(term.matched) = \(bind(value, toColumn: term.column, into: ¶meters))" } - parameters.append(value.asAny) + parameters.append(value.asAny, typeName: typeName(of: term.column)) let bound = placeholder(at: parameters.count - 1) - parameters.append(value.asAny) + // Compared with the column's text, not the column. + parameters.append(value.asAny, typeName: nil) let read = placeholder(at: parameters.count - 1) return "(\(term.matched) IN (\(text), \(bound)) AND CAST(\(term.matched) AS TEXT) = \(read))" }.joined(separator: " AND ") @@ -554,12 +589,12 @@ struct SQLStatementGenerator { private func generateDeleteStatements(for changes: [RowChange]) -> [AttributedStatement] { let rowMatches = changes.compactMap { change in - deleteRowMatches(for: change).map { (rowID: change.rowID, matches: $0) } + deleteRowMatches(for: change).map { (rowID: change.rowID, changeID: change.id, matches: $0) } } guard !rowMatches.isEmpty else { return [] } var statements: [AttributedStatement] = [] - var chunk: [(rowID: RowID, matches: [DeleteColumnMatch])] = [] + var chunk: [(rowID: RowID, changeID: UUID, matches: [DeleteColumnMatch])] = [] var chunkParameterCount = 0 func flush() { @@ -568,6 +603,7 @@ struct SQLStatementGenerator { statement: deleteStatement(for: chunk.map(\.matches)), kind: .delete, rowIDs: chunk.map(\.rowID), + changeIDs: chunk.map(\.changeID), refusesSharedKey: chunk.contains { $0.matches.contains { matchesEitherWay($0.boundValue) } } ) ) @@ -627,7 +663,7 @@ struct SQLStatementGenerator { } private func deleteStatement(for rows: [[DeleteColumnMatch]]) -> ParameterizedStatement { - var parameters: [Any?] = [] + var parameters = Bindings() let rowClauses = rows.map { matches -> String in let terms = matches.map { match in RowMatchTerm( @@ -648,6 +684,6 @@ struct SQLStatementGenerator { let whereClause = rowClauses.joined(separator: " OR ") let sql = "DELETE FROM \(qualifiedTableName) WHERE \(whereClause)" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } } diff --git a/apple/TablePro/Core/ChangeTracking/SQLiteRowidAlias.swift b/apple/TablePro/Core/ChangeTracking/SQLiteRowidAlias.swift new file mode 100644 index 0000000000..52fcfe2331 --- /dev/null +++ b/apple/TablePro/Core/ChangeTracking/SQLiteRowidAlias.swift @@ -0,0 +1,20 @@ +// +// SQLiteRowidAlias.swift +// TablePro +// + +import Foundation + +// SQLite's INTEGER PRIMARY KEY is the rowid: an INSERT that leaves it out takes the next one. A WITHOUT ROWID table has none. +enum SQLiteRowidAlias { + static func isAlias( + column: String, + declaredType: String?, + primaryKey: [String], + family: SQLTypeFamily, + hasRowID: Bool = true + ) -> Bool { + guard family == .sqlite, hasRowID, primaryKey == [column], let declaredType else { return false } + return declaredType.trimmingCharacters(in: .whitespaces).uppercased() == "INTEGER" + } +} diff --git a/apple/TablePro/Core/Compare/CompareRowService.swift b/apple/TablePro/Core/Compare/CompareRowService.swift index 909a199607..86ec0e5923 100644 --- a/apple/TablePro/Core/Compare/CompareRowService.swift +++ b/apple/TablePro/Core/Compare/CompareRowService.swift @@ -39,9 +39,11 @@ internal enum ConcurrentReadRefusal: Equatable, Sendable { @MainActor internal struct CompareRowService { private let manager: DatabaseManager + private let gate: any ExecutionGate - internal init(manager: DatabaseManager = .shared) { + internal init(manager: DatabaseManager = .shared, gate: any ExecutionGate = ExecutionGateProvider.shared) { self.manager = manager + self.gate = gate } internal func concurrentReadRefusal( @@ -132,6 +134,42 @@ internal struct CompareRowService { // MARK: - Plumbing + // A filter is the user's own text, read on a connection that may be at Read-Only: it meets the gate as a Run would. + private func authorizeReadOnlyFilter( + _ filter: String?, + on endpoint: DatabaseEndpoint, + schema: String?, + table: String + ) async throws { + let driver = manager.driver(for: endpoint.connectionId) + let statement = ReadOnlyReadGate.filterStatement( + table: table, schema: schema, filter: filter, databaseType: endpoint.databaseType + ) { name in + driver?.quoteIdentifier(name) ?? name + } + try await ReadOnlyReadGate.authorize( + statement.map { [$0] } ?? [], + connectionId: endpoint.connectionId, + databaseType: endpoint.databaseType, + gate: gate, + operationDescription: String(localized: "Execute Query"), + manager: manager, + runsIn: endpoint.scope + ) + } + + // The sides are built inside the lift: a stream starts its statement when it is built, and a bounded one parks + // its connection once its buffer is full, so a lift sent after it would wait behind that statement for good. + internal static func readingBothSides( + liftingTimeoutOn drivers: [DatabaseDriver], + sides makeSides: () -> Sides, + _ body: (Sides) async throws -> T + ) async throws -> T { + try await BulkReadTimeout.lifted(on: drivers) { + try await body(makeSides()) + } + } + private func withBothSides( source: DatabaseEndpoint, sourceConnection: DatabaseConnection, @@ -149,6 +187,10 @@ internal struct CompareRowService { } try await manager.ensureConnected(sourceConnection) try await manager.ensureConnected(targetConnection) + try await authorizeReadOnlyFilter(plan.scope.effectiveSourceFilter, on: source, schema: source.schema ?? plan.schema, table: plan.table) + try await authorizeReadOnlyFilter(plan.scope.effectiveTargetFilter, on: target, schema: plan.targetSchema, table: plan.table) + let sourceIsReadOnly = ExecutionGateProvider.runsAtReadOnly(source.connectionId, in: manager) + let targetIsReadOnly = ExecutionGateProvider.runsAtReadOnly(target.connectionId, in: manager) let engine = DataDiffEngine(options: options, shape: plan.comparisonShape) let sourceRead = SideRead( @@ -182,22 +224,27 @@ internal struct CompareRowService { String(localized: "The source driver cannot stream rows for a comparison.") ) } + let sourceSide = sourceRead.carrying(sourceDriver.readOnlyRunContext(safeModeIsReadOnly: sourceIsReadOnly)) return try await DatabaseManager.shared.withMetadataDriver(scope: target.scope) { targetDriver in guard let targetPlugin = CompareMetadataService.pluginDriver(from: targetDriver) else { throw CompareSyncError.unsupportedOperation( String(localized: "The target driver cannot stream rows for a comparison.") ) } - let sides = RowSides( - source: sourceRead.provider(driver: sourcePlugin), - target: targetRead.provider(driver: targetPlugin), - resolver: KeyLookupResolver( - source: (sourceRead, sourcePlugin), - target: (targetRead, targetPlugin) - ), - targetDriver: targetPlugin - ) - return try await body(engine, sides) + let targetSide = targetRead.carrying(targetDriver.readOnlyRunContext(safeModeIsReadOnly: targetIsReadOnly)) + return try await Self.readingBothSides(liftingTimeoutOn: [sourceDriver, targetDriver]) { + RowSides( + source: sourceSide.provider(driver: sourcePlugin), + target: targetSide.provider(driver: targetPlugin), + resolver: KeyLookupResolver( + source: (sourceSide, sourcePlugin), + target: (targetSide, targetPlugin) + ), + targetDriver: targetPlugin + ) + } _: { sides in + try await body(engine, sides) + } } } } @@ -214,6 +261,20 @@ private struct SideRead: Sendable { let rowLimit: Int? let dialect: SQLDialectDescriptor? let side: ComparisonSide + // Read-Only's, when the side's connection runs at it and its driver holds the engine to it. + var context: PluginStatementContext? + + func carrying(_ context: PluginStatementContext?) -> SideRead { + var read = self + read.context = context + return read + } + + private func stream(_ query: String, driver: any PluginDatabaseDriver) -> AsyncThrowingStream { + guard var context else { return driver.streamRows(query: query) } + context.readFunctionCalls(in: query, databaseType: endpoint.databaseType) + return driver.streamRows(query: query, context: context) + } func provider(driver: any PluginDatabaseDriver) -> StreamingRowProvider { let query = KeyOrderedQuery.build( @@ -228,7 +289,7 @@ private struct SideRead: Sendable { dialect: dialect ) return StreamingRowProvider( - stream: driver.streamRows(query: query), + stream: stream(query, driver: driver), columns: columns, rowLimit: rowLimit, context: RowReadContext(side: side, isFiltered: filter != nil) @@ -247,7 +308,7 @@ private struct SideRead: Sendable { databaseType: endpoint.databaseType ) return StreamingRowProvider( - stream: driver.streamRows(query: query), + stream: stream(query, driver: driver), columns: columns, context: RowReadContext(side: side, isFiltered: false) ) diff --git a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift index 24cc1a74a8..40fc624f42 100644 --- a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift +++ b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift @@ -77,17 +77,20 @@ internal enum CompareSQLLiteral { return SQLStringLiteralPrefix.forDatabaseType(databaseType) + literal } + // Shared by Compare & Sync and Copy as SQL. `SQLExportBinaryLiteral` in the SQL export plugin holds the + // measurements behind each spelling; the plugin cannot link app code, so it keeps its own copy. internal static func binaryLiteral(for data: Data, databaseType: DatabaseType) -> String? { - let hex = data.map { String(format: "%02X", $0) }.joined() + let hex = data.uppercaseHex() switch binaryStyle(for: databaseType) { case .bitString: return "X'\(hex)'" case .postgresBytea: - return "'\\x\(hex.lowercased())'::bytea" + return postgresFamilyBinaryLiteral(hex: hex, databaseType: databaseType) case .zeroX: return "0x\(hex)" case .hexToRaw: - return "HEXTORAW('\(hex)')" + // `HEXTORAW('')` stores NULL, which a `BLOB NOT NULL` refuses. + return data.isEmpty ? "EMPTY_BLOB()" : "HEXTORAW('\(hex)')" case .unhexFunction: return "unhex('\(hex)')" case .unknown: @@ -95,6 +98,19 @@ internal enum CompareSQLLiteral { } } + // `'\x…'::bytea` follows `standard_conforming_strings`: with it off the server stores the text's bytes. KingbaseES + // documents `decode` only as Oracle's DECODE, so it keeps the bytea literal; Redshift has no bytea and reads VARBYTE. + private static func postgresFamilyBinaryLiteral(hex: String, databaseType: DatabaseType) -> String { + switch databaseType { + case .kingbasees: + return "'\\x\(hex)'::bytea" + case .redshift: + return "from_hex('\(hex)')" + default: + return "decode('\(hex)', 'hex')" + } + } + internal enum BinaryStyle { case bitString case postgresBytea diff --git a/apple/TablePro/Core/Compare/DataDiffEngine.swift b/apple/TablePro/Core/Compare/DataDiffEngine.swift index 882c746b62..657ddbe084 100644 --- a/apple/TablePro/Core/Compare/DataDiffEngine.swift +++ b/apple/TablePro/Core/Compare/DataDiffEngine.swift @@ -30,10 +30,16 @@ internal struct DataRow: Hashable, Sendable { internal protocol DataRowProviding: AnyObject { var endedAtRowLimit: Bool { get } + // The limit the driver stopped this side's stream at, once it has ended there. + var driverRowCut: Int? { get } func nextRow() async throws -> DataRow? func drain() async throws } +internal extension DataRowProviding { + var driverRowCut: Int? { nil } +} + internal protocol OneSidedRowResolving: AnyObject { func rows(matching keys: [[PluginCellValue]], on side: ComparisonSide) async throws -> [DataRow] } @@ -111,6 +117,10 @@ internal struct DataDiffSummary: Hashable, Sendable { /// as `id > ` resumes from without skipping a key or comparing one twice. internal let resumeKey: [PluginCellValue]? + // The limit a driver stopped a side's stream at, so the answer covers only the rows read before it. + internal let sourceRowCut: Int? + internal let targetRowCut: Int? + /// One comparison's own identity. Two runs that produce the same counts and the same digest are /// still two answers, and a view that caches on the answer has to be able to tell them apart: /// matching rows are counted rather than digested, so both can move while the digest stands. @@ -130,6 +140,8 @@ internal struct DataDiffSummary: Hashable, Sendable { stoppedAtRowLimit: Bool = false, differenceDigest: String = "", resumeKey: [PluginCellValue]? = nil, + sourceRowCut: Int? = nil, + targetRowCut: Int? = nil, runIdentity: UUID = UUID() ) { self.insertCount = insertCount @@ -145,6 +157,8 @@ internal struct DataDiffSummary: Hashable, Sendable { self.stoppedAtRowLimit = stoppedAtRowLimit self.differenceDigest = differenceDigest self.resumeKey = resumeKey + self.sourceRowCut = sourceRowCut + self.targetRowCut = targetRowCut self.runIdentity = runIdentity } @@ -359,7 +373,9 @@ internal struct DataDiffEngine { return accumulator.summary( comparedKeyCount: positions, stoppedAtRowLimit: stoppedAtRowLimit, - resumeKey: stoppedAtRowLimit ? lastClassifiedKey : nil + resumeKey: stoppedAtRowLimit ? lastClassifiedKey : nil, + sourceRowCut: source.driverRowCut, + targetRowCut: target.driverRowCut ) } @@ -536,7 +552,9 @@ internal extension DataDiffEngine { func summary( comparedKeyCount: Int, stoppedAtRowLimit: Bool, - resumeKey: [PluginCellValue]? = nil + resumeKey: [PluginCellValue]? = nil, + sourceRowCut: Int? = nil, + targetRowCut: Int? = nil ) -> DataDiffSummary { DataDiffSummary( insertCount: counts[.insert] ?? 0, @@ -551,7 +569,9 @@ internal extension DataDiffEngine { comparedKeyCount: comparedKeyCount, stoppedAtRowLimit: stoppedAtRowLimit, differenceDigest: digest.finalize().map { String(format: "%02x", $0) }.joined(), - resumeKey: resumeKey + resumeKey: resumeKey, + sourceRowCut: sourceRowCut, + targetRowCut: targetRowCut ) } @@ -671,31 +691,3 @@ private final class KeyedRowReader { ) } } - -internal final class ArrayRowProvider: DataRowProviding { - private let rows: [DataRow] - private let rowLimit: Int? - private let holdsRowPastTheLimit: Bool - private var index = 0 - - internal init(rows: [DataRow], rowLimit: Int? = nil) { - self.rows = rowLimit.map { Array(rows.prefix($0)) } ?? rows - self.rowLimit = rowLimit - self.holdsRowPastTheLimit = rowLimit.map { rows.count > $0 } ?? false - } - - internal var endedAtRowLimit: Bool { - guard let rowLimit, index >= rows.count else { return false } - return holdsRowPastTheLimit && rows.count >= rowLimit - } - - internal func nextRow() async throws -> DataRow? { - guard index < rows.count else { return nil } - defer { index += 1 } - return rows[index] - } - - internal func drain() async throws { - index = rows.count - } -} diff --git a/apple/TablePro/Core/Compare/StreamingRowProvider.swift b/apple/TablePro/Core/Compare/StreamingRowProvider.swift index 979cabe6fb..b6fb6f44cf 100644 --- a/apple/TablePro/Core/Compare/StreamingRowProvider.swift +++ b/apple/TablePro/Core/Compare/StreamingRowProvider.swift @@ -5,14 +5,15 @@ // Feeds the merge join from a driver's row stream. It holds one batch of rows // at a time, and hands them out one row at a time. // -// It is not what bounds a comparison's memory. A driver's stream buffers -// `.unbounded` and its producer starts at the moment the stream is built, not -// at the first read, so a side the walk consumes slowly runs ahead inside the -// stream's own buffer. The row limit, the filter, and the key order the two -// sides share are what keep a comparison's footprint down. +// It is not what bounds a comparison's memory. Most drivers' streams hold a +// few batches and wait for the walk; ClickHouse, Elasticsearch, Trino, MongoDB, +// Spanner and SQLite over SSH run ahead of a side the walk consumes slowly, +// inside the stream's own buffer. The row limit, the filter, and the key order +// the two sides share are what keep a comparison's footprint down there. // import Foundation +import os import TableProPluginKit internal struct RowReadContext: Sendable { @@ -55,6 +56,7 @@ internal final class StreamingRowProvider: DataRowProviding { private var deliveredCount = 0 private var isFinished = false private var sawRowPastTheLimit = false + private let rowCut: OSAllocatedUnfairLock internal init( stream: AsyncThrowingStream, @@ -62,17 +64,25 @@ internal final class StreamingRowProvider: DataRowProviding { rowLimit: Int? = nil, context: RowReadContext? = nil ) { - self.iterator = stream.makeAsyncIterator() + let rowCut = OSAllocatedUnfairLock(initialState: nil) + self.rowCut = rowCut + self.iterator = ExportRowCut.reporting(stream, engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + }.makeAsyncIterator() self.columns = columns self.rowLimit = rowLimit self.context = context } - /// The read asks for one row more than the limit and never hands that row out, so this is the - /// server's own answer to whether anything was left rather than a guess from the row count: a - /// table holding exactly the limit is not a truncated read. + // The read asks for one row past the limit, so a table holding exactly the limit is not a cut read. A stream the + // driver stopped at a limit it declares ends here too: a key only the other side holds was never read on this one. internal var endedAtRowLimit: Bool { - sawRowPastTheLimit + sawRowPastTheLimit || driverRowCut != nil + } + + internal var driverRowCut: Int? { + guard isFinished else { return nil } + return rowCut.withLock { $0 } } internal func nextRow() async throws -> DataRow? { diff --git a/apple/TablePro/Core/Coordinators/ExactRowCounter.swift b/apple/TablePro/Core/Coordinators/ExactRowCounter.swift index cae7d98f63..b262e649dc 100644 --- a/apple/TablePro/Core/Coordinators/ExactRowCounter.swift +++ b/apple/TablePro/Core/Coordinators/ExactRowCounter.swift @@ -42,12 +42,15 @@ internal enum ExactRowCounter { return try await driver.fetchExactRowCount(table: table, browseFilters: search.pluginQueryFilters) } + // `safeModeIsReadOnly` reaches the host's COUNT(*), which carries the user's raw filter. A driver's own count builds + // its query from the filters behind a call that takes no context. internal static func count( on driver: DatabaseDriver, table: String, filters: [TableFilter], logicMode: FilterLogicMode, - countSQL: String? + countSQL: String?, + safeModeIsReadOnly: Bool = false ) async throws -> Int? { let chosen = route( countSQL: countSQL, @@ -58,14 +61,14 @@ internal enum ExactRowCounter { case .driverCount: return try await driver.fetchExactRowCount(table: table, filters: filters, logicMode: logicMode) case .hostCountSQL(let sql): - return try await hostCount(sql, on: driver) + return try await hostCount(sql, on: driver, safeModeIsReadOnly: safeModeIsReadOnly) case .driverCountThenHostSQL(let sql): if let counted = try await driverCountAllowingFallback( on: driver, table: table, filters: filters, logicMode: logicMode ) { return counted } - return try await hostCount(sql, on: driver) + return try await hostCount(sql, on: driver, safeModeIsReadOnly: safeModeIsReadOnly) } } @@ -84,8 +87,13 @@ internal enum ExactRowCounter { } } - private static func hostCount(_ sql: String, on driver: DatabaseDriver) async throws -> Int? { - let result = try await driver.execute(query: sql) + private static func hostCount(_ sql: String, on driver: DatabaseDriver, safeModeIsReadOnly: Bool) async throws -> Int? { + let result: QueryResult + if let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) { + result = try await driver.execute(query: sql, context: context) + } else { + result = try await driver.execute(query: sql) + } guard let countText = result.rows.first?.first?.asText else { return nil } return Int(countText) } diff --git a/apple/TablePro/Core/Coordinators/FindCoordinator.swift b/apple/TablePro/Core/Coordinators/FindCoordinator.swift index 146c4ac2af..0d6b7f247a 100644 --- a/apple/TablePro/Core/Coordinators/FindCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/FindCoordinator.swift @@ -19,7 +19,8 @@ final class FindCoordinator { var hasActiveFind: Bool { guard let find = model?.find else { return false } - return find.isVisible && !find.matches.isEmpty + // A typed term's search has no matches yet; Find Next waits for it rather than going to the editor. + return find.isVisible && (!find.matches.isEmpty || find.isAwaitingLoadedRows) } func showFind(replace: Bool = false) { diff --git a/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift b/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift index b86898ccc0..7836d93b37 100644 --- a/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/PaginationCoordinator.swift @@ -172,6 +172,10 @@ final class PaginationCoordinator: ObservableObject { tableName: tableName, schemaName: schemaName, filters: filters, logicMode: logicMode, columns: queryColumns.columns, columnTypes: queryColumns.columnTypes ) + let safeModeIsReadOnly = parent.runsAtReadOnly + let gate = browseFilters == nil && parent.rawFilterNeedsReadOnlyGate(filters) + ? countSQL.map { parent.rawFilterRequest(sql: $0, canPrompt: true, scope: scope) } + : nil // Claims the superseded automatic count's flag too: its task finds the token changed and won't clear a successor's state. parent.tabManager.mutate(at: index) { tab in @@ -183,14 +187,26 @@ final class PaginationCoordinator: ObservableObject { let token = UUID() parent.claimExactCount(for: tabId, token: token) let task = Task(priority: .userInitiated) { [parent] in - let outcome = await Self.exactRowCount( - scope: scope, - tableName: tableName, - filters: filters, - logicMode: logicMode, - browseFilters: browseFilters, - countSQL: countSQL - ) + var refusal: ExecutionGateError? + if let gate { + refusal = await parent.executionGate.authorize(gate).denialError + } + let outcome: Result + if case .cancelledByUser? = refusal { + outcome = .success(nil) + } else if let refusal { + outcome = .failure(refusal) + } else { + outcome = await Self.exactRowCount( + scope: scope, + tableName: tableName, + filters: filters, + logicMode: logicMode, + browseFilters: browseFilters, + countSQL: countSQL, + safeModeIsReadOnly: safeModeIsReadOnly + ) + } // Clears on every exit: returning early on cancel left a page turn during COUNT(*) with an endless spinner and no Count Exactly. let isCurrent = !Task.isCancelled && parent.tabExecution.isSameContent(contentEpoch, for: tabId) @@ -233,7 +249,8 @@ final class PaginationCoordinator: ObservableObject { filters: [TableFilter], logicMode: FilterLogicMode, browseFilters: [PluginQueryFilter]?, - countSQL: String? + countSQL: String?, + safeModeIsReadOnly: Bool ) async -> Result { do { let count = try await DatabaseManager.shared.withRowCountDriver(scope: scope, filtered: !filters.isEmpty) { driver in @@ -241,7 +258,12 @@ final class PaginationCoordinator: ObservableObject { return try await driver.fetchExactRowCount(table: tableName, browseFilters: browseFilters) } return try await ExactRowCounter.count( - on: driver, table: tableName, filters: filters, logicMode: logicMode, countSQL: countSQL + on: driver, + table: tableName, + filters: filters, + logicMode: logicMode, + countSQL: countSQL, + safeModeIsReadOnly: safeModeIsReadOnly ) } return .success(count) @@ -310,6 +332,7 @@ final class PaginationCoordinator: ObservableObject { let isTableTab = parent.tabManager.tabs[idx].tabType == .table let lengthColumns = isTableTab ? parent.unloadedValueLengthColumns(for: parent.tabManager.tabs[idx]) : [] let failureOutput = ServerOutputBox() + let safeModeIsReadOnly = parent.runsAtReadOnly let startedAt = ContinuousClock.Instant.now // Both releases cover the whole task: a cancelled bare return left a finished fetch installed, which `installQueryTask` then ended. @@ -321,6 +344,25 @@ final class PaginationCoordinator: ObservableObject { guard let self, !parent.isTearingDown else { return } do { + // The level may have changed since the statement first ran, so the gate reads it again, as a sort's rerun does. + let decision = await parent.executionGate.authorize(OperationRequest( + connectionId: scope.connectionId, + databaseType: parent.connection.type, + sql: baseQuery, + kind: .readQuery, + caller: .userInterface, + capabilities: .interactiveUser, + operationDescription: String(localized: "Fetch All"), + scope: scope + )) + if case .cancelledByUser? = decision.denialError { + await MainActor.run { [weak self] in + self?.parent.tabManager.mutate(tabId: tabId) { $0.pagination.isLoadingMore = false } + } + return + } + if let denial = decision.denialError { throw denial } + let start = CFAbsoluteTimeGetCurrent() progressLog.info("[fetchAll] executing full query: \(baseQuery.prefix(100), privacy: .private)") let result = try await parent.withExecutionDriver( @@ -328,12 +370,15 @@ final class PaginationCoordinator: ObservableObject { isTableTab: isTableTab, lease: lease ) { driver in - let rerun = { - try await driver.executeUserQuery( - query: baseQuery, - rowCap: nil, - parameters: storedParamValues.map { $0.map { $0 as Any? } } - ) + let parameters = storedParamValues.map { $0.map { $0 as Any? } } + let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) + let rerun: () async throws -> QueryResult = { + if let context { + return try await driver.executeUserQuery( + query: baseQuery, rowCap: nil, parameters: parameters, context: context + ) + } + return try await driver.executeUserQuery(query: baseQuery, rowCap: nil, parameters: parameters) } let result: QueryResult if isTableTab { @@ -347,6 +392,15 @@ final class PaginationCoordinator: ObservableObject { let fetchTime = CFAbsoluteTimeGetCurrent() - start progressLog.info("[fetchAll] rows=\(result.rows.count) fetchTime=\(String(format: "%.3f", fetchTime))s") + let prepared = await TableRows.preparing( + result.rows, + rowLocators: isTableTab ? result.rowLocators : nil, + exportDocuments: result.exportDocuments, + absentCells: result.absentCells, + unloadedCells: result.unloadedCells, + shape: parent.tabSessionRegistry.tableRows(for: tabId).rowShape + ) + guard !Task.isCancelled else { // Clears the flag like every exit: a bare return stuck the tab on Loading after a late cancel (no-op `cancelQuery()` drivers). await MainActor.run { [weak self] in @@ -368,7 +422,8 @@ final class PaginationCoordinator: ObservableObject { guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return } parent.mutateActiveTableRows(for: tabId) { rows in - rows.replace( + if let installed = rows.install(prepared) { return installed } + return rows.replace( rows: result.rows, rowLocators: isTableTab ? result.rowLocators : nil, exportDocuments: result.exportDocuments, diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift index aec7e5162d..46fd174885 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift @@ -26,6 +26,7 @@ private struct BatchOutput: Sendable { let rowsAffected: Int let errors: [PluginBatchError] let discardedResultSetCount: Int + let supersededResultSetCount: Int let executionTime: TimeInterval let errorDescription: String? var serverOutput: PluginServerOutput = .none @@ -67,6 +68,14 @@ private struct PrintedBatchAnswer: CarriesServerOutput { var serverOutput: PluginServerOutput = .none } +// Which query each batch's result sets belong to (nil: they are numbered), and their rows built off the main actor. +private struct BatchResultPlan { + let queries: [[SQLStatementScanner.ExecutableStatement]?] + let built: StatementRowsBuild + // Over every result the batches produce, in order: each result set, and one count result per batch with none. + let results: RunResultPlan +} + private struct BatchRun { let outcome: BatchStatementOutcome let plan: BatchTransactionPlan @@ -97,6 +106,7 @@ extension QueryExecutionCoordinator { /// The app opens no transaction around the run, as SQL Server's own tools do not: a batch carries on past most /// errors, so an app-owned rollback would take back statements that ran after the failure, and the script's own /// `BEGIN TRAN`, `TRY...CATCH` and procedures are what decide. A transaction the script leaves open is reported. + /// Manual Commit is the exception: the run is held, and a failure or a Stop rolls it back. func executeBatches( _ batches: [ExecutableBatch], parameters: [QueryParameter], @@ -146,20 +156,35 @@ extension QueryExecutionCoordinator { guard let self else { return } let run = await runBatches(prepared, scope: scope, mode: mode, claim: claim, lease: lease) postRanStatements(of: prepared, run: run, scope: scope, connection: conn) + let plan = await batchResultPlan(prepared, outputs: Self.outputs(of: run.outcome), tabId: tabId) let sessionNotice = Self.runNotice(outcome: run.outcome, sessionState: run.sessionState) + guard parent.tabExecution.settle(claim) || reportsUnknownCommitAfterStop(run.outcome, claim: claim) else { + await keepBatchesAfterStop( + prepared, run: run, plan: plan, parameters: parameters, claim: claim, sessionNotice: sessionNotice + ) + return + } + parent.retireQueryTask(.claim(claim)) + switch run.outcome { case .cancelled(let outputs): - guard parent.tabExecution.settle(claim) else { return } - parent.retireQueryTask(.claim(claim)) - keepStoppedBatches(prepared, outputs: outputs, parameters: parameters, tabId: tabId, sessionNotice: sessionNotice) + keepStoppedBatches( + prepared, + outputs: outputs, + parameters: parameters, + tabId: tabId, + sessionNotice: sessionNotice, + plan: plan, + presentsResults: true + ) parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) case .completed(let outputs): applyCompletedBatches( prepared, outputs: outputs, parameters: parameters, - tabId: tabId, claim: claim, sessionNotice: sessionNotice + tabId: tabId, claim: claim, sessionNotice: sessionNotice, plan: plan ) case .failed(let outputs, let failure, let errorDescription): handleBatchFailure( @@ -177,7 +202,8 @@ extension QueryExecutionCoordinator { parameters: parameters, tabId: tabId, claim: claim, - failureOutput: run.failureOutput + failureOutput: run.failureOutput, + plan: plan ) } } @@ -240,6 +266,10 @@ extension QueryExecutionCoordinator { lease: DriverLeaseOwner ) async -> BatchRun { let failureOutput = ServerOutputBox() + let commitMode = DatabaseManager.shared.commitMode(for: scope.connectionId) + let databaseType = parent.connection.type + let grammar = parent.lexicalGrammar + let statementTexts = prepared.flatMap { $0.batch.statements.map(\.sql) } do { var run = try await DatabaseManager.shared.withScopedDriver( scope: scope, @@ -247,7 +277,16 @@ extension QueryExecutionCoordinator { cancellation: .cancellableRead(lease) ) { driver in let startState = await driver.heldSessionTransactionState() - let plan = BatchTransactionPlan.autocommit.joining(startState) + let manual = ManualCommit.decision( + commitMode: commitMode, + statements: statementTexts, + databaseType: databaseType, + grammar: grammar, + supportsTransactions: driver.supportsTransactions, + sessionState: startState + ) + if let refusal = manual.refusal { throw refusal } + let plan = ManualCommit.runPlan(text: .autocommit, decision: manual, sessionState: startState) let outcome = await BatchStatementRun.run( prepared, plan: plan, @@ -261,6 +300,11 @@ extension QueryExecutionCoordinator { ) { batch in try await Self.runBatch(batch, driver: driver, failureOutput: failureOutput) } + await DatabaseManager.shared.noteSessionStatements( + prepared.prefix(outcome.ranCount(of: prepared.count)).flatMap { $0.batch.statements.map(\.sql) }, + ranOn: driver, + scope: scope + ) let sessionState = await driver.heldSessionTransactionState() return BatchRun(outcome: outcome, plan: plan, sessionState: sessionState, startState: startState) } @@ -291,7 +335,8 @@ extension QueryExecutionCoordinator { driver: DatabaseDriver, failureOutput: ServerOutputBox ) async throws -> BatchOutput { - var combined = QueryBatchResult.empty + var earliest = QueryBatchResult.empty + var latest = QueryBatchResult.empty var printed = BatchPrintedOutput() for _ in 0.. [ResultSet] { - let databaseType = parent.connection.type let grammar = parent.lexicalGrammar let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 - var resultSets: [ResultSet] = [] - for (entry, output) in zip(prepared, outputs) { - let isPlainQuery = { (sql: String) in - QueryExecutor.qualifiesForRowCap(sql: sql, tabType: .query, databaseType: databaseType) - } - let produced: [ResultSet] - if BatchResultMapping.mapsToStatements( - entry.batch, - resultSetCount: output.resultSets.count, - hasErrors: !output.errors.isEmpty, - grammar: grammar, - isPlainQuery: isPlainQuery - ) { - let queries = entry.batch.statements.filter { isPlainQuery($0.sql) } - produced = zip(queries, output.resultSets).map { statement, result in - let replayable = entry.parameterValues == nil - && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) - return makeStatementResultSet( - result: result, - sql: statement.sql, - index: firstNumber + resultSets.count, - baseQuery: replayable ? statement.sql : nil, - tabId: tabId, - anchor: StatementAnchor(statement) + let reading = parent.tableNameReading + let runPlan = plan.results + var builtIndex = 0 + // Taken in the order `batchResultPlan` built them: batch by batch, each batch's result sets in order. + func nextBuilt() -> StatementRows? { + defer { builtIndex += 1 } + return plan.built.statement(at: builtIndex, readBy: reading) + } + var own: [Int: ResultSet] = [:] + var parts: [Int: RunSummaryPart] = [:] + var candidate = 0 + // A result left out by the plan is never built; one the summary covers keeps only its counts and output. + func place(_ result: QueryResult?, count output: BatchOutput? = nil, build: (Int) -> ResultSet) { + defer { candidate += 1 } + if let position = runPlan.position(ofStatement: candidate) { + own[candidate] = build(firstNumber + position) + } else if runPlan.summarizes(statement: candidate) { + if let result { + parts[candidate] = RunSummaryPart(result) + } else if let output { + parts[candidate] = RunSummaryPart( + rowsAffected: output.rowsAffected, + executionTime: output.executionTime, + serverOutput: .none, + statusMessage: nil ) } + } + } + for ((entry, output), queries) in zip(zip(prepared, outputs), plan.queries) { + candidate += output.supersededResultSetCount + let firstCandidate = candidate + if let queries { + for (statement, result) in zip(queries, output.resultSets) { + let built = nextBuilt() + place(result) { index in + let replayable = entry.parameterValues == nil + && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) + return makeStatementResultSet( + result: result, + sql: statement.sql, + index: index, + baseQuery: replayable ? statement.sql : nil, + tabId: tabId, + anchor: StatementAnchor(statement), + built: built + ) + } + } } else if output.resultSets.isEmpty { - produced = output.errors.isEmpty - ? [countResultSet(output, anchor: entry.batch.anchor, index: firstNumber + resultSets.count)] - : [] + if output.errors.isEmpty { + place(nil, count: output) { index in countResultSet(output, anchor: entry.batch.anchor, index: index) } + } } else { - produced = output.resultSets.enumerated().map { offset, result in - numberedResultSet( - result, anchor: entry.batch.anchor, index: firstNumber + resultSets.count + offset, tabId: tabId - ) + for result in output.resultSets { + let built = nextBuilt() + place(result) { index in + numberedResultSet(result, anchor: entry.batch.anchor, index: index, tabId: tabId, built: built) + } } } - produced.last?.serverOutput = output.serverOutput - resultSets += produced + // The batch's printed output belongs to its last result, wherever that result went. + guard candidate > firstCandidate else { continue } + let last = candidate - 1 + if let resultSet = own[last] { + resultSet.serverOutput = output.serverOutput + } else if let part = parts[last] { + parts[last] = part.withServerOutput(output.serverOutput) + } + } + let resultSets = runPlan.entries.compactMap { entry -> ResultSet? in + switch entry { + case .statement(let index): + return own[index] + case .summary(let indices): + return makeRunSummaryResultSet(indices.compactMap { parts[$0] }) + } } + appendDroppedNotice(of: runPlan, to: resultSets) return resultSets } + // Decided once, before the settle: the mapping costs a classification per statement and the rows a pass each. + private func batchResultPlan(_ prepared: [PreparedBatch], outputs: [BatchOutput], tabId: UUID) async -> BatchResultPlan { + let databaseType = parent.connection.type + let grammar = parent.lexicalGrammar + let queries = zip(prepared, outputs).map { entry, output in + Self.mappedQueries(of: entry, output: output, databaseType: databaseType, grammar: grammar) + } + // One entry per result in the order `batchResultSets` places them; a count result has no rows to build. + var shapes: [Bool] = [] + var superseded: Set = [] + var sources: [(sql: String?, result: QueryResult)] = [] + var sourceCandidates: [Int] = [] + for (mapped, output) in zip(queries, outputs) { + if mapped == nil, output.resultSets.isEmpty { + if output.errors.isEmpty { + shapes.append(false) + } + continue + } + // Result sets the batch let go still count toward the run's results, and are never shown. + superseded.formUnion(shapes.count..<(shapes.count + output.supersededResultSetCount)) + shapes += repeatElement(true, count: output.supersededResultSetCount) + let pairs: [(sql: String?, result: QueryResult)] = mapped.map { mapped in + zip(mapped, output.resultSets).map { (sql: $0.sql as String?, result: $1) } + } ?? output.resultSets.map { (sql: nil, result: $0) } + for pair in pairs { + sourceCandidates.append(shapes.count) + shapes.append(!pair.result.columns.isEmpty) + sources.append(pair) + } + } + let results = RunResultPlan( + producesRows: shapes, + keepingRowResults: AppSettingsManager.shared.dataGrid.resultSetLimitPerRun, + unavailable: superseded + ) + let shown = zip(sources, sourceCandidates).map { source, candidate in + results.showsRows(ofStatement: candidate) ? source : nil + } + return BatchResultPlan(queries: queries, built: await buildStatementRows(shown, tabId: tabId), results: results) + } + + // The query each of a batch's result sets belongs to, or nil when the batch's shape does not pin them. + private static func mappedQueries( + of entry: PreparedBatch, + output: BatchOutput, + databaseType: DatabaseType, + grammar: SQLLexicalGrammar + ) -> [SQLStatementScanner.ExecutableStatement]? { + let isPlainQuery = { (sql: String) in + QueryExecutor.qualifiesForRowCap(sql: sql, tabType: .query, databaseType: databaseType) + } + guard output.supersededResultSetCount == 0, BatchResultMapping.mapsToStatements( + entry.batch, + resultSetCount: output.resultSets.count, + hasErrors: !output.errors.isEmpty, + grammar: grammar, + isPlainQuery: isPlainQuery + ) else { return nil } + return entry.batch.statements.filter { isPlainQuery($0.sql) } + } + + private static func outputs(of outcome: BatchStatementOutcome) -> [BatchOutput] { + switch outcome { + case .completed(let results), .cancelled(let results), .failed(let results, _, _): + return results + } + } + private static func runNotice( outcome: BatchStatementOutcome, sessionState: PluginSessionTransactionState ) -> String? { - let outputs: [BatchOutput] - switch outcome { - case .completed(let results), .cancelled(let results), .failed(let results, _, _): - outputs = results - } - return BatchRunNotice.text( - discardedResultSetCount: outputs.reduce(0) { $0 + $1.discardedResultSetCount }, + BatchRunNotice.text( + discardedResultSetCount: outputs(of: outcome).reduce(0) { $0 + $1.discardedResultSetCount }, sessionState: sessionState ) } - private func numberedResultSet(_ result: QueryResult, anchor: StatementAnchor?, index: Int, tabId: UUID) -> ResultSet { - let columns = result.columns.map { String($0) } + private func numberedResultSet( + _ result: QueryResult, + anchor: StatementAnchor?, + index: Int, + tabId: UUID, + built: StatementRows? + ) -> ResultSet { let resultSet = ResultSet( label: ResultSet.label(tableName: nil, anchor: nil, index: index), - tableRows: TableRows.from( - queryRows: result.rows, - columns: columns, - columnTypes: result.columnTypes, - exportDocuments: result.exportDocuments, - absentCells: result.absentCells, - masks: parent.columnMasks(forTabId: tabId, table: nil, columns: columns, columnTypes: result.columnTypes), - exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: nil, - columnOrigins: result.columnOrigins - ) + tableRows: statementTableRows(result, tableName: nil, tabId: tabId, built: built?.prepared) ) resultSet.statementAnchor = anchor resultSet.executionTime = result.executionTime diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 4289d16183..ecad7c5c02 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -183,6 +183,19 @@ extension QueryExecutionCoordinator { } } + // Read before and inside `applyPhase1Result`, so rows built off the main actor install as built. A Reveal stays only on its table. + func phase1RowShape(tabId: UUID, tableName: String?, columns: [String], columnTypes: [ColumnType]) -> TableRows.RowShape { + let tab = parent.tabManager.tabs.first { $0.id == tabId } + let keepsReveal = tab?.revealsMaskedValues == true && tab?.tableContext.tableName == tableName + let policy = keepsReveal ? MaskingPolicy.none : parent.maskingPolicy + return TableRows.RowShape( + columns: columns, + masks: policy.columnMasks(table: tableName, columns: columns, columnTypes: columnTypes), + exportPolicy: policy, + exportTable: tableName + ) + } + func applyPhase1Result( // swiftlint:disable:this function_parameter_count tabId: UUID, columns: [String], @@ -210,7 +223,9 @@ extension QueryExecutionCoordinator { exportDocuments: [PluginExportDocument]? = nil, absentCells: [Int: Set] = [:], unloadedCells: [Int: [Int: Int]] = [:], - columnOrigins: [PluginColumnOrigin?]? = nil + columnOrigins: [PluginColumnOrigin?]? = nil, + prepared: TableRows.PreparedRows? = nil, + trigger: TableLoadTrigger = .userInitiated ) { guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return } @@ -250,11 +265,18 @@ extension QueryExecutionCoordinator { if parent.tabManager.tabs[idx].revealsMaskedValues, parent.tabManager.tabs[idx].tableContext.tableName != tableName { parent.tabManager.mutate(at: idx) { $0.revealsMaskedValues = false } } - let masks = parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: columnTypes) + let shape = phase1RowShape(tabId: tabId, tableName: tableName, columns: columns, columnTypes: columnTypes) + let builtRows = prepared.flatMap { $0.fits(shape) ? $0 : nil } ?? TableRows.prepare( + rows, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: shape + ) let newTableRows = TableRows.from( - queryRows: rows, - columns: columns, + prepared: builtRows, columnTypes: columnTypes, columnDefaults: resolved.columnDefaults, columnForeignKeys: resolved.columnForeignKeys, @@ -267,13 +289,6 @@ extension QueryExecutionCoordinator { rowMatchPolicy: resolved.rowMatchPolicy, hasAuthoritativeSchema: resolved.hasAuthoritativeSchema, foreignKeysFetched: resolved.foreignKeysFetched, - rowLocators: rowLocators, - exportDocuments: exportDocuments, - absentCells: absentCells, - unloadedCells: unloadedCells, - masks: masks, - exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: tableName, columnOrigins: columnOrigins ) let previousTableName = parent.tabManager.tabs[idx].tableContext.tableName @@ -369,20 +384,12 @@ extension QueryExecutionCoordinator { } parent.dropParkedEdits(ofBackgroundTab: tabId) - recordHistory( - QueryHistoryRecordRequest( - query: historySQL ?? sql, - connectionId: conn.id, - databaseName: historyDatabaseName(tabId: tabId), - databaseType: conn.type, - schemaName: historySchemaName(tabId: tabId), - source: historySource(tabId: tabId), - executionTime: executionTime, - rowCount: rows.count, - wasSuccessful: true, - timing: timing + if trigger.recordsHistory { + recordSuccessfulRun( + query: historySQL ?? sql, tabId: tabId, connection: conn, + executionTime: executionTime, rowCount: rows.count, timing: timing ) - ) + } if parent.tabManager.selectedTabId == tabId, isEditable, !parent.changeManager.hasChanges { parent.changeManager.clearChangesAndUndoHistory() @@ -465,11 +472,36 @@ extension QueryExecutionCoordinator { ) } + private func recordSuccessfulRun( + query: String, + tabId: UUID, + connection conn: DatabaseConnection, + executionTime: TimeInterval, + rowCount: Int, + timing: PluginQueryTiming? + ) { + recordHistory( + QueryHistoryRecordRequest( + query: query, + connectionId: conn.id, + databaseName: historyDatabaseName(tabId: tabId), + databaseType: conn.type, + schemaName: historySchemaName(tabId: tabId), + source: historySource(tabId: tabId), + executionTime: executionTime, + rowCount: rowCount, + wasSuccessful: true, + timing: timing + ) + ) + } + func launchPhase2Work( tableName: String, tabId: UUID, connectionType: DatabaseType, - schemaTask: Task? + schemaTask: Task?, + countsRows: Bool = true ) { let isNonSQL = PluginManager.shared.editorLanguage(for: connectionType) != .sql let contentEpoch = parent.tabExecution.contentEpoch(for: tabId) @@ -477,7 +509,7 @@ extension QueryExecutionCoordinator { Task(priority: .utility) { [weak self, parent] in // The caller marked the total pending; every exit that skips `resolveRowCount` must release it, or Count Exactly never returns. guard let self else { - parent.releaseCountPending(for: tabId) + if countsRows { parent.releaseCountPending(for: tabId) } return } guard !parent.isTearingDown else { return } @@ -489,14 +521,14 @@ extension QueryExecutionCoordinator { await MainActor.run { [weak self] in guard let self else { - parent.releaseCountPending(for: tabId) + if countsRows { parent.releaseCountPending(for: tabId) } return } if let schema { applySchemaMetadata(schema, tabId: tabId, tableName: tableName, resultSetId: resultSetId) } // A retarget or newer run owns the tab now with its own pending mark; clearing here would release the successor's. - guard parent.tabExecution.isSameContent(contentEpoch, for: tabId) else { return } + guard countsRows, parent.tabExecution.isSameContent(contentEpoch, for: tabId) else { return } resolveRowCount( tableName: tableName, tabId: tabId, @@ -730,11 +762,26 @@ extension QueryExecutionCoordinator { columns: queryColumns.columns, columnTypes: queryColumns.columnTypes ) - return (plan, ExactCountInput(sql: sql, filters: filters, logicMode: logicMode), scope) + let input = ExactCountInput( + sql: sql, + filters: filters, + logicMode: logicMode, + safeModeIsReadOnly: parent.runsAtReadOnly, + gate: parent.rawFilterNeedsReadOnlyGate(filters) + ? sql.map { parent.rawFilterRequest(sql: $0, canPrompt: false, scope: scope) } + : nil + ) + return (plan, input, scope) + } + + // Nobody asked for this count, so nothing may ask the user about it: a refusal leaves the total unknown. + var plan = prepared.plan + if let count = prepared.count, let gate = count.gate, !(await parent.executionGate.authorize(gate).isAuthorized) { + plan = .clear } let outcome = await Self.rowCountOutcome( - plan: prepared.plan, + plan: plan, count: prepared.count, scope: prepared.scope, tableName: tableName @@ -771,7 +818,7 @@ extension QueryExecutionCoordinator { case .clear: return .clear case .approximate: - guard let count = try? await DatabaseManager.shared.withMetadataDriver(scope: scope, { driver in + guard let count = try? await DatabaseManager.shared.withMetadataRead(scope: scope, workload: .interactive, { driver in try await driver.fetchApproximateRowCount(table: tableName, schema: scope.schema) }) else { return nil } return .count(count, isApproximate: true) @@ -801,7 +848,8 @@ extension QueryExecutionCoordinator { table: tableName, filters: exactCount.filters, logicMode: exactCount.logicMode, - countSQL: sql + countSQL: sql, + safeModeIsReadOnly: exactCount.safeModeIsReadOnly ) } return count.map { RowCountOutcome.count($0, isApproximate: false) } @@ -846,7 +894,8 @@ extension QueryExecutionCoordinator { sql: String, tabId: UUID, connection conn: DatabaseConnection, - serverOutput: PluginServerOutput = .none + serverOutput: PluginServerOutput = .none, + trigger: TableLoadTrigger = .userInitiated ) { let diagnosis = DatabaseWriteRejectionDiagnosis.formatted(error) let message = ServerOutputCapture.failureMessage(diagnosis, output: serverOutput) @@ -859,6 +908,7 @@ extension QueryExecutionCoordinator { tab.execution.executionTime = nil } + guard trigger.recordsHistory else { return } recordHistory( QueryHistoryRecordRequest( query: sql, @@ -907,6 +957,9 @@ internal struct ExactCountInput: Sendable { internal let sql: String? internal let filters: [TableFilter] internal let logicMode: FilterLogicMode + internal var safeModeIsReadOnly = false + // What the gate is asked before a raw filter that calls a function runs at Read-Only. + internal var gate: OperationRequest? } enum RowCountPlan: Equatable { diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 08d9ddbe9a..e9d278d3dc 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -7,6 +7,24 @@ import Foundation import TableProPluginKit import TableProSQLGrammar +// A statement's table name and rows, built away from the main actor for the result it becomes. +struct StatementRows: Sendable { + let tableName: String? + let prepared: TableRows.PreparedRows +} + +// The names hold only while the session still reads them the same way, so the build keeps the reading it used. +// A statement whose rows no result shows has nothing built. +struct StatementRowsBuild: Sendable { + let reading: TableNameReading + let statements: [StatementRows?] + + func statement(at index: Int, readBy current: TableNameReading) -> StatementRows? { + guard current == reading, statements.indices.contains(index) else { return nil } + return statements[index] + } +} + extension QueryExecutionCoordinator { func executeMultipleStatements( _ statements: [SQLStatementScanner.ExecutableStatement], @@ -21,15 +39,12 @@ extension QueryExecutionCoordinator { // The rest of a run, cut again once a statement changed how the session reads literals, in tab coordinates. nonisolated static func remainingStatements( in text: String, - after ran: NSRange, + after ran: SQLStatementScanner.ExecutableStatement, through end: Int, grammar: SQLLexicalGrammar ) -> [SQLStatementScanner.ExecutableStatement] { - let source = text as NSString - let start = NSMaxRange(ran) - guard start < end, end <= source.length else { return [] } - let rest = source.substring(with: NSRange(location: start, length: end - start)) - return SQLStatementScanner.executableStatements(in: rest, grammar: grammar).map { $0.offset(by: start) } + guard end <= (text as NSString).length else { return [] } + return SQLStatementScanner.executableStatements(in: text, after: ran, through: end, grammar: grammar) } // False once the editor's text no longer holds each statement where the run found it. @@ -40,12 +55,26 @@ extension QueryExecutionCoordinator { } } + // A context is Read-Only's, and every call below carries it. A COPY with rows never goes as a header alone: a driver + // without copy-in refuses before anything is sent, where some servers drop the connection on the header. func executeStatement( rowCap: Int?, originalSQL: String, driver: DatabaseDriver, - parameters: [Any?]? = nil + parameters: [Any?]? = nil, + copyData: String? = nil, + context: PluginStatementContext? = nil ) async throws -> QueryResult { + if let copyData { + return try await driver.copyFromStandardInput( + statement: originalSQL, rows: CopyDataPieces.reading(copyData), context: context + ) + } + if let context { + return try await executeStatement( + rowCap: rowCap, originalSQL: originalSQL, driver: driver, parameters: parameters, carrying: context + ) + } if rowCap != nil { if parameters == nil, let cap = rowCap, cap > 0, let bounded = try await driver.executeBoundedQuery(query: originalSQL, rowCap: cap) { @@ -59,32 +88,43 @@ extension QueryExecutionCoordinator { return try await driver.execute(query: originalSQL) } + private func executeStatement( + rowCap: Int?, + originalSQL: String, + driver: DatabaseDriver, + parameters: [Any?]?, + carrying context: PluginStatementContext + ) async throws -> QueryResult { + if rowCap != nil { + if parameters == nil, let cap = rowCap, cap > 0, + let bounded = try await driver.executeBoundedQuery(query: originalSQL, rowCap: cap, context: context) { + return bounded + } + return try await driver.executeUserQuery( + query: originalSQL, rowCap: rowCap, parameters: parameters, context: context + ) + } + if let parameters { + return try await driver.executeParameterized(query: originalSQL, parameters: parameters, context: context) + } + return try await driver.execute(query: originalSQL, context: context) + } + func makeStatementResultSet( result: QueryResult, sql: String, index: Int, baseQuery: String?, - baseQueryParameterValues: [String?]? = nil, + baseQueryParameterValues: [TypedParameterValue?]? = nil, namedParameterStatement: NamedParameterStatement? = nil, tabId: UUID, - anchor: StatementAnchor? = nil + anchor: StatementAnchor? = nil, + built: StatementRows? = nil ) -> ResultSet { - let tableName = parent.extractTableName(from: sql) - let columns = result.columns.map { String($0) } - let rows = TableRows.from( - queryRows: result.rows, - columns: columns, - columnTypes: result.columnTypes, - exportDocuments: result.exportDocuments, - absentCells: result.absentCells, - masks: parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: result.columnTypes), - exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: tableName, - columnOrigins: result.columnOrigins - ) + let tableName = if let built { built.tableName } else { parent.extractTableName(from: sql) } let resultSet = ResultSet( label: ResultSet.label(tableName: tableName, anchor: anchor, index: index), - tableRows: rows + tableRows: statementTableRows(result, tableName: tableName, tabId: tabId, built: built?.prepared) ) resultSet.statementAnchor = anchor resultSet.executionTime = result.executionTime @@ -101,6 +141,105 @@ extension QueryExecutionCoordinator { return resultSet } + // The rows built off the main actor when the masks they were built with still apply, else built here. + func statementTableRows(_ result: QueryResult, tableName: String?, tabId: UUID, built: TableRows.PreparedRows?) -> TableRows { + let columns = result.columns.map { String($0) } + let shape = TableRows.RowShape( + columns: columns, + masks: parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: result.columnTypes), + exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), + exportTable: tableName + ) + let prepared = built.flatMap { $0.fits(shape) && $0.rows.count == result.rows.count ? $0 : nil } ?? TableRows.prepare( + result.rows, + exportDocuments: result.exportDocuments, + absentCells: result.absentCells, + shape: shape + ) + return TableRows.from(prepared: prepared, columnTypes: result.columnTypes, columnOrigins: result.columnOrigins) + } + + // A nil sql is a result no statement names, which reads no table. A nil entry builds nothing. + func buildStatementRows(_ statements: [(sql: String?, result: QueryResult)?], tabId: UUID) async -> StatementRowsBuild { + let reveals = parent.tabManager.tabs.first { $0.id == tabId }?.revealsMaskedValues == true + return await Self.statementRows( + statements, + reading: parent.tableNameReading, + policy: reveals ? .none : parent.maskingPolicy + ) + } + + // A run of 300 SELECTs holds 300 capped results, too many rows to build on the main actor. A Stop keeps them, so + // this build runs to the end. + @concurrent + nonisolated static func statementRows( + _ statements: [(sql: String?, result: QueryResult)?], + reading: TableNameReading, + policy: MaskingPolicy + ) async -> StatementRowsBuild { + StatementRowsBuild(reading: reading, statements: statements.map { entry -> StatementRows? in + guard let entry else { return nil } + let (sql, result) = entry + let tableName = sql.flatMap(reading.tableName(in:)) + let shape = TableRows.RowShape( + columns: result.columns, + masks: policy.columnMasks(table: tableName, columns: result.columns, columnTypes: result.columnTypes), + exportPolicy: policy, + exportTable: tableName + ) + return StatementRows( + tableName: tableName, + prepared: TableRows.prepare( + result.rows, exportDocuments: result.exportDocuments, absentCells: result.absentCells, shape: shape + ) + ) + }) + } + + static let summaryMessageLimit = 5 + + // The statements of one run that returned no rows, as one result: rows affected and time added + // up, server output in the order it was printed. + func makeRunSummaryResultSet(_ parts: [RunSummaryPart]) -> ResultSet { + let count = parts.count.formatted() + let resultSet = ResultSet(label: String(format: String(localized: "%@ Statements"), count)) + resultSet.rowsAffected = parts.reduce(0) { $0 + $1.rowsAffected } + resultSet.executionTime = parts.reduce(0) { $0 + $1.executionTime } + // A driver's own message (a warning, a server reply) stays, each once, after the count; a long script's + // varied replies are cut to the first few. + var seen: Set = [] + var messages: [String] = [] + var unshown = 0 + for message in parts.compactMap(\.statusMessage) where !message.isEmpty && seen.insert(message).inserted { + if messages.count < Self.summaryMessageLimit { + messages.append(message) + } else { + unshown += 1 + } + } + if unshown == 1 { + messages.append(String(localized: "1 more message.")) + } else if unshown > 1 { + messages.append(String(format: String(localized: "%@ more messages."), unshown.formatted())) + } + resultSet.statusMessage = ([String(format: String(localized: "Ran %@ statements that returned no rows."), count)] + messages) + .joined(separator: " ") + resultSet.serverOutput = PluginServerOutput( + lines: parts.flatMap(\.serverOutput.lines), + isTruncated: parts.contains { $0.serverOutput.isTruncated } + ) + return resultSet + } + + func appendDroppedNotice(of plan: RunResultPlan, to resultSets: [ResultSet]) { + guard let notice = plan.droppedNotice else { return } + for resultSet in resultSets { + resultSet.statusMessage = [resultSet.statusMessage, notice] + .compactMap { $0?.isEmpty == false ? $0 : nil } + .joined(separator: " ") + } + } + /// Each statement in a multi-statement run targets its own table, so each result carries its /// own identity. Nothing fetches key columns for these statements, so the origin says so and /// the result is read-only until something does. Inheriting the previous run's keys made an @@ -188,8 +327,8 @@ extension QueryExecutionCoordinator { parent.tabManager.mutate(at: idx) { tab in if tab.tabType == .query { + // The schema stays: on a query tab it is the picker's binding, and a run summary has no origin to give. tab.tableContext.tableName = activeOrigin?.tableName - tab.tableContext.schemaName = activeOrigin?.schemaName tab.tableContext.primaryKeyColumns = activeOrigin?.primaryKeyColumns ?? [] tab.tableContext.isEditable = activeOrigin?.isEditable ?? false } else if activeOrigin?.tableName == nil { diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index b3492b2e3b..b6dcf300b5 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -44,9 +44,13 @@ private struct PreparedStatement: @unchecked Sendable { let parameterValues: [Any?]? let rowCap: Int? let anchor: StatementAnchor? - /// Whether this is the script's own `COMMIT`, read from the text before the lease is taken so - /// the run never lexes inside it. + /// Where the script was cut after this statement, which a replan of the rest starts from. + let statement: SQLStatementScanner.ExecutableStatement? + // Read from the text before the lease is taken, so the run never lexes inside it. let isCommitPoint: Bool + let endsItsTransaction: Bool + let closesTransaction: Bool + let movesDatabase: Bool } // What a run sends: the rest is cut again from the script once a statement changes how the session reads literals. @@ -74,10 +78,10 @@ private final class RunningStatements { func replanned(after ran: Int, session: SQLLexicalGrammar) -> [PreparedStatement]? { guard session != grammar, let script, prepared.indices.contains(ran - 1), - let last = prepared[ran - 1].anchor + let last = prepared[ran - 1].statement else { return nil } let rest = QueryExecutionCoordinator.remainingStatements( - in: script, after: last.range, through: end, grammar: session + in: script, after: last, through: end, grammar: session ).map { prepare($0, session) } grammar = session prepared = Array(prepared.prefix(ran)) + rest @@ -96,6 +100,34 @@ private struct ExecutedStatementHistory { let results: [QueryResult] let parameters: [QueryParameter] let connection: DatabaseConnection + // Read once per run: the rows built off the main actor and the results shown take the same plan. + let rowResultLimit: Int? + + func keeping(_ kept: [QueryResult]) -> ExecutedStatementHistory { + ExecutedStatementHistory( + prepared: prepared, results: kept, parameters: parameters, connection: connection, rowResultLimit: rowResultLimit + ) + } +} + +extension BatchStatementOutcome { + // The units a run the user stopped keeps: those that completed, and none that its plan rolled back. + func keptAfterStop(plan: BatchTransactionPlan) -> [Output] { + switch self { + case .completed(let results), .cancelled(let results): + return results + case .failed(_, .commitOutcomeUnknown, _): + // Nothing can say whether these were saved, so they are not recorded as plain successes. + return [] + case .failed(let results, _, _): + return plan.keepsExecutedStatements ? Array(results.prefix(succeededCount)) : [] + } + } + + var leftCommitOutcomeUnknown: Bool { + guard case .failed(_, .commitOutcomeUnknown, _) = self else { return false } + return true + } } extension QueryExecutionCoordinator { @@ -202,6 +234,8 @@ extension QueryExecutionCoordinator { let boundValues = BoundParameterValues(values: parameters) let failureOutput = ServerOutputBox() let grammar = parent.lexicalGrammar + let safeModeIsReadOnly = parent.runsAtReadOnly + let held = parent.manualCommitRun(skipped: tab.tabType != .query) let parameterizedTask = Task { [weak self, parent] in guard let self else { return } @@ -215,13 +249,18 @@ extension QueryExecutionCoordinator { route: DatabaseManager.shared.executionRoute(for: scope), cancellation: .cancellableRead(lease) ) { [queryExecutor = parent.queryExecutor, boundValues] driver in - let fetched = try await queryExecutor.executeQuery( - driver: driver, - sql: statement.sql, - parameters: boundValues.values, - rowCap: rowCap, - capturingOutputInto: failureOutput - ) + let fetched = try await held.running([statement.sql], on: driver) { + // Before it runs, so a statement that fails or a connection lost under it is still noted. + await DatabaseManager.shared.noteSessionStatements([statement.sql], ranOn: driver, scope: scope) + return try await queryExecutor.executeQuery( + driver: driver, + sql: statement.sql, + parameters: boundValues.values, + rowCap: rowCap, + safeModeIsReadOnly: safeModeIsReadOnly, + capturingOutputInto: failureOutput + ) + } let edits = await SucceededStatements.single( statement.sql, scope: scope, databaseType: conn.type, grammar: grammar, ranOn: driver ) @@ -286,7 +325,10 @@ extension QueryExecutionCoordinator { ) await MainActor.run { [weak self] in guard let self else { return } - guard parent.tabExecution.settle(claim) else { return } + guard parent.tabExecution.settle(claim) else { + parent.reportStopRollback(error, tabId: tabId) + return + } parent.tabManager.mutate(tabId: tabId) { tab in tab.pagination.isLoadingMore = false } @@ -295,6 +337,7 @@ extension QueryExecutionCoordinator { parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) + parent.reportStopRollback(error, tabId: tabId) return } handleQueryExecutionError( @@ -358,7 +401,7 @@ extension QueryExecutionCoordinator { let statementTexts = statements.map(\.sql) let transactionKind = OperationKind.worst(of: statementTexts, databaseType: conn.type) let grammar = parent.lexicalGrammar - let plan = BatchTransactionPolicy.plan(for: statementTexts, databaseType: conn.type, grammar: grammar) + let plan = BatchTransactionPolicy.plan(for: statementTexts, databaseType: conn.type, grammar: grammar, safeModeIsReadOnly: parent.runsAtReadOnly) let prepared = statements.map { statement in prepareStatement( statement: statement, @@ -377,7 +420,8 @@ extension QueryExecutionCoordinator { prepared: prepared, grammar: grammar, script: Self.scriptHolds(statements, in: script) ? script : nil, - end: statements.last.map { NSMaxRange($0.range) } ?? 0 + // The segment's end, so a last COPY cut again keeps its rows. + end: statements.last?.segmentEnd ?? 0 ) { statement, grammar in self.prepareStatement( statement: statement, @@ -404,7 +448,8 @@ extension QueryExecutionCoordinator { ) let outcome = run.outcome let sent = running.prepared - let sessionNotice = run.plan == .sessionTransaction ? run.sessionState.openTransactionNotice : nil + let readsEndState = run.plan.readsSessionEndState || run.plan == .autocommit + let sessionNotice = readsEndState ? run.sessionState.openTransactionNotice : nil let ranStatements: [String] switch outcome { @@ -428,28 +473,45 @@ extension QueryExecutionCoordinator { ) ))) + let results: [QueryResult] switch outcome { - case .cancelled(let results): - guard parent.tabExecution.settle(claim) else { return } - parent.retireQueryTask(.claim(claim)) + case .completed(let ran), .cancelled(let ran), .failed(let ran, _, _): + results = ran + } + let rowResultLimit = AppSettingsManager.shared.dataGrid.resultSetLimitPerRun + let resultPlan = RunResultPlan(producesRows: results.map { !$0.columns.isEmpty }, keepingRowResults: rowResultLimit) + let built = await buildStatementRows( + zip(sent, results).enumerated().map { index, pair in + resultPlan.showsRows(ofStatement: index) ? (sql: pair.0.originalSQL as String?, result: pair.1) : nil + }, + tabId: tabId + ) + + let history = ExecutedStatementHistory( + prepared: sent, results: results, parameters: parameters, connection: conn, rowResultLimit: rowResultLimit + ) + guard parent.tabExecution.settle(claim) || reportsUnknownCommitAfterStop(outcome, claim: claim) else { + keepStatementsAfterStop( + history.keeping(outcome.keptAfterStop(plan: run.plan)), + claim: claim, + sessionNotice: sessionNotice, + built: built + ) + return + } + parent.retireQueryTask(.claim(claim)) + + switch outcome { + case .cancelled: keepStoppedStatements( - history: ExecutedStatementHistory( - prepared: sent, results: results, parameters: parameters, connection: conn - ), - tabId: tabId, - sessionNotice: sessionNotice + history: history, tabId: tabId, sessionNotice: sessionNotice, built: built, presentsResults: true ) parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) - case .completed(let results): + case .completed: applyCompletedStatements( - history: ExecutedStatementHistory( - prepared: sent, results: results, parameters: parameters, connection: conn - ), - tabId: tabId, - claim: claim, - sessionNotice: sessionNotice + history: history, tabId: tabId, claim: claim, sessionNotice: sessionNotice, built: built ) case .failed(let results, let failure, let errorDescription): handleMultiStatementError( @@ -461,14 +523,13 @@ extension QueryExecutionCoordinator { plan: run.plan, sessionState: run.sessionState ), - history: ExecutedStatementHistory( - prepared: sent, results: results, parameters: parameters, connection: conn - ), + history: history, tabId: tabId, claim: claim, anchors: sent.map(\.anchor), timing: PluginQueryTiming.batch(of: results), - failureOutput: run.failureOutput + failureOutput: run.failureOutput, + built: built ) } } @@ -492,6 +553,8 @@ extension QueryExecutionCoordinator { : SQLParameterExtractor.convertToNativeStyle(sql: sql, parameters: parameters, style: style, grammar: grammar) let executableSQL = conversion?.sql ?? sql let bounded = resolveStatement(sql: executableSQL, tabType: tabType, bypassLimit: bypassRowLimit) + let family = TransactionEngineFamily.of(parent.connection.type) + let definitionEnds = family.definitionEndsTransaction return PreparedStatement( originalSQL: sql, executableSQL: executableSQL, @@ -499,7 +562,11 @@ extension QueryExecutionCoordinator { parameterValues: conversion?.values, rowCap: bounded.rowCap, anchor: StatementAnchor(statement), - isCommitPoint: BatchCommitStatement.matches(sql, grammar: grammar) + statement: statement, + isCommitPoint: BatchCommitStatement.matches(sql, grammar: grammar), + endsItsTransaction: definitionEnds && AutocommitOnlyStatement.matches(sql, family: family, grammar: grammar), + closesTransaction: definitionEnds && TransactionEndStatement.matches(sql, grammar: grammar), + movesDatabase: AutocommitOnlyStatement.movesDatabase(sql, family: family, grammar: grammar) ) } @@ -514,17 +581,32 @@ extension QueryExecutionCoordinator { claim: TabExecutionClaim, lease: DriverLeaseOwner ) async -> MultiStatementRun { + let commitMode = DatabaseManager.shared.commitMode(for: scope.connectionId) let failureOutput = ServerOutputBox() let commitOutput = ServerOutputBox() let databaseType = parent.connection.type + let definitionEnds = TransactionEngineFamily.of(databaseType).definitionEndsTransaction + let safeModeIsReadOnly = parent.runsAtReadOnly + let grammar = parent.lexicalGrammar + let statementTexts = prepared.map(\.sentSQL) do { var run = try await DatabaseManager.shared.withScopedDriver( scope: scope, route: DatabaseManager.shared.executionRoute(for: scope), cancellation: .cancellableRead(lease) ) { driver in + let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) let startState = await driver.heldSessionTransactionState() - let sessionPlan = plan.joining(startState) + let manual = ManualCommit.decision( + commitMode: commitMode, + statements: statementTexts, + databaseType: databaseType, + grammar: grammar, + supportsTransactions: driver.supportsTransactions, + sessionState: startState + ) + if let refusal = manual.refusal { throw refusal } + let sessionPlan = ManualCommit.runPlan(text: plan, decision: manual, sessionState: startState) let outcome = await BatchStatementRun.run( prepared, plan: sessionPlan, @@ -534,6 +616,9 @@ extension QueryExecutionCoordinator { gate: self.claimGate(for: claim), failureSQL: \.executableSQL, isCommitPoint: \.isCommitPoint, + endsItsTransaction: \.endsItsTransaction, + closesTransaction: \.closesTransaction, + movesDatabase: \.movesDatabase, replanned: { ran in running.replanned( after: ran, @@ -549,18 +634,29 @@ extension QueryExecutionCoordinator { rowCap: statement.rowCap, originalSQL: statement.sentSQL, driver: driver, - parameters: statement.parameterValues + parameters: statement.parameterValues, + copyData: statement.statement?.copyData, + context: context ) } } - guard sessionPlan == .sessionTransaction else { + let sent = await running.prepared + await DatabaseManager.shared.noteSessionStatements( + sent.prefix(outcome.ranCount(of: sent.count)).map(\.sentSQL), ranOn: driver, scope: scope + ) + guard sessionPlan.readsSessionEndState else { let endState: PluginSessionTransactionState = asksEndState ? await driver.heldSessionTransactionState() : .unknown + // An unwrapped run can still leave a transaction open: a Teradata ANSI session holds its DDL in one, completed or not. + let readsHeldWork = sessionPlan == .autocommit && (!outcome.isCompleted || definitionEnds) + let heldState: PluginSessionTransactionState = readsHeldWork + ? (asksEndState ? endState : await driver.heldSessionTransactionState()) + : .idle return MultiStatementRun( outcome: outcome, plan: sessionPlan, - sessionState: .idle, + sessionState: heldState, startState: startState, endState: endState ) @@ -611,36 +707,69 @@ extension QueryExecutionCoordinator { ) } - /// A Stop cannot take back what a plan without a transaction already committed, so the results - /// and the history of the statements that ran stay rather than being dropped with the run. + // A commit nobody can confirm is reported as one even after a Stop, rather than kept as plain successes. + func reportsUnknownCommitAfterStop(_ outcome: BatchStatementOutcome, claim: TabExecutionClaim) -> Bool { + outcome.leftCommitOutcomeUnknown + && stoppedRunCanReplaceResults(onTab: claim.tabId) + && parent.tabExecution.settleStopped(claim, replacingContent: true) + } + + // A stopped run comes back after the tab went idle. Edits staged since then keep the result they were made on. + func stoppedRunCanReplaceResults(onTab tabId: UUID) -> Bool { + guard let tab = parent.tabManager.tabs.first(where: { $0.id == tabId }) else { return false } + return !parent.holdsUnsavedWork(tab) + } + + // A Stop during a statement reads as one between statements: the completed ones are kept, with no error result. + private func keepStatementsAfterStop( + _ history: ExecutedStatementHistory, + claim: TabExecutionClaim, + sessionNotice: String?, + built: StatementRowsBuild + ) { + guard !history.results.isEmpty else { return } + let presentsResults = stoppedRunCanReplaceResults(onTab: claim.tabId) + guard parent.tabExecution.settleStopped(claim, replacingContent: presentsResults) else { return } + keepStoppedStatements( + history: history, + tabId: claim.tabId, + sessionNotice: sessionNotice, + built: built, + presentsResults: presentsResults + ) + parent.reportEndedExecutions([ + EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) + ]) + } + + // A Stop cannot take back what a plan without a transaction already committed, so those statements keep their history. private func keepStoppedStatements( history: ExecutedStatementHistory, tabId: UUID, - sessionNotice: String? + sessionNotice: String?, + built: StatementRowsBuild, + presentsResults: Bool ) { guard !history.results.isEmpty else { return } recordExecutedStatements(history, tabId: tabId, commitOutcomeIsUnknown: false) + guard presentsResults else { return } presentMultiStatementResults( tabId: tabId, timing: PluginQueryTiming.batch(of: history.results), totalRowsAffected: history.results.reduce(0) { $0 + $1.rowsAffected }, - newResultSets: statementResultSets(history, tabId: tabId), + newResultSets: statementResultSets(history, tabId: tabId, built: built), sessionNotice: sessionNotice ) } - /// Settles first, then writes. Everything below the gate belongs to a batch that still owns its - /// tab: its history rows, its outcome notification and its result sets. A superseded batch - /// writes none of them, which is what the single-statement path has always done. + // Below the caller's settle gate: a superseded batch writes no history rows, outcome or result sets. private func applyCompletedStatements( history: ExecutedStatementHistory, tabId: UUID, claim: TabExecutionClaim, - sessionNotice: String? + sessionNotice: String?, + built: StatementRowsBuild ) { - guard parent.tabExecution.settle(claim) else { return } - parent.retireQueryTask(.claim(claim)) - let totalRowsAffected = history.results.reduce(0) { $0 + $1.rowsAffected } reportOperation( kind: .queryBatch, @@ -654,28 +783,40 @@ extension QueryExecutionCoordinator { tabId: tabId, timing: PluginQueryTiming.batch(of: history.results), totalRowsAffected: totalRowsAffected, - newResultSets: statementResultSets(history, tabId: tabId), + newResultSets: statementResultSets(history, tabId: tabId, built: built), sessionNotice: sessionNotice ) } - private func statementResultSets(_ history: ExecutedStatementHistory, tabId: UUID) -> [ResultSet] { + // Only the statements the plan keeps become results, so a long script builds a few, not one per statement. + private func statementResultSets(_ history: ExecutedStatementHistory, tabId: UUID, built: StatementRowsBuild) -> [ResultSet] { let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 - return zip(history.prepared, history.results).enumerated().map { index, pair in - let (statement, result) = pair - return makeStatementResultSet( - result: result, - sql: statement.originalSQL, - index: firstNumber + index, - baseQuery: statement.executableSQL, - baseQueryParameterValues: statement.parameterValues?.map { $0 as? String }, - namedParameterStatement: statement.parameterValues == nil - ? nil - : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), - tabId: tabId, - anchor: statement.anchor - ) + let reading = parent.tableNameReading + let ran = Array(zip(history.prepared, history.results)) + let plan = RunResultPlan(producesRows: ran.map { !$0.1.columns.isEmpty }, keepingRowResults: history.rowResultLimit) + let resultSets = plan.entries.enumerated().map { position, entry -> ResultSet in + switch entry { + case .summary(let indices): + return makeRunSummaryResultSet(indices.map { RunSummaryPart(ran[$0].1) }) + case .statement(let index): + let (statement, result) = ran[index] + return makeStatementResultSet( + result: result, + sql: statement.originalSQL, + index: firstNumber + position, + baseQuery: statement.executableSQL, + baseQueryParameterValues: statement.parameterValues?.map(TypedParameterValue.kept), + namedParameterStatement: statement.parameterValues == nil + ? nil + : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), + tabId: tabId, + anchor: statement.anchor, + built: built.statement(at: index, readBy: reading) + ) + } } + appendDroppedNotice(of: plan, to: resultSets) + return resultSets } /// A commit whose connection died before the answer leaves every statement of the batch in a @@ -716,7 +857,15 @@ extension QueryExecutionCoordinator { anchor: StatementAnchor? = nil, install: ResultInstall = .newResult ) async -> Bool { - await MainActor.run { [weak self] in + let prepared = await TableRows.preparing( + fetchResult.rows, + exportDocuments: fetchResult.exportDocuments, + absentCells: fetchResult.absentCells, + shape: phase1RowShape( + tabId: tabId, tableName: tableName, columns: fetchResult.columns, columnTypes: fetchResult.columnTypes + ) + ) + return await MainActor.run { [weak self] in guard let self else { return false } guard parent.tabExecution.settle(claim) else { return false } parent.retireQueryTask(.claim(claim)) @@ -764,10 +913,11 @@ extension QueryExecutionCoordinator { serverOutput: fetchResult.serverOutput, exportDocuments: fetchResult.exportDocuments, absentCells: fetchResult.absentCells, - columnOrigins: fetchResult.columnOrigins + columnOrigins: fetchResult.columnOrigins, + prepared: prepared ) - let parameterValues = nativeParameters.map { $0 as? String } + let parameterValues = nativeParameters.map(TypedParameterValue.kept) let statement = originalSQL.map { NamedParameterStatement(sql: $0, parameters: originalParameters) } parent.tabManager.mutate(tabId: tabId) { $0.pagination.baseQueryParameterValues = parameterValues @@ -778,7 +928,7 @@ extension QueryExecutionCoordinator { } } - // Reports only, below the settle gate: the lease already rolled back, and a superseded batch writes no history. + // Below the caller's settle gate: the lease already rolled back, and a superseded batch writes no history. private func handleMultiStatementError( _ context: MultiStatementFailureContext, history: ExecutedStatementHistory, @@ -786,7 +936,8 @@ extension QueryExecutionCoordinator { claim: TabExecutionClaim, anchors: [StatementAnchor?], timing: PluginQueryTiming, - failureOutput: PluginServerOutput + failureOutput: PluginServerOutput, + built: StatementRowsBuild ) { let cumulativeTime = timing.total let errorDescription = context.errorDescription @@ -800,13 +951,10 @@ extension QueryExecutionCoordinator { .flatMap { anchors.indices.contains($0) ? anchors[$0] : nil } let failedStatementSQL = report.failedSQL - guard parent.tabExecution.settle(claim) else { return } - parent.retireQueryTask(.claim(claim)) - - /// Below the settle gate for the same reason the success arm is: a superseded batch - /// has its error dropped here, so announcing it would report on work the user has - /// already navigated away from. - reportOperation(kind: .queryBatch, claim: claim, outcome: .failed(reason: errorDescription)) + // A run stopped to wait for a commit has no server error, so the notification carries the report. + reportOperation( + kind: .queryBatch, claim: claim, outcome: .failed(reason: errorDescription.isEmpty ? report.message : errorDescription) + ) recordExecutedStatements( history, tabId: tabId, @@ -821,7 +969,7 @@ extension QueryExecutionCoordinator { tab.execution.executionTime = cumulativeTime tab.execution.lastExecutedAt = Date() - tab.display.replaceUnpinnedResults(with: statementResultSets(history, tabId: tabId) + [errorRS]) + tab.display.replaceUnpinnedResults(with: statementResultSets(history, tabId: tabId, built: built) + [errorRS]) if tab.display.isResultsCollapsed { tab.display.isResultsCollapsed = false } diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift index 9911aa248a..243da63feb 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift @@ -32,6 +32,7 @@ final class QueryExecutionCoordinator: ObservableObject { let batches = executionBatches(in: tab.content.query) let statements = batches.flatMap(\.statements) guard !statements.isEmpty else { return } + parent.tabVersions.record(tab, connectionId: parent.connectionId, reason: .run) if AppSettingsManager.shared.editor.queryParametersEnabled, parent.bindsNamedParameters { let combinedSQL = SQLParameterExtractor.parameterSource(of: statements) @@ -74,6 +75,7 @@ final class QueryExecutionCoordinator: ObservableObject { parent.isShowingSafeModePrompt = true let request = makeExecuteRequest( statements: batches.flatMap(\.statements), + tabIndex: index, extraCapabilities: extraCapabilities ) Task { [parent] in @@ -131,9 +133,11 @@ final class QueryExecutionCoordinator: ObservableObject { private func makeExecuteRequest( statements: [SQLStatementScanner.ExecutableStatement], + tabIndex index: Int, extraCapabilities: CallerCapabilities = [] ) -> OperationRequest { let sql = statements.map(\.sql) + let tabs = parent.tabManager.tabs return OperationRequest( connectionId: parent.connectionId, databaseType: parent.connection.type, @@ -141,7 +145,8 @@ final class QueryExecutionCoordinator: ObservableObject { kind: OperationKind.worst(of: sql, databaseType: parent.connection.type), caller: .userInterface, capabilities: CallerCapabilities.interactiveUser.union(extraCapabilities), - operationDescription: String(localized: "Execute Query") + operationDescription: String(localized: "Execute Query"), + runsIn: tabs.indices.contains(index) ? parent.scope(for: tabs[index]) : nil ) } @@ -158,6 +163,7 @@ final class QueryExecutionCoordinator: ObservableObject { parent.isShowingSafeModePrompt = true let request = makeExecuteRequest( statements: batches.flatMap(\.statements), + tabIndex: index, extraCapabilities: extraCapabilities ) Task { [parent] in @@ -206,7 +212,7 @@ final class QueryExecutionCoordinator: ObservableObject { } } - /// A run the driver cannot carry out as written says so before anything is sent, rather than running part of it. + // A run the driver cannot carry out as written says so before anything is sent, rather than running part of it. private func refuses(_ route: QueryExecutionRoute, tabIndex index: Int) -> Bool { guard case .needsBatchDriver = route else { return false } parent.tabManager.mutate(at: index) { diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator+Discard.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator+Discard.swift index b74650db33..408ed53bfa 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator+Discard.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator+Discard.swift @@ -12,7 +12,7 @@ extension RowEditingCoordinator { // MARK: - Sidebar Transaction // Scope is read before the authorization prompt (sheet and Touch ID), which gives the selection time to move. - func executeSidebarChanges(steps: [DataWriteStep]) async throws { + func executeSidebarChanges(steps: [DataWriteStep], viewTarget: PluginViewWriteTarget?) async throws { guard let scope = parent.selectedTabScope else { throw DatabaseError.notConnected } @@ -35,13 +35,16 @@ extension RowEditingCoordinator { } let mode: PluginTransactionAccessMode = kind.declaresWrite ? .readWrite : .serverDefault - let plan = DataWritePlan(scope: scope, databaseType: parent.connection.type, steps: steps) + var built = DataWritePlan(scope: scope, databaseType: parent.connection.type, steps: steps) + built.viewTarget = viewTarget + let plan = built + let commitMode = DatabaseManager.shared.commitMode(for: parent.connectionId) _ = try await DatabaseManager.shared.withScopedDriver( scope: scope, route: DatabaseManager.shared.executionRoute(for: scope), cancellation: .protectedWrite ) { driver in - _ = try await DataWriteExecutor.run(plan, mode: mode, on: driver) + _ = try await DataWriteExecutor.run(plan, mode: mode, commitMode: commitMode, on: driver) } } diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift index bcb80872e1..805110f096 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift @@ -52,10 +52,11 @@ extension RowEditingCoordinator { return } + let savingTabId = parent.tabManager.selectedTabId /// One change manager serves the whole window, so a second save cannot start while one is /// in flight. Say so: over the slow link this guard exists for, a Save that silently does /// nothing is indistinguishable from a Save that is broken. - guard beginSaveInFlight() else { + guard beginSaveInFlight(tabId: savingTabId) else { saveChangesLogger.debug("Save already in flight, refusing the repeat") if let index = parent.tabManager.selectedTabIndex { parent.tabManager.mutate(at: index) { @@ -66,7 +67,6 @@ extension RowEditingCoordinator { return } - let savingTabId = parent.tabManager.selectedTabId guard parent.selectedTabScope != nil else { failSave(message: String(localized: "Not connected to database"), tabId: savingTabId, waiter: waiter) return @@ -260,6 +260,7 @@ extension RowEditingCoordinator { } let route = DatabaseManager.shared.executionRoute(for: scope) + let commitMode = DatabaseManager.shared.commitMode(for: conn.id) let serverOwned = serverOwnedColumns(of: savingTabId) let columnTypeNames = parent.changeManager.columnTypeNames @@ -277,7 +278,7 @@ extension RowEditingCoordinator { route: route, cancellation: .protectedWrite ) { driver in - try await DataWriteExecutor.run(plan, mode: .readWrite, on: driver) + try await DataWriteExecutor.run(plan, mode: .readWrite, commitMode: commitMode, on: driver) } let history = recordSuccessHistory( @@ -351,18 +352,19 @@ extension RowEditingCoordinator { } if clearTableOps { - /// On an engine whose DDL commits as it runs, a TRUNCATE before the failure has - /// already emptied its table, and nothing reports which ones ran. Queuing them - /// again would empty the table a second time on the next Save, rows added since - /// included, so they are left for the user to stage again. + // The executor throws every refusal before its first statement. + let ranNothing = error is any ManualCommitRefusing + // Where DDL commits as it runs a failed save may have emptied a table, and a requeued truncate empties it again. let ddlRolledBack = DatabaseManager.shared.driver(for: conn.id)?.supportsTransactionalDDL ?? false restorePendingTableOperations( connectionId: conn.id, - truncates: ddlRolledBack ? truncatedTables : [], + truncates: ranNothing || ddlRolledBack ? truncatedTables : [], deletes: deletedTables, options: capturedOptions ) - reconcileDropsAfterFailedSave(deletedTables, scope: scope) + if !ranNothing { + reconcileDropsAfterFailedSave(deletedTables, scope: scope) + } } failSave( diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift index a3baa8c780..fa81cadf9a 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift @@ -13,19 +13,29 @@ final class RowEditingCoordinator: ObservableObject { // Nothing clears the pending changes until the write returns, so a second Cmd+S in the round trip would commit twice. private(set) var isSaveInFlight = false + // The landing save clears this tab's staged edits and reloads its rows, so nothing may stage there meanwhile. + private(set) var savingTabId: UUID? init(parent: MainContentCoordinator) { self.parent = parent } - func beginSaveInFlight() -> Bool { + func beginSaveInFlight(tabId: UUID?) -> Bool { guard !isSaveInFlight else { return false } + parent.objectWillChange.send() isSaveInFlight = true + savingTabId = tabId return true } func endSaveInFlight() { + parent.objectWillChange.send() isSaveInFlight = false + savingTabId = nil + } + + func isSaving(tabId: UUID?) -> Bool { + tabId != nil && savingTabId == tabId } // A row command selects the row it made. The JSON view reads the selection as "show only these rows". @@ -104,43 +114,52 @@ final class RowEditingCoordinator: ObservableObject { guard parent.canEditActiveResult(.delete), let tabId = parent.tabManager.selectedTabId else { return } let deleted = rowIDs.filter { parent.changeManager.isRowDeleted($0) } guard !deleted.isEmpty else { return } - for rowID in deleted { - parent.changeManager.undoRowDeletion(rowID: rowID) - } + parent.changeManager.restoreDeletedRows(deleted) parent.existingGridModel(forTab: tabId)?.repaintRows(deleted) } func duplicateRow(_ rowID: RowID) { + duplicateRows([rowID]) + } + + func duplicateRows(_ rowIDs: [RowID]) { guard parent.canEditActiveResult(.insert), let (tab, tabIndex) = parent.tabManager.selectedTabAndIndex, tab.tableContext.tableName != nil else { return } let tabId = tab.id let tableRows = parent.tabSessionRegistry.tableRows(for: tabId) - guard tableRows.hasAuthoritativeSchema, - let storageIndex = tableRows.index(of: rowID) else { return } - guard parent.unloadedValuesAreLoaded(rowIDs: [rowID], retry: { [weak self] in - self?.duplicateRow(rowID) + let storageIndices = rowIDs.compactMap { tableRows.index(of: $0) } + guard tableRows.hasAuthoritativeSchema, !storageIndices.isEmpty else { return } + guard parent.unloadedValuesAreLoaded(rowIDs: rowIDs, retry: { [weak self] in + self?.duplicateRows(rowIDs) }) else { return } parent.commitActiveGridEdit() let uuidKeys = uuidKeyForms(in: tableRows, tab: tab) - var dupResult: RowOperationsManager.AddNewRowResult? + var duplicated: [RowOperationsManager.AddNewRowResult] = [] parent.mutateActiveTableRows(for: tabId) { rows in - let result = parent.rowOperationsManager.duplicateRow( - sourceRowIndex: storageIndex, + let result = parent.rowOperationsManager.duplicateRows( + sourceRowIndices: storageIndices, tableRows: &rows, uuidKeyForms: uuidKeys ) - dupResult = result - return result?.delta ?? .none + duplicated = result.rows + return result.delta } - guard let result = dupResult else { return } + guard let first = duplicated.first else { return } parent.tabManager.mutate(at: tabIndex) { $0.hasUserInteraction = true } - selectAndEditInsertedRow(result.rowID, tabId: tabId, skipping: result.prefilledColumns) + guard duplicated.count > 1 else { + selectAndEditInsertedRow(first.rowID, tabId: tabId, skipping: first.prefilledColumns) + return + } + if selectionPointsTheGrid, let model = parent.existingGridModel(forTab: tabId) { + model.selectRows(duplicated.map(\.rowID)) + model.reveal(rowID: first.rowID) + } } // The caret skips a key the row was given a fresh UUID for, so the first keystroke does not replace it. @@ -218,7 +237,7 @@ final class RowEditingCoordinator: ObservableObject { private static func sampleText(atColumn index: Int, in tableRows: TableRows) -> String? { for row in tableRows.rows where row.values.indices.contains(index) { - if case .text(let text) = row.values[index], text != "__DEFAULT__" { return text } + if case .text(let text) = row.values[index], row.values[index] != .columnDefault { return text } } return nil } @@ -286,7 +305,8 @@ final class RowEditingCoordinator: ObservableObject { if continuity == .discrete { endInspectorEditRun() } - guard let (tab, tabIndex) = parent.tabManager.selectedTabAndIndex, !valuesByRow.isEmpty else { return } + guard !parent.isSavingSelectedTab, + let (tab, tabIndex) = parent.tabManager.selectedTabAndIndex, !valuesByRow.isEmpty else { return } let tabId = tab.id let tableRows = parent.tabSessionRegistry.tableRows(for: tabId) guard tableRows.columns.indices.contains(columnIndex) else { return } @@ -296,46 +316,48 @@ final class RowEditingCoordinator: ObservableObject { var edits: [(row: Int, column: Int, value: PluginCellValue)] = [] var removedCells: Set = [] - for (rowID, value) in valuesByRow { - guard let storageRow = tableRows.index(of: rowID) else { continue } - let row = tableRows.rows[storageRow] - guard row.unloadedByteCount(columnIndex) == nil else { continue } - let values = Array(row.values) - let removesField = removedRowIDs.contains(rowID) && parent.changeManager.supportsFieldRemoval - let absence = FieldAbsence( - wasAbsent: row.isAbsent(columnIndex), isAbsent: removesField, originalRow: row.absentColumns - ) - let isExpression = expressionRowIDs.contains(rowID) && !removesField - let wasExpression = parent.changeManager.isExpression(rowID: rowID, columnIndex: columnIndex) - guard values.indices.contains(columnIndex), - values[columnIndex] != value || absence.wasAbsent != absence.isAbsent - || wasExpression != isExpression else { continue } - if continuity == .typing { - parent.changeManager.recordTypedCellChange( - rowID: rowID, - columnIndex: columnIndex, - columnName: columnName, - oldValue: values[columnIndex], - newValue: value, - originalRow: values, - absence: absence, - isExpression: isExpression - ) - } else { - parent.changeManager.recordCellChange( - rowID: rowID, - columnIndex: columnIndex, - columnName: columnName, - oldValue: values[columnIndex], - newValue: value, - originalRow: values, - absence: absence, - isExpression: isExpression + parent.changeManager.recordAsOneAction { + for (rowID, value) in valuesByRow { + guard let storageRow = tableRows.index(of: rowID) else { continue } + let row = tableRows.rows[storageRow] + guard row.unloadedByteCount(columnIndex) == nil else { continue } + let values = Array(row.values) + let removesField = removedRowIDs.contains(rowID) && parent.changeManager.supportsFieldRemoval + let absence = FieldAbsence( + wasAbsent: row.isAbsent(columnIndex), isAbsent: removesField, originalRow: row.absentColumns ) - } - edits.append((row: storageRow, column: columnIndex, value: value)) - if removesField { - removedCells.insert(CellPosition(row: storageRow, column: columnIndex)) + let isExpression = expressionRowIDs.contains(rowID) && !removesField + let wasExpression = parent.changeManager.isExpression(rowID: rowID, columnIndex: columnIndex) + guard values.indices.contains(columnIndex), + values[columnIndex] != value || absence.wasAbsent != absence.isAbsent + || wasExpression != isExpression else { continue } + if continuity == .typing { + parent.changeManager.recordTypedCellChange( + rowID: rowID, + columnIndex: columnIndex, + columnName: columnName, + oldValue: values[columnIndex], + newValue: value, + originalRow: values, + absence: absence, + isExpression: isExpression + ) + } else { + parent.changeManager.recordCellChange( + rowID: rowID, + columnIndex: columnIndex, + columnName: columnName, + oldValue: values[columnIndex], + newValue: value, + originalRow: values, + absence: absence, + isExpression: isExpression + ) + } + edits.append((row: storageRow, column: columnIndex, value: value)) + if removesField { + removedCells.insert(CellPosition(row: storageRow, column: columnIndex)) + } } } guard !edits.isEmpty else { return } @@ -351,21 +373,23 @@ final class RowEditingCoordinator: ObservableObject { let tableRows = parent.tabSessionRegistry.tableRows(for: tabId) var edits: [(row: Int, column: Int, value: PluginCellValue)] = [] var removedCells: Set = [] - for edit in resolved { - guard let storageRow = tableRows.index(of: edit.rowID) else { continue } - changeManager.recordCellChange( - rowID: edit.rowID, - columnIndex: edit.dataIndex, - columnName: edit.columnName, - oldValue: edit.oldValue, - newValue: edit.newValue, - originalRow: edit.originalRow, - absence: edit.absence, - isExpression: edit.isExpression - ) - edits.append((row: storageRow, column: edit.dataIndex, value: edit.newValue)) - if edit.absence.isAbsent { - removedCells.insert(CellPosition(row: storageRow, column: edit.dataIndex)) + changeManager.recordAsOneAction { + for edit in resolved { + guard let storageRow = tableRows.index(of: edit.rowID) else { continue } + changeManager.recordCellChange( + rowID: edit.rowID, + columnIndex: edit.dataIndex, + columnName: edit.columnName, + oldValue: edit.oldValue, + newValue: edit.newValue, + originalRow: edit.originalRow, + absence: edit.absence, + isExpression: edit.isExpression + ) + edits.append((row: storageRow, column: edit.dataIndex, value: edit.newValue)) + if edit.absence.isAbsent { + removedCells.insert(CellPosition(row: storageRow, column: edit.dataIndex)) + } } } guard !edits.isEmpty else { return false } @@ -393,11 +417,24 @@ final class RowEditingCoordinator: ObservableObject { } parent.tabManager.mutate(at: tabIndex) { $0.hasUserInteraction = true } - model?.repaintAllRows() + // Undoing a command over many cells runs one step per cell, so a step repaints only its own rows. + model?.repaintRows(Self.rowIDs(touchedBy: result.action)) // The inspector reads its fields from the rows this rewrote, and nothing else it watches moves on an undo. parent.gridDisplayRevision &+= 1 } + // A step moves the pending marks of the rows it names and of no other row. + static func rowIDs(touchedBy action: UndoAction) -> [RowID] { + switch action { + case .cellEdit(let rowID, _, _, _, _, _, _, _), .rowInsertion(let rowID, _), .rowDeletion(let rowID, _, _): + return [rowID] + case .batchRowDeletion(let rows, _): + return rows.map { $0.rowID } + case .batchRowInsertion(let rows, _, _, _): + return rows.map { $0.rowID } + } + } + func copyRowsToClipboard(_ rowIDs: [RowID], includeHeaders: Bool = false) { guard let (tab, _) = parent.tabManager.selectedTabAndIndex, !rowIDs.isEmpty else { return } guard parent.unloadedValuesAreLoaded(rowIDs: rowIDs, retry: { [weak self] in @@ -417,10 +454,12 @@ final class RowEditingCoordinator: ObservableObject { self?.copyRowsAsJson(rowIDs) }) else { return } let model = parent.gridModel(forTab: tab.id) + let tableRows = parent.tabSessionRegistry.tableRows(for: tab.id) let output = ResultJsonSerializer.serialize( - tableRows: parent.tabSessionRegistry.tableRows(for: tab.id), + tableRows: tableRows, displayIDs: model.displayedRowIDs, selectedDisplayIndices: Set(rowIDs.compactMap { model.displayIndex(of: $0) }), + defaultedCells: parent.changeManager.defaultedCells(in: tableRows), columns: VisibleColumnProjection(indices: presentedColumnIndices(tabId: tab.id)) ) guard output.rowCount > 0 else { return } diff --git a/apple/TablePro/Core/Coordinators/RunResultPlan.swift b/apple/TablePro/Core/Coordinators/RunResultPlan.swift new file mode 100644 index 0000000000..a8e079d8a3 --- /dev/null +++ b/apple/TablePro/Core/Coordinators/RunResultPlan.swift @@ -0,0 +1,105 @@ +// +// RunResultPlan.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// Which statements of a run become results of their own. Two or more statements that return no +// rows share one summary, placed where the last of them ran, so the result of the last statement +// stays the last result. With a limit, only that many of the newest results that return rows are kept. +struct RunResultPlan: Equatable, Sendable { + enum Entry: Equatable, Sendable { + case statement(Int) + case summary([Int]) + } + + let entries: [Entry] + let rowResultCount: Int + let keptRowResultCount: Int + private let ownPositions: [Int: Int] + private let summarized: Set + + var droppedRowResultCount: Int { rowResultCount - keptRowResultCount } + + // `unavailable` are results that return rows but are already gone, so they count as dropped whatever the limit. + init(producesRows: [Bool], keepingRowResults limit: Int?, unavailable: Set = []) { + let rowIndices = producesRows.indices.filter { producesRows[$0] } + let available = rowIndices.filter { !unavailable.contains($0) } + let kept = Set(limit.map { available.suffix(max(1, $0)) } ?? available[...]) + let summarized = producesRows.indices.filter { !producesRows[$0] } + let collapses = summarized.count >= 2 + + var entries: [Entry] = [] + for index in producesRows.indices { + if producesRows[index] { + if kept.contains(index) { entries.append(.statement(index)) } + } else if !collapses { + entries.append(.statement(index)) + } else if index == summarized.last { + entries.append(.summary(summarized)) + } + } + self.entries = entries + var ownPositions: [Int: Int] = [:] + var covered: Set = [] + for (position, entry) in entries.enumerated() { + switch entry { + case .statement(let index): ownPositions[index] = position + case .summary(let indices): covered.formUnion(indices) + } + } + self.ownPositions = ownPositions + self.summarized = covered + rowResultCount = rowIndices.count + keptRowResultCount = kept.count + } + + // True when the statement's rows are shown on a result of its own, so they are worth building. + func showsRows(ofStatement index: Int) -> Bool { + ownPositions[index] != nil + } + + // Where the statement's own result sits among the run's results. + func position(ofStatement index: Int) -> Int? { + ownPositions[index] + } + + func summarizes(statement index: Int) -> Bool { + summarized.contains(index) + } + + var droppedNotice: String? { + switch droppedRowResultCount { + case ...0: + return nil + case 1: + return String(localized: "1 earlier result was dropped.") + default: + return String(format: String(localized: "%@ earlier results were dropped."), droppedRowResultCount.formatted()) + } + } +} + +struct RunSummaryPart { + let rowsAffected: Int + let executionTime: TimeInterval + let serverOutput: PluginServerOutput + let statusMessage: String? + + func withServerOutput(_ output: PluginServerOutput) -> RunSummaryPart { + RunSummaryPart(rowsAffected: rowsAffected, executionTime: executionTime, serverOutput: output, statusMessage: statusMessage) + } +} + +extension RunSummaryPart { + init(_ result: QueryResult) { + self.init( + rowsAffected: result.rowsAffected, + executionTime: result.executionTime, + serverOutput: result.serverOutput, + statusMessage: result.statusMessage + ) + } +} diff --git a/apple/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift b/apple/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift index f87f31e94f..0f3395d82e 100644 --- a/apple/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift +++ b/apple/TablePro/Core/CrossEngine/CrossEngineDefaultValue.swift @@ -66,6 +66,18 @@ internal enum CrossEngineDefaultValue { return head.trimmingCharacters(in: .whitespaces) } + // A default built on the current date or time, UTC forms and time zone conversions included. + internal static func readsTheClock(_ value: String) -> Bool { + let upper = value.uppercased() + return clockNames.contains { upper.contains($0) } + } + + private static let clockNames = [ + "NOW(", "CURRENT_TIMESTAMP", "LOCALTIMESTAMP", "CURRENT_DATE", "CURDATE(", "CURTIME(", "TODAY(", + "GETDATE(", "GETUTCDATE(", "SYSDATETIME(", "SYSUTCDATETIME(", "SYSDATETIMEOFFSET(", "UTC_TIMESTAMP", + "SYSDATE", "SYSTIMESTAMP", "STATEMENT_TIMESTAMP(", "TRANSACTION_TIMESTAMP(", "CLOCK_TIMESTAMP(", + ] + private static func currentTimestamp(_ upper: String, target: SQLTypeFamily) -> String? { let names = [ "CURRENT_TIMESTAMP", "CURRENT_TIMESTAMP()", "NOW()", "GETDATE()", "GETUTCDATE()", diff --git a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift index 8ff206ca42..af13e95a3d 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift @@ -95,18 +95,20 @@ extension DataGridModel { searchDataset(revealsFirstMatch: true) return } - let source = findSource - let result = GridFindEngine.search(find.term, options: find.options, in: source) - let current = GridFindEngine.index( - atOrAfter: selection.activeCell, - in: GridFindEngine.positions(of: result.matches, in: source) - ) - find.setResults(result.matches, current: current, problem: result.problem) - revealCurrentFindMatch() + searchLoadedRows(GridFindState.Landing(revealsFirstMatch: true, cursorAtStart: activeCellIdentity)) } // The rows changed under the bar: keep the current match when it survives and leave the selection alone. func refreshFind() { + refreshFind(rereading: nil) + } + + // Only these cells (by data index) were written and these rows added, and others may have left. + func refreshFind(writtenCells: [RowID: IndexSet], addedRows: [RowID] = []) { + refreshFind(rereading: (writtenCells: writtenCells, addedRows: addedRows)) + } + + private func refreshFind(rereading changes: (writtenCells: [RowID: IndexSet], addedRows: [RowID])?) { guard find.isVisible else { return } if datasetFinder != nil { // Another page changes the grid's rows, not the owner's; only another column set searches again. @@ -114,19 +116,24 @@ extension DataGridModel { searchDataset(revealsFirstMatch: false) return } - let source = findSource - let previous = find.currentMatch - let result = GridFindEngine.search(find.term, options: find.options, in: source) - let current: Int? - if let previous, let kept = result.matches.firstIndex(of: previous) { - current = kept - } else { - current = GridFindEngine.index( - atOrAfter: selection.activeCell, - in: GridFindEngine.positions(of: result.matches, in: source) - ) + // A running search leaves no basis, so a change meanwhile searches again. + if let changes { + let source = findSource + let basis = findBasis(of: source) + if find.loadedRowsBasis == basis, + let refreshed = GridFindEngine.refreshed( + find.matches, + writtenCells: changes.writtenCells, + addedRows: changes.addedRows, + term: find.term, + options: find.options, + in: source + ) { + landLoadedRowsResult(refreshed, landing: GridFindState.Landing(), basis: basis, source: source) + return + } } - find.setResults(result.matches, current: current, problem: result.problem) + searchLoadedRows(GridFindState.Landing()) } // The owner's rows changed (an edit, an undo, a filter): search them again, keeping the current match. @@ -147,11 +154,11 @@ extension DataGridModel { find.stopDatasetSearch() } - // Returns once no whole-dataset search is waiting or running. + // Returns once no search of the loaded rows or the whole dataset is waiting or running. func findSearchSettled() async { - while let task = find.datasetSearch { + while let task = find.datasetSearch ?? find.loadedRowsSearch { await task.value - if find.datasetSearch == task { return } + if find.datasetSearch == task || find.loadedRowsSearch == task { return } } } @@ -192,7 +199,12 @@ extension DataGridModel { // MARK: - Replacing func replaceCurrentMatch() { - guard canReplaceInFind, find.problem == nil, let match = find.currentMatch, commitEditing() else { return } + guard canReplaceInFind else { return } + // Replace acts on the match the bar is about to show. + if find.isAwaitingLoadedRows { + searchLoadedRowsNow() + } + guard find.problem == nil, let match = find.currentMatch, commitEditing() else { return } if let finder = datasetFinder { replaceDatasetMatch(match, finder: finder) return @@ -212,7 +224,7 @@ extension DataGridModel { } let replaced = GridFindEngine.position(of: match, in: source) guard commit(replacement.edits, actionName: String(localized: "Replace")) else { return } - refreshFind() + refreshFind(writtenCells: writtenCells(of: replacement.edits)) guard !find.matches.isEmpty else { return } let positions = GridFindEngine.positions(of: find.matches, in: findSource) find.setCurrentIndex(replaced.flatMap { GridFindEngine.index(after: $0, in: positions) } ?? 0) @@ -229,10 +241,12 @@ extension DataGridModel { return nil } let source = findSource - let result = GridFindEngine.search(find.term, options: find.options, in: source) - guard result.problem == nil else { return nil } + if find.isAwaitingLoadedRows || find.loadedRowsBasis != findBasis(of: source) { + searchLoadedRowsNow() + } + guard find.problem == nil else { return nil } let replacement = GridFindEngine.replacements( - for: result.matches, + for: find.matches, term: find.term, options: find.options, template: find.replacement, @@ -241,7 +255,7 @@ extension DataGridModel { ) guard !replacement.edits.isEmpty else { return replacement } guard commit(replacement.edits, actionName: String(localized: "Replace All")) else { return nil } - refreshFind() + refreshFind(writtenCells: writtenCells(of: replacement.edits)) return replacement } @@ -258,15 +272,43 @@ extension DataGridModel { ) } + // A replacement writes only cells the bar found, all in presented columns. + private func writtenCells(of edits: [DataGridCellEdit]) -> [RowID: IndexSet] { + var written: [RowID: IndexSet] = [:] + for edit in edits { + guard let column = columns.column(id: edit.columnID) else { continue } + written[edit.rowID, default: []].insert(column.dataIndex) + } + return written + } + + private func findBasis(of source: GridFindEngine.Source) -> GridFindEngine.Basis { + GridFindEngine.Basis( + term: find.term, + options: find.options, + source: source, + valueFilter: store.valueFilter, + gridSettings: AppSettingsManager.shared.dataGrid, + timeZoneGeneration: DateFormattingService.shared.systemTimeZoneGeneration + ) + } + private func stepFind(forward: Bool) { guard find.isVisible else { return } if let finder = datasetFinder { stepDatasetFind(finder, forward: forward) return } - if find.matches.isEmpty { - refreshFind() + if find.isAwaitingLoadedRows { + find.deferStep(forward: forward) + } else if find.matches.isEmpty { + searchLoadedRows(GridFindState.Landing(steps: forward ? 1 : -1)) + } else { + stepThroughFindMatches(forward: forward) } + } + + private func stepThroughFindMatches(forward: Bool) { guard !find.matches.isEmpty else { return } let index = GridFindEngine.step( from: find.currentIndex, @@ -278,6 +320,78 @@ extension DataGridModel { revealCurrentFindMatch() } + // MARK: - Loaded rows + + // Off the main actor. A term with nothing to scan is answered here, as is an invalid pattern. + private func searchLoadedRows(_ landing: GridFindState.Landing) { + let source = findSource + let basis = findBasis(of: source) + let term = find.term + let options = find.options + let generation = find.beginLoadedRowsSearch(landing) + if let answered = GridFindEngine.resultWithoutScanning(term, options: options, in: source) { + if let merged = find.endLoadedRowsSearch(generation: generation) { + landLoadedRowsResult(answered, landing: merged, basis: basis, source: source) + } + return + } + let dateFormat = DateFormattingService.shared.currentFormat + find.attachLoadedRowsSearch(Task { [weak self] in + let result = await GridFindEngine.searchOffTheMainActor(term, options: options, in: source, dateFormat: dateFormat) + self?.loadedRowsSearchDidFinish(result, basis: basis, generation: generation) + }, generation: generation) + } + + private func loadedRowsSearchDidFinish(_ result: GridFindEngine.Result?, basis: GridFindEngine.Basis, generation: Int) { + guard let landing = find.endLoadedRowsSearch(generation: generation), let result, find.isVisible else { return } + let source = findSource + // A setting the cell text follows changed while it ran, and nothing refreshed the bar. + guard findBasis(of: source) == basis else { + searchLoadedRows(landing) + return + } + landLoadedRowsResult(result, landing: landing, basis: basis, source: source) + } + + // On the main actor, landing as a running search would have. + private func searchLoadedRowsNow() { + let source = findSource + let basis = findBasis(of: source) + let generation = find.beginLoadedRowsSearch(GridFindState.Landing()) + guard let landing = find.endLoadedRowsSearch(generation: generation) else { return } + let result = GridFindEngine.search(find.term, options: find.options, in: source) + landLoadedRowsResult(result, landing: landing, basis: basis, source: source) + } + + private func landLoadedRowsResult( + _ result: GridFindEngine.Result, + landing: GridFindState.Landing, + basis: GridFindEngine.Basis, + source: GridFindEngine.Source + ) { + let current: Int? + if !landing.revealsFirstMatch, let previous = find.currentMatch, let kept = result.matches.firstIndex(of: previous) { + current = kept + } else { + current = GridFindEngine.index( + atOrAfter: selection.activeCell, + in: GridFindEngine.positions(of: result.matches, in: source) + ) + } + find.setResults(result.matches, current: current, problem: result.problem, basis: basis) + if landing.revealsFirstMatch, landing.cursorAtStart == activeCellIdentity { + revealCurrentFindMatch() + } + for _ in 0.. 0) + } + } + + // By identity, so a row change that moves the cursor's position is not a move by the user. + private var activeCellIdentity: GridFindState.Match? { + activeCell.map { GridFindState.Match(rowID: $0.rowID, columnID: $0.column.id) } + } + // The match becomes the active cell, so Copy, Return and the arrow keys act on the found column. private func revealCurrentFindMatch() { guard let match = find.currentMatch else { return } diff --git a/apple/TablePro/Core/DataGrid/DataGridModel.swift b/apple/TablePro/Core/DataGrid/DataGridModel.swift index 608496a96f..86fb966261 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel.swift @@ -15,6 +15,10 @@ protocol DataGridMountedView: AnyObject { func viewportSample() -> (row: Int, offset: CGFloat)? @discardableResult func beginEditing(row: Int, displayColumn: Int) -> Bool var isEditing: Bool { get } + // A viewer, the value filter or the foreign key preview: a reload closes or repoints each of them. + var holdsTransientUI: Bool { get } + // A live scroll or a selection drag, which a reload would move rows under. + var isTrackingInteraction: Bool { get } func commitEditing() -> Bool func discardEditing() @discardableResult func focus() -> Bool @@ -22,6 +26,11 @@ protocol DataGridMountedView: AnyObject { func anchor(forColumn displayColumn: Int) -> (view: NSView, rect: NSRect)? } +extension DataGridMountedView { + var holdsTransientUI: Bool { false } + var isTrackingInteraction: Bool { false } +} + @MainActor final class DataGridModel: ObservableObject { let id = UUID() @@ -144,7 +153,7 @@ final class DataGridModel: ObservableObject { restoreSelection(in: current) displayRevision &+= 1 mountedView?.apply(change) - refreshFind() + refreshFind(after: delta, in: current, previousOrder: previous) return } let change = DataGridChange.from(delta, order: order, rows: current, columns: columns) @@ -158,7 +167,37 @@ final class DataGridModel: ObservableObject { } // An edited row stays where the filter put it until the filter or the row set changes (#2667). mountedView?.apply(change) - refreshFind() + refreshFind(after: delta, in: current) + } + + // An open find bar reads again only the cells the delta wrote and the rows that now show and did not before; + // one that left drops its matches. + private func refreshFind(after delta: Delta, in current: TableRows, previousOrder: DataGridRowOrder? = nil) { + guard find.isVisible else { return } + var written: [RowID: IndexSet] = [:] + var added: [RowID] = [] + switch delta { + case .cellChanged(let row, let column): + if current.rows.indices.contains(row) { + written[current.rows[row].id] = IndexSet(integer: column) + } + case .cellsChanged(let positions): + for position in positions where current.rows.indices.contains(position.row) { + written[current.rows[position.row].id, default: []].insert(position.column) + } + case .rowsInserted(let inserted): + added = inserted.compactMap { current.rows.indices.contains($0) ? current.rows[$0].id : nil } + case .rowsRemoved: + break + case .columnsReplaced, .fullReplace: + refreshFind() + return + } + // A filtered order resolved again can bring back a row hidden since an edit, which no refresh read then. + if let previousOrder, let shown = order.filteredIDs { + added += shown.filter { previousOrder.displayIndex(of: $0, in: current) == nil } + } + refreshFind(writtenCells: written, addedRows: added) } private func resolveOrder(in rows: TableRows) -> DataGridRowOrder { @@ -210,6 +249,12 @@ final class DataGridModel: ObservableObject { setSelection(GridSelection(rectangles: rects, activeCell: active, anchor: active, mode: rects.isEmpty ? .cells : .rows)) } + // Takes back a selection kept by identity, so a cell cursor stays a cursor on its own column. + func select(_ snapshot: GridSelectionSnapshot) { + let resolved = snapshot.resolved(order: order, rows: rows, columns: columns) + setSelection(resolved.rectangles.isEmpty && resolved.activeCell == nil ? .empty : resolved) + } + func clearSelection() { setSelection(.empty) } @@ -548,6 +593,8 @@ final class DataGridModel: ObservableObject { } var isEditing: Bool { mountedView?.isEditing ?? false } + var holdsTransientUI: Bool { mountedView?.holdsTransientUI ?? false } + var isTrackingInteraction: Bool { mountedView?.isTrackingInteraction ?? false } // Owners that deferred work while an editor was open resume here. var onEditingEnded: (@MainActor () -> Void)? diff --git a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift index b9fbfbed40..0890bd9e78 100644 --- a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift +++ b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift @@ -120,7 +120,7 @@ struct GridFindMatcher { @MainActor enum GridFindEngine { - struct Source { + struct Source: Sendable { let rows: TableRows let order: DataGridRowOrder let columns: DataGridColumnSet @@ -129,11 +129,50 @@ enum GridFindEngine { var revealsInvisibleCharacters = false } - struct Result: Equatable { + struct Result: Equatable, Sendable { var matches: [GridFindState.Match] = [] var problem: String? } + private enum Plan { + case answered(Result) + case scan(GridFindMatcher, columns: [DataGridColumn]) + } + + nonisolated static let cancellationInterval = 4_096 + + // Everything a loaded-rows search read besides the rows themselves, the settings the cell text follows included. + struct Basis: Equatable { + let term: String + let options: GridFindState.Options + let columns: DataGridColumnSet + let displayFormats: [ValueDisplayFormat?] + let databaseType: DatabaseType? + let revealsInvisibleCharacters: Bool + let valueFilter: GridValueFilterState + let gridSettings: DataGridSettings + let timeZoneGeneration: Int + + init( + term: String, + options: GridFindState.Options, + source: Source, + valueFilter: GridValueFilterState, + gridSettings: DataGridSettings, + timeZoneGeneration: Int + ) { + self.term = term + self.options = options + columns = source.columns + displayFormats = source.displayFormats + databaseType = source.databaseType + revealsInvisibleCharacters = source.revealsInvisibleCharacters + self.valueFilter = valueFilter + self.gridSettings = gridSettings + self.timeZoneGeneration = timeZoneGeneration + } + } + struct Replacement: Equatable { var edits: [DataGridCellEdit] = [] var replacements = 0 @@ -141,7 +180,7 @@ enum GridFindEngine { var cells: Int { edits.count } } - static var invalidPatternProblem: String { + nonisolated static var invalidPatternProblem: String { String(localized: "Invalid regular expression") } @@ -155,22 +194,39 @@ enum GridFindEngine { } static func search(_ term: String, options: GridFindState.Options, in source: Source) -> Result { - let matcher: GridFindMatcher? - do { - matcher = try GridFindMatcher.make(term: term, options: options) - } catch { - return Result(problem: invalidPatternProblem) + search(term, options: options, in: source, dates: DateFormattingService.shared.renderer, isCancelled: { false }) ?? Result() + } + + // Nil once cancelled, which is asked every 4,096 cells. + nonisolated static func search( + _ term: String, + options: GridFindState.Options, + in source: Source, + dates: DateDisplayRenderer, + isCancelled: () -> Bool + ) -> Result? { + let matcher: GridFindMatcher + let searched: [DataGridColumn] + switch plan(term, options: options, in: source) { + case .answered(let result): + return result + case .scan(let planned, let columns): + matcher = planned + searched = columns } - guard let matcher else { return Result() } - let searched = searchedColumns(options: options, in: source) - guard !searched.isEmpty else { return Result() } var found: [GridFindState.Match] = [] + var budget = 0 let rowCount = source.order.count(in: source.rows) for displayRow in 0.. Result? { + search( + term, + options: options, + in: source, + dates: DateDisplayRenderer(format: dateFormat), + isCancelled: { Task.isCancelled } + ) + } + + // What a search answers without reading a cell: an invalid pattern, a term that finds nothing, no column to search. + nonisolated static func resultWithoutScanning(_ term: String, options: GridFindState.Options, in source: Source) -> Result? { + guard case .answered(let result) = plan(term, options: options, in: source) else { return nil } + return result + } + + nonisolated private static func plan(_ term: String, options: GridFindState.Options, in source: Source) -> Plan { + let matcher: GridFindMatcher? + do { + matcher = try GridFindMatcher.make(term: term, options: options) + } catch { + return .answered(Result(problem: invalidPatternProblem)) + } + guard let matcher else { return .answered(Result()) } + let searched = searchedColumns(options: options, in: source) + guard !searched.isEmpty else { return .answered(Result()) } + return .scan(matcher, columns: searched) + } + + // What `search` finds after only these cells (by data index) were written and these rows added, others having + // left, on the same basis. Nil when the term cannot match, which `search` answers as cheaply. + static func refreshed( + _ matches: [GridFindState.Match], + writtenCells: [RowID: IndexSet], + addedRows: [RowID], + term: String, + options: GridFindState.Options, + in source: Source + ) -> Result? { + guard let matcher = try? GridFindMatcher.make(term: term, options: options) else { return nil } + let searched = searchedColumns(options: options, in: source) + guard !searched.isEmpty else { return nil } + + let added = Set(addedRows) + let reread = added.union(writtenCells.keys) + var keptByRow: [RowID: [GridFindState.Match]] = [:] + for match in matches where writtenCells[match.rowID] != nil && !added.contains(match.rowID) { + keptByRow[match.rowID, default: []].append(match) + } + + var rereadMatches: [(displayRow: Int, matches: [GridFindState.Match])] = [] + for rowID in reread { + guard let displayRow = source.order.displayIndex(of: rowID, in: source.rows), + let row = source.rows.row(withID: rowID) else { continue } + let written = added.contains(rowID) ? nil : writtenCells[rowID] + let kept = keptByRow[rowID] ?? [] + var found: [GridFindState.Match] = [] + for column in searched { + if let written, !written.contains(column.dataIndex) { + if kept.contains(where: { $0.columnID == column.id }) { + found.append(GridFindState.Match(rowID: rowID, columnID: column.id)) + } + continue + } + guard let text = searchableText(in: row, column: column, source: source), + matcher.matches(text) else { continue } + found.append(GridFindState.Match(rowID: rowID, columnID: column.id)) + } + if !found.isEmpty { + rereadMatches.append((displayRow: displayRow, matches: found)) + } + } + rereadMatches.sort { $0.displayRow < $1.displayRow } + + var merged: [GridFindState.Match] = [] + merged.reserveCapacity(matches.count + rereadMatches.count) + var next = rereadMatches.startIndex + for match in matches where !reread.contains(match.rowID) { + guard let displayRow = source.order.displayIndex(of: match.rowID, in: source.rows) else { continue } + while next < rereadMatches.endIndex, rereadMatches[next].displayRow < displayRow { + merged.append(contentsOf: rereadMatches[next].matches) + next += 1 + } + merged.append(match) + } + for entry in rereadMatches[next...] { + merged.append(contentsOf: entry.matches) + } + return Result(matches: merged) + } + // A scope naming a column that is no longer presented searches every presented column. - static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { + nonisolated static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { let candidates: [DataGridColumn] if let scope = options.columnScope, let scoped = source.columns.column(id: scope) { candidates = [scoped] @@ -191,8 +345,17 @@ enum GridFindEngine { } } - // Nil for NULL, an empty cell, and bytes the column's format could not decode, which show as hex. static func searchableText(in row: Row, column: DataGridColumn, source: Source) -> String? { + searchableText(in: row, column: column, source: source, dates: DateFormattingService.shared.renderer) + } + + // Nil for NULL, an empty cell, and bytes the column's format could not decode, which show as hex. + nonisolated static func searchableText( + in row: Row, + column: DataGridColumn, + source: Source, + dates: DateDisplayRenderer + ) -> String? { let dataIndex = column.dataIndex guard dataIndex < row.values.count else { return nil } let rawValue = row.values[dataIndex] @@ -211,7 +374,8 @@ enum GridFindEngine { rawValue, columnType: column.type, displayFormat: format, - databaseType: source.databaseType + databaseType: source.databaseType, + dates: dates ) guard !shown.isEmpty else { return nil } return source.revealsInvisibleCharacters ? shown : shown.shownWithoutMarks @@ -345,7 +509,7 @@ enum GridFindEngine { return String(format: String(localized: "Replaced %1$@ in %2$@."), matchPhrase, cellPhrase) } - private static func displayFormat(for column: DataGridColumn, in source: Source) -> ValueDisplayFormat? { + nonisolated private static func displayFormat(for column: DataGridColumn, in source: Source) -> ValueDisplayFormat? { column.dataIndex < source.displayFormats.count ? source.displayFormats[column.dataIndex] : nil } diff --git a/apple/TablePro/Core/DataGrid/Engine/GridSelectionSnapshot.swift b/apple/TablePro/Core/DataGrid/Engine/GridSelectionSnapshot.swift index 14e8eb047c..507d2d4f46 100644 --- a/apple/TablePro/Core/DataGrid/Engine/GridSelectionSnapshot.swift +++ b/apple/TablePro/Core/DataGrid/Engine/GridSelectionSnapshot.swift @@ -57,6 +57,35 @@ struct GridSelectionSnapshot: Equatable { self.mode = selection.mode } + var namedRowIDs: Set { + var named = Set(rowIDs) + for rectangle in rectangles { named.formUnion(rectangle.rowIDs) } + if let activeCell { named.insert(activeCell.rowID) } + if let anchor { named.insert(anchor.rowID) } + return named + } + + // A row the transform cannot place leaves, and a cell on it goes with it; the mode and columns stay. + func mappingRows(_ transform: (RowID) -> RowID?) -> GridSelectionSnapshot { + func cell(_ cell: Cell?) -> Cell? { + guard let cell, let rowID = transform(cell.rowID) else { return nil } + return Cell(rowID: rowID, columnID: cell.columnID) + } + var mapped = GridSelectionSnapshot( + rowIDs: rowIDs.compactMap(transform), + rectangles: rectangles.compactMap { rectangle in + let rows = rectangle.rowIDs.compactMap(transform) + return rows.isEmpty ? nil : Rectangle(rowIDs: rows, columnIDs: rectangle.columnIDs) + }, + activeCell: cell(activeCell), + anchor: cell(anchor) + ) + guard !mapped.isEmpty else { return .empty } + mapped.pickedColumnIDs = pickedColumnIDs + mapped.mode = mode + return mapped + } + func resolved(order: DataGridRowOrder, rows: TableRows, columns: DataGridColumnSet) -> GridSelection { func coord(_ cell: Cell?) -> GridCoord? { guard let cell, diff --git a/apple/TablePro/Core/DataGrid/GridFindState.swift b/apple/TablePro/Core/DataGrid/GridFindState.swift index 955e70e629..e297c1740d 100644 --- a/apple/TablePro/Core/DataGrid/GridFindState.swift +++ b/apple/TablePro/Core/DataGrid/GridFindState.swift @@ -3,18 +3,28 @@ import Foundation @MainActor final class GridFindState: ObservableObject { - struct Options: Equatable { + struct Options: Equatable, Sendable { var matchesCase = false var matchesWholeWord = false var usesRegularExpression = false var columnScope: DataGridColumnID? } - struct Match: Hashable { + struct Match: Hashable, Sendable { let rowID: RowID let columnID: DataGridColumnID } + // What a search of the loaded rows does once its matches land. + struct Landing: Equatable { + // A typed term selects the first match from the cursor; a refresh keeps the current match. + var revealsFirstMatch = false + // The active cell when the typed term's search began. A user who moved it since keeps their place. + var cursorAtStart: Match? + // Find Next presses minus Find Previous presses while the search ran. + var steps = 0 + } + @Published var isVisible = false @Published var showsReplace = false @Published var term = "" @@ -26,21 +36,36 @@ final class GridFindState: ObservableObject { @Published private(set) var isSearching = false // The user stopped a whole-dataset search before it finished. @Published private(set) var searchStopped = false + // An empty match list means nothing yet; unlike isSearching it shows no progress. + @Published private(set) var isAwaitingLoadedRows = false weak var datasetFinder: (any DataGridDatasetFinding)? private(set) var datasetQuery: GridDatasetFindQuery? private(set) var datasetSearch: Task? private var datasetGeneration = 0 + // What the matches were searched on, while they came from the loaded rows; nil once anything else set them. + private(set) var loadedRowsBasis: GridFindEngine.Basis? + // Runs off the main actor and sets no isSearching: a refresh keeps the matches shown until the new ones land. + private(set) var loadedRowsSearch: Task? + private var loadedRowsGeneration = 0 + private var loadedRowsLanding = Landing() + + // A closed grid's scan would otherwise run to the end and hold its rows until then. + deinit { + loadedRowsSearch?.cancel() + datasetSearch?.cancel() + } var currentMatch: Match? { guard let currentIndex, matches.indices.contains(currentIndex) else { return nil } return matches[currentIndex] } - func setResults(_ matches: [Match], current: Int?, problem: String?) { + func setResults(_ matches: [Match], current: Int?, problem: String?, basis: GridFindEngine.Basis? = nil) { self.matches = matches self.currentIndex = current.flatMap { matches.indices.contains($0) ? $0 : nil } self.problem = problem + loadedRowsBasis = basis } func setCurrentIndex(_ index: Int?) { @@ -51,28 +76,95 @@ final class GridFindState: ObservableObject { func removeMatch(at index: Int) { guard matches.indices.contains(index) else { return } matches.remove(at: index) + loadedRowsBasis = nil currentIndex = matches.isEmpty ? nil : (index < matches.count ? index : 0) } func reset() { cancelDatasetSearch() + cancelLoadedRowsSearch() datasetQuery = nil searchStopped = false term = "" replacement = "" matches = [] + loadedRowsBasis = nil currentIndex = nil problem = nil } + // MARK: - Loaded-rows search + + // The search hands this generation back when it lands. Any other search stops, and a running one this + // replaces hands on what it was to do on landing. + func beginLoadedRowsSearch(_ landing: Landing) -> Int { + var merged = landing + if loadedRowsSearch != nil { + if !landing.revealsFirstMatch, loadedRowsLanding.revealsFirstMatch { + merged.revealsFirstMatch = true + merged.cursorAtStart = loadedRowsLanding.cursorAtStart + } + merged.steps += loadedRowsLanding.steps + } + cancelLoadedRowsSearch() + if datasetSearch != nil { + cancelDatasetSearch() + } + loadedRowsLanding = merged + loadedRowsBasis = nil + // The old term's count and current match would read as the new term's until the scan lands. + if merged.revealsFirstMatch { + matches = [] + currentIndex = nil + problem = nil + } + return loadedRowsGeneration + } + + func attachLoadedRowsSearch(_ task: Task, generation: Int) { + guard generation == loadedRowsGeneration else { + task.cancel() + return + } + loadedRowsSearch = task + // Set here, not in begin, so a term answered without scanning never publishes it. + isAwaitingLoadedRows = true + } + + // Nil when a later search, a reset or a dataset search took this one's place. + func endLoadedRowsSearch(generation: Int) -> Landing? { + guard generation == loadedRowsGeneration else { return nil } + let landing = loadedRowsLanding + loadedRowsSearch = nil + if isAwaitingLoadedRows { isAwaitingLoadedRows = false } + loadedRowsLanding = Landing() + loadedRowsGeneration &+= 1 + return landing + } + + func deferStep(forward: Bool) { + loadedRowsLanding.steps += forward ? 1 : -1 + } + + func cancelLoadedRowsSearch() { + loadedRowsSearch?.cancel() + loadedRowsSearch = nil + // @Published fires on every write, so skip the redraw when nothing changes. + if isAwaitingLoadedRows { isAwaitingLoadedRows = false } + loadedRowsGeneration &+= 1 + loadedRowsLanding = Landing() + } + // MARK: - Whole-dataset search // The search hands this generation back when it finishes; any later start or cancel makes it stale. func beginDatasetSearch(_ query: GridDatasetFindQuery?, clearsResults: Bool) -> Int { cancelDatasetSearch() + cancelLoadedRowsSearch() datasetQuery = query searchStopped = false problem = nil + loadedRowsBasis = nil if clearsResults { matches = [] currentIndex = nil diff --git a/apple/TablePro/Core/DataGrid/GridViewportResolver.swift b/apple/TablePro/Core/DataGrid/GridViewportResolver.swift index 91dd9d2d77..ddfb7daee0 100644 --- a/apple/TablePro/Core/DataGrid/GridViewportResolver.swift +++ b/apple/TablePro/Core/DataGrid/GridViewportResolver.swift @@ -11,37 +11,74 @@ internal struct GridViewportSnapshot: Equatable { let firstVisiblePosition: Int let firstVisibleOffset: CGFloat let firstVisibleKey: [String: String]? - var selectedKeys: [[String: String]] = [] + // Row ids are positional per page, so each row the selection names travels as its key. + var selection: GridSelectionSnapshot = .empty + var selectionKeys: [RowID: [String: String]] = [:] static let top = GridViewportSnapshot(firstVisiblePosition: 0, firstVisibleOffset: 0, firstVisibleKey: nil) + var selectedKeys: [[String: String]] { + selection.rowIDs.compactMap { selectionKeys[$0] } + } + // For a re-read that starts at the first row but keeps what is selected. var selectionOnly: GridViewportSnapshot { - GridViewportSnapshot(firstVisiblePosition: 0, firstVisibleOffset: 0, firstVisibleKey: nil, selectedKeys: selectedKeys) + GridViewportSnapshot( + firstVisiblePosition: 0, + firstVisibleOffset: 0, + firstVisibleKey: nil, + selection: selection, + selectionKeys: selectionKeys + ) } } internal struct GridViewportPlacement: Equatable { let firstVisibleRow: RowID? let firstVisibleOffset: CGFloat - let selectedRows: [RowID] + let selection: GridSelectionSnapshot let revealsSelection: Bool + init( + firstVisibleRow: RowID?, + firstVisibleOffset: CGFloat, + selection: GridSelectionSnapshot, + revealsSelection: Bool + ) { + self.firstVisibleRow = firstVisibleRow + self.firstVisibleOffset = firstVisibleOffset + self.selection = selection + self.revealsSelection = revealsSelection + } + + init(firstVisibleRow: RowID?, firstVisibleOffset: CGFloat, selectedRows: [RowID], revealsSelection: Bool) { + self.init( + firstVisibleRow: firstVisibleRow, + firstVisibleOffset: firstVisibleOffset, + selection: selectedRows.isEmpty + ? .empty + : GridSelectionSnapshot(rowIDs: selectedRows, rectangles: [], activeCell: nil, anchor: nil), + revealsSelection: revealsSelection + ) + } + + var selectedRows: [RowID] { selection.rowIDs } + static let firstRow = GridViewportPlacement( firstVisibleRow: nil, firstVisibleOffset: 0, - selectedRows: [], + selection: .empty, revealsSelection: false ) } extension GridViewportPlacement { - func selecting(_ rows: [RowID]) -> GridViewportPlacement { - guard !rows.isEmpty else { return self } + func selecting(_ selection: GridSelectionSnapshot) -> GridViewportPlacement { + guard selection != .empty else { return self } return GridViewportPlacement( firstVisibleRow: firstVisibleRow, firstVisibleOffset: firstVisibleOffset, - selectedRows: rows, + selection: selection, revealsSelection: revealsSelection ) } @@ -60,21 +97,27 @@ internal enum GridViewportResolver { displayIDs: [RowID]?, firstVisibleDisplayRow: Int, firstVisibleOffset: CGFloat, - selectedDisplayRows: [Int] = [], + selection: GridSelectionSnapshot = .empty, keyColumns: [String], isCellModified: (RowID, Int) -> Bool = { _, _ in false } ) -> GridViewportSnapshot { let isKeyed = !keyColumns.isEmpty && tableRows.count <= keyedRowLimit let order = DataGridRowOrder(filteredIDs: displayIDs, revision: 0) - func key(_ displayRow: Int) -> [String: String]? { - order.row(at: displayRow, in: tableRows) - .flatMap { savedKey(of: $0, in: tableRows, keyColumns: keyColumns, isCellModified: isCellModified) } + func key(of row: Row?) -> [String: String]? { + row.flatMap { savedKey(of: $0, in: tableRows, keyColumns: keyColumns, isCellModified: isCellModified) } + } + var selectionKeys: [RowID: [String: String]] = [:] + if isKeyed { + for rowID in selection.namedRowIDs { + selectionKeys[rowID] = key(of: tableRows.row(withID: rowID)) + } } return GridViewportSnapshot( firstVisiblePosition: max(0, firstVisibleDisplayRow), firstVisibleOffset: max(0, firstVisibleOffset), - firstVisibleKey: isKeyed ? key(firstVisibleDisplayRow) : nil, - selectedKeys: isKeyed ? selectedDisplayRows.sorted().compactMap { key($0) } : [] + firstVisibleKey: isKeyed ? key(of: order.row(at: firstVisibleDisplayRow, in: tableRows)) : nil, + selection: isKeyed ? selection : .empty, + selectionKeys: selectionKeys ) } @@ -89,10 +132,10 @@ internal enum GridViewportResolver { switch intent { case .firstRow: return GridViewportPlacement.firstRow - .selecting(rows(matching: snapshot.selectedKeys, in: incoming, keyColumns: keyColumns)) + .selecting(remappedSelection(snapshot, in: incoming, keyColumns: keyColumns)) case .keepPlace: return keptPlace(from: snapshot, in: incoming, displayIDs: displayIDs, keyColumns: keyColumns) - .selecting(rows(matching: snapshot.selectedKeys, in: incoming, keyColumns: keyColumns)) + .selecting(remappedSelection(snapshot, in: incoming, keyColumns: keyColumns)) case .restoreRow(let anchor): guard let row = uniqueRow(matching: anchor, in: incoming) else { return .firstRow } return GridViewportPlacement( @@ -153,14 +196,14 @@ internal enum GridViewportResolver { } // One pass, so a large selection does not rescan per row; a repeated key matches nothing, as in uniqueRow(matching:in:). - private static func rows( - matching keys: [[String: String]], + private static func remappedSelection( + _ snapshot: GridViewportSnapshot, in incoming: TableRows, keyColumns: [String] - ) -> [RowID] { - guard !keys.isEmpty, !keyColumns.isEmpty, incoming.count <= keyedRowLimit else { return [] } + ) -> GridSelectionSnapshot { + guard !snapshot.selectionKeys.isEmpty, !keyColumns.isEmpty, incoming.count <= keyedRowLimit else { return .empty } let indices = keyColumns.compactMap { incoming.columns.firstIndex(of: $0) } - guard indices.count == keyColumns.count else { return [] } + guard indices.count == keyColumns.count else { return .empty } var rowByKey: [[String]: RowID] = [:] var repeated = Set<[String]>() @@ -171,11 +214,13 @@ internal enum GridViewportResolver { repeated.insert(values) } } - return keys.compactMap { key in + var incomingRows: [RowID: RowID] = [:] + for (outgoing, key) in snapshot.selectionKeys { let values = keyColumns.compactMap { key[$0] } - guard values.count == keyColumns.count, !repeated.contains(values) else { return nil } - return rowByKey[values] + guard values.count == keyColumns.count, !repeated.contains(values), let row = rowByKey[values] else { continue } + incomingRows[outgoing] = row } + return snapshot.selection.mappingRows { incomingRows[$0] } } private static func uniqueRow(matching values: [String: String], in incoming: TableRows) -> RowID? { diff --git a/apple/TablePro/Core/DataGrid/RowDisplayBox.swift b/apple/TablePro/Core/DataGrid/RowDisplayBox.swift index 209b20d4f5..590df90467 100644 --- a/apple/TablePro/Core/DataGrid/RowDisplayBox.swift +++ b/apple/TablePro/Core/DataGrid/RowDisplayBox.swift @@ -7,18 +7,48 @@ import Foundation final class RowDisplayBox { var values: ContiguousArray + // A bit per column whose display format produced its text, so the draw reads it without formatting again. + private var formattedBits: ContiguousArray = [] init(_ values: ContiguousArray) { self.values = values } + + func setText(_ text: String?, at index: Int, isFormatted: Bool) { + values[index] = text + let word = index >> 6 + let bit = UInt64(1) << UInt64(index & 63) + if isFormatted { + if word >= formattedBits.count { + formattedBits.append(contentsOf: repeatElement(0, count: word + 1 - formattedBits.count)) + } + formattedBits[word] |= bit + } else if word < formattedBits.count { + formattedBits[word] &= ~bit + } + } + + func isFormatted(at index: Int) -> Bool { + let word = index >> 6 + return index >= 0 && word < formattedBits.count && formattedBits[word] & (UInt64(1) << UInt64(index & 63)) != 0 + } + + func clearText(at index: Int) { + guard index >= 0, index < values.count else { return } + setText(nil, at: index, isFormatted: false) + } + + func clearAllText() { + for index in values.indices { + values[index] = nil + } + formattedBits.removeAll() + } } @MainActor final class RowDisplayCache { - /// A box is a reference and callers mutate one in place before handing it back, so the cost - /// recorded at insertion is the only number that still describes what was added. Recomputing it - /// from the box on removal subtracts a different figure than was added and drifts `totalCost` - /// away from the budget it is there to enforce. + // Callers mutate a box in place, so only the cost recorded at insertion matches what was added to `totalCost`. private struct Entry { let box: RowDisplayBox let cost: Int @@ -80,17 +110,12 @@ final class RowDisplayCache { totalCost = 0 } - /// Drops one row's formatted values while keeping its box, so the next read - /// reformats from the current cell values. Row ids are positional, so a row - /// whose content changed in place keeps its id and would otherwise be served - /// its pre-edit text. + // Row ids are positional: a row edited in place keeps its id and would otherwise be served its pre-edit text. func clearValues(forID id: RowID) { highlights.removeValue(forKey: id) guard let existing = storage[id] else { return } totalCost -= existing.cost - for index in existing.box.values.indices { - existing.box.values[index] = nil - } + existing.box.clearAllText() storage[id] = Entry(box: existing.box, cost: 0) } @@ -99,8 +124,8 @@ final class RowDisplayCache { guard !dataIndexes.isEmpty else { return } highlights.removeAll(keepingCapacity: true) storage = storage.mapValues { entry in - for index in dataIndexes where index < entry.box.values.count { - entry.box.values[index] = nil + for index in dataIndexes { + entry.box.clearText(at: index) } let cost = Self.rowCost(entry.box.values) totalCost += cost - entry.cost diff --git a/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift b/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift index 2dda435413..394d9c34e8 100644 --- a/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift +++ b/apple/TablePro/Core/DataWrite/DataWriteExecutor.swift @@ -11,20 +11,22 @@ // affected count against the count the plan expected is what catches it, and inside a transaction // it is caught before it lands. // -// For a keyed write the comparison is `actual > expected`, never `actual != expected`. MySQL reports -// zero affected rows for an UPDATE that writes a value a row already holds, which is a normal save, -// not a fault. +// For a keyed write the comparison is `actual > expected`, and for a keyed UPDATE also `actual < +// expected` where the count is the rows it matched (`DataWriteRowCounts`, or a driver's own +// `PluginRowWrite.expectedRowCount`): a short one means the row was not written, deleted or +// rekeyed since it was read, kept out by row-level security, or skipped by a trigger. // // A keyless write is held to both ends. It finds its row by matching every column against the text // the grid read, and that match can simply find nothing: measured on MySQL 8.4.11 and OceanBase // 4.4.2.1, a row holding a FLOAT or a JSON column matched zero rows, the statement wrote nothing, // and the save reported success while the edit disappeared on the next reload. Only the engines -// whose drivers report a real count can be held to it, which `reportsRowsAffected` decides. +// whose drivers report a real count can be held to it, which `DataWriteRowCounts` decides. // import Foundation import os import TableProPluginKit +import TableProSQLGrammar struct DataWriteStepResult: Sendable { let executionTime: TimeInterval @@ -103,12 +105,25 @@ enum DataWriteExecutor { nonisolated static func run( _ plan: DataWritePlan, mode: PluginTransactionAccessMode = .readWrite, + commitMode: ConnectionCommitMode, on driver: DatabaseDriver, context: PluginStatementContext? = nil ) async throws -> DataWriteRun { let steps = plan.steps.filter { !$0.statement.sql.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } guard !steps.isEmpty || !plan.prologue.isEmpty else { return DataWriteRun(results: [], sideStatements: []) } + // A write Manual Commit can't hold would commit with no Commit pressed, so it is refused before anything runs. + if commitMode == .manual { + guard plan.prologue.isEmpty, plan.epilogue.isEmpty else { throw ManualCommitSessionOptionsRefusal() } + let state = await driver.heldSessionTransactionState() + try ManualCommit.refuseUnheldWrite(supportsTransactions: driver.supportsTransactions, sessionState: state) + } + + let endsSegment = definitionSteps(steps, of: plan) + if endsSegment.contains(true) { + try await refuseDefinitions(on: driver, commitMode: commitMode) + } + // A row a lookup cannot find on its own refuses the save before anything is written. Each step reads its // lookup again right before it runs, since the steps ahead of it can move or change its rows. for step in steps { @@ -154,20 +169,27 @@ enum DataWriteExecutor { } } - /// Asked inside the same lease that runs the statements, so nothing can open a transaction - /// between the answer and the first write. + // Asked inside the lease that runs the statements, so nothing opens a transaction before the first write. let owner = WriteTransactionOwner.resolve( supportsTransactions: driver.supportsTransactions, - sessionState: await driver.heldSessionTransactionState() + sessionState: await driver.heldSessionTransactionState(), + commitMode: commitMode ) var results: [DataWriteStepResult] = [] + let engineCountsMatchedRows = DataWriteRowCounts.countsMatchedRows(for: plan.databaseType) + // A delete earlier in the save can cascade to a row a later update edits, which then matches nothing. + var ranDelete = false + let closesSegments = endsSegment.contains(true) && owner == .app + var committedCount = 0 + var lastSegmentClosed = false do { - if owner.opensTransaction { + if owner.sendsBegin { try await driver.beginTransaction(mode: mode) + if owner == .held { ManualCommit.turnHold?.mark(true) } } - for step in steps { + for (index, step) in steps.enumerated() { let start = Date() var statement = step.statement if let lookup = step.lookup { @@ -198,12 +220,16 @@ enum DataWriteExecutor { ) } + let countsMatchedRows = engineCountsMatchedRows || step.countsMatchedRows try verify( step, rowsAffected: result.rowsAffected, owner: owner, - countsAreMeaningful: DataWriteRowCounts.areMeaningful(for: plan.databaseType) + countsAreMeaningful: countsMatchedRows, + // Through a view the count is whatever its trigger or rule reports. + holdsKeyedUpdates: countsMatchedRows && plan.viewTarget == nil && !ranDelete ) + ranDelete = ranDelete || step.rowKind == .delete || step.kind == .tableOperation results.append( DataWriteStepResult( executionTime: Date().timeIntervalSince(start), @@ -215,16 +241,27 @@ enum DataWriteExecutor { if let check = step.lookup?.check { try await confirmLanded(check, for: step, on: driver) } + + if closesSegments, endsSegment[index] { + try await TaskCancellationShield.run { try await driver.commitTransaction() } + committedCount = results.count + lastSegmentClosed = index == steps.count - 1 + if !lastSegmentClosed { + try await driver.beginTransaction(mode: mode) + } + } } - if owner.opensTransaction { + // Teradata mode answers an END TRANSACTION with no BT open with an error. + if owner.commitsAtEnd, !lastSegmentClosed { try await driver.commitTransaction() } } catch { var rollbackSucceeded = owner.canRollBack - if owner.opensTransaction { + if owner.canRollBack { do { try await driver.rollbackTransaction() + if owner == .held { ManualCommit.turnHold?.mark(false) } } catch { rollbackSucceeded = false logger.error("Rollback failed: \(error.publicLogShape, privacy: .public)") @@ -232,17 +269,22 @@ enum DataWriteExecutor { } await drainEpilogue() - /// Without a transaction the statements that already ran are on the server for good, - /// and so they are when the rollback itself failed. Reporting that as a plain failure - /// tells the user to try again, and trying again writes them a second time. A run that - /// joined the user's own transaction is the third case: the statements are pending in - /// it, and only the user can commit or roll it back. + if committedCount > 0 { + throw DataWritePartialCommitError( + committed: rollbackSucceeded ? Array(results.prefix(committedCount)) : results, + totalStatements: steps.count, + engine: plan.databaseType.rawValue, + disposition: .written, + underlying: error + ) + } + // What ran stays written, or pending in a transaction only the user ends, so a plain failure would invite a retry. if !rollbackSucceeded, !results.isEmpty { throw DataWritePartialCommitError( committed: results, totalStatements: steps.count, engine: plan.databaseType.rawValue, - disposition: owner == .session ? .pendingInSessionTransaction : .written, + disposition: owner == .session || owner == .held ? .pendingInSessionTransaction : .written, underlying: error ) } @@ -253,6 +295,31 @@ enum DataWriteExecutor { return DataWriteRun(results: results, sideStatements: sideStatements) } + // Teradata takes only COMMIT or ROLLBACK after a definition statement; its truncate is a DELETE, which joins as DML. + private static func definitionSteps(_ steps: [DataWriteStep], of plan: DataWritePlan) -> [Bool] { + let family = TransactionEngineFamily.of(plan.databaseType) + guard family.definitionEndsTransaction else { return Array(repeating: false, count: steps.count) } + let grammar = SQLLexicalResolver.readings(for: plan.databaseType, session: plan.sessionLexicalState).execution + return steps.map { AutocommitOnlyStatement.matches($0.statement.sql, family: family, grammar: grammar) } + } + + // Refused before anything runs: the commit after a definition statement would end the transaction the session holds. + private static func refuseDefinitions(on driver: DatabaseDriver, commitMode: ConnectionCommitMode) async throws { + let owner = WriteTransactionOwner.resolve( + supportsTransactions: driver.supportsTransactions, + sessionState: await driver.heldSessionTransactionState(), + commitMode: commitMode + ) + switch owner { + case .session: + throw DefinitionInOpenTransactionRefusal() + case .held: + throw ManualCommitRefusal() + case .app, .none: + return + } + } + private static func openPreservingScope(on driver: DatabaseDriver) async -> PluginSessionScope? { do { return try await driver.openSessionScope(for: .preserve) @@ -291,7 +358,13 @@ enum DataWriteExecutor { ) ) } - guard rows.count != lookup.rowCount else { return rows } + guard rows.count != lookup.rowCount else { + guard let row = rows.first, lookup.rowCount == 1, + !lookup.heldValues.allSatisfy({ row.indices.contains($0.key) && row[$0.key] == $0.value }) + else { return rows } + logger.error("Lookup on '\(table, privacy: .private(mask: .hash))' found the row changed since it was read") + throw DataWriteError.changeRefused(table: table, kind: step.rowKind, reason: rowNotFoundReason) + } logger.error( "Lookup on '\(table, privacy: .private(mask: .hash))' found \(rows.count, privacy: .public) rows, expected \(lookup.rowCount, privacy: .public)" ) @@ -300,7 +373,7 @@ enum DataWriteExecutor { // A lookup that expects no row is an insert checking its key is free. reason = String(localized: "A row with this key already exists, so the insert would not add one.") } else if rows.count < lookup.rowCount { - reason = String(localized: "No row holds the values it was loaded with. Another client may have changed or deleted it.") + reason = rowNotFoundReason } else { reason = String(localized: "Other rows hold exactly the same values, so the change cannot reach this row alone.") } @@ -324,6 +397,10 @@ enum DataWriteExecutor { ) } + private static var rowNotFoundReason: String { + String(localized: "No row holds the values it was loaded with. Another client may have changed or deleted it.") + } + private static func fetch(_ statement: ParameterizedStatement, on driver: DatabaseDriver) async throws -> QueryResult { statement.parameters.isEmpty ? try await driver.execute(query: statement.sql) @@ -334,7 +411,8 @@ enum DataWriteExecutor { _ step: DataWriteStep, rowsAffected: Int, owner: WriteTransactionOwner, - countsAreMeaningful: Bool + countsAreMeaningful: Bool, + holdsKeyedUpdates: Bool ) throws { // A row inserted between a lookup without a pin and its write can still be reached, and a real count shows it. guard let expected = step.expectedRowCount ?? step.lookup?.rowCount else { return } @@ -354,6 +432,18 @@ enum DataWriteExecutor { ) throw DataWriteError.keyNoLongerMatchesOneRow(table: table, expected: expected, actual: rowsAffected) } + if holdsKeyedUpdates, step.rowKind == .update, !step.matchesRowsWithoutKey { + logger.error( + "Keyed update on '\(table, privacy: .private(mask: .hash))' matched \(rowsAffected, privacy: .public) rows, expected \(expected, privacy: .public)" + ) + throw DataWriteError.changeRefused( + table: table, + kind: .update, + reason: String( + localized: "No row took the change. Another client may have changed or deleted it, or a security policy or trigger skipped it." + ) + ) + } guard step.matchesRowsWithoutKey else { return } logger.error( "Keyless statement on '\(table, privacy: .private(mask: .hash))' affected \(rowsAffected, privacy: .public) rows, expected \(expected, privacy: .public)" @@ -370,7 +460,7 @@ enum DataWriteExecutor { actual: Int ) -> DataWriteError { switch owner { - case .app: + case .app, .held: return .tooManyRowsAffected(table: table, expected: expected, actual: actual) case .session: return .tooManyRowsAffectedInSessionTransaction(table: table, expected: expected, actual: actual) diff --git a/apple/TablePro/Core/DataWrite/DataWritePlan.swift b/apple/TablePro/Core/DataWrite/DataWritePlan.swift index 0c0e29ccdd..3fa90222df 100644 --- a/apple/TablePro/Core/DataWrite/DataWritePlan.swift +++ b/apple/TablePro/Core/DataWrite/DataWritePlan.swift @@ -66,6 +66,10 @@ struct RowWriteOperation: Codable, Sendable, Equatable { // What the driver found the row by when it was read, for an update or a delete. A rewind writes to that row and // no other. Nil in a record saved before it was kept, and on an engine that keeps none. var rowLocator: String? + // Where the row's first write ran in the save. Nil in a record saved before it was kept, which runs by list order. + var order: Int? + // Only for an update written in two or more steps. An older build ignores it and rewinds the net images. + var steps: [RowWriteStep]? var isReversible: Bool { refusal == nil } @@ -85,6 +89,82 @@ struct RowWriteOperation: Codable, Sendable, Equatable { guard let image else { return [] } return Set(image.indices.filter { image[$0].isNull }) } + + // The row as each step left it, applying the steps to the pre-image in turn. + func stepImages() -> [RowWriteStepImage]? { + guard let steps, var row = preImage else { return nil } + var absent = absentColumnsBeforeWrite + return steps.map { step in + for (offset, column) in step.columns.enumerated() { + if row.indices.contains(column), step.after.indices.contains(offset) { + row[column] = step.after[offset] + } + if step.afterAbsent.contains(column) { + absent.insert(column) + } else { + absent.remove(column) + } + } + return RowWriteStepImage(step: step, after: row, absentAfter: absent) + } + } +} + +// `before` and `after` line up with `columns`, which are indices into the operation's columns. +struct RowWriteStep: Codable, Sendable, Equatable { + let order: Int + let columns: [Int] + let before: [PluginCellValue] + let after: [PluginCellValue] + let beforeAbsent: Set + let afterAbsent: Set + + init( + order: Int, + columns: [Int], + before: [PluginCellValue], + after: [PluginCellValue], + beforeAbsent: Set = [], + afterAbsent: Set = [] + ) { + self.order = order + self.columns = columns + self.before = before + self.after = after + self.beforeAbsent = beforeAbsent + self.afterAbsent = afterAbsent + } + + init(_ step: RowChange) { + self.init( + order: step.sequence, + columns: step.cellChanges.map(\.columnIndex), + before: step.cellChanges.map(\.oldValue), + after: step.cellChanges.map(\.newValue), + beforeAbsent: Set(step.cellChanges.filter(\.oldIsAbsent).map(\.columnIndex)), + afterAbsent: Set(step.cellChanges.filter(\.newIsAbsent).map(\.columnIndex)) + ) + } + + var inverted: RowWriteStep { + RowWriteStep( + order: order, columns: columns, before: after, after: before, + beforeAbsent: afterAbsent, afterAbsent: beforeAbsent + ) + } + + func reordered(_ order: Int) -> RowWriteStep { + RowWriteStep( + order: order, columns: columns, before: before, after: after, + beforeAbsent: beforeAbsent, afterAbsent: afterAbsent + ) + } +} + +struct RowWriteStepImage { + let step: RowWriteStep + let after: [PluginCellValue] + let absentAfter: Set } // A driver's `PluginRowLookup`, bound for the executor. @@ -93,12 +173,21 @@ struct RowLookup: Sendable { let rowCount: Int let pinsRows: Bool let check: ParameterizedStatement? + // By position in the query's columns: what the one row found must still hold, or another writer changed it. + let heldValues: [Int: PluginCellValue] - init(query: ParameterizedStatement, rowCount: Int, pinsRows: Bool, check: ParameterizedStatement? = nil) { + init( + query: ParameterizedStatement, + rowCount: Int, + pinsRows: Bool, + check: ParameterizedStatement? = nil, + heldValues: [Int: PluginCellValue] = [:] + ) { self.query = query self.rowCount = rowCount self.pinsRows = pinsRows self.check = check + self.heldValues = heldValues } init(_ lookup: PluginRowLookup) { @@ -125,7 +214,7 @@ struct DataWriteStep: Sendable { let statement: ParameterizedStatement /// The most rows this statement may legitimately touch, or nil when the engine's count for it /// carries no meaning: DDL, a session setting, or a statement a plugin wrote itself for a table - /// with a primary key, where nothing tells the host which rows went into it. + /// with a primary key without saying how many rows it matches. let expectedRowCount: Int? let tableName: String? /// Whether this statement finds its rows by matching every column, because the table declares no @@ -137,6 +226,8 @@ struct DataWriteStep: Sendable { let rowKind: RowWriteKind? // Finds the statement's rows right before it runs, for a driver whose engine has no key that names one row. let lookup: RowLookup? + // The driver vouched for `expectedRowCount` as the rows the statement matches, whatever engine it runs on. + let countsMatchedRows: Bool init( kind: Kind, @@ -146,7 +237,8 @@ struct DataWriteStep: Sendable { matchesRowsWithoutKey: Bool = false, refusesSharedKey: Bool = false, rowKind: RowWriteKind? = nil, - lookup: RowLookup? = nil + lookup: RowLookup? = nil, + countsMatchedRows: Bool = false ) { self.kind = kind self.statement = statement @@ -156,6 +248,7 @@ struct DataWriteStep: Sendable { self.refusesSharedKey = refusesSharedKey self.rowKind = rowKind self.lookup = lookup + self.countsMatchedRows = countsMatchedRows } } @@ -174,7 +267,8 @@ extension DataWriteStep { expectedRowCount: $0.rowCount, tableName: tableName, matchesRowsWithoutKey: primaryKeyColumns.isEmpty && $0.kind != .insert, - refusesSharedKey: $0.refusesSharedKey + refusesSharedKey: $0.refusesSharedKey, + rowKind: $0.kind ) } case .driverWritten(let statements): @@ -182,11 +276,13 @@ extension DataWriteStep { DataWriteStep( kind: .rowWrite, statement: $0.statement, - expectedRowCount: $0.keylessRowCount ?? $0.keyedRowCount, + // The lowest: a driver's count never loosens the one row the host holds a statement to. + expectedRowCount: [$0.matchedRowCount, $0.keylessRowCount, $0.keyedRowCount].compactMap { $0 }.min(), tableName: tableName, matchesRowsWithoutKey: $0.keylessRowCount != nil, rowKind: $0.kind, - lookup: $0.lookup + lookup: $0.lookup, + countsMatchedRows: $0.matchedRowCount != nil ) } } diff --git a/apple/TablePro/Core/DataWrite/DataWriteRowCounts.swift b/apple/TablePro/Core/DataWrite/DataWriteRowCounts.swift index 7c584aff47..13bd4787fc 100644 --- a/apple/TablePro/Core/DataWrite/DataWriteRowCounts.swift +++ b/apple/TablePro/Core/DataWrite/DataWriteRowCounts.swift @@ -5,22 +5,24 @@ import Foundation -/// Whether a statement's reported row count means what it says on a given engine. +/// Whether a statement's reported row count is the rows it matched on a given engine, a row that +/// already held the value included. /// -/// A keyless save is only held to the count it expected where the count is real. Most drivers read -/// one from the server, but a driver that reports a constant would fail every such save, so an -/// engine has to earn its way onto this list by a driver that actually asks: MySQL's -/// `mysql_affected_rows`, libpq's `PQcmdTuples`, `sqlite3_changes`, db-lib's `dbcount`, and the -/// count `OracleNIO` carries on a finished stream. +/// A save is only held to the count it expected where the count is real. Most drivers read one +/// from the server, but a driver that reports a constant would fail every such save, so an engine +/// has to earn its way onto this list by a driver that actually asks: MySQL's `mysql_affected_rows` +/// on a session opened with `CLIENT_FOUND_ROWS`, libpq's `PQcmdTuples`, `sqlite3_changes`, db-lib's +/// `dbcount`, and the count `OracleNIO` carries on a finished stream. A driver can also vouch for one +/// statement's count through `PluginRowWrite.expectedRowCount`. enum DataWriteRowCounts { - private static let enginesReportingRealCounts: Set = [ + private static let enginesCountingMatchedRows: Set = [ .mysql, .mariadb, .tidb, .oceanbase, .postgresql, .redshift, .cockroachdb, .pglite, .kingbasees, .sqlite, .libsql, .turso, .duckdb, .mssql, .oracle, ] - static func areMeaningful(for databaseType: DatabaseType) -> Bool { - enginesReportingRealCounts.contains(databaseType) + static func countsMatchedRows(for databaseType: DatabaseType) -> Bool { + enginesCountingMatchedRows.contains(databaseType) } } diff --git a/apple/TablePro/Core/DataWrite/KeyedWriteVerification.swift b/apple/TablePro/Core/DataWrite/KeyedWriteVerification.swift index fc82d15ad3..0b7ef0d62c 100644 --- a/apple/TablePro/Core/DataWrite/KeyedWriteVerification.swift +++ b/apple/TablePro/Core/DataWrite/KeyedWriteVerification.swift @@ -2,9 +2,9 @@ // KeyedWriteVerification.swift // TablePro // -// The rule every keyed write is held to: more rows than expected is a fault, fewer is not. -// MySQL reports zero affected rows for an UPDATE that writes a value a row already holds, which -// is a normal outcome, so a keyed write is never held to `actual != expected`. +// The rule every keyed write is held to: more rows than expected is a fault. Fewer is one only +// for an UPDATE whose count is the rows it matched (`DataWriteRowCounts`): a DELETE that found its +// row already gone left the table as the user wanted it. // import Foundation diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift index a4c255ea3b..f1f1e8c49d 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift @@ -134,12 +134,23 @@ struct RewindExecutor { ) } + // In the order the restore ran them, each step ordered by when the restore ran it, since undoing the restore + // runs its record backwards. static func inverseOperations(of plan: RewindPlan) -> [RowWriteOperation] { - plan.rows.filter { $0.outcome.restores }.map { row -> RowWriteOperation in - let source = row.operation + let restored = plan.rows.filter { $0.outcome.restores }.map(\.operation) + var rank: [RewindPlanner.RunEntry: Int] = [:] + for (position, entry) in RewindPlanner.runOrder(of: restored).enumerated() { + rank[entry] = position + } + return restored.enumerated().reversed().map { index, source -> RowWriteOperation in let kind = source.kind.inverted // The same stored refusal a save through a view writes, so a build without the view flag refuses too. let throughView = plan.record.writtenThroughView == true && kind == .delete + let steps = source.steps.map { sourceSteps in + sourceSteps.indices.reversed().map { stepIndex in + sourceSteps[stepIndex].inverted.reordered(rank[.init(operation: index, step: stepIndex)] ?? 0) + } + } return RowWriteOperation( kind: kind, target: source.target, @@ -151,7 +162,9 @@ struct RewindExecutor { refusal: throughView ? .identityNotPreservable : nil, preImageAbsentColumns: source.postImageAbsentColumns, postImageAbsentColumns: source.preImageAbsentColumns, - rowLocator: source.rowLocator + rowLocator: source.rowLocator, + order: steps?.first?.order ?? rank[.init(operation: index, step: nil)], + steps: steps ) } } @@ -177,6 +190,7 @@ struct RewindExecutor { ) let route = DatabaseManager.shared.executionRoute(for: scope) + let commitMode = DatabaseManager.shared.commitMode(for: connection.id) let run = try await ExecutionGateProvider.shared.authorizing( OperationRequest( connectionId: connection.id, @@ -193,7 +207,7 @@ struct RewindExecutor { route: route, cancellation: .protectedWrite ) { driver in - try await DataWriteExecutor.run(writePlan, on: driver) + try await DataWriteExecutor.run(writePlan, commitMode: commitMode, on: driver) } } diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift index 0438224191..914ea8cb13 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift @@ -56,38 +56,31 @@ struct RewindPlanner { let byLocator = indexByLocator(currentRows) let matchesByLocator = locatorQuery() != nil || !byLocator.isEmpty - var rows: [RewindRowPlan] = [] - var restorable: [RowWriteOperation] = [] - for operation in record.operations { + var found: [FoundRow] = record.operations.map { operation in if let refusal = record.refusal(of: operation) { - rows.append( - RewindRowPlan( - id: UUID(), operation: operation, - outcome: .notReversible(refusal), currentImage: nil - ) - ) - continue + return FoundRow(outcome: .notReversible(refusal), current: nil) } guard keyIsComparable(operation) else { - rows.append( - RewindRowPlan( - id: UUID(), operation: operation, - outcome: .notReversible(.keyNotComparable), currentImage: nil - ) - ) - continue + return FoundRow(outcome: .notReversible(.keyNotComparable), current: nil) } let currentRow: RewindCurrentRow? if let locator = operation.rowLocator, matchesByLocator { currentRow = byLocator[Array(locator.utf8)] } else { currentRow = primaryKey(of: operation).flatMap { current[$0] } + ?? rowRestoredUnderKeyBefore(operation, in: current) } - let outcome = classify(operation, current: currentRow) + return FoundRow(outcome: classify(operation, current: currentRow), current: currentRow) + } + restoreDeletesWhoseKeyAnInsertFrees(&found) + + var rows: [RewindRowPlan] = [] + var restorable: [RowWriteOperation] = [] + for (operation, row) in zip(record.operations, found) { rows.append( - RewindRowPlan(id: UUID(), operation: operation, outcome: outcome, currentImage: currentRow?.values) + RewindRowPlan(id: UUID(), operation: operation, outcome: row.outcome, currentImage: row.current?.values) ) - if outcome.restores { + if row.outcome.restores { restorable.append(operation) } } @@ -105,6 +98,36 @@ struct RewindPlanner { // MARK: - Classification + private struct FoundRow { + var outcome: RewindRowOutcome + var current: RewindCurrentRow? + } + + // A new row that took a deleted row's key is removed before the deleted row is put back, so that key is free. + private func restoreDeletesWhoseKeyAnInsertFrees(_ found: inout [FoundRow]) { + var rank: [Int: Int] = [:] + for (position, entry) in Self.runOrder(of: record.operations).enumerated() where rank[entry.operation] == nil { + rank[entry.operation] = position + } + let operations = record.operations + for (index, delete) in operations.enumerated() + where delete.kind == .delete && found[index].outcome == .rowAlreadyPresent { + guard let key = primaryKey(of: delete), + let occupant = found[index].current, + primaryKey(of: occupant.values, columns: delete.columns, keyColumns: delete.primaryKeyColumns) == key, + let deleteRank = rank[index] + else { continue } + let freed = operations.indices.contains { other in + operations[other].kind == .insert && found[other].outcome == .willRestore + && primaryKey(of: operations[other]) == key + && (rank[other] ?? .max) < deleteRank + } + if freed { + found[index] = FoundRow(outcome: .willRestore, current: nil) + } + } + } + private var readsThroughView: Bool { record.writtenThroughView == true } private func classify(_ operation: RowWriteOperation, current: RewindCurrentRow?) -> RewindRowOutcome { @@ -135,7 +158,7 @@ struct RewindPlanner { } } - /// Whether the row holds what the image says on every column the save wrote: the same value, + /// Whether the row holds what the image says on every compared column: the same value, /// and the field there or missing as it was. Set NULL on a missing field and Remove Field on a /// NULL one leave the value alone, so comparing values alone reads either as already restored. /// @@ -147,7 +170,7 @@ struct RewindPlanner { absent expectedAbsent: Set?, on operation: RowWriteOperation ) -> Bool { - operation.writtenColumns.allSatisfy { column in + comparedColumns(of: operation).allSatisfy { column in guard let index = operation.columns.firstIndex(of: column), index < current.values.count, index < expected.count, current.values[index] == expected[index] @@ -157,6 +180,14 @@ struct RewindPlanner { } } + // A row found by a key the save wrote holds that key whatever row it is, so a moved row is compared on every + // column the server does not compute; a trigger's stamp then reads as changed, which leaves the row alone. + private func comparedColumns(of operation: RowWriteOperation) -> [String] { + guard keyBeforeMove(of: operation) != nil else { return operation.writtenColumns } + let generated = Set(record.generatedColumns) + return operation.columns.filter { !generated.contains($0) } + } + // MARK: - Reading private func indexByKey(_ rows: [RewindCurrentRow]) -> [RewindRowKey: RewindCurrentRow] { @@ -194,9 +225,11 @@ struct RewindPlanner { guard !keyColumns.isEmpty else { return [] } if keyColumns.count == 1, let keyColumn = keyColumns.first { - let filters = operations.compactMap { operation -> TableFilter? in - guard let value = keyValue(of: operation, column: keyColumn) else { return nil } - return TableFilter(columnName: keyColumn, filterOperator: .equal, value: value) + let filters = operations.flatMap { operation in + readImages(of: operation).compactMap { image -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: keyColumn) else { return nil } + return TableFilter(columnName: keyColumn, filterOperator: .equal, value: value) + } } guard !filters.isEmpty, let query = queryBuilder.buildFilteredQuery( @@ -211,47 +244,141 @@ struct RewindPlanner { return [query] } - return operations.compactMap { operation in - let filters = keyColumns.compactMap { column -> TableFilter? in - guard let value = keyValue(of: operation, column: column) else { return nil } - return TableFilter(columnName: column, filterOperator: .equal, value: value) + return operations.flatMap { operation in + readImages(of: operation).compactMap { image -> String? in + let filters = keyColumns.compactMap { column -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: column) else { return nil } + return TableFilter(columnName: column, filterOperator: .equal, value: value) + } + guard filters.count == keyColumns.count else { return nil } + return queryBuilder.buildFilteredQuery( + tableName: record.target.table, + schemaName: record.target.schema, + filters: filters, + logicMode: .and, + columns: columns, + selectColumns: columns, + limit: 1 + ) } - guard filters.count == keyColumns.count else { return nil } - return queryBuilder.buildFilteredQuery( - tableName: record.target.table, - schemaName: record.target.schema, - filters: filters, - logicMode: .and, - columns: columns, - selectColumns: columns, - limit: 1 - ) } } // MARK: - Statements - /// The inverse operations, newest first. - /// - /// Reverse order matters when one row in the save freed a value another row took: putting the - /// second row's change back before the first row's is the only order that does not collide. - /// The forward generator groups by verb, which loses that order, so the inverse is generated - /// one operation at a time. + struct RunEntry: Hashable { + let operation: Int + let step: Int? + } + + // Newest first, step by step, since a value one write freed may be one a later write took. Ties, and a record + // saved before orders were kept, run by list position, the later first. + static func runOrder(of operations: [RowWriteOperation]) -> [RunEntry] { + var entries: [(order: Int?, entry: RunEntry)] = [] + for (index, operation) in operations.enumerated() { + if let steps = operation.steps, !steps.isEmpty { + for (stepIndex, step) in steps.enumerated() { + entries.append((step.order, RunEntry(operation: index, step: stepIndex))) + } + } else { + entries.append((operation.order, RunEntry(operation: index, step: nil))) + } + } + return entries.sorted { lhs, rhs in + (lhs.order ?? .min, lhs.entry.operation, lhs.entry.step ?? 0) + > (rhs.order ?? .min, rhs.entry.operation, rhs.entry.step ?? 0) + }.map(\.entry) + } + private func statements(for operations: [RowWriteOperation]) throws -> RestoreStatements { var statements: [ParameterizedStatement] = [] var lookups: [RowLookup?] = [] var prologue: [String] = [] var epilogue: [String] = [] - for operation in operations.reversed() { - let inverse = try inverseStatements(for: operation) + var checked: Set = [] + for entry in Self.runOrder(of: operations) { + let operation = operations[entry.operation] + let inverse: RestoreStatements + if let step = entry.step { + inverse = try inverseStatements(for: operation, step: step) + } else { + inverse = try inverseStatements(for: operation) + } statements.append(contentsOf: inverse.statements) - lookups.append(contentsOf: inverse.lookups) + // The operation's first statement to run finds the row as the save left it, so it is the one checked. + if checked.insert(entry.operation).inserted, let lookup = heldRowLookup(for: operation, inverse) { + lookups.append(lookup) + } else { + lookups.append(contentsOf: inverse.lookups) + } prologue.append(contentsOf: inverse.prologue.filter { !prologue.contains($0) }) epilogue.append(contentsOf: inverse.epilogue.filter { !epilogue.contains($0) }) } return RestoreStatements(statements: statements, lookups: lookups, prologue: prologue, epilogue: epilogue) } + // A statement with no lookup of its own finds the row by its key alone, so the sheet's check runs again inside the + // write: a row another writer changed since the review refuses the restore instead of being overwritten. + private func heldRowLookup(for operation: RowWriteOperation, _ inverse: RestoreStatements) -> RowLookup? { + guard inverse.statements.count == 1, inverse.lookups.allSatisfy({ $0 == nil }), + operation.kind == .update || operation.kind == .insert, + operation.rowLocator == nil, + !PluginManager.shared.supportsFieldRemoval(for: record.databaseType), + let image = operation.postImage, + let query = keyedRead(of: image, for: operation) + else { return nil } + var held: [Int: PluginCellValue] = [:] + for column in operation.writtenColumns { + guard let index = operation.columns.firstIndex(of: column), index < image.count else { return nil } + held[index] = image[index] + } + return RowLookup( + query: ParameterizedStatement(sql: query, parameters: []), rowCount: 1, pinsRows: false, heldValues: held + ) + } + + private func keyedRead(of image: [PluginCellValue], for operation: RowWriteOperation) -> String? { + let filters = operation.primaryKeyColumns.compactMap { column -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: column) else { return nil } + return TableFilter(columnName: column, filterOperator: .equal, value: value) + } + guard !filters.isEmpty, filters.count == operation.primaryKeyColumns.count else { return nil } + return queryBuilder.buildFilteredQuery( + tableName: record.target.table, + schemaName: record.target.schema, + filters: filters, + logicMode: .and, + columns: operation.columns, + selectColumns: operation.columns, + limit: 1 + ) + } + + // Sets the step's columns back, keyed on the row as that step left it. + private func inverseStatements(for operation: RowWriteOperation, step index: Int) throws -> RestoreStatements { + guard let image = operation.stepImages()?[safe: index] else { return RestoreStatements(statements: []) } + var factory = factory + factory.rowLocators = operation.rowLocator.map { [.existing(0): $0] } ?? [:] + let step = image.step + let cellChanges = step.columns.enumerated().compactMap { offset, column -> CellChange? in + guard operation.columns.indices.contains(column), image.after.indices.contains(column), + step.before.indices.contains(offset) + else { return nil } + return CellChange( + columnIndex: column, columnName: operation.columns[column], + oldValue: image.after[column], newValue: step.before[offset], + oldIsAbsent: image.absentAfter.contains(column), newIsAbsent: step.beforeAbsent.contains(column) + ) + } + guard !cellChanges.isEmpty else { return RestoreStatements(statements: []) } + return try factory.lookedUpStatements( + for: [RowChange( + rowID: .existing(0), type: .update, cellChanges: cellChanges, + originalRow: image.after, absentColumns: image.absentAfter + )] + ) + } + private func inverseStatements(for operation: RowWriteOperation) throws -> RestoreStatements { var factory = factory factory.rowLocators = operation.rowLocator.map { [.existing(0): $0] } ?? [:] @@ -293,12 +420,40 @@ struct RewindPlanner { // MARK: - Keys + // Where the save left the row, or for a delete where it was: an edit that rewrote the key moved it. + private func savedImage(of operation: RowWriteOperation) -> [PluginCellValue]? { + operation.postImage ?? operation.preImage + } + private func primaryKey(of operation: RowWriteOperation) -> RewindRowKey? { - let source = operation.preImage ?? operation.postImage - guard let source else { return nil } + guard let source = savedImage(of: operation) else { return nil } return primaryKey(of: source, columns: operation.columns, keyColumns: operation.primaryKeyColumns) } + private func keyBeforeMove(of operation: RowWriteOperation) -> RewindRowKey? { + guard operation.kind == .update, let preImage = operation.preImage else { return nil } + let before = primaryKey(of: preImage, columns: operation.columns, keyColumns: operation.primaryKeyColumns) + return before == primaryKey(of: operation) ? nil : before + } + + // A rewind that already ran put a moved row back under its old key, where it reads as restored. + private func readImages(of operation: RowWriteOperation) -> [[PluginCellValue]] { + guard let saved = savedImage(of: operation) else { return [] } + guard keyBeforeMove(of: operation) != nil, let preImage = operation.preImage else { return [saved] } + return [saved, preImage] + } + + // Only a row holding the old values, so another row that took the old key since is not mistaken for this one. + private func rowRestoredUnderKeyBefore( + _ operation: RowWriteOperation, + in current: [RewindRowKey: RewindCurrentRow] + ) -> RewindCurrentRow? { + guard let key = keyBeforeMove(of: operation), let row = current[key], let preImage = operation.preImage, + matches(row, preImage, absent: operation.preImageAbsentColumns, on: operation) + else { return nil } + return row + } + private func primaryKey( of row: [PluginCellValue], columns: [String], @@ -319,16 +474,17 @@ struct RewindPlanner { /// compare against, and a delete would be reported as restorable with no conflict check at all. private func keyIsComparable(_ operation: RowWriteOperation) -> Bool { !operation.primaryKeyColumns.isEmpty - && operation.primaryKeyColumns.allSatisfy { keyValue(of: operation, column: $0) != nil } + && operation.primaryKeyColumns.allSatisfy { + keyValue(in: savedImage(of: operation), of: operation, column: $0) != nil + } } - private func keyValue(of operation: RowWriteOperation, column: String) -> String? { - let source = operation.preImage ?? operation.postImage - guard let source, + private func keyValue(in image: [PluginCellValue]?, of operation: RowWriteOperation, column: String) -> String? { + guard let image, let index = operation.columns.firstIndex(of: column), - index < source.count + index < image.count else { return nil } - return source[index].asText + return image[index].asText } } diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 968b67c8df..4b936fe665 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -53,6 +53,8 @@ struct DriverWrittenStatement { var lookup: RowLookup? // A keyed write through a view, held to its one row: the key is the view's answer, not the table's own. var keyedRowCount: Int? + // The driver's own count of the rows the statement matches. + var matchedRowCount: Int? } @MainActor @@ -131,22 +133,22 @@ struct RowChangeStatementFactory { } } - /// Every statement the changes need, or a throw naming the changes that would be left out. - /// - /// The host's statements carry the rows they touch, so the executor can hold the server to that - /// count. A driver's carry one only for a keyless update or delete of one row. + // Split into save steps first, so the refusals, every writer, the preview and the coverage see what runs. func rowWriteStatements( for changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] = [:], deletedRowIDs: Set = [], insertedRowIDs: Set = [] ) throws -> RowWriteStatements { - try refuseServerOwnedEdits(in: changes) - try refuseDefaultOnUpdate(in: changes) - try refuseUnwritableExpressions(in: changes, insertedRowData: insertedRowData) - try refuseMisdimensionedVectors(in: changes, insertedRowData: insertedRowData) + let steps = RowChange.saveSteps( + changes, keyColumns: Set(primaryKeyColumns.compactMap { columns.firstIndex(of: $0) }) + ) + try refuseServerOwnedEdits(in: steps) + try refuseDefaultOnUpdate(in: steps) + try refuseUnwritableExpressions(in: steps, insertedRowData: insertedRowData) + try refuseMisdimensionedVectors(in: steps, insertedRowData: insertedRowData) let insertedRowData = insertedRowData.mapValues(leavingServerOwnedColumnsToTheServer) - let changes = changes.map(droppingServerOwnedExpressions) + let changes = steps.map { markingServerOwnedDefaults($0, in: insertedRowData) } if let driverStatements = try pluginRowWrites( for: changes, insertedRowData: insertedRowData, @@ -190,7 +192,9 @@ struct RowChangeStatementFactory { private func refuseDefaultOnUpdate(in changes: [RowChange]) throws { guard !CellExpressionVocabulary.updateTakesDefault(databaseType) else { return } for change in changes where change.type == .update { - guard let defaulted = change.cellChanges.first(where: { $0.newValue.isDefaultMarker }) else { continue } + guard let defaulted = change.cellChanges.first(where: { + $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) + }) else { continue } throw DataWriteError.changeRefused( table: tableName, kind: .update, @@ -209,9 +213,10 @@ struct RowChangeStatementFactory { in changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] ) throws { + // A default reaches a driver in the spelling it reads, so only SQL proper has to be SQL this engine runs. let marked = changes.flatMap { change in Self.markedCells(of: change, insertedRowData: insertedRowData, columns: columns) - } + }.filter { $0.value != .columnDefault } guard !marked.isEmpty else { return } for cell in marked { guard case .text(let sql) = cell.value, CellExpressionVocabulary.isCanonical(sql, for: databaseType) @@ -309,21 +314,65 @@ struct RowChangeStatementFactory { guard !generatedColumns.isEmpty else { return values } return values.enumerated().map { index, value in guard columns.indices.contains(index), generatedColumns.contains(columns[index]) else { return value } - return Self.defaultMarker + return .columnDefault } } - /// The cell now holds the default marker, so it is no longer SQL to write. - private func droppingServerOwnedExpressions(_ change: RowChange) -> RowChange { - guard change.type == .insert, !change.expressionColumns.isEmpty, !generatedColumns.isEmpty else { return change } - var kept = change - kept.expressionColumns = change.expressionColumns.filter { - !(columns.indices.contains($0) && generatedColumns.contains(columns[$0])) + // Whatever SQL the cell held, it now holds the default, which only the mark makes one. + private func markingServerOwnedDefaults(_ change: RowChange, in insertedRowData: [RowID: [PluginCellValue]]) -> RowChange { + guard change.type == .insert, !generatedColumns.isEmpty else { return change } + let owned = Set(columns.indices.filter { generatedColumns.contains(columns[$0]) }) + var marked = change + marked.expressionColumns.subtract(owned) + if let values = insertedRowData[change.rowID] { + marked.expressionColumns.formUnion(owned.filter { values.indices.contains($0) }) } - return kept + return marked } - private static let defaultMarker = PluginCellValue.text("__DEFAULT__") + // A driver built before `PluginRowChange.defaultColumns` reads its own spelling for a default, so a default reaches + // every driver spelled that way, with the mark beside it for one that reads the mark. + private static func spellingDefaultsForDrivers( + _ changes: [RowChange], + insertedRowData: [RowID: [PluginCellValue]] + ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], defaults: [RowID: Set]) { + var rows = insertedRowData + var defaults: [RowID: Set] = [:] + let spelledChanges = changes.map { change -> RowChange in + var spelled = change + if change.type == .insert, var values = rows[change.rowID] { + for index in change.expressionColumns where values.indices.contains(index) && values[index] == .columnDefault { + values[index] = .pluginDefaultMarker + spelled.expressionColumns.remove(index) + defaults[change.rowID, default: []].insert(index) + } + rows[change.rowID] = values + } + let defaulted = change.cellChanges.filter { $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) } + guard !defaulted.isEmpty else { return spelled } + // A new row's stored values are what a driver writes, so only they mark its defaults. + if !(change.type == .insert && insertedRowData[change.rowID] != nil) { + defaults[change.rowID, default: []].formUnion(defaulted.map(\.columnIndex)) + } + spelled.cellChanges = change.cellChanges.map { cell in + guard cell.newValue.isColumnDefault(isExpression: cell.newIsExpression) else { return cell } + return CellChange( + columnIndex: cell.columnIndex, + columnName: cell.columnName, + oldValue: cell.oldValue, + newValue: .pluginDefaultMarker, + oldIsAbsent: cell.oldIsAbsent, + newIsAbsent: cell.newIsAbsent, + oldIsExpression: cell.oldIsExpression + ) + } + if change.type == .insert, rows[change.rowID] == nil { + spelled.expressionColumns.subtract(defaulted.map(\.columnIndex)) + } + return spelled + } + return (spelledChanges, rows, defaults) + } private func rowWriteContext(for keyed: PluginKeyedChanges) -> PluginRowWriteContext { var context = PluginRowWriteContext() @@ -352,7 +401,7 @@ struct RowChangeStatementFactory { changes, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs, - writtenRowIDs: Set(statements.flatMap(\.rowIDs)) + writtenChangeIDs: Set(statements.flatMap(\.changeIDs)) ) if let kind = UnwrittenRowCounts(unwritten).leadingKind { throw DataWriteError.rowsNotIdentifiable(tableName, kind) @@ -386,6 +435,9 @@ struct RowChangeStatementFactory { }, context: restoreContext(rowCount: rows.count) ) { + if !pluginDriver.capabilities.contains(.columnDefaultMarks) { + try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) + } try refuseUnfilteredWrites(restored.map { $0.statement }) return RestoreStatements(statements: restored.map { ParameterizedStatement(sql: $0.statement, parameters: $0.parameters.map(\.asAny)) @@ -489,11 +541,13 @@ struct RowChangeStatementFactory { insertedRowIDs: Set ) throws -> [DriverWrittenStatement]? { guard let pluginDriver else { return nil } + let spelled = Self.spellingDefaultsForDrivers(changes, insertedRowData: insertedRowData) let keyed = PluginKeyedChanges( - changes: changes, - insertedRowData: insertedRowData, + changes: spelled.changes, + insertedRowData: spelled.insertedRowData, deletedRowIDs: deletedRowIDs, - insertedRowIDs: insertedRowIDs + insertedRowIDs: insertedRowIDs, + defaultColumns: spelled.defaults ) let writes: [PluginRowWrite] do { @@ -524,12 +578,15 @@ struct RowChangeStatementFactory { throw DataWriteError.changeRefused(table: tableName, kind: nil, reason: error.localizedDescription) } + if !pluginDriver.capabilities.contains(.columnDefaultMarks) { + try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) + } try refuseUnfilteredWrites(writes.map(\.statement)) let unwritten = RowWriteCoverage.unwrittenChanges( changes, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs, - writtenRowIDs: Set(writes.flatMap(\.rowIndices).compactMap(keyed.rowID(forIndex:))) + writtenChangeIDs: Set(writes.flatMap(\.rowIndices).compactMap { keyed.change(forIndex: $0)?.id }) ) guard unwritten.isEmpty else { throw DataWriteError.changesNotWritable(table: tableName, unwritten: UnwrittenRowCounts(unwritten)) @@ -542,11 +599,52 @@ struct RowChangeStatementFactory { lookup: write.lookup.map(RowLookup.init), keyedRowCount: write.lookup == nil && holdsKeyedWritesToOneRow ? oneRowCount(of: write, keyed: keyed) - : nil + : nil, + matchedRowCount: write.expectedRowCount ) } } + // A driver that does not read `PluginRowChange.defaultColumns` may read the text `__DEFAULT__` as the column default, + // so the text is never handed to one as a value. + private func refuseTextDriversReadAsDefault( + in changes: [RowChange], + insertedRowData: [RowID: [PluginCellValue]] + ) throws { + for change in changes where change.type != .delete { + var written = change.cellChanges.filter { !$0.newIsExpression }.map { (column: $0.columnName, value: $0.newValue) } + if change.type == .insert, let stored = insertedRowData[change.rowID] { + written = zip(columns, stored).enumerated() + .filter { !change.expressionColumns.contains($0.offset) } + .map { (column: $0.element.0, value: $0.element.1) } + } + guard let literal = written.first(where: { $0.value == .pluginDefaultMarker }) else { continue } + throw textReadAsDefault(in: literal.column, kind: change.type == .insert ? .insert : .update) + } + } + + // A rewind puts a deleted row back with the values it held, which can be that text. + private func refuseRestoredTextDriversReadAsDefault(_ rows: [[PluginCellValue]], restorable: [Int]) throws { + for row in rows { + guard let index = restorable.first(where: { row.indices.contains($0) && row[$0] == .pluginDefaultMarker }) + else { continue } + throw textReadAsDefault(in: columns[index], kind: .insert) + } + } + + private func textReadAsDefault(in column: String, kind: RowWriteKind) -> DataWriteError { + .changeRefused( + table: tableName, + kind: kind, + reason: String( + format: String( + localized: "This database's driver may read the text __DEFAULT__ in %@ as the column default, so it cannot be saved as text." + ), + column + ) + ) + } + // The host's own generator never writes one, so only a driver's statements are read. private func refuseUnfilteredWrites(_ statements: [String]) throws { guard PluginManager.shared.editorLanguage(for: databaseType) == .sql else { return } @@ -571,7 +669,7 @@ struct RowChangeStatementFactory { } private func oneRowCount(of write: PluginRowWrite, keyed: PluginKeyedChanges) -> Int? { - guard DataWriteRowCounts.areMeaningful(for: databaseType), + guard DataWriteRowCounts.countsMatchedRows(for: databaseType), write.rowIndices.count == 1, let row = write.rowIndices.first else { return nil } @@ -614,38 +712,45 @@ struct RowChangeStatementFactory { } } +// One key per change, so a row written in two steps reaches the driver as two changes. struct PluginKeyedChanges { let changes: [PluginRowChange] let insertedRowData: [Int: [PluginCellValue]] let deletedRowIndices: Set let insertedRowIndices: Set - /// The row each key stands for, indexed by key. + // Indexed by key. let rowIDs: [RowID] + private let steps: [RowChange] init( changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], deletedRowIDs: Set, - insertedRowIDs: Set + insertedRowIDs: Set, + defaultColumns: [RowID: Set] = [:] ) { let ordered = changes.sorted { $0.sequence < $1.sequence } - var keys: [RowID: Int] = [:] - var rowIDs: [RowID] = [] - for change in ordered where keys[change.rowID] == nil { - keys[change.rowID] = rowIDs.count - rowIDs.append(change.rowID) + var insertKeys: [RowID: Int] = [:] + var deleteKeys: [RowID: Int] = [:] + for (key, change) in ordered.enumerated() { + switch change.type { + case .insert: insertKeys[change.rowID] = key + case .delete: deleteKeys[change.rowID] = key + case .update: break + } } - self.rowIDs = rowIDs - self.changes = ordered.compactMap { change in - keys[change.rowID].map { PluginRowChange(change, key: $0) } + self.steps = ordered + self.rowIDs = ordered.map(\.rowID) + self.changes = ordered.enumerated().map { key, change in + PluginRowChange(change, key: key, defaultColumns: defaultColumns[change.rowID]) } self.insertedRowData = Dictionary( uniqueKeysWithValues: insertedRowData.compactMap { rowID, values in - keys[rowID].map { ($0, values) } + insertKeys[rowID].map { ($0, values) } } ) - self.deletedRowIndices = Set(deletedRowIDs.compactMap { keys[$0] }) - self.insertedRowIndices = Set(insertedRowIDs.compactMap { keys[$0] }) + self.deletedRowIndices = Set(deletedRowIDs.compactMap { deleteKeys[$0] }) + self.insertedRowIndices = Set(insertedRowIDs.compactMap { insertKeys[$0] }) } func locators(_ byRow: [RowID: String]) -> [Int: String] { @@ -656,14 +761,14 @@ struct PluginKeyedChanges { return keyed } - /// The row a driver's `rowIndex` names, or nil when it names none of these changes. - func rowID(forIndex index: Int) -> RowID? { - rowIDs.indices.contains(index) ? rowIDs[index] : nil + // Nil when a driver's `rowIndex` names none of these changes. + func change(forIndex index: Int) -> RowChange? { + steps.indices.contains(index) ? steps[index] : nil } func writeKind(ofRowIndex index: Int) -> RowWriteKind? { - guard let change = changes.first(where: { $0.rowIndex == index }) else { return nil } - switch change.type { + guard let type = change(forIndex: index)?.type else { return nil } + switch type { case .insert: return .insert case .update: return .update case .delete: return .delete @@ -690,7 +795,7 @@ private extension SQLUnfilteredWrite { } private extension PluginRowChange { - init(_ change: RowChange, key: Int) { + init(_ change: RowChange, key: Int, defaultColumns: Set?) { self.init( rowIndex: key, type: { @@ -707,6 +812,7 @@ private extension PluginRowChange { ) absentColumns = Self.absentColumns(after: change) expressionColumns = Self.expressionColumns(of: change) + self.defaultColumns = change.type == .delete || defaultColumns?.isEmpty != false ? nil : defaultColumns } static func expressionColumns(of change: RowChange) -> Set? { diff --git a/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift b/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift index fef6c92260..188f5e7e30 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift @@ -23,20 +23,21 @@ enum RowWriteCoverage { } } + // By change, not by row: a row edited around another change is written in steps, and each one has to be. static func unwrittenChanges( _ changes: [RowChange], deletedRowIDs: Set, insertedRowIDs: Set, - writtenRowIDs: Set + writtenChangeIDs: Set ) -> [RowChange] { changes.filter { change in isPending(change, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs) - && !writtenRowIDs.contains(change.rowID) + && !writtenChangeIDs.contains(change.id) } } } -/// How many changes of each kind a save could not write, which is what the refusal names. +// How many rows of each kind a save could not write, which is what the refusal names. struct UnwrittenRowCounts: Equatable, Sendable { let updates: Int let inserts: Int @@ -49,11 +50,10 @@ struct UnwrittenRowCounts: Equatable, Sendable { } init(_ changes: [RowChange]) { - self.init( - updates: changes.count { $0.type == .update }, - inserts: changes.count { $0.type == .insert }, - deletes: changes.count { $0.type == .delete } - ) + func rows(_ type: ChangeType) -> Int { + Set(changes.lazy.filter { $0.type == type }.map(\.rowID)).count + } + self.init(updates: rows(.update), inserts: rows(.insert), deletes: rows(.delete)) } var isEmpty: Bool { updates == 0 && inserts == 0 && deletes == 0 } diff --git a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift index 457424223b..77503f51b6 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift @@ -34,19 +34,36 @@ enum RowWriteOperationBuilder { writesThroughView: Bool = false, rowLocators: [RowID: String] = [:] ) -> [RowWriteOperation] { - changes.compactMap { change in + // The same steps the save runs, so the record matches what ran. + let keyColumns = Set(primaryKeyColumns.compactMap { columns.firstIndex(of: $0) }) + var stepsByRow: [RowID: [RowChange]] = [:] + for step in RowChange.saveSteps(changes, keyColumns: keyColumns) where step.type == .update { + stepsByRow[step.rowID, default: []].append(step) + } + func firstOrder(of change: RowChange) -> Int { + guard change.type == .update else { return change.sequence } + return stepsByRow[change.rowID]?.first?.sequence ?? change.sequence + } + + // A rewind undoes them newest first, and `changes` stops being in edit order once a change is cancelled. + let ordered = changes.enumerated() + .sorted { (firstOrder(of: $0.element), $0.offset) < (firstOrder(of: $1.element), $1.offset) } + .map(\.element) + return ordered.compactMap { change -> RowWriteOperation? in + var operation: RowWriteOperation switch change.type { case .update: - var operation = update( - change, target: target, columns: columns, + let steps = stepsByRow[change.rowID] ?? [change] + operation = update( + change, steps: steps, target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, generatedColumns: generatedColumns, containsTableOperation: containsTableOperation ) operation.rowLocator = rowLocators[change.rowID] - return operation + operation.steps = steps.count > 1 ? steps.map { RowWriteStep($0) } : nil case .delete: guard deletedRowIDs.contains(change.rowID) else { return nil } - var operation = delete( + operation = delete( change, target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, containsTableOperation: containsTableOperation, @@ -54,21 +71,23 @@ enum RowWriteOperationBuilder { throughView: writesThroughView ) operation.rowLocator = rowLocators[change.rowID] - return operation case .insert: guard insertedRowIDs.contains(change.rowID) else { return nil } - return insert( + operation = insert( change, values: insertedRowData[change.rowID], target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, containsTableOperation: containsTableOperation ) } + operation.order = firstOrder(of: change) + return operation } } private static func update( _ change: RowChange, + steps: [RowChange], target: DataWriteTarget, columns: [String], primaryKeyColumns: [String], @@ -94,16 +113,18 @@ enum RowWriteOperationBuilder { postImage: postImage, writtenColumns: writtenColumns, refusal: updateRefusal( - change, preImage: preImage, primaryKeyColumns: primaryKeyColumns, - generatedColumns: generatedColumns, containsTableOperation: containsTableOperation + writing: change.cellChanges + steps.flatMap(\.cellChanges), preImage: preImage, + primaryKeyColumns: primaryKeyColumns, generatedColumns: generatedColumns, + containsTableOperation: containsTableOperation ), preImageAbsentColumns: change.absentColumns, postImageAbsentColumns: change.absentColumnsAfterUpdate ) } + // Every step's cells count: the inverse of a later step would write an earlier step's SQL back as a literal. private static func updateRefusal( - _ change: RowChange, + writing cellChanges: [CellChange], preImage: [PluginCellValue]?, primaryKeyColumns: [String], generatedColumns: Set, @@ -111,14 +132,14 @@ enum RowWriteOperationBuilder { ) -> RewindRefusal? { if containsTableOperation { return .destructiveTableOperation } if primaryKeyColumns.isEmpty || preImage == nil { return .noPrimaryKey } - if change.cellChanges.contains(where: { $0.newIsExpression || $0.newValue.isDefaultMarker }) { + if cellChanges.contains(where: \.newIsExpression) { return .serverComputedValue } - if change.cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { + if cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { return .serverComputedValue } if let preImage, preImage.contains(where: exceedsCapture) { return .valueTooLarge } - if change.cellChanges.contains(where: { exceedsCapture($0.newValue) }) { return .valueTooLarge } + if cellChanges.contains(where: { exceedsCapture($0.newValue) }) { return .valueTooLarge } return nil } @@ -172,11 +193,11 @@ enum RowWriteOperationBuilder { return .serverAssignedKey } let value = values[index] - if value.isNull || value.isDefaultMarker || change.expressionColumns.contains(index) { + if value.isNull || change.expressionColumns.contains(index) { return .serverAssignedKey } } - if values.contains(where: \.isDefaultMarker) || !change.expressionColumns.isEmpty { + if !change.expressionColumns.isEmpty { return .serverComputedValue } if values.contains(where: exceedsCapture) { return .valueTooLarge } diff --git a/apple/TablePro/Core/Database/AWS/RedshiftIAMSignIn.swift b/apple/TablePro/Core/Database/AWS/RedshiftIAMSignIn.swift index 9ba0729632..0755cb44cc 100644 --- a/apple/TablePro/Core/Database/AWS/RedshiftIAMSignIn.swift +++ b/apple/TablePro/Core/Database/AWS/RedshiftIAMSignIn.swift @@ -9,15 +9,32 @@ import TableProPluginKit // Redshift issues the user and a temporary password together, so a sign-in mints both. struct RedshiftIAMSignIn: Sendable { + // A custom domain is asked for by name: AWS answers only for a domain associated with a cluster or workgroup in + // the caller's account, so the password still goes only to that database. enum Endpoint: Equatable, Sendable { case cluster(identifier: String, region: String) case serverless(workgroup: String, region: String) + case clusterDomain(name: String, region: String) + case serverlessDomain(name: String, region: String) var region: String { switch self { - case .cluster(_, let region), .serverless(_, let region): region + case .cluster(_, let region), .serverless(_, let region), + .clusterDomain(_, let region), .serverlessDomain(_, let region): + region } } + + var isServerless: Bool { + switch self { + case .serverless, .serverlessDomain: true + case .cluster, .clusterDomain: false + } + } + + func onCustomDomain(_ name: String) -> Endpoint { + isServerless ? .serverlessDomain(name: name, region: region) : .clusterDomain(name: name, region: region) + } } static let endpointFieldId = "awsRedshiftEndpoint" @@ -28,9 +45,10 @@ struct RedshiftIAMSignIn: Sendable { let fields: [String: String] let configuredHost: String let preTunnelHost: String? + var aliasResolver: any CNAMEResolving = SystemCNAMEResolver() func signIn(deadline: ConnectionDeadline?) async throws -> ResolvedSignIn { - let endpoint = try resolveEndpoint() + let endpoint = try await resolveEndpoint(within: deadline?.remainingDuration) let region = nonEmpty(fields["awsRegion"]) ?? endpoint.region return try await AWSDeadlineSession.run(within: deadline) { session in let credentials = try await AWSCredentialResolver.resolve(source: source, fields: fields, session: session) @@ -50,32 +68,59 @@ struct RedshiftIAMSignIn: Sendable { } } - func resolveEndpoint() throws -> Endpoint { + func resolveEndpoint(within timeout: Duration?) async throws -> Endpoint { + let dialledHost = AWSEndpointHost.normalized(configuredHost) + let dialsLoopback = AWSEndpointHost.isLoopback(configuredHost) + // The password goes to this name: the far end of a forward, else the host the driver dials. + let reachedHost = dialsLoopback ? preTunnelHost.map(AWSEndpointHost.normalized) : dialledHost if let override = nonEmpty(fields[Self.endpointFieldId]) { - let overrideHost = Self.hostName(in: override) + let overrideHost = AWSEndpointHost.normalized(Self.hostName(in: override)) guard let endpoint = Self.endpoint(forHost: overrideHost) else { throw RedshiftIAMError.endpointInvalid(override) } - // The password goes to the host the driver dials, so it must be a forward or that endpoint itself. - guard Self.isLoopback(configuredHost) - || overrideHost.caseInsensitiveCompare(configuredHost) == .orderedSame - else { + if dialsLoopback || overrideHost == dialledHost { + return endpoint + } + // Only a DNS name can be a custom domain; an address or another AWS endpoint is not the one named. + guard AWSEndpointHost.isAliasCandidate(dialledHost), Self.endpoint(forHost: dialledHost) == nil else { throw RedshiftIAMError.endpointNotDialed(host: configuredHost) } - return endpoint + return endpoint.onCustomDomain(dialledHost) } for host in [preTunnelHost, configuredHost].compactMap({ $0 }) { if let endpoint = Self.endpoint(forHost: host) { return endpoint } } + if let reachedHost, AWSEndpointHost.isAliasCandidate(reachedHost) { + let lookup = await aliasResolver.firstName( + behind: reachedHost, + within: timeout, + where: { Self.endpoint(forHost: $0) != nil } + ) + switch lookup { + case .found(let target): + if let endpoint = Self.endpoint(forHost: target) { + return endpoint.onCustomDomain(reachedHost) + } + case .failed: + throw AWSAuthError.endpointLookupFailed(host: reachedHost) + case .notFound: + break + } + } throw RedshiftIAMError.endpointUnresolved(host: preTunnelHost ?? configuredHost) } func request(for endpoint: Endpoint, region: String, credentials: AWSCredentials, now: Date = Date()) throws -> URLRequest { switch endpoint { - case .cluster(let identifier, _): - var parameters = ["Version": "2012-12-01", "ClusterIdentifier": identifier] + case .cluster, .clusterDomain: + var parameters = ["Version": "2012-12-01"] + switch endpoint { + case .clusterDomain(let name, _): parameters["CustomDomainName"] = name + case .cluster(let identifier, _): parameters["ClusterIdentifier"] = identifier + case .serverless, .serverlessDomain: break + } if let databaseUser = nonEmpty(databaseUser) { parameters["Action"] = "GetClusterCredentials" parameters["DbUser"] = databaseUser @@ -87,8 +132,13 @@ struct RedshiftIAMSignIn: Sendable { } return try AWSQueryRequest(service: "redshift", region: region, parameters: parameters) .signedURLRequest(credentials: credentials, now: now) - case .serverless(let workgroup, _): - var body = ["workgroupName": workgroup] + case .serverless, .serverlessDomain: + var body: [String: String] = [:] + switch endpoint { + case .serverlessDomain(let name, _): body["customDomainName"] = name + case .serverless(let workgroup, _): body["workgroupName"] = workgroup + case .cluster, .clusterDomain: break + } if let database = nonEmpty(database) { body["dbName"] = database } @@ -104,10 +154,7 @@ struct RedshiftIAMSignIn: Sendable { // Only a name AWS issued, under the region's own DNS suffix: anything else is someone else's host. static func endpoint(forHost host: String) -> Endpoint? { - var name = host.trimmingCharacters(in: .whitespaces).lowercased() - if name.hasSuffix(".") { - name.removeLast() - } + let name = AWSEndpointHost.normalized(host) let labels = name.split(separator: ".", omittingEmptySubsequences: false).map(String.init) guard labels.count > 4, !labels[0 ..< 3].contains(where: \.isEmpty) else { return nil } let region = labels[2] @@ -122,12 +169,11 @@ struct RedshiftIAMSignIn: Sendable { static func signIn(from data: Data, for endpoint: Endpoint) throws -> ResolvedSignIn { let user: String? let password: String? - switch endpoint { - case .cluster: + if !endpoint.isServerless { let parser = ClusterCredentialsXMLParser() guard parser.parse(data) else { throw RedshiftIAMError.unreadableResponse } (user, password) = (parser.databaseUser, parser.databasePassword) - case .serverless: + } else { let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] (user, password) = (object?["dbUser"] as? String, object?["dbPassword"] as? String) } @@ -138,21 +184,13 @@ struct RedshiftIAMSignIn: Sendable { } static func errorMessage(in data: Data, for endpoint: Endpoint) -> String? { - switch endpoint { - case .cluster: + guard endpoint.isServerless else { return AWSQueryErrorResponse.parse(data)?.summary - case .serverless: - guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } - let code = (object["__type"] as? String).map { $0.split(separator: "#").last.map(String.init) ?? $0 } - let message = object["message"] as? String ?? object["Message"] as? String - return AWSQueryErrorResponse(code: code, message: message).summary } - } - - private static func isLoopback(_ host: String) -> Bool { - let normalized = host.trimmingCharacters(in: .whitespaces).lowercased() - return ["localhost", "localhost.localdomain", "::1", "[::1]", "0.0.0.0"].contains(normalized) - || normalized.hasPrefix("127.") + guard let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } + let code = (object["__type"] as? String).map { $0.split(separator: "#").last.map(String.init) ?? $0 } + let message = object["message"] as? String ?? object["Message"] as? String + return AWSQueryErrorResponse(code: code, message: message).summary } private static func hostName(in value: String) -> String { @@ -203,8 +241,8 @@ enum RedshiftIAMError: LocalizedError, Equatable { return String( format: String(localized: """ TablePro cannot tell which Redshift cluster or workgroup "%@" is. Set Host to the \ - endpoint AWS issued, such as my-cluster.abc123.us-east-1.redshift.amazonaws.com, or \ - enter that endpoint as the Redshift Endpoint when you connect through a port forward. + endpoint AWS issued, such as my-cluster.abc123.us-east-1.redshift.amazonaws.com, or to a \ + custom domain name that points to it, or enter that endpoint as the Redshift Endpoint. """), host ) @@ -216,8 +254,8 @@ enum RedshiftIAMError: LocalizedError, Equatable { case .endpointNotDialed(let host): return String( format: String(localized: """ - The Redshift Endpoint applies only when Host is a local port forward or that same \ - endpoint, and "%@" is neither. + The Redshift Endpoint applies only when Host is a local port forward, a custom domain \ + name or that same endpoint, and "%@" is none of them. """), host ) diff --git a/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift b/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift new file mode 100644 index 0000000000..bc9ed2421c --- /dev/null +++ b/apple/TablePro/Core/Database/Access/AgentTransactionGuard.swift @@ -0,0 +1,186 @@ +// +// AgentTransactionGuard.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit +import TableProSQLGrammar + +// What an agent's statement or script may do to the transaction its session holds, decided inside the lease. +internal struct AgentTransactionGuard: Sendable { + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "AgentTransactionGuard") + internal static let savepointStatement = "SAVEPOINT tablepro_agent" + internal static let releaseStatement = "RELEASE SAVEPOINT tablepro_agent" + internal static let rollbackStatement = "ROLLBACK TO SAVEPOINT tablepro_agent" + + let commitMode: ConnectionCommitMode + let databaseType: DatabaseType + let grammar: SQLLexicalGrammar + let connectionId: UUID + + internal enum Protection: Equatable { + case none + case savepoint + case refuse + } + + internal struct Savepoint: Equatable, Sendable { + let open: String + let release: String? + let rollback: String + + static let standard = Savepoint(open: savepointStatement, release: releaseStatement, rollback: rollbackStatement) + + // SQL Server and Oracle have no RELEASE; a savepoint lasts until the transaction ends. + nonisolated static func of(_ family: TransactionEngineFamily) -> Savepoint? { + switch family { + case .postgres, .cockroach, .mysql, .sqlite: + return .standard + case .sqlServer: + return Savepoint( + open: "SAVE TRANSACTION tablepro_agent", release: nil, rollback: "ROLLBACK TRANSACTION tablepro_agent" + ) + case .oracle: + return Savepoint(open: savepointStatement, release: nil, rollback: rollbackStatement) + // Teradata has no savepoint. + case .redshift, .duckdb, .redis, .teradata, .other: + return nil + } + } + } + + // A call of several writes joining the user's open transaction is taken back whole when one fails, so no half + // of it waits for the user's Commit. + nonisolated internal static func savepoint( + family: TransactionEngineFamily, + protection: Protection, + sessionState: PluginSessionTransactionState, + asOneUnit: Bool + ) -> Savepoint? { + switch protection { + case .savepoint: + return Savepoint.of(family) + case .refuse: + return nil + case .none: + return asOneUnit && sessionState == .inTransaction ? Savepoint.of(family) : nil + } + } + + // A failed statement inside an open block aborts it on these engines, which takes the user's pending work with it. + nonisolated internal static func protection( + family: TransactionEngineFamily, + sessionState: PluginSessionTransactionState, + controlsTransaction: Bool + ) -> Protection { + guard !controlsTransaction, sessionState == .inTransaction, family.failedStatementAbortsTransaction else { + return .none + } + return family.takesBackAFailedStatement ? .savepoint : .refuse + } + + // Runs `body`, told whether this call opened the hold, and answers what the reply should say about the transaction. + nonisolated func running( + _ statements: [String], + writes: Bool, + asOneUnit: Bool = false, + on driver: DatabaseDriver, + _ body: (_ openedHold: Bool) async throws -> T + ) async throws -> (value: T, notice: String?) { + let controls = ManualCommit.controlsTransaction(statements, databaseType: databaseType, grammar: grammar) + if commitMode == .manual, controls { throw ManualCommitTransactionControlRefusal() } + let family = TransactionEngineFamily.of(databaseType) + let unseen = ManualCommit.mayEndTransactionUnseen(statements, databaseType: databaseType, grammar: grammar) + let readsState = asOneUnit || commitMode == .manual || controls || unseen + || family.failedStatementAbortsTransaction + let startState: PluginSessionTransactionState = readsState ? await driver.heldSessionTransactionState() : .unknown + // MySQL's START TRANSACTION commits the transaction already open; PostgreSQL's BEGIN warns and joins it. + if controls, startState.holdsUserTransaction, + ManualCommit.startsTransaction(statements, databaseType: databaseType, grammar: grammar) { + throw OpenTransactionRefusal() + } + // On an idle session the statement's own COMMIT would end the hold this call opens for its write. + if commitMode == .manual, unseen, writes || startState.holdsUserTransaction { + throw ManualCommitTransactionControlRefusal() + } + let protection = Self.protection(family: family, sessionState: startState, controlsTransaction: controls) + if protection == .refuse { throw AbortingTransactionRefusal() } + + let savepoint = Self.savepoint( + family: family, protection: protection, sessionState: startState, asOneUnit: asOneUnit + ) + + let opened = try await ManualCommit.openIfNeeded( + on: driver, statements: statements, databaseType: databaseType, grammar: grammar, commitMode: commitMode + ) + let value: T + do { + if let savepoint { + value = try await savepointed(savepoint, on: driver) { try await body(opened) } + } else { + value = try await body(opened) + } + } catch { + if opened { await ManualCommit.rollBackOpened(on: driver, connectionId: connectionId) } + throw error + } + let pendingMatters = commitMode == .manual && writes + // A Teradata ANSI session holds every change in a transaction it opens, which then refuses the next statement. + let mayHoldWrite = writes && !controls && startState == .idle + && family.failedStatementAbortsTransaction && !family.takesBackAFailedStatement + guard pendingMatters || startState.holdsUserTransaction || mayHoldWrite else { return (value, nil) } + let endState = await driver.heldSessionTransactionState() + let notice = Self.notice( + startState: startState, endState: endState, pendingMatters: pendingMatters, openedHold: opened + ) + return (value, notice) + } + + nonisolated internal static func notice( + startState: PluginSessionTransactionState, + endState: PluginSessionTransactionState, + pendingMatters: Bool, + openedHold: Bool + ) -> String? { + if startState.holdsUserTransaction || openedHold, endState == .idle { return ManualCommit.transactionEndedNotice } + if pendingMatters { return endState.holdsUserTransaction ? ManualCommit.pendingNotice : nil } + return startState == .idle && !openedHold && endState.holdsUserTransaction ? ManualCommit.statementOpenedNotice : nil + } + + nonisolated private func savepointed( + _ savepoint: Savepoint, + on driver: DatabaseDriver, + _ body: () async throws -> T + ) async throws -> T { + _ = try await driver.execute(query: savepoint.open) + let value: T + do { + value = try await body() + } catch { + await Self.rollBack(to: savepoint, on: driver, connectionId: connectionId) + throw error + } + guard let release = savepoint.release else { return value } + do { + _ = try await driver.execute(query: release) + } catch { + Self.logger.warning("Releasing the agent savepoint failed: \(error.publicLogShape, privacy: .public)") + } + return value + } + + @MainActor + private static func rollBack(to savepoint: Savepoint, on driver: DatabaseDriver, connectionId: UUID) async { + let token = DatabaseManager.shared.beginProtectedWrite(on: driver, for: connectionId) + defer { DatabaseManager.shared.endProtectedWrite(token, for: connectionId) } + do { + try await TaskCancellationShield.run { @Sendable in + _ = try await driver.execute(query: savepoint.rollback) + } + } catch { + logger.error("Rolling back to the agent savepoint failed: \(error.publicLogShape, privacy: .public)") + } + } +} diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift index fb726e0bd5..557cfda009 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift @@ -21,26 +21,36 @@ extension DatabaseAccessBridge { internal let executionTimeMs: Double + // What the answer spent, nil when it ran with none. + internal var budget: ExternalAnswerBudget? + internal var rowsReturned: Int { resultSets.reduce(0) { $0 + $1.rows.count } } - internal static func statement(_ result: QueryResult, executionTimeMs: Double) -> ScriptOutcome { - ScriptOutcome(resultSets: [result], primary: result, executionTimeMs: executionTimeMs) + internal static func statement( + _ result: QueryResult, + executionTimeMs: Double, + budget: ExternalAnswerBudget? = nil + ) -> ScriptOutcome { + ScriptOutcome(resultSets: [result], primary: result, executionTimeMs: executionTimeMs, budget: budget) } } - // An approved statement as a statement; an approved SQL Server script batch by batch at its GO lines. + // An approved statement as a statement; an approved SQL Server script batch by batch at its GO lines. One + // `budget` holds every result set; AppleScript and Shortcuts pass none. internal func runScript( _ operation: ApprovedOperation, maxRows: Int, timeoutSeconds: Int, - cancellation: (any StatementCancellationSignal)? + cancellation: (any StatementCancellationSignal)?, + budget: ExternalAnswerBudget? = nil ) async throws -> ScriptOutcome { let reach = DriverReach() do { return try await runApprovedScript( - operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, reach: reach + operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, budget: budget, + reach: reach ) } catch { throw await externalError(error, on: operation.scope.connectionId, reach: reach) @@ -52,6 +62,7 @@ extension DatabaseAccessBridge { maxRows: Int, timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, + budget: ExternalAnswerBudget?, reach: DriverReach ) async throws -> ScriptOutcome { guard operation.unit == .script else { @@ -59,9 +70,10 @@ extension DatabaseAccessBridge { operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + budget: budget ) - return .statement(outcome.result, executionTimeMs: outcome.executionTimeMs) + return .statement(outcome.result, executionTimeMs: outcome.executionTimeMs, budget: outcome.budget) } guard operation.statements.count == 1, let script = operation.statements.first, script.parameters.isEmpty else { throw Self.notApproved } @@ -80,14 +92,34 @@ extension DatabaseAccessBridge { rowCap: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - context: operation.context, + context: operation.context(readingAtMost: budget?.remainingBytes), + budget: budget, reach: reach ) - let policy = await maskingPolicy(for: scope.connectionId) + guard var spent = outcome.budget else { + let policy = await maskingPolicy(for: scope.connectionId) + return ScriptOutcome( + resultSets: outcome.resultSets.map { $0.maskedForExternalCaller(by: policy) }, + primary: outcome.primary.maskedForExternalCaller(by: policy), + executionTimeMs: outcome.executionTimeMs + ) + } + // Masked and held to the budget batch by batch. The payload writes the first of several sets twice, so the sets + // are counted again with it doubled, their values as already cut. + guard outcome.resultSets.count > 1 else { return outcome } + var recount = ExternalAnswerBudget(bytes: spent.limitBytes, longValues: .whole) + var kept: [QueryResult] = [] + for (index, set) in outcome.resultSets.enumerated() { + kept.append(recount.keep(set, weight: index == 0 ? 2 : 1)) + } + if recount.sizeLimited { + spent.markSizeLimited() + } return ScriptOutcome( - resultSets: outcome.resultSets.map { $0.maskedForExternalCaller(by: policy) }, - primary: outcome.primary.maskedForExternalCaller(by: policy), - executionTimeMs: outcome.executionTimeMs + resultSets: kept, + primary: outcome.primary.keepingLeadingRows(kept[0].rows, truncated: kept[0].isTruncated), + executionTimeMs: outcome.executionTimeMs, + budget: spent ) } @@ -102,6 +134,7 @@ extension DatabaseAccessBridge { timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, context: PluginStatementContext, + budget: ExternalAnswerBudget?, reach: DriverReach ) async throws -> ScriptOutcome { let classification = QueryClassifier.classify(text, databaseType: databaseType) @@ -114,6 +147,13 @@ extension DatabaseAccessBridge { let manager = await databaseManager let route = await MainActor.run { manager.executionRoute(for: scope) } + let commitMode = await MainActor.run { manager.commitMode(for: connectionId) } + let statementTexts = batches.flatMap { $0.statements.map(\.sql) } + let agentGuard = AgentTransactionGuard( + commitMode: commitMode, databaseType: databaseType, grammar: databaseType.lexicalGrammar, + connectionId: connectionId + ) + let writes = classification.tier != .safe let startLines = BatchErrorText.lines(of: batches.map(\.range.location), in: text) let startTime = CFAbsoluteTimeGetCurrent() let statementsRan = CatalogEvent.statementsRan( @@ -122,6 +162,13 @@ extension DatabaseAccessBridge { databaseType: databaseType ) + // With a budget each answer is masked as it arrives, so it can be held to the budget before the next batch runs. + var masking = MaskingPolicy.none + if budget != nil { + masking = await maskingPolicy(for: connectionId) + } + let answerMasking = masking + let progress = ScriptBatchProgress() let run: ScriptBatchRun do { @@ -134,9 +181,23 @@ extension DatabaseAccessBridge { onLateCompletion: { CatalogChangeService.post(statementsRan) } ) { driver in reach.mark() - return try await ScriptBatchRun.run( - batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context - ) + let ran = try await agentGuard.running(statementTexts, writes: writes, on: driver) { opened in + do { + let answered = try await ScriptBatchRun.run( + batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, + context: context, commitMode: commitMode, heldByApp: opened, + budget: budget, masking: answerMasking + ) + await manager.noteSessionStatements(Self.ranStatements(of: batches, progress), ranOn: driver, scope: scope) + return answered + } catch { + await manager.noteSessionStatements(Self.ranStatements(of: batches, progress), ranOn: driver, scope: scope) + throw error + } + } + var run = ran.value + run.transactionNotice = ran.notice + return run } } catch { if classification.tier != .safe { @@ -151,6 +212,11 @@ extension DatabaseAccessBridge { return run.outcome(executionTimeMs: (CFAbsoluteTimeGetCurrent() - startTime) * 1_000) } + // The batch a script stopped at ran too, and can have created a temporary table before its error. + private static func ranStatements(of batches: [ExecutableBatch], _ progress: ScriptBatchProgress) -> [String] { + batches.prefix(progress.completedBatchCount + 1).flatMap { $0.statements.map(\.sql) } + } + private static func postSucceededBatches( of batches: [ExecutableBatch], progress: ScriptBatchProgress, @@ -222,6 +288,10 @@ final class ScriptBatchProgress: Sendable { struct ScriptBatchRun: Sendable { let answers: [QueryBatchResult] let sessionState: PluginSessionTransactionState + // With a budget, every answer was masked and held to it as it came in; this is what they spent. + var budget: ExternalAnswerBudget? + // What the agent is told instead of "commit or roll it back": the work is pending, or the transaction ended. + var transactionNotice: String? /// The session is asked what it holds before and after, as the editor asks: the first answer decides how a /// failure reads, and the second whether the script left a transaction open. A batch that raised an error ends the @@ -232,15 +302,24 @@ struct ScriptBatchRun: Sendable { rowCap: Int, driver: DatabaseDriver, progress: ScriptBatchProgress, - context: PluginStatementContext? = nil + context: PluginStatementContext? = nil, + commitMode: ConnectionCommitMode = .automatic, + heldByApp: Bool = false, + budget: ExternalAnswerBudget? = nil, + masking: MaskingPolicy = .none ) async throws -> ScriptBatchRun { let startState = await driver.heldSessionTransactionState() progress.begin(in: startState) - let plan = BatchTransactionPlan.autocommit.joining(startState) + let plan: BatchTransactionPlan = heldByApp + ? .heldTransaction + : BatchTransactionPlan.autocommit.joining(startState, commitMode: commitMode) var answers: [QueryBatchResult] = [] + var held = budget for (batch, startLine) in zip(batches, startLines) { try Task.checkCancellation() - let answer = try await repeatedAnswer(to: batch, rowCap: rowCap, driver: driver, context: context) + let answer = try await repeatedAnswer( + to: batch, rowCap: rowCap, driver: driver, context: context, held: &held, masking: masking + ) answers.append(answer) guard answer.errors.isEmpty else { let endState = await driver.heldSessionTransactionState() @@ -260,29 +339,61 @@ struct ScriptBatchRun: Sendable { } let endState = await driver.heldSessionTransactionState() progress.end(in: endState) - return ScriptBatchRun(answers: answers, sessionState: endState) + return ScriptBatchRun(answers: answers, sessionState: endState, budget: held) } /// `GO 5` sends the batch five times and keeps every answer. A repetition that raised an error ends the - /// repeating, as it ends the run. + /// repeating, as it ends the run. With a budget, each answer is cut down to it before the next is read, so rows + /// past it are not held between batches. Once it is spent a batch still runs, since it may write and what was + /// approved runs to its end, but it reads one row per result set and keeps none. private static func repeatedAnswer( to batch: ExecutableBatch, rowCap: Int, driver: DatabaseDriver, - context: PluginStatementContext? + context: PluginStatementContext?, + held: inout ExternalAnswerBudget?, + masking: MaskingPolicy ) async throws -> QueryBatchResult { var combined = QueryBatchResult.empty for _ in 0.. QueryBatchResult { + var sets: [QueryResult] = [] + sets.reserveCapacity(answer.resultSets.count) + for (index, set) in answer.resultSets.enumerated() { + sets.append(index < room ? budget.keep(set.maskedForExternalCaller(by: masking)) : set) + } + return QueryBatchResult( + resultSets: sets, + rowsAffected: answer.rowsAffected, + errors: answer.errors, + discardedResultSetCount: answer.discardedResultSetCount, + executionTime: answer.executionTime + ) + } + func outcome(executionTimeMs: Double) -> DatabaseAccessBridge.ScriptOutcome { let resultSets = answers.flatMap(\.resultSets) let first = resultSets.first @@ -298,16 +409,18 @@ struct ScriptBatchRun: Sendable { primary.columnOrigins = first?.columnOrigins let notes = [ first?.statusMessage, + transactionNotice, BatchRunNotice.text( discardedResultSetCount: answers.reduce(0) { $0 + $1.discardedResultSetCount }, - sessionState: sessionState + sessionState: transactionNotice == nil ? sessionState : .idle ) ].compactMap { $0 } primary.statusMessage = notes.isEmpty ? nil : notes.joined(separator: " ") return DatabaseAccessBridge.ScriptOutcome( resultSets: resultSets, primary: primary, - executionTimeMs: executionTimeMs + executionTimeMs: executionTimeMs, + budget: budget ) } } diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index d3577f51c1..6e390bec4c 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -290,6 +290,10 @@ internal actor DatabaseAccessBridge { ) case .batches: return PlannedText(statement: OperationStatement(sql: query), unit: .script) + case .statements(let statements) where statements.contains(where: { $0.copyData != nil }): + throw DatabaseAccessError.invalidArgument( + String(localized: "COPY FROM STDIN takes its rows from the lines after it, which only the query editor and SQL import send. Run it there.") + ) case .statements: throw DatabaseAccessError.invalidArgument( String(localized: "Update the database driver in Settings > Extensions to run several statements in one call.") @@ -340,21 +344,25 @@ internal actor DatabaseAccessBridge { internal struct StatementOutcome: Sendable { internal let result: QueryResult internal let executionTimeMs: Double + // What the answer spent, nil when it ran with none. + internal let budget: ExternalAnswerBudget? } - // Runs the one approved statement as it was approved, its rows masked as `masking` says. + // Runs the one approved statement as it was approved, its rows masked as `masking` says, then held to `budget`. + // AppleScript and Shortcuts pass none. internal func runStatement( _ operation: ApprovedOperation, maxRows: Int, timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, - masking: ExternalRowMasking = .byOrigin + masking: ExternalRowMasking = .byOrigin, + budget: ExternalAnswerBudget? = nil ) async throws -> StatementOutcome { let reach = DriverReach() do { return try await runApprovedStatement( operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, masking: masking, - reach: reach + budget: budget, reach: reach ) } catch { throw await externalError(error, on: operation.scope.connectionId, reach: reach) @@ -367,6 +375,7 @@ internal actor DatabaseAccessBridge { timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, masking: ExternalRowMasking, + budget: ExternalAnswerBudget?, reach: DriverReach ) async throws -> StatementOutcome { guard operation.unit == .statement, operation.statements.count == 1, let statement = operation.statements.first @@ -384,7 +393,7 @@ internal actor DatabaseAccessBridge { let ceiling = await MainActor.run { PluginManager.shared.paginationCapability(for: databaseType).maximumRows } rowCap = ceiling.map { min(maxRows, $0) } ?? maxRows } - let context = operation.context + let context = operation.context(readingAtMost: budget?.remainingBytes) let fetchCap = rowCap let connectionId = scope.connectionId // One owner per statement, so a cancel or a timeout reaches this statement's lease only. @@ -397,16 +406,22 @@ internal actor DatabaseAccessBridge { let manager = await databaseManager let route = await MainActor.run { manager.executionRoute(for: scope) } + let commitMode = await MainActor.run { manager.commitMode(for: connectionId) } + let agentGuard = AgentTransactionGuard( + commitMode: commitMode, databaseType: databaseType, grammar: grammar, connectionId: connectionId + ) + let writes = classification.tier != .safe let startTime = CFAbsoluteTimeGetCurrent() let statementRan = CatalogEvent.statementsRan( connectionId: connectionId, statements: [sql], databaseType: databaseType ) // A write that timed out or failed may still have committed, so the catalog is refreshed rather than trusted. - let result: QueryResult + var result: QueryResult let tableEdits: SucceededStatements? + let transactionNotice: String? do { - (result, tableEdits) = try await runRacingTimeout( + (result, tableEdits, transactionNotice) = try await runRacingTimeout( scope: scope, route: route, policy: policy, @@ -415,16 +430,20 @@ internal actor DatabaseAccessBridge { onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in reach.mark() - let answer = try await driver.executeUserQuery( - query: sql, - rowCap: fetchCap, - parameters: statement.parameters.isEmpty ? nil : statement.boundValues, - context: context - ) + let ran = try await agentGuard.running([sql], writes: writes, on: driver) { _ in + // Before it runs, so a statement that fails or a connection lost under it is still noted. + await manager.noteSessionStatements([sql], ranOn: driver, scope: scope) + return try await driver.executeUserQuery( + query: sql, + rowCap: fetchCap, + parameters: statement.parameters.isEmpty ? nil : statement.boundValues, + context: context + ) + } let edits = await SucceededStatements.single( sql, scope: scope, databaseType: databaseType, grammar: grammar, ranOn: driver ) - return (answer, edits) + return (ran.value, edits, ran.notice) } } catch { if classification.tier != .safe { @@ -437,10 +456,23 @@ internal actor DatabaseAccessBridge { if let tableEdits { CatalogChangeService.post(.statementsSucceeded(tableEdits)) } + if let transactionNotice { + result.statusMessage = [result.statusMessage, transactionNotice].compactMap { $0 }.joined(separator: " ") + } let executionTimeMs = (CFAbsoluteTimeGetCurrent() - startTime) * 1_000 let rules = await maskingPolicy(for: connectionId) let applied = await MainActor.run { masking.resolved(in: scope, databaseType: databaseType) } - return StatementOutcome(result: applied.apply(rules, to: result), executionTimeMs: executionTimeMs) + // Masked first, so a partial mask reads the whole value before any cut. + let masked = applied.apply(rules, to: result) + guard var spent = budget else { + return StatementOutcome(result: masked, executionTimeMs: executionTimeMs, budget: nil) + } + let kept = spent.keep(masked) + // Short of the row cap and truncated anyway: the driver stopped at the byte limit. + if context.resultByteLimit != nil, let fetchCap, masked.isTruncated, masked.rows.count < fetchCap { + spent.markSizeLimited() + } + return StatementOutcome(result: kept, executionTimeMs: executionTimeMs, budget: spent) } // A plan the driver builds for the approved statement, masked as `masking` says. @@ -457,7 +489,9 @@ internal actor DatabaseAccessBridge { operation, step: step, timeoutSeconds: timeoutSeconds, cancellation: cancellation, reach: reach ) let rules = await maskingPolicy(for: operation.scope.connectionId) - return StatementOutcome(result: masking.apply(rules, to: outcome.result), executionTimeMs: outcome.executionTimeMs) + return StatementOutcome( + result: masking.apply(rules, to: outcome.result), executionTimeMs: outcome.executionTimeMs, budget: nil + ) } catch { throw await externalError(error, on: operation.scope.connectionId, reach: reach) } @@ -522,11 +556,14 @@ internal actor DatabaseAccessBridge { if writes { CatalogChangeService.post(statementRan) } - return StatementOutcome(result: result, executionTimeMs: (CFAbsoluteTimeGetCurrent() - startTime) * 1_000) + return StatementOutcome( + result: result, executionTimeMs: (CFAbsoluteTimeGetCurrent() - startTime) * 1_000, budget: nil + ) } - // One approved INSERT per row, on one write lease, each with the values it was approved with. - internal func runInserts(_ operation: ApprovedOperation) async throws -> Int { + // One approved INSERT per row, on one write lease, each with the values it was approved with. The notice says the + // rows are pending in a transaction. + internal func runInserts(_ operation: ApprovedOperation) async throws -> (affected: Int, notice: String?) { let reach = DriverReach() do { return try await runApprovedInserts(operation, reach: reach) @@ -535,7 +572,10 @@ internal actor DatabaseAccessBridge { } } - private func runApprovedInserts(_ operation: ApprovedOperation, reach: DriverReach) async throws -> Int { + private func runApprovedInserts( + _ operation: ApprovedOperation, + reach: DriverReach + ) async throws -> (affected: Int, notice: String?) { guard operation.kind == .writeQuery, operation.unit == .statement, !operation.statements.isEmpty else { throw Self.notApproved } let scope = operation.scope @@ -544,24 +584,38 @@ internal actor DatabaseAccessBridge { let manager = await databaseManager // No cancel handler: the inserts run under a `.protectedWrite` lease, which nothing may abort. let route = await MainActor.run { manager.executionRoute(for: scope) } + let agentGuard = await MainActor.run { + AgentTransactionGuard( + commitMode: manager.commitMode(for: scope.connectionId), + databaseType: databaseType, + grammar: SQLLexicalResolver.executionGrammar(for: databaseType, connectionId: scope.connectionId), + connectionId: scope.connectionId + ) + } let statements = operation.statements let context = operation.context return try await manager.withScopedDriver( scope: scope, route: route, cancellation: .protectedWrite - ) { driver -> Int in + ) { driver -> (affected: Int, notice: String?) in reach.mark() - var affected = 0 - for statement in statements { - let result = try await driver.executeParameterized( - query: statement.sql, - parameters: statement.boundValues, - context: context - ) - affected += max(result.rowsAffected, 0) + // The hold and savepoint execute_query gets, so a Manual Commit insert waits for Commit too. + let ran = try await agentGuard.running( + statements.map(\.sql), writes: true, asOneUnit: statements.count > 1, on: driver + ) { _ in + var affected = 0 + for statement in statements { + let result = try await driver.executeParameterized( + query: statement.sql, + parameters: statement.boundValues, + context: context + ) + affected += max(result.rowsAffected, 0) + } + return affected } - return affected + return (ran.value, ran.notice) } } @@ -630,6 +684,10 @@ internal actor DatabaseAccessBridge { guard supported else { throw DatabaseAccessError.dataSourceError(String(localized: "This engine does not support transactions.")) } + // The user commits a Manual Commit connection's work; an agent's begin, commit or rollback would end or bury it. + guard await MainActor.run(body: { manager.commitMode(for: scope.connectionId) }) == .automatic else { + throw DatabaseAccessError.dataSourceError(ManualCommit.transactionControlRefusalMessage) + } try await manager.withScopedDriver( scope: scope, route: .sessionDriver, @@ -637,7 +695,12 @@ internal actor DatabaseAccessBridge { ) { driver in reach.mark() switch action { - case "begin": try await driver.beginTransaction(mode: .readWrite) + case "begin": + // MySQL's START TRANSACTION commits a transaction already open; PostgreSQL's BEGIN warns and joins it. + guard await driver.heldSessionTransactionState() == .idle else { + throw DatabaseAccessError.dataSourceError(ManualCommit.openTransactionRefusalMessage) + } + try await driver.beginTransaction(mode: .readWrite) case "commit": try await driver.commitTransaction() case "rollback": try await driver.rollbackTransaction() default: @@ -729,7 +792,7 @@ internal actor DatabaseAccessBridge { ) { driver in reach.mark() try await before(driver, ownsConnection) - return try await DataWriteExecutor.run(plan, on: driver, context: context) + return try await DataWriteExecutor.run(plan, commitMode: .automatic, on: driver, context: context) } } @@ -820,6 +883,7 @@ internal actor DatabaseAccessBridge { _ work: @Sendable @escaping (_ enterDriver: @escaping @Sendable () throws -> Void) async throws -> Output ) async throws -> Output { let phase = OSAllocatedUnfairLock(initialState: ProtectedWritePhase.queued) + let callerLeft = OSAllocatedUnfairLock(initialState: false) let abandonIfQueued: @Sendable () -> Bool = { phase.withLock { current -> Bool in guard current == .queued else { return current == .abandoned } @@ -853,8 +917,9 @@ internal actor DatabaseAccessBridge { continuation.finish(throwing: error) } deadline.cancel() - let ranInDriver = phase.withLock { $0 == .inDriver } - if await deadline.value, ranInDriver { + guard phase.withLock({ $0 == .inDriver }) else { return } + let deadlinePassed = await deadline.value + if deadlinePassed || callerLeft.withLock({ $0 }) { onLateCompletion?() } } @@ -876,16 +941,31 @@ internal actor DatabaseAccessBridge { context: ["outcome": "unknown"] ) } + if phase.withLock({ $0 == .inDriver }) { + throw stoppedWhileRunning + } try Task.checkCancellation() throw DatabaseAccessError.dataSourceError("No result from query execution") } onCancel: { if abandonIfQueued() { workTask.cancel() + } else { + callerLeft.withLock { $0 = true } } deadline.cancel() } } + // Not "timed out": the caller stopped it, and a retry would apply it twice. + static var stoppedWhileRunning: DatabaseAccessError { + .stoppedWhileRunning(String( + localized: """ + The statement was still running in the database when it was stopped, and it runs to the end. \ + It may have been applied; check before running it again. + """ + )) + } + /// The text an external client's statement reaches the driver as. /// /// Trailing separators come off and a terminator that belongs to the statement stays, as the editor decides it: @@ -968,7 +1048,8 @@ internal enum ServerErrorText { // The app's own refusals, timeouts and cancels name no value; whatever a driver or a server wrote might. static func mayRepeatAValue(_ error: Error) -> Bool { switch error { - case is CancellationError, is DatabaseAccessError, is ExternalStatementGateError, is ShortcutsActionError: + case is CancellationError, is DatabaseAccessError, is ExternalStatementGateError, is ShortcutsActionError, + is any ManualCommitRefusing: return false case let failure as DatabaseError: if case .queryFailed = failure { return true } diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessError.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessError.swift index b4c3fedfcd..f072fc1f6e 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessError.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessError.swift @@ -9,6 +9,8 @@ public enum DatabaseAccessError: Error, Sendable { case expired(String) case userCancelled case dataSourceError(String) + // The caller stopped waiting while a write it sent was in the driver, so the write may have been applied. + case stoppedWhileRunning(String) public var message: String { switch self { @@ -28,6 +30,8 @@ public enum DatabaseAccessError: Error, Sendable { "User cancelled" case .dataSourceError(let detail): "Data source error: \(detail)" + case .stoppedWhileRunning(let detail): + detail } } diff --git a/apple/TablePro/Core/Database/Access/ExternalAnswerBudget.swift b/apple/TablePro/Core/Database/Access/ExternalAnswerBudget.swift new file mode 100644 index 0000000000..e950945d17 --- /dev/null +++ b/apple/TablePro/Core/Database/Access/ExternalAnswerBudget.swift @@ -0,0 +1,210 @@ +// +// ExternalAnswerBudget.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// The bytes one answer to an MCP client or the assistant may hold, measured after masking. The limits, the +// measure and the mark are the core's (`core/crates/mcp/src/collected_results.rs`), so both apps answer alike. +internal struct ExternalAnswerBudget: Sendable, Equatable { + internal enum LongValues: Sendable, Equatable { + case cut + // An export is data to keep exactly, so a row that does not fit is left out instead. + case whole + } + + internal static let answerBytes = 4 << 20 + internal static let valueBytes = 64 << 10 + internal static let assistantAnswerBytes = 256 << 10 + + // A cell's quotes, colon and comma, and a row's brackets and comma. + private static let cellBytes = 6 + private static let rowBytes = 3 + private static let nullBytes = 4 + + internal let limitBytes: Int + internal let longValues: LongValues + internal private(set) var spentBytes = 0 + internal private(set) var valuesCut = false + internal private(set) var sizeLimited = false + + internal init(bytes: Int, longValues: LongValues = .cut) { + limitBytes = max(bytes, 0) + self.longValues = longValues + } + + internal static var mcpAnswer: ExternalAnswerBudget { + ExternalAnswerBudget(bytes: answerBytes) + } + + internal static var assistantAnswer: ExternalAnswerBudget { + ExternalAnswerBudget(bytes: assistantAnswerBytes) + } + + internal static var export: ExternalAnswerBudget { + ExternalAnswerBudget(bytes: answerBytes, longValues: .whole) + } + + internal var remainingBytes: Int { + max(limitBytes - spentBytes, 0) + } + + // The leading rows of `result` that fit, each long value cut when values are cut. Once one row does not fit, + // no later row of this or any later result is kept. `weight` counts a result the answer writes more than once. + internal mutating func keep(_ result: QueryResult, weight: Int = 1) -> QueryResult { + guard !result.rows.isEmpty else { return result } + let nameBytes = result.columns.map { $0.utf8.count + Self.cellBytes } + var kept: [[PluginCellValue]] = [] + kept.reserveCapacity(result.rows.count) + var changed = false + for row in result.rows { + guard !sizeLimited else { + changed = true + break + } + var cells = row + var cut = false + var size = Self.rowBytes + for index in cells.indices { + if longValues == .cut, let shortened = Self.cut(cells[index]) { + cells[index] = shortened + cut = true + } + size += (index < nameBytes.count ? nameBytes[index] : Self.cellBytes) + Self.encodedBytes(cells[index]) + } + let cost = size.multipliedReportingOverflow(by: max(weight, 1)) + guard !cost.overflow, cost.partialValue <= remainingBytes else { + sizeLimited = true + changed = true + break + } + spentBytes += cost.partialValue + valuesCut = valuesCut || cut + changed = changed || cut + kept.append(cells) + } + guard changed else { return result } + return result.keepingLeadingRows(kept, truncated: result.isTruncated || kept.count < result.rows.count) + } + + internal mutating func keep(text: String) -> String { + let size = text.utf8.count + guard size > remainingBytes else { + spentBytes += size + return text + } + sizeLimited = true + let kept = Self.cut(text, toBytes: remainingBytes) + spentBytes = limitBytes + return kept + } + + // A part its caller measured, kept only whole. Leaving it out does not stop the parts after it, which may fit. + internal mutating func keepWhole(bytes: Int) -> Bool { + guard !sizeLimited, bytes <= remainingBytes else { return false } + spentBytes += bytes + return true + } + + // A source that stopped reading at its own byte limit left rows out too. + internal mutating func markSizeLimited() { + sizeLimited = true + } + + internal var notes: [String] { + var notes: [String] = [] + if valuesCut { + notes.append(String(localized: """ + Some values were longer than 64 KB and were cut there; each one cut ends in a mark with its full \ + length. Read such a value in parts with a narrower query. + """)) + } + if sizeLimited { + let limit = ByteCountFormatter.string(fromByteCount: Int64(limitBytes), countStyle: .memory) + let text = longValues == .whole + ? String(localized: """ + The rows reached TablePro's %@ limit for one answer, so an export marked truncated holds only the \ + rows that fit, and the exports after it may hold none. Export fewer tables, columns or rows per call. + """) + : String(localized: """ + The rows reached TablePro's %@ limit for one answer, so a result marked truncated holds only the \ + rows that fit. Ask for fewer columns or rows. + """) + notes.append(String(format: text, limit)) + } + return notes + } + + internal static func cut(_ text: String) -> String { + cut(text, toBytes: valueBytes) + } + + private static func cut(_ text: String, toBytes limit: Int) -> String { + let utf8 = text.utf8 + let total = utf8.count + guard total > limit else { return text } + var end = utf8.index(utf8.startIndex, offsetBy: max(limit, 0)) + while end > utf8.startIndex, utf8[end] & 0xC0 == 0x80 { + end = utf8.index(before: end) + } + return String(String.UnicodeScalarView(text.unicodeScalars[.. PluginCellValue? { + switch cell { + case .null: + return nil + case .text(let text): + guard text.utf8.count > valueBytes else { return nil } + return .text(cut(text)) + case .bytes(let data): + guard base64Bytes(data.count) > valueBytes else { return nil } + return .text(data.prefix(valueBytes / 4 * 3).base64EncodedString() + mark(data.count)) + } + } + + private static func mark(_ total: Int) -> String { + " [TablePro cut this value: \(total) bytes in all]" + } + + // Measured without rendering, since rendering is what the limit is there to avoid. + private static func encodedBytes(_ cell: PluginCellValue) -> Int { + switch cell { + case .null: return nullBytes + case .text(let text): return text.utf8.count + case .bytes(let data): return base64Bytes(data.count) + } + } + + private static func base64Bytes(_ count: Int) -> Int { + 4 * ((count + 2) / 3) + } +} + +extension QueryResult { + // `rows` is a prefix of this result's rows, so what the driver keeps per row keeps its place. + func keepingLeadingRows(_ rows: [[PluginCellValue]], truncated: Bool) -> QueryResult { + var kept = QueryResult( + columns: columns, + columnTypes: columnTypes, + rows: rows, + rowsAffected: rowsAffected, + executionTime: executionTime, + error: error + ) + kept.timing = timing + kept.isTruncated = truncated + kept.statusMessage = statusMessage + kept.columnMeta = columnMeta + kept.columnOrigins = columnOrigins + kept.rowLocators = rowLocators.map { Array($0.prefix(rows.count)) } + kept.serverOutput = serverOutput + kept.exportDocuments = exportDocuments.map { Array($0.prefix(rows.count)) } + kept.absentCells = absentCells.filter { $0.key < rows.count } + kept.unloadedCells = unloadedCells.filter { $0.key < rows.count } + return kept + } +} diff --git a/apple/TablePro/Core/Database/AppDefinitionCommit.swift b/apple/TablePro/Core/Database/AppDefinitionCommit.swift new file mode 100644 index 0000000000..c31da06e56 --- /dev/null +++ b/apple/TablePro/Core/Database/AppDefinitionCommit.swift @@ -0,0 +1,101 @@ +// +// AppDefinitionCommit.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// Refused before anything ran, so an external caller reads it as it is. +internal struct DefinitionInOpenTransactionRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "A transaction is open on this connection, and this change would have to end it. Commit or roll back the transaction, then try again." + ) + } +} + +// One statement the app runs with no BEGIN of its own; a Teradata ANSI session keeps it pending until a COMMIT. +internal enum AppDefinitionCommit { + internal static func run( + family: TransactionEngineFamily, + refusesOpenTransaction: Bool, + readState: () async -> PluginSessionTransactionState, + commit: @escaping @Sendable () async throws -> Void, + rollback: @escaping @Sendable () async throws -> Void, + body: () async throws -> T + ) async throws -> T { + guard family.unwrappedWorkStaysOpen else { return try await body() } + let before = await readState() + if refusesOpenTransaction, before.holdsUserTransaction { + throw DefinitionInOpenTransactionRefusal() + } + do { + let value = try await body() + try await commitIfOpen(family: family, readState: readState, commit: commit) + return value + } catch { + // Teradata keeps a failed request's locks until the transaction ends. + if before == .idle { + _ = try? await TaskCancellationShield.run(rollback) + } + throw error + } + } + + internal static func commitIfOpen( + family: TransactionEngineFamily, + readState: () async -> PluginSessionTransactionState, + commit: @escaping @Sendable () async throws -> Void + ) async throws { + guard family.unwrappedWorkStaysOpen, await readState() == .inTransaction else { return } + // The work it ends has already run, so a cancelled task must not refuse it. + try await TaskCancellationShield.run(commit) + } +} + +extension AppDefinitionCommit { + internal static func run( + on driver: DatabaseDriver, + refusesOpenTransaction: Bool, + body: () async throws -> T + ) async throws -> T { + try await run( + family: TransactionEngineFamily.of(driver.connection.type), + refusesOpenTransaction: refusesOpenTransaction, + readState: { await driver.heldSessionTransactionState() }, + commit: { try await driver.commitTransaction() }, + rollback: { try await driver.rollbackTransaction() }, + body: body + ) + } + + internal static func run( + on driver: any PluginDatabaseDriver, + family: TransactionEngineFamily, + refusesOpenTransaction: Bool, + body: () async throws -> T + ) async throws -> T { + try await run( + family: family, + refusesOpenTransaction: refusesOpenTransaction, + readState: { await Self.heldState(of: driver) }, + commit: { try await driver.commitTransaction() }, + rollback: { try await driver.rollbackTransaction() }, + body: body + ) + } + + internal static func commitIfOpen(on driver: any PluginDatabaseDriver, family: TransactionEngineFamily) async throws { + try await commitIfOpen( + family: family, + readState: { await Self.heldState(of: driver) }, + commit: { try await driver.commitTransaction() } + ) + } + + private static func heldState(of driver: any PluginDatabaseDriver) async -> PluginSessionTransactionState { + guard driver.supportsTransactions else { return .unknown } + return await driver.sessionTransactionState() + } +} diff --git a/apple/TablePro/Core/Database/CLIToolVersionProbe.swift b/apple/TablePro/Core/Database/CLIToolVersionProbe.swift index 56551ce1f8..38ce4a21f3 100644 --- a/apple/TablePro/Core/Database/CLIToolVersionProbe.swift +++ b/apple/TablePro/Core/Database/CLIToolVersionProbe.swift @@ -6,7 +6,6 @@ import Foundation import os import TableProProcess -import TableProUtilities enum CLIToolVersionProbe { private static let logger = Logger(subsystem: "com.TablePro", category: "CLIToolVersionProbe") diff --git a/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift b/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift index 74e46c8b23..9cf22a2c33 100644 --- a/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift +++ b/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift @@ -29,12 +29,14 @@ enum ConnectionCredentialResolver { } // A nil username keeps the connection's own; Redshift IAM and a Vault database role hand back another. - // A sign-in that carries a lease belongs to whoever uses it, which releases it when done. + // A sign-in that carries a lease belongs to whoever uses it, which releases it when done. `leaseOwner` replaces + // `credentialOwner(for:)`, for a Vault user that must not be shared. static func resolveSignIn( for connection: DatabaseConnection, fields: [String: String], override: String? = nil, - deadline: ConnectionDeadline? = nil + deadline: ConnectionDeadline? = nil, + leaseOwner: UUID? = nil ) async throws -> ResolvedSignIn { if let minter = iamPasswordMinter(for: connection, fields: fields) { return try await minter.mint(deadline: deadline) @@ -43,7 +45,11 @@ enum ConnectionCredentialResolver { if KerberosAuthFields.isSelected(in: connection.additionalFields) { return ResolvedSignIn(username: nil, password: "") } - return try await resolveStoredSignIn(for: connection, override: override) + return try await resolveStoredSignIn( + for: connection, + override: override, + leaseOwner: leaseOwner ?? credentialOwner(for: connection) + ) } // A lease is a database user Vault creates, so a caller that only wants to know whether a password exists asks this. @@ -60,17 +66,21 @@ enum ConnectionCredentialResolver { return connection.passwordSource } - private static func resolveStoredSignIn(for connection: DatabaseConnection, override: String?) async throws -> ResolvedSignIn { + private static func resolveStoredSignIn( + for connection: DatabaseConnection, + override: String?, + leaseOwner: UUID + ) async throws -> ResolvedSignIn { if let override { return ResolvedSignIn(username: nil, password: override) } if case .profile(let profileId) = connection.credentialMode, let profile = CredentialProfileStorage.shared.profile(for: profileId) { - return try await resolveProfileSignIn(profile, for: connection) + return try await resolveProfileSignIn(profile, for: connection, leaseOwner: leaseOwner) } if let passwordSource = connection.passwordSource { guard ConnectionStorage.shared.storeIsTrusted else { throw PasswordSourceResolver.ResolutionError.storeNotTrusted } - return try await resolveSource(passwordSource, for: connection) + return try await resolveSource(passwordSource, for: connection, leaseOwner: leaseOwner) } if connection.usePgpass, readsPgpass(connection) { return ResolvedSignIn(username: nil, password: pgpassPassword(for: connection, username: resolveUsername(for: connection))) @@ -78,9 +88,18 @@ enum ConnectionCredentialResolver { return ResolvedSignIn(username: nil, password: ConnectionStorage.shared.loadPassword(for: connection.id) ?? "") } - private static func resolveSource(_ source: PasswordSource, for connection: DatabaseConnection) async throws -> ResolvedSignIn { + private static func resolveSource( + _ source: PasswordSource, + for connection: DatabaseConnection, + leaseOwner: UUID + ) async throws -> ResolvedSignIn { if case .vaultDatabaseRole(let role) = source { - return try await VaultDatabaseCredentials.shared.signIn(role, interactive: false) + return try await VaultDatabaseCredentials.shared.signIn( + role, + owner: leaseOwner, + for: connection.id, + interactive: false + ) } let password = try await PasswordSourceResolver.resolve(source, for: ConnectionScriptContext(connection: connection)) return ResolvedSignIn(username: nil, password: password) @@ -99,10 +118,8 @@ enum ConnectionCredentialResolver { return profile.username } - /// Which record a prompted password is remembered against. A profile answers for every - /// connection linked to it, so one profile in "ask every time" mode asks once per launch - /// rather than once per connection. - static func promptCacheKey(for connection: DatabaseConnection) -> UUID { + // A profile answers for every connection linked to it: one prompt per launch, one Vault user while its lease lives. + static func credentialOwner(for connection: DatabaseConnection) -> UUID { connection.credentialMode.profileId ?? connection.id } @@ -125,7 +142,8 @@ enum ConnectionCredentialResolver { private static func resolveProfileSignIn( _ profile: CredentialProfile, - for connection: DatabaseConnection + for connection: DatabaseConnection, + leaseOwner: UUID ) async throws -> ResolvedSignIn { switch profile.passwordMode { case .stored: @@ -141,7 +159,7 @@ enum ConnectionCredentialResolver { guard CredentialProfileStorage.shared.storeIsTrusted else { throw PasswordSourceResolver.ResolutionError.profileStoreNotTrusted } - return try await resolveSource(source, for: connection) + return try await resolveSource(source, for: connection, leaseOwner: leaseOwner) } } @@ -171,6 +189,12 @@ enum ConnectionCredentialResolver { ) } + // The connections `iamPasswordMinter` gives an RDS token, which names the host it is signed for. + nonisolated static func signsRDSToken(for connection: DatabaseConnection) -> Bool { + guard case .aws = connection.iamSignIn else { return false } + return connection.type != .redshift && connection.type != .redis && !connection.resolvesAWSIAMInDriver + } + // The AWS token is signed for the resolved username: `connection.username` can lag behind its profile. static func iamPasswordMinter( for connection: DatabaseConnection, @@ -208,9 +232,9 @@ enum ConnectionCredentialResolver { struct ResolvedSignIn: Equatable, Sendable { let username: String? let password: String - let lease: VaultLease? + let lease: VaultLeaseHold? - init(username: String?, password: String, lease: VaultLease? = nil) { + init(username: String?, password: String, lease: VaultLeaseHold? = nil) { self.username = username self.password = password self.lease = lease diff --git a/apple/TablePro/Core/Database/ConnectionTunnelError.swift b/apple/TablePro/Core/Database/ConnectionTunnelError.swift index 234178b0e0..6f8912c462 100644 --- a/apple/TablePro/Core/Database/ConnectionTunnelError.swift +++ b/apple/TablePro/Core/Database/ConnectionTunnelError.swift @@ -5,6 +5,7 @@ enum ConnectionTunnelError: Error, LocalizedError, Equatable { case remoteFileUnsupported(String) case remoteFilePathMissing case srvLookupBehindTunnel + case sshProfileMoved static func refusal(for connection: DatabaseConnection) -> ConnectionTunnelError? { guard connection.enabledTunnelKinds.contains(where: { $0 != .remoteFile && $0 != .remoteDatabaseSession }) @@ -36,6 +37,8 @@ enum ConnectionTunnelError: Error, LocalizedError, Equatable { An SRV record names servers the tunnel cannot reach, so the connection would go around it. Turn the tunnel \ off, or list the servers under Hosts and turn off Use SRV Record. """) + case .sshProfileMoved: + return String(localized: "The SSH profile this connection uses now points to another server. Click Reconnect in its window to use it.") } } } diff --git a/apple/TablePro/Core/Database/DatabaseDriver.swift b/apple/TablePro/Core/Database/DatabaseDriver.swift index f2dae7e00a..8a7ee251c0 100644 --- a/apple/TablePro/Core/Database/DatabaseDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseDriver.swift @@ -116,6 +116,10 @@ protocol DatabaseDriver: AnyObject, Sendable { // Whether this driver runs a statement marked read-only in the engine's read-only mode. var runsReadOnlyStatements: Bool { get } + // Whether this driver runs a statement whose context `enforcesReadOnly` in the engine's read-only mode, or refuses + // it, from every call that takes a context. + var enforcesReadOnlyMode: Bool { get } + /// Run a read that stops once `rowCap` rows are known to be exceeded, rather than fetching the /// whole result and discarding the tail. Returns nil when the driver cannot bound its own fetch. /// @@ -123,6 +127,10 @@ protocol DatabaseDriver: AnyObject, Sendable { /// rest of the fetch, which for some drivers cancels the statement on the server. func executeBoundedQuery(query: String, rowCap: Int) async throws -> QueryResult? + // Nil also when the driver cannot honor `context` on a bounded read: the caller then runs `executeUserQuery` with the + // same context and cap. + func executeBoundedQuery(query: String, rowCap: Int, context: PluginStatementContext) async throws -> QueryResult? + /// Runs a statement the app wrote to read, telling the driver it may only read. A driver that /// evaluates statements in a scripting engine with state refuses any write the script reaches, /// whatever an earlier statement redefined. @@ -136,6 +144,17 @@ protocol DatabaseDriver: AnyObject, Sendable { /// text statement by statement instead. A server error inside the batch is part of the answer, not a throw. func executeBatch(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryBatchResult? + /// Whether ``copyFromStandardInput(statement:rows:context:)`` sends a `COPY ... FROM STDIN` its rows. + var supportsCopyFromStandardInput: Bool { get } + + /// Runs `statement`, a `COPY ... FROM STDIN`, and sends it `rows` a piece at a time, holding the session until the + /// copy ends. Throws when the driver cannot. + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult + func explainPlan(statement: String, variantId: String) async throws -> QueryResult // MARK: - Schema Operations @@ -174,9 +193,12 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Default implementation falls back to per-table fetchColumns. func fetchAllColumns() async throws -> [String: [ColumnInfo]] - // False when `fetchAllColumns` degrades to one round trip per table. + // One query that also reports every column `fetchColumns` does, so Compare can trust it. var providesBulkColumnFetch: Bool { get } + // Only whether `fetchAllColumns` reads every table in one request, whatever its completeness. + var fetchesAllColumnsInOneRequest: Bool { get } + /// Dotted field paths a document store exposes for a collection, for query authoring. /// Default implementation returns nothing, which is correct for every SQL driver. func sampleFieldPaths(table: String, limit: Int) async throws -> [PluginFieldPath] @@ -253,8 +275,6 @@ protocol DatabaseDriver: AnyObject, Sendable { /// per table, which is too expensive to run ahead of the user. var providesBulkForeignKeyFetch: Bool { get } - /// Fetch foreign keys for a specific set of tables. - /// Default implementation calls fetchAllForeignKeys and filters, or falls back to per-table. func fetchForeignKeys(forTables tableNames: [String]) async throws -> [String: [ForeignKeyInfo]] /// Fetch an approximate row count using fast database-specific metadata. @@ -350,6 +370,11 @@ protocol DatabaseDriver: AnyObject, Sendable { /// Fetch the source of one trigger. func fetchTriggerDDL(_ trigger: TriggerInfo) async throws -> String + // Mirrors `PluginDatabaseDriver.serverObjectKinds`: asked of the live driver, never of the type. + var serverObjectKinds: [PluginServerObjectKind] { get } + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [ServerObjectInfo] + func fetchServerObject(_ object: ServerObjectInfo) async throws -> ServerObjectInfo + /// Fetch metadata for a specific database (table count, size, etc.) func fetchDatabaseMetadata(_ database: String) async throws -> DatabaseMetadata @@ -552,6 +577,24 @@ extension DatabaseDriver { func executeBoundedQuery(query: String, rowCap: Int) async throws -> QueryResult? { nil } + // A driver without its own overload ignores every field, so a read-only mode, a statement policy or a catalog pin + // it was asked for would silently not happen. + func executeBoundedQuery(query: String, rowCap: Int, context: PluginStatementContext) async throws -> QueryResult? { + guard !context.enforcesReadOnly, !context.readOnly, !context.readsCatalog, + context.allowsDestructive, context.allowsServerCode + else { return nil } + return try await executeBoundedQuery(query: query, rowCap: rowCap) + } + + var enforcesReadOnlyMode: Bool { false } + + // What a run carries when Safe Mode is Read-Only and this driver can hold the engine to it. Nil otherwise, so the run + // keeps the calls it always made. + func readOnlyRunContext(safeModeIsReadOnly: Bool) -> PluginStatementContext? { + guard safeModeIsReadOnly, enforcesReadOnlyMode else { return nil } + return PluginStatementContext(enforcesReadOnly: true) + } + func executeRead(query: String) async throws -> QueryResult { try await execute(query: query) } @@ -592,6 +635,16 @@ extension DatabaseDriver { func executeBatch(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryBatchResult? { nil } + var supportsCopyFromStandardInput: Bool { false } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + func explainPlan(statement: String, variantId: String) async throws -> QueryResult { throw PluginDriverUnsupportedOperation.explainPlan } @@ -816,16 +869,16 @@ extension DatabaseDriver { ) } - /// Default fetchAllDatabaseMetadata: falls back to per-database calls (N+1). - /// Drivers should override with a single bulk query where possible. func fetchAllDatabaseMetadata() async throws -> [DatabaseMetadata] { let dbNames = try await fetchDatabases() var results: [DatabaseMetadata] = [] for dbName in dbNames { + try Task.checkCancellation() do { let metadata = try await fetchDatabaseMetadata(dbName) results.append(metadata) } catch { + try Self.rethrowIfCancelled(error) results.append(DatabaseMetadata.minimal(name: dbName)) } } @@ -834,6 +887,8 @@ extension DatabaseDriver { var providesBulkColumnFetch: Bool { false } + var fetchesAllColumnsInOneRequest: Bool { false } + var providesBulkForeignKeyFetch: Bool { false } func fetchAllForeignKeys(schema: String?) async throws -> [String: [ForeignKeyInfo]] { @@ -842,72 +897,67 @@ extension DatabaseDriver { func fetchAllForeignKeys() async throws -> [String: [ForeignKeyInfo]] { let allTables = try await fetchTables() - var result: [String: [ForeignKeyInfo]] = [:] - for table in allTables { - do { - let fks = try await fetchForeignKeys(table: table.name) - if !fks.isEmpty { result[table.name] = fks } - } catch { - Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Failed to fetch foreign keys for \(table.name): \(error.localizedDescription)") - } + return try await readEachTable(allTables.map(\.name), skippingFailures: true, reading: "foreign keys") { name in + let keys = try await fetchForeignKeys(table: name) + return keys.isEmpty ? nil : keys } - return result } func fetchForeignKeys(forTables tableNames: [String]) async throws -> [String: [ForeignKeyInfo]] { - // For small subsets, per-table fetch avoids scanning the entire schema - if tableNames.count <= 5 { - var result: [String: [ForeignKeyInfo]] = [:] - for tableName in tableNames { - do { - let fks = try await fetchForeignKeys(table: tableName) - if !fks.isEmpty { result[tableName] = fks } - } catch { - Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Failed to fetch foreign keys for \(tableName): \(error.localizedDescription)") - } - } - return result + if providesBulkForeignKeyFetch, tableNames.count > 5 { + let all = try await fetchAllForeignKeys() + let nameSet = Set(tableNames) + return all.filter { nameSet.contains($0.key) } + } + // Without a bulk query the schema-wide read is a round trip per schema table; past five names an error still fails it. + return try await readEachTable(tableNames, skippingFailures: tableNames.count <= 5, reading: "foreign keys") { name in + let keys = try await fetchForeignKeys(table: name) + return keys.isEmpty ? nil : keys } - let all = try await fetchAllForeignKeys() - let nameSet = Set(tableNames) - return all.filter { nameSet.contains($0.key) } } func fetchIndexes(forTables tableNames: [String]) async throws -> [String: [IndexInfo]] { - var result: [String: [IndexInfo]] = [:] - for tableName in tableNames { + try await readEachTable(tableNames, skippingFailures: true, reading: "indexes") { name in + let indexes = try await fetchIndexes(table: name) + return indexes.isEmpty ? nil : indexes + } + } + + private func readEachTable( + _ tableNames: [String], + skippingFailures: Bool, + reading what: String, + _ read: (String) async throws -> Value? + ) async throws -> [String: Value] { + var result: [String: Value] = [:] + for name in tableNames { + try Task.checkCancellation() do { - let indexes = try await fetchIndexes(table: tableName) - if !indexes.isEmpty { result[tableName] = indexes } + if let value = try await read(name) { result[name] = value } } catch { + try Self.rethrowIfCancelled(error) + guard skippingFailures else { throw error } Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Failed to fetch indexes for \(tableName): \(error.localizedDescription)") + .debug("Skipping \(what) for \(name): \(error.localizedDescription)") } } return result } + private static func rethrowIfCancelled(_ error: any Error) throws { + try Task.checkCancellation() + if error is CancellationError { throw error } + } + func sampleFieldPaths(table: String, limit: Int) async throws -> [PluginFieldPath] { [] } - /// Default fetchAllColumns: falls back to per-table fetchColumns (N+1). - /// Drivers should override with a single bulk query where possible. func fetchAllColumns() async throws -> [String: [ColumnInfo]] { let allTables = try await fetchTables() - var result: [String: [ColumnInfo]] = [:] - for table in allTables { - do { - let columns = try await fetchColumns(table: table.name) - result[table.name] = columns - } catch { - Logger(subsystem: "com.TablePro", category: "DatabaseDriver") - .debug("Skipping columns for table '\(table.name)': \(error.localizedDescription)") - } + return try await readEachTable(allTables.map(\.name), skippingFailures: true, reading: "columns") { name in + try await fetchColumns(table: name) } - return result } /// Default: no dependent types (MySQL/SQLite don't have standalone enum types) @@ -1039,6 +1089,17 @@ extension DatabaseDriver { throw PluginObjectSourceError.unsupported(trigger.name) } + var serverObjectKinds: [PluginServerObjectKind] { [] } + + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [ServerObjectInfo] { [] } + + func fetchServerObject(_ object: ServerObjectInfo) async throws -> ServerObjectInfo { + guard let definition = object.definition, !definition.isEmpty else { + throw PluginObjectSourceError.unsupported(object.name) + } + return object + } + var supportsTransactions: Bool { true } var hasLostConnection: Bool { false } diff --git a/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift b/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift index 147c37dfaf..f4550da094 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift @@ -59,10 +59,7 @@ extension DatabaseManager { } } - /// Runs a container operation the driver performs itself, with no statement to show. - /// - /// `kind` is the caller's to state because there is no text to classify: a drop is - /// `.destructiveQuery` and everything else is `.schemaMutation`. + // No statement to classify, so the caller states `kind`: a drop is destructive, anything else a schema mutation. func runContainerOperation( description: String, kind: OperationKind, @@ -70,6 +67,7 @@ extension DatabaseManager { databaseType: DatabaseType, event: CatalogEvent?, isConfirmationPreCleared: Bool = false, + gate: any ExecutionGate = ExecutionGateProvider.shared, perform: @Sendable @escaping (any DatabaseDriver) async throws -> Void ) async throws { try await authorizeContainerOperation( @@ -78,7 +76,8 @@ extension DatabaseManager { connectionId: scope.connectionId, databaseType: databaseType, description: description, - isConfirmationPreCleared: isConfirmationPreCleared + isConfirmationPreCleared: isConfirmationPreCleared, + gate: gate ) try await withScopedDriver( scope: scope, @@ -90,16 +89,14 @@ extension DatabaseManager { if let event { CatalogChangeService.post(event) } } - /// Runs a container operation the driver expressed as statements. - /// - /// The kind comes from the statements rather than from the action's name, so an edit that - /// revokes a privilege is authorized at the destructive tier the same way a drop is. + // The kind comes from the statements, so an edit that revokes a privilege is authorized like a drop. func runContainerStatements( _ plan: ContainerDDLPlan, description: String, scope: DatabaseScope, databaseType: DatabaseType, - event: CatalogEvent? + event: CatalogEvent?, + gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { let statements = plan.statements guard !statements.isEmpty else { return } @@ -113,7 +110,8 @@ extension DatabaseManager { connectionId: scope.connectionId, databaseType: databaseType, description: description, - isConfirmationPreCleared: false + isConfirmationPreCleared: false, + gate: gate ) let route = schemaChangeRoute(for: scope) @@ -139,21 +137,19 @@ extension DatabaseManager { if let event { CatalogChangeService.post(event) } } - /// `isConfirmationPreCleared` is for a caller that has already shown the user a destructive - /// confirmation of its own. Without it the sidebar's Drop dialog is followed by the gate's, - /// asking twice for one gesture and once per target for a batch. It clears the confirmation - /// only: Touch ID, the read-only refusal and the audit record all still apply. + // Pre-cleared skips only a confirmation the caller already showed; Touch ID, Read-Only and the audit still apply. private func authorizeContainerOperation( sql: String?, kind: OperationKind, connectionId: UUID, databaseType: DatabaseType, description: String, - isConfirmationPreCleared: Bool + isConfirmationPreCleared: Bool, + gate: any ExecutionGate ) async throws { var capabilities = CallerCapabilities.interactiveUser if isConfirmationPreCleared { capabilities.insert(.confirmationPreCleared) } - let decision = await ExecutionGateProvider.shared.authorize( + let decision = await gate.authorize( OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -164,11 +160,7 @@ extension DatabaseManager { operationDescription: description ) ) - guard case .authorized = decision else { - throw DatabaseError.queryFailed( - decision.deniedReason ?? String(localized: "This change was not authorized") - ) - } + if let denial = decision.denialError { throw denial } } private static func execute( diff --git a/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift b/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift index 164eba3a84..a0d45461fe 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift @@ -48,15 +48,19 @@ extension DatabaseManager { } } - // The lane reopens on its next use, signing in with a new lease. A transaction it held is reported as lost. + // The lane reopens on its next use, signing in with a new lease. A transaction or session state it held is + // reported as lost. private func closeLane(_ database: String, for ending: CredentialLeaseEnding) async { let connectionId = ending.connectionId guard let driver = sessionLanes.parkedDriver(for: connectionId, database: database) else { return } let held = await driver.heldSessionTransactionState() guard credentialLeaseAction(for: ending) == .closeLane(database: database) else { return } - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) Self.logger.info("Closing a lane whose Vault lease is ending for \(connectionId, privacy: .public)") sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) } @@ -73,11 +77,13 @@ extension DatabaseManager { guard credentialLeaseAction(for: ending) == .reconnectSession else { return } try? await sessionDriverGate.withExclusiveAccess(browsedGateKey(for: connectionId)) { guard credentialLeaseAction(for: ending) == .reconnectSession else { return } - if holdsTransaction(connectionId, driver: ending.driver) { - // Reported to the next statement on this database, as a lane's lost transaction is. - let database = activeSessions[connectionId]?.resolvedBrowseDatabase ?? "" - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + // Reported to the next statement on this database, as a lane's lost transaction is. What the session set up + // is noted by the reconnect, just before it closes this driver. + sessionLanes.markLost( + database: activeSessions[connectionId]?.resolvedBrowseDatabase ?? "", + transaction: holdsTransaction(connectionId, driver: ending.driver), + for: connectionId + ) Self.logger.info("Vault lease ending, reconnecting \(connectionId, privacy: .public) with a new one") let outcome = await performHealthMonitorReconnect(connectionId: connectionId, failedDriver: ending.driver) guard case .retry = outcome, credentialLeaseAction(for: ending) == .reconnectSession else { return } diff --git a/apple/TablePro/Core/Database/DatabaseManager+EnsureConnected.swift b/apple/TablePro/Core/Database/DatabaseManager+EnsureConnected.swift index 2b4af026a3..71d309b4f9 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+EnsureConnected.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+EnsureConnected.swift @@ -11,12 +11,10 @@ extension DatabaseManager { _ connection: DatabaseConnection, passwordOverride: String? = nil, sshPasswordOverride: String? = nil, - trigger: ConnectTrigger = .request + trigger: ConnectTrigger = .request, + adoptsSSHProfileEdits: Bool = false ) async throws { - /// An installed driver is only a reason to skip while it still answers. A reconnect that - /// gave up leaves one behind, and returning here made Reconnect a button that did nothing - /// on the one connection that needed it. A driver nobody has heard from in a while is the - /// same problem one step earlier, which is what the check answers. + // A driver is a reason to skip only while it answers: one a reconnect gave up on made Reconnect do nothing. await verifyBeforeUse(connection.id) if let session = activeSessions[connection.id], session.driver != nil, session.liveness == .live { return @@ -26,7 +24,8 @@ extension DatabaseManager { connection, passwordOverride: passwordOverride, sshPasswordOverride: sshPasswordOverride, - trigger: trigger + trigger: trigger, + adoptsSSHProfileEdits: adoptsSSHProfileEdits ) } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index b42ed9eae7..8a01955d0f 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -149,10 +149,17 @@ extension DatabaseManager { result.driver.disconnect() return .abort } + let warning = openingWarning( + driver: result.driver, + startupFailures: result.startupFailures, + restoreWarning: result.restoreWarning, + for: connectionId + ) updateSession(connectionId) { session in session.driver = result.driver session.effectiveConnection = result.effectiveConnection session.status = .connected + session.connectionWarning = warning if let schemaDriver = result.driver as? SchemaSwitchable { session.browseSchema = schemaDriver.currentSchema } @@ -179,17 +186,25 @@ extension DatabaseManager { } } - // Read before the driver is closed and replaced, which prunes the hold. The user's next statement there - // is refused once, as on a lane, instead of running outside the transaction it was written for. - internal func noteTransactionLost(with driver: DatabaseDriver?, for connectionId: UUID) { + // Read before the driver is closed and replaced, which prunes the hold and the ledger. The user's next statement + // there is refused once, as on a lane, instead of running without the transaction or session it was written for. + // A switch the user asked for passes `includingState: false`: they chose to leave that session. + internal func noteSessionLost(with driver: DatabaseDriver?, for connectionId: UUID, includingState: Bool = true) { guard let driver, let session = activeSessions[connectionId] else { return } let lost = sessionTransactions.holds(for: connectionId).filter { sessionTransactions.driver(for: $0) === driver } for hold in lost { - sessionLanes.markTransactionLost( + sessionLanes.markLost( database: hold.key.lane ?? session.resolvedBrowseDatabase, + transaction: true, for: connectionId ) } + guard includingState else { return } + sessionLanes.markLost( + database: session.resolvedBrowseDatabase, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) } internal func reconnectVerdict(for error: Error) -> ConnectionFailureClassifier.ReconnectVerdict { @@ -197,10 +212,20 @@ extension DatabaseManager { return ConnectionFailureClassifier.reconnectVerdict(for: tlsError) } // A refusal that needs a person gets the same answer on every retry. - if error is EnvVarReferenceConsentError || error is PreConnectScriptConsentError { + if error is EnvVarReferenceConsentError || error is PreConnectScriptConsentError + || error is LocalCredentialConsentError || error is ConnectionTunnelError { return .stop(ConnectionFailureClassifier.info(for: error)) } - guard isAuthenticationFailure(error) else { return .retry(nil) } + // The lease renewal stops on the same Vault errors; an unreachable or failing Vault is worth another try. + if error is VaultError || error is VaultRequestError { + let info = ConnectionFailureClassifier.info(for: error) + return VaultLease.isDefinitive(error) ? .stop(info) : .retry(info) + } + // The new user is kept, and the next attempt may find it loaded. + if error is VaultUserNotLoaded { + return .retry(ConnectionFailureClassifier.info(for: error)) + } + guard ConnectionFailureClassifier.isAuthenticationFailure(error) else { return .retry(nil) } let message = String(format: String(localized: "Reconnect failed: %@"), error.localizedDescription) return .stop(ConnectionFailureInfo(message: message)) } @@ -229,10 +254,18 @@ extension DatabaseManager { message: String(localized: "The connection was closed and reconnecting was cancelled.") ) + // A direct dial reaches another host when the CNAME of an AWS IAM connection now leads to another RDS endpoint. + internal func movesEndpoint(of session: ConnectionSession, to rebuilt: DatabaseConnection) -> Bool { + let previous = session.effectiveConnection ?? session.connection + return rebuilt.host.caseInsensitiveCompare(previous.host) != .orderedSame + } + internal struct ReconnectResult { let driver: DatabaseDriver let effectiveConnection: DatabaseConnection let cachedPassword: String? + let startupFailures: [StartupCommandFailure] + let restoreWarning: String? } // Rebuilds the tunnel for a tunneled session before connecting the new driver. Tunnels are keyed by @@ -243,26 +276,30 @@ extension DatabaseManager { generation: Int ) async throws -> ReconnectResult? { let connectionId = session.connection.id - noteTransactionLost(with: session.driver, for: connectionId) + noteSessionLost(with: session.driver, for: connectionId) session.driver?.disconnect() // Resolved as the connect resolved it; a reference that lost its approval needs a person. - let connection = try await envVarReferenceConsent.resolve(session.connection, trigger: .background) - let preparedConfiguration = try await DatabaseDriverFactory.prepareConfiguration(for: connection) + let resolved = try await envVarReferenceConsent.resolve(session.connection, trigger: .background) + // Followed again, for a direct dial too: the CNAME may lead to another RDS endpoint than at the connect. + let connection = try await resolvingRDSEndpointAlias(of: resolved, trigger: .background) + let preparedConfiguration = try await DatabaseDriverFactory.prepareConfiguration(for: resolved) let deadline = ConnectionDeadline(configuredSeconds: connection.connectTimeoutSeconds) try requireCurrentAttempt(generation, for: connectionId) - let connectionForDriver: DatabaseConnection - if connection.activeTunnelKind != nil { + let tunneled = connection.activeTunnelKind != nil + if tunneled { // A rebuilt tunnel moves to a new local port, stranding every other database's connection. await closeTransportLanes(for: connectionId) { self.isCurrentAttempt(generation, for: connectionId) } - connectionForDriver = try await buildEffectiveConnection( - for: connection, - deadline: deadline, - ownsAttempt: { self.isCurrentAttempt(generation, for: connectionId) } - ) - try requireCurrentAttempt(generation, for: connectionId) - } else { - connectionForDriver = session.effectiveConnection ?? session.connection + } + let connectionForDriver = try await buildEffectiveConnection( + for: connection, + pinnedSSHRoute: session.connection.destinationTunnel, + deadline: deadline, + ownsAttempt: { self.isCurrentAttempt(generation, for: connectionId) } + ) + try requireCurrentAttempt(generation, for: connectionId) + if !tunneled, movesEndpoint(of: session, to: connectionForDriver) { + await closeTransportLanes(for: connectionId) { self.isCurrentAttempt(generation, for: connectionId) } } guard let connectResult = try await connectReconnectDriver( @@ -282,13 +319,13 @@ extension DatabaseManager { throw CancellationError() } - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: session.effectiveQueryTimeoutSeconds, startupCommands: session.connection.startupCommands, connectionName: session.connection.name ) - await restoreSchemaAndDatabase( + let restoreWarning = await restoreSchemaAndDatabase( on: driver, savedSchema: session.browseSchema, savedDatabase: databaseSwitchRequiresReconnect(session.connection) ? nil : session.browseDatabase @@ -297,7 +334,9 @@ extension DatabaseManager { return ReconnectResult( driver: driver, effectiveConnection: connectionForDriver, - cachedPassword: connectResult.cachedPassword + cachedPassword: connectResult.cachedPassword, + startupFailures: startupFailures, + restoreWarning: restoreWarning ) } @@ -306,7 +345,7 @@ extension DatabaseManager { queryTimeoutSeconds: Int, startupCommands: String?, connectionName: String - ) async { + ) async -> [StartupCommandFailure] { do { try await driver.applyQueryTimeout(queryTimeoutSeconds) } catch { @@ -315,7 +354,7 @@ extension DatabaseManager { ) } - await executeStartupCommands(startupCommands, on: driver, connectionName: connectionName) + return await executeStartupCommands(startupCommands, on: driver, connectionName: connectionName) } private func databaseSwitchRequiresReconnect(_ connection: DatabaseConnection) -> Bool { @@ -323,17 +362,23 @@ extension DatabaseManager { .capabilities.requiresReconnectForDatabaseSwitch ?? false } + // The reconnected session stays on the server's default when a restore fails, which the window would not show. func restoreSchemaAndDatabase( on driver: DatabaseDriver, savedSchema: String?, savedDatabase: String? - ) async { + ) async -> String? { + var warnings: [String?] = [] // The schema lives inside the database, and a database switch can leave the driver on none. if let savedDatabase, let adapter = driver as? PluginDriverAdapter { do { try await adapter.switchDatabase(to: savedDatabase) } catch { Self.logger.warning("Failed to restore database '\(savedDatabase)' on reconnect: \(error.localizedDescription)") + warnings.append(String( + format: String(localized: "Reconnecting could not reopen the database %1$@: %2$@"), + savedDatabase, error.localizedDescription + )) } } @@ -342,8 +387,13 @@ extension DatabaseManager { try await schemaDriver.switchSchema(to: savedSchema) } catch { Self.logger.warning("Failed to restore schema '\(savedSchema)' on reconnect: \(error.localizedDescription)") + warnings.append(String( + format: String(localized: "Reconnecting could not reopen the schema %1$@: %2$@"), + savedSchema, error.localizedDescription + )) } } + return Self.joinedWarning(warnings) } internal func stopHealthMonitor(for connectionId: UUID) async { @@ -383,7 +433,8 @@ extension DatabaseManager { // A debug target dialed through this transport would be left holding its locks once it goes. await RoutineDebugSessions.shared.stopAll(connectionId: sessionId) MetadataConnectionPool.shared.beginTransportReplacement(connectionId: sessionId) - await closeTransportLanes(for: sessionId) { self.isCurrentAttempt(generation, for: sessionId) } + // The user asked for this reconnect, so what the lanes' sessions set up is not reported as lost. + await closeTransportLanes(for: sessionId, includingState: false) { self.isCurrentAttempt(generation, for: sessionId) } } defer { if replacesPooledTransport { @@ -417,14 +468,35 @@ extension DatabaseManager { } try requireCurrentAttempt(generation, for: sessionId) - let preparedConfiguration = try await DatabaseDriverFactory.prepareConfiguration(for: session.connection) - let deadline = ConnectionDeadline(configuredSeconds: session.connection.connectTimeoutSeconds) + // Resolved as the connect resolved it: the session keeps the row as saved, `${VAR}` references included. + let resolved = try await envVarReferenceConsent.resolve(session.connection, trigger: .request) + try requireCurrentAttempt(generation, for: sessionId) + let connection = try await resolvingRDSEndpointAlias(of: resolved, trigger: .request) + try requireCurrentAttempt(generation, for: sessionId) + let preparedConfiguration = try await DatabaseDriverFactory.prepareConfiguration(for: resolved) + let deadline = ConnectionDeadline(configuredSeconds: connection.connectTimeoutSeconds) let effectiveConnection = try await buildEffectiveConnection( - for: session.connection, + for: connection, + pinnedSSHRoute: session.connection.destinationTunnel, deadline: deadline, ownsAttempt: { self.isCurrentAttempt(generation, for: sessionId) } ) try requireCurrentAttempt(generation, for: sessionId) + // Pooled and parked connections stand on the old endpoint, so they go the way a rebuilt tunnel's do. + let movesPooledEndpoint = !replacesPooledTransport && movesEndpoint(of: session, to: effectiveConnection) + if movesPooledEndpoint { + await RoutineDebugSessions.shared.stopAll(connectionId: sessionId) + MetadataConnectionPool.shared.beginTransportReplacement(connectionId: sessionId) + await closeTransportLanes(for: sessionId, includingState: false) { + self.isCurrentAttempt(generation, for: sessionId) + } + } + defer { + if movesPooledEndpoint { + MetadataConnectionPool.shared.endTransportReplacement(connectionId: sessionId) + RoutineDebugSessions.shared.reopenAdmission(for: sessionId) + } + } // Nil means the user declined the re-prompt after an auth failure: a cancellation, not a refusal. guard let connectResult = try await connectReconnectDriver( @@ -440,13 +512,13 @@ extension DatabaseManager { } let driver = connectResult.driver - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: session.effectiveQueryTimeoutSeconds, startupCommands: session.connection.startupCommands, connectionName: session.connection.name ) - await restoreSchemaAndDatabase( + let restoreWarning = await restoreSchemaAndDatabase( on: driver, savedSchema: activeSessions[sessionId]?.browseSchema, savedDatabase: databaseSwitchRequiresReconnect(session.connection) ? nil : activeSessions[sessionId]?.browseDatabase @@ -456,10 +528,14 @@ extension DatabaseManager { throw CancellationError() } + let warning = openingWarning( + driver: driver, startupFailures: startupFailures, restoreWarning: restoreWarning, for: sessionId + ) updateSession(sessionId) { session in session.driver = driver session.status = .connected session.effectiveConnection = effectiveConnection + session.connectionWarning = warning if let schemaDriver = driver as? SchemaSwitchable { session.browseSchema = schemaDriver.currentSchema } @@ -566,7 +642,7 @@ extension DatabaseManager { guard allowsCredentialPrompt else { return .fail } guard ConnectionCredentialResolver.promptsForPassword(session.connection), !pluginManager.hidesPassword(for: session.connection), - isAuthenticationFailure(error) + ConnectionFailureClassifier.isAuthenticationFailure(error) else { return .fail } @@ -587,29 +663,6 @@ extension DatabaseManager { return .retry(prompted) } - private static let invalidAuthorizationSQLState = "28000" - private static let mysqlAccessDeniedErrorCode = 1_045 - - internal func isAuthenticationFailure(_ error: Error) -> Bool { - if let pluginError = error as? any PluginDriverError { - if pluginError.pluginSqlState == Self.invalidAuthorizationSQLState { - return true - } - if pluginError.pluginErrorCode == Self.mysqlAccessDeniedErrorCode { - return true - } - return messageIndicatesAuthenticationFailure(pluginError.pluginErrorMessage) - } - return messageIndicatesAuthenticationFailure(error.localizedDescription) - } - - private func messageIndicatesAuthenticationFailure(_ message: String) -> Bool { - let lowered = message.lowercased() - return lowered.contains("access denied") - || lowered.contains("authentication failed") - || lowered.contains("invalid credentials") - } - // A superseded attempt leaves the slot alone: the tunnel under this connection's id is the newer attempt's. private func closeReconnectTunnels(for connection: DatabaseConnection, ownsAttempt: () -> Bool) async { guard ownsAttempt(), let tunnelManager = activeTunnelManager(for: connection) else { return } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift b/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift index d10ea0b38b..643ff155dc 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift @@ -16,6 +16,12 @@ protocol ScopedMetadataProviding: AnyObject, Sendable { _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T + func withMetadataRead( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T + func browseScope(for connectionId: UUID) -> DatabaseScope? } @@ -27,6 +33,15 @@ extension ScopedMetadataProviding { try await withMetadataDriver(scope: scope, workload: .interactive, body) } + // A conformer with no session turns has no transaction revision to keep. + func withMetadataRead( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + try await withMetadataDriver(scope: scope, workload: workload, body) + } + /// For reads that belong to the sidebar rather than to a tab: the object list, the /// database and quick switchers, and the AI schema context. Completion is not one of /// them any more: a query tab completes against its own scope. @@ -38,7 +53,7 @@ extension ScopedMetadataProviding { guard let scope = browseScope(for: connectionId) else { throw DatabaseError.notConnected } - return try await withMetadataDriver(scope: scope, workload: workload, body) + return try await withMetadataRead(scope: scope, workload: workload, body) } } @@ -59,4 +74,21 @@ extension DatabaseManager { body ) } + + // Reads only: on the session driver its turn leaves an open transaction's revision alone, unless the state moved. + func withMetadataRead( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload = .interactive, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + try await withScopedDriver( + scope: scope, + route: metadataRoute(for: scope), + workload: workload, + cancellation: .untracked, + ownsSessionState: false, + readsOnly: true, + body + ) + } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+OpenTransactions.swift b/apple/TablePro/Core/Database/DatabaseManager+OpenTransactions.swift index cdd8268df7..e922e93eb5 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+OpenTransactions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+OpenTransactions.swift @@ -8,10 +8,8 @@ import Foundation import TableProPluginKit extension DatabaseManager { - /// A connection running a turn is skipped, because asking it waits for that turn. One that has - /// not answered within a second counts as holding nothing, so a close never hangs on a busy or - /// dead server. An aborted transaction counts: a savepoint taken before the failure can still - /// bring its earlier work back to a commit. + // A busy lane is not asked and a slow one counts as idle, so a close never hangs; what the tracker + // last read covers both. An aborted transaction counts: a savepoint can still bring earlier work back. internal func databasesHoldingTransaction( for connectionId: UUID, among databases: Set? = nil, @@ -37,6 +35,7 @@ extension DatabaseManager { state: Task { await Self.boundedTransactionState(of: candidate.driver, clock: clock) } ) } + let browsedDatabase = session.resolvedBrowseDatabase var holding: [String] = [] for read in reads { let state = await read.state.value @@ -44,6 +43,12 @@ extension DatabaseManager { holding.append(read.database) } } + let tracked = sessionTransactions.holds(for: connectionId) + .map { $0.key.lane ?? browsedDatabase } + .filter { databases?.contains($0) ?? true } + for database in tracked where !holding.contains(database) { + holding.append(database) + } return holding } @@ -98,6 +103,13 @@ extension DatabaseManager { return safety } + // A turn running on the hold's own connection, which a commit would have to wait behind. + internal func isLaneBusy(holding hold: SessionTransactionTracker.Hold) -> Bool { + let key = hold.key.lane.map { SessionDriverGate.Key(connectionId: hold.connectionId, database: $0) } + ?? browsedGateKey(for: hold.connectionId) + return sessionDriverGate.isHeld(key) + } + /// Ends the transaction a hold shows, on the connection that holds it. Refused when that /// connection was replaced, or when other work ran there after the hold was read: the user /// confirmed what they saw, not a transaction someone opened since. diff --git a/apple/TablePro/Core/Database/DatabaseManager+Principals.swift b/apple/TablePro/Core/Database/DatabaseManager+Principals.swift index 7cf505ddcd..d3b63817f1 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Principals.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Principals.swift @@ -14,7 +14,8 @@ extension DatabaseManager { func executePrincipalChanges( changes: [PrincipalChange], databaseType: DatabaseType, - connectionId: UUID + connectionId: UUID, + gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { guard let driver = driver(for: connectionId) else { throw DatabaseError.notConnected @@ -39,7 +40,7 @@ extension DatabaseManager { ? .destructiveQuery : .schemaMutation - let authorization = await ExecutionGateProvider.shared.authorize( + let authorization = await gate.authorize( OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -50,12 +51,7 @@ extension DatabaseManager { operationDescription: String(localized: "Apply User and Role Changes") ) ) - guard case .authorized = authorization else { - throw DatabaseError.queryFailed( - authorization.deniedReason - ?? String(localized: "User and role changes were not authorized") - ) - } + if let denial = authorization.denialError { throw denial } /// Reported whether or not every statement ran: an engine that cannot roll principal /// statements back keeps the ones before a failure, and `DROP OWNED` among them drops objects. @@ -83,15 +79,14 @@ extension DatabaseManager { rollsBack: Bool, connectionId: UUID ) async throws { - /// No transaction is opened over one the session already holds: on this shared session an - /// app-owned `COMMIT` commits the user's pending work, and MySQL's `START TRANSACTION` - /// commits it implicitly. Joining it instead leaves the statements pending, which is what - /// the failure then reports. + // Joins a transaction the session holds: an app COMMIT, or MySQL's START TRANSACTION, would commit the user's work. + // `.automatic`: this runs outside a session turn, so a hold it opened would never reach the bar. let owner = WriteTransactionOwner.resolve( supportsTransactions: driver.supportsTransactions, - sessionState: await driver.heldSessionTransactionState() + sessionState: await driver.heldSessionTransactionState(), + commitMode: .automatic ) - let useTransaction = owner.opensTransaction && rollsBack + let useTransaction = owner.sendsBegin && owner.commitsAtEnd && rollsBack if useTransaction { try await driver.beginTransaction(mode: .readWrite) } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Queries.swift b/apple/TablePro/Core/Database/DatabaseManager+Queries.swift index 16d495e9d6..c22ebed391 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Queries.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Queries.swift @@ -59,13 +59,19 @@ extension DatabaseManager { } } + let dialled: DatabaseConnection + do { + dialled = try await resolvingRDSEndpointAlias(of: connection, trigger: .request) + } catch { + throw await diagnosedConnectFailure(error, for: connection) + } let preparedConfiguration = try await DatabaseDriverFactory.prepareConfiguration(for: connection) let deadline = ConnectionDeadline(configuredSeconds: connection.connectTimeoutSeconds) let testConnection: DatabaseConnection do { testConnection = try await buildEffectiveConnection( - for: connection, + for: dialled, sshPasswordOverride: sshPassword, deadline: deadline ) @@ -74,6 +80,13 @@ extension DatabaseManager { } let tunnelManager = activeTunnelManager(for: connection) + // The form tests under a throwaway id, so a role of the connection's own leases a user no connect reuses. + defer { + let owner = ConnectionCredentialResolver.credentialOwner(for: connection) + if owner == connection.id, activeSessions[connection.id] == nil { + VaultDatabaseCredentials.shared.endUse(connection: connection.id, live: Set(activeSessions.keys)) + } + } let result: ConnectionTestResult do { diff --git a/apple/TablePro/Core/Database/DatabaseManager+RDSEndpoint.swift b/apple/TablePro/Core/Database/DatabaseManager+RDSEndpoint.swift new file mode 100644 index 0000000000..d985d6a459 --- /dev/null +++ b/apple/TablePro/Core/Database/DatabaseManager+RDSEndpoint.swift @@ -0,0 +1,53 @@ +// +// DatabaseManager+RDSEndpoint.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +extension DatabaseManager { + // RDS accepts a token only for the endpoint AWS issued, never a CNAME of the user's own. The connection dials + // that endpoint as well: signing it while dialling the CNAME would let whoever answers the CNAME's next lookup + // receive the token. The found name is never saved, so each connect follows the CNAME again. + // + // Only a direct dial: behind a tunnel the far end resolves the name, its DNS can disagree with this Mac's, and a + // bastion's PermitOpen matches the name as typed. The RDS Endpoint field covers a forward. + // + // Runs before the connect's deadline starts, as the first sign-in approval does, because it can ask a person. + internal func resolvingRDSEndpointAlias( + of connection: DatabaseConnection, + trigger: ConnectTrigger + ) async throws -> DatabaseConnection { + guard ConnectionCredentialResolver.signsRDSToken(for: connection), + connection.enabledTunnelKinds.isEmpty, + AWSEndpointHost.isAliasCandidate(connection.host) + else { return connection } + let lookupDeadline = ConnectionDeadline(configuredSeconds: connection.connectTimeoutSeconds) + let lookup = await AWSEndpointHost.rdsEndpoint( + behind: connection.host, + resolver: cnameResolver, + within: lookupDeadline.remainingDuration + ) + try Task.checkCancellation() + switch lookup { + case .notFound: + return connection + case .failed: + // Signing the typed name instead only earns an access denied, which reads as a credential problem and + // ends a reconnect that a working DNS would have let through. + throw AWSAuthError.endpointLookupFailed(host: connection.host) + case .found(let endpoint): + Self.logger.info("AWS IAM signs in as RDS endpoint \(endpoint, privacy: .private), which \(connection.host, privacy: .private) points to") + var resolved = connection + resolved.host = endpoint + // A definition from someone else was approved for the name it carries, and whoever runs that name's DNS + // can point it at another database at any time, so each endpoint behind it is approved on its own. + if connection.origin != .thisMac { + try await localCredentialConsent.check(resolved, trigger: trigger) + } + return resolved + } + } +} diff --git a/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift b/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift index 59fab36098..3d657c0891 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift @@ -1,5 +1,6 @@ import Foundation import os +import TableProSQLGrammar import TableProUtilities extension DatabaseManager { @@ -239,7 +240,12 @@ extension DatabaseManager { // Closes only the lanes parked when it starts, each while still parked as the same driver and only while // `owns` holds: a connect that took the session over parks lanes, and transactions, that are not this teardown's. - internal func closeTransportLanes(for connectionId: UUID, while owns: () -> Bool) async { + // A reconnect the user asked for passes `includingState: false`, as their own database switch does. + internal func closeTransportLanes( + for connectionId: UUID, + includingState: Bool = true, + while owns: () -> Bool + ) async { guard owns() else { return } let lanes = sessionLanes.parkedDatabases(for: connectionId).compactMap { database in sessionLanes.parkedDriver(for: connectionId, database: database).map { (database, $0) } @@ -250,9 +256,12 @@ extension DatabaseManager { let held = await driver.heldSessionTransactionState() guard owns() else { return } guard sessionLanes.parkedDriver(for: connectionId, database: database) === driver else { continue } - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: includingState ? sessionStateLedger.state(heldBy: driver, for: connectionId) : [], + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+SSH.swift b/apple/TablePro/Core/Database/DatabaseManager+SSH.swift index 629955c883..bbf27fc1b4 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+SSH.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+SSH.swift @@ -7,6 +7,7 @@ import Foundation +import TableProConnectionLibrary import TableProSSHConfig import TableProSSHTransport @@ -16,13 +17,15 @@ extension DatabaseManager { // `sshPasswordOverride` is the password Test Connection typed, used instead of the stored one. // `ownsAttempt` is asked with nothing suspending before the tunnel is created: tunnels are keyed by // connection, and creating one closes whatever a newer attempt opened under the same id. + // `pinnedSSHRoute` is the route an open session dialed: a linked profile moved since then is refused, not followed. internal func buildEffectiveConnection( for original: DatabaseConnection, sshPasswordOverride: String? = nil, + pinnedSSHRoute: ConnectionDestination.Tunnel? = nil, deadline: ConnectionDeadline, ownsAttempt: () -> Bool = { true } ) async throws -> DatabaseConnection { - var connection = SSHProfileStorage.shared.refreshingLinkedProfile(original) + var connection = try routedThroughLinkedProfile(original, pinnedSSHRoute: pinnedSSHRoute) // Only the tunnel this connect opens says it secures the transport or names the server behind it; a saved or // imported field never does. connection.additionalFields[DatabaseConnection.tunnelSecuresTransportKey] = nil @@ -127,6 +130,17 @@ extension DatabaseManager { ) } + internal func routedThroughLinkedProfile( + _ connection: DatabaseConnection, + pinnedSSHRoute: ConnectionDestination.Tunnel? + ) throws -> DatabaseConnection { + let routed = SSHProfileStorage.shared.refreshingLinkedProfile(connection) + if let pinnedSSHRoute, case .profile = routed.sshTunnelMode, routed.destinationTunnel != pinnedSSHRoute { + throw ConnectionTunnelError.sshProfileMoved + } + return routed + } + // MARK: - SSH Tunnel Recovery func handleSSHTunnelDied(connectionId: UUID) async { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift index fd4b884352..5b3342fe02 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift @@ -42,7 +42,8 @@ extension DatabaseManager { _ script: SchemaChangeScript, databaseType: DatabaseType, scope: DatabaseScope, - gate: any ExecutionGate = ExecutionGateProvider.shared + gate: any ExecutionGate = ExecutionGateProvider.shared, + holdConfirmation: SchemaChangeHoldConfirmation = DatabaseManager.confirmSchemaChangeBesideHold ) async throws { let route = schemaChangeRoute(for: scope) let statements = script.statements @@ -52,6 +53,8 @@ extension DatabaseManager { if let denial = await gate.authorize(request).denialError { throw denial } + try await confirmSchemaChange(on: route, connectionId: scope.connectionId, confirmation: holdConfirmation) + let isPooled = route.isPooled let executionTimes: [TimeInterval] do { @@ -61,7 +64,7 @@ extension DatabaseManager { cancellation: .protectedWrite ) { driver in try await Self.refuseBeforeWriting(script, scope: scope, on: driver) - let useTransaction = Self.wrapsDDLInTransaction(driver) + let useTransaction = await Self.wrapsSchemaChange(on: driver, isPooled: isPooled) if useTransaction { try await driver.beginTransaction(mode: schemaKind.declaresWrite ? .readWrite : .serverDefault) } @@ -69,7 +72,7 @@ extension DatabaseManager { do { for stmt in statements { let startedAt = Date() - _ = try await driver.execute(query: stmt.sql) + try await Self.runSchemaStatement(stmt.sql, on: driver, wrapped: useTransaction, isPooled: isPooled) measured.append(Date().timeIntervalSince(startedAt)) } if useTransaction { @@ -83,6 +86,9 @@ extension DatabaseManager { Self.logger.error("Rollback failed after schema change error: \(error.localizedDescription)") } } + if measured.isEmpty, error is DefinitionInOpenTransactionRefusal { + throw error + } let applied: [SchemaStatement] = useTransaction ? [] : statements.prefix(measured.count).filter { !$0.carriesCredentials } @@ -96,6 +102,8 @@ extension DatabaseManager { } } catch let refusal as SchemaOperationRefusedError { throw refusal + } catch let refusal as DefinitionInOpenTransactionRefusal { + throw refusal } catch let failure as SchemaChangeFailedAfterWriting { Self.reportCatalogChangeAfterFailure(in: scope) reportSavedChange(of: script, in: scope) @@ -134,6 +142,22 @@ extension DatabaseManager { driver.supportsTransactions && driver.supportsTransactionalDDL } + // On the session driver the commit after an unwrapped statement would end the user's open transaction too. + nonisolated private static func runSchemaStatement( + _ sql: String, + on driver: DatabaseDriver, + wrapped: Bool, + isPooled: Bool + ) async throws { + guard !wrapped else { + _ = try await driver.execute(query: sql) + return + } + _ = try await AppDefinitionCommit.run(on: driver, refusesOpenTransaction: !isPooled) { + try await driver.execute(query: sql) + } + } + /// A save that only set the comment changed no column, so it is announced as a comment change. private func reportSavedChange(of script: SchemaChangeScript, in scope: DatabaseScope) { guard script.setsCommentOnly else { @@ -226,21 +250,17 @@ extension DatabaseManager { } } - /// Run a Create Table draft's statements, on the same isolated route and in the same shape as - /// `executeSchemaChanges`. - /// - /// The route is what matters. The view used to reach for `driver(for:)`, which is the session - /// driver, so a `BEGIN` opened here joined whatever transaction a query tab had left open and - /// the matching `COMMIT` took that tab's uncommitted work with it. `schemaChangeRoute` exists - /// for exactly that, and the app's own DDL has no business on the user's connection. + // On the schema change route: a BEGIN on the session driver would join a query tab's open transaction. func executeCreateTable( statements: [String], databaseType: DatabaseType, scope: DatabaseScope, - gate: any ExecutionGate = ExecutionGateProvider.shared + gate: any ExecutionGate = ExecutionGateProvider.shared, + holdConfirmation: SchemaChangeHoldConfirmation = DatabaseManager.confirmSchemaChangeBesideHold ) async throws { guard !statements.isEmpty else { return } let route = schemaChangeRoute(for: scope) + let isPooled = route.isPooled let script = statements.map { $0.hasSuffix(";") ? $0 : $0 + ";" }.joined(separator: "\n\n") let authorization = await gate.authorize( @@ -254,11 +274,8 @@ extension DatabaseManager { operationDescription: String(localized: "Create Table") ) ) - guard case .authorized = authorization else { - throw DatabaseError.queryFailed( - authorization.deniedReason ?? String(localized: "Operation not permitted") - ) - } + if let denial = authorization.denialError { throw denial } + try await confirmSchemaChange(on: route, connectionId: scope.connectionId, confirmation: holdConfirmation) let executionTimes: [TimeInterval] do { @@ -267,7 +284,8 @@ extension DatabaseManager { route: route, cancellation: .protectedWrite ) { driver in - let useTransaction = Self.wrapsDDLInTransaction(driver) && statements.count > 1 + let wraps = await Self.wrapsSchemaChange(on: driver, isPooled: isPooled) + let useTransaction = wraps && statements.count > 1 if useTransaction { try await driver.beginTransaction(mode: .readWrite) } @@ -275,7 +293,7 @@ extension DatabaseManager { do { for statement in statements { let startedAt = Date() - _ = try await driver.execute(query: statement) + try await Self.runSchemaStatement(statement, on: driver, wrapped: useTransaction, isPooled: isPooled) measured.append(Date().timeIntervalSince(startedAt)) } if useTransaction { diff --git a/apple/TablePro/Core/Database/DatabaseManager+SchemaChangeHold.swift b/apple/TablePro/Core/Database/DatabaseManager+SchemaChangeHold.swift new file mode 100644 index 0000000000..3b5711b168 --- /dev/null +++ b/apple/TablePro/Core/Database/DatabaseManager+SchemaChangeHold.swift @@ -0,0 +1,45 @@ +// +// DatabaseManager+SchemaChangeHold.swift +// TablePro +// + +import AppKit +import Foundation + +typealias SchemaChangeHoldConfirmation = @MainActor @Sendable () async -> Bool + +extension DatabaseManager { + // On its own connection a schema change queues behind the open transaction's locks, and Stop cannot reach it. + static let confirmSchemaChangeBesideHold: SchemaChangeHoldConfirmation = { + await AlertHelper.confirm( + title: String(localized: "A Transaction Is Open"), + message: String( + localized: "This change runs on its own connection and can wait until you commit or roll back the open transaction." + ), + confirmButton: String(localized: "Continue") + ) + } + + func confirmSchemaChange( + on route: ScopedDriverRoute, + connectionId: UUID, + confirmation: SchemaChangeHoldConfirmation + ) async throws { + guard route.isPooled, !sessionTransactions.holds(for: connectionId).isEmpty else { return } + guard await confirmation() else { + throw ExecutionGateError.cancelledByUser(String(localized: "The change did not run.")) + } + } + + // A BEGIN on the session driver joins what a tab left open and its COMMIT ends that, so the DDL joins it instead. + // A trigger plan already chose its transactional strategy, so it skips the DDL check. + nonisolated static func wrapsSchemaChange( + on driver: DatabaseDriver, + isPooled: Bool, + checksTransactionalDDL: Bool = true + ) async -> Bool { + if checksTransactionalDDL, !wrapsDDLInTransaction(driver) { return false } + guard !isPooled else { return true } + return await driver.heldSessionTransactionState().permitsAppTransaction + } +} diff --git a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift index ad1ba9c387..fc16b57818 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar /// Where an operation bound to a `DatabaseScope` runs. enum ScopedDriverRoute: Equatable { @@ -23,8 +24,8 @@ enum ScopedDriverRoute: Equatable { } /// What a table read's turn on the session driver came to. -private enum TableReadTurn: Sendable { - case ran(T) +private enum TableReadTurn: Sendable { + case ran(any Sendable) /// The route decided once the turn came, which is never the session driver. case moved(ScopedDriverRoute) } @@ -76,16 +77,15 @@ extension DatabaseManager { else { return .sessionDriver } - /// A database the user has browsed keeps its own session connection, so its tabs carry on in - /// the transaction, temp tables and settings they left there. - /// A database whose connection ended with a transaction still open takes the session path - /// once more, where that loss is reported before anything runs on a replacement. + // A database the user has browsed keeps its own session connection, so its tabs carry on in the transaction, + // temp tables and settings they left there. A database whose connection ended holding a transaction or session + // state takes the session path once more, where that loss is reported before anything runs on a replacement. if usesDatabaseLanes(session) { if sessionLanes.parkedDriver(for: scope.connectionId, database: scope.database) != nil { return .sessionDriver } - // A table read depends on no transaction, so a lost one is not its to hear about. - if opensLane, sessionLanes.hasTransactionLoss(database: scope.database, for: scope.connectionId) { + // A table read depends on no transaction or session state, so a lost one is not its to hear about. + if opensLane, sessionLanes.hasLoss(database: scope.database, for: scope.connectionId) { return .sessionDriver } // A temporary table lives in the session that made it, which a new connection is not. @@ -106,18 +106,42 @@ extension DatabaseManager { return .pooled } - /// `cancellation` decides whether Stop, or a navigation that supersedes a tab, can reach the - /// leased driver. The parameter has no default on purpose: a new call site must state which it - /// is, because getting it wrong silently either makes a query uncancellable or makes a commit - /// killable. + // `cancellation` decides whether Stop, or a navigation that supersedes a tab, can reach the leased driver. The + // parameter has no default on purpose: a new call site must state which it is, because getting it wrong silently + // either makes a query uncancellable or makes a commit killable. func withScopedDriver( scope: DatabaseScope, route: ScopedDriverRoute, workload: MetadataConnectionPool.Workload = .interactive, cancellation: DriverCancellationPolicy, ownsSessionState: Bool = true, + readsOnly: Bool = false, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { + try await Self.unboxed( + withBoxedScopedDriver( + scope: scope, + route: route, + workload: workload, + cancellation: cancellation, + ownsSessionState: ownsSessionState, + readsOnly: readsOnly + ) { driver in + try await body(driver) + } + ) + } + + // The lease chain under the generic entry points runs boxed, so it is compiled once, not once per result type. + private func withBoxedScopedDriver( + scope: DatabaseScope, + route: ScopedDriverRoute, + workload: MetadataConnectionPool.Workload, + cancellation: DriverCancellationPolicy, + ownsSessionState: Bool, + readsOnly: Bool, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> any Sendable + ) async throws -> any Sendable { let leased = trackedLease(for: scope.connectionId, cancellation: cancellation, body) switch route { case .unavailable(let message): @@ -127,8 +151,17 @@ extension DatabaseManager { scope: scope, workload: workload, leased ) case .sessionDriver: - return try await withPinnedSessionDriver(scope: scope, ownsSessionState: ownsSessionState, leased) + return try await withPinnedSessionDriver( + scope: scope, ownsSessionState: ownsSessionState, readsOnly: readsOnly, leased + ) + } + } + + nonisolated private static func unboxed(_ value: any Sendable) -> T { + guard let result = value as? T else { + preconditionFailure("A lease returned \(type(of: value)) where \(T.self) was expected") } + return result } // A read that stops on the server once the task awaiting it is cancelled. It runs under a lease owner of its own, @@ -138,17 +171,40 @@ extension DatabaseManager { route: ScopedDriverRoute, workload: MetadataConnectionPool.Workload = .interactive, ownsSessionState: Bool = true, + readsOnly: Bool = false, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { + try await Self.unboxed( + withBoxedCancellableRead( + scope: scope, + route: route, + workload: workload, + ownsSessionState: ownsSessionState, + readsOnly: readsOnly + ) { driver in + try await body(driver) + } + ) + } + + private func withBoxedCancellableRead( + scope: DatabaseScope, + route: ScopedDriverRoute, + workload: MetadataConnectionPool.Workload, + ownsSessionState: Bool, + readsOnly: Bool, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> any Sendable + ) async throws -> any Sendable { let owner = DriverLeaseOwner() let connectionId = scope.connectionId return try await withTaskCancellationHandler { - try await withScopedDriver( + try await withBoxedScopedDriver( scope: scope, route: route, workload: workload, cancellation: .cancellableRead(owner), ownsSessionState: ownsSessionState, + readsOnly: readsOnly, body ) } onCancel: { @@ -173,7 +229,7 @@ extension DatabaseManager { guard rowCountRunsWithPage(scope, filtered: filtered) else { return try await withMetadataDriver(scope: scope, workload: .bulk, body) } - return try await withCancellableRead(scope: scope, route: tableReadRoute(for: scope), body) + return try await withCancellableRead(scope: scope, route: tableReadRoute(for: scope), readsOnly: true, body) } /// A table tab's read is a SELECT the app built from the tab's own table, so it depends on @@ -192,6 +248,18 @@ extension DatabaseManager { cancellation: DriverCancellationPolicy, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { + try await Self.unboxed( + withBoxedTableReadDriver(scope: scope, cancellation: cancellation) { driver in + try await body(driver) + } + ) + } + + private func withBoxedTableReadDriver( + scope: DatabaseScope, + cancellation: DriverCancellationPolicy, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> any Sendable + ) async throws -> any Sendable { let leased = trackedLease(for: scope.connectionId, cancellation: cancellation, body) var route = tableReadRoute(for: scope) while true { @@ -372,9 +440,10 @@ extension DatabaseManager { private func withPinnedSessionDriver( scope: DatabaseScope, ownsSessionState: Bool, + readsOnly: Bool, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { - try await withSessionDriverTurn(scope: scope, ownsSessionState: ownsSessionState) { driver in + try await withSessionDriverTurn(scope: scope, ownsSessionState: ownsSessionState, readsOnly: readsOnly) { driver in try await pin(driver, to: scope) return try await body(driver) } @@ -383,11 +452,11 @@ extension DatabaseManager { /// The route is asked again through `tableReadRoute` itself, so it reads the same inputs the /// caller's first answer did, the browsed database among them. The driver's own connection is no /// substitute: it carries the database name the connection resolved to, not the one browsed. - private func withTableReadSessionDriver( + private func withTableReadSessionDriver( scope: DatabaseScope, - _ body: @Sendable @escaping (DatabaseDriver) async throws -> T - ) async throws -> TableReadTurn { - try await withSessionDriverTurn(scope: scope, ownsSessionState: false) { driver in + _ body: @Sendable @escaping (DatabaseDriver) async throws -> any Sendable + ) async throws -> TableReadTurn { + try await withSessionDriverTurn(scope: scope, ownsSessionState: false, readsOnly: true) { driver in let route = tableReadRoute(for: scope) guard route == .sessionDriver else { return .moved(route) } try await pin(driver, to: scope) @@ -395,31 +464,32 @@ extension DatabaseManager { } } - /// `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the - /// session holds. Such a turn neither opens a session connection nor takes the report of a lost - /// transaction, which is kept for the user's next statement. + // `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the session holds. Such + // a turn neither opens a session connection nor takes the report of a lost session, which is kept for the next + // turn that owns session state. That turn is refused once with the whole loss, whatever work it is. private func withSessionDriverTurn( scope: DatabaseScope, ownsSessionState: Bool, + readsOnly: Bool = false, _ turn: (DatabaseDriver) async throws -> R ) async throws -> R { let connectionId = scope.connectionId let lane = activeSessions[connectionId].flatMap { laneDatabase(for: scope, in: $0) } let startsOnBrowsed = lane == nil || lane == activeSessions[connectionId]?.resolvedBrowseDatabase - /// Decided before the turn is queued. Work written against a connection that was there when - /// it queued and is gone when its turn comes is refused, never moved onto a new one: the - /// transaction it meant to continue ended with the connection it was on. + // Decided before the turn is queued. Work written against a connection that was there when it queued and is + // gone when its turn comes is refused, never moved onto a new one: the transaction it meant to continue ended + // with the connection it was on. let startsWithoutLane = !startsOnBrowsed && !scope.isSessionBound && lane.flatMap { sessionLanes.parkedDriver(for: connectionId, database: $0) } == nil if startsOnBrowsed { - /// Outside the gate on purpose. A verification that has to reconnect runs the whole - /// reconnect, which restores the schema and the database on the new driver, and doing that - /// while holding the gate would deadlock the very thing waiting to be pinned. + // Outside the gate on purpose. A verification that has to reconnect runs the whole reconnect, which + // restores the schema and the database on the new driver, and doing that while holding the gate would + // deadlock the very thing waiting to be pinned. await verifyBeforeUse(connectionId) } - /// A check that failed and could not recover left the driver installed and disconnected, - /// so the presence of a driver below is not enough. Refusing here is the point of checking - /// at all: without it the user's own work runs on a handle the app already knows is dead. + // A check that failed and could not recover left the driver installed and disconnected, so the presence of a + // driver below is not enough. Refusing here is the point of checking at all: without it the user's own work + // runs on a handle the app already knows is dead. guard isUsable(connectionId) else { throw DatabaseError.notConnected } @@ -428,21 +498,23 @@ extension DatabaseManager { ) { try await trackOperation(sessionId: connectionId) { try Task.checkCancellation() - /// Asked again once the lease has its turn, because a database switch that held the - /// gate can have left the driver the same way while this lease waited, or moved the - /// database this turn belongs to from the browsed connection to a parked one. - /// A database whose own connection is gone, its entry closed, falls back to the browsed - /// driver only to be turned away: a table read re-routes to the pool and anything else - /// is refused by `pin` before it runs a statement there. + // Asked again once the lease has its turn, because a database switch that held the gate can have left + // the driver the same way while this lease waited, or moved the database this turn belongs to from the + // browsed connection to a parked one. A database whose own connection is gone, its entry closed, falls + // back to the browsed driver only to be turned away: a table read re-routes to the pool and anything + // else is refused by `pin` before it runs a statement there. guard isUsable(connectionId), let browsedDriver = activeSessions[connectionId]?.driver else { throw DatabaseError.notConnected } guard let lane else { - // A driver replaced while it held a transaction, by a reconnect or a Vault lease rotation. - if ownsSessionState, let lost = sessionLanes.takeAnyTransactionLoss(for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lost)) + // A driver replaced while it held a transaction or session state, by a reconnect or a Vault lease + // rotation. + if ownsSessionState, let lost = sessionLanes.takeAnyLoss(for: connectionId) { + throw DatabaseError.connectionFailed(Self.sessionLostMessage(database: lost.database, loss: lost.loss)) } - return try await recordingTransaction(on: browsedDriver, for: .init(connectionId: connectionId, lane: nil)) { + return try await recordingTransaction( + on: browsedDriver, for: .init(connectionId: connectionId, lane: nil), readsOnly: readsOnly + ) { try await turn(browsedDriver) } } @@ -451,7 +523,7 @@ extension DatabaseManager { driver = held } else if ownsSessionState, !scope.isSessionBound, startsWithoutLane || sessionLanes.wasClosedIdle(database: lane, for: connectionId), - !sessionLanes.hasTransactionLoss(database: lane, for: connectionId) { + !sessionLanes.hasLoss(database: lane, for: connectionId) { driver = try await openedLaneDriver(database: lane, for: connectionId) } else { driver = browsedDriver @@ -459,12 +531,14 @@ extension DatabaseManager { let laneDriver = sessionLanes.isParked(driver, for: connectionId) ? try await verifiedParkedDriver(driver, database: lane, for: connectionId) : driver - if ownsSessionState, sessionLanes.takeTransactionLoss(database: lane, for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lane)) + if ownsSessionState, let loss = sessionLanes.takeLoss(database: lane, for: connectionId) { + throw DatabaseError.connectionFailed(Self.sessionLostMessage(database: lane, loss: loss)) } sessionLanes.noteOpenedUse(database: lane, for: connectionId) defer { sessionLanes.noteOpenedUse(database: lane, for: connectionId) } - return try await recordingTransaction(on: laneDriver, for: .init(connectionId: connectionId, lane: lane)) { + return try await recordingTransaction( + on: laneDriver, for: .init(connectionId: connectionId, lane: lane), readsOnly: readsOnly + ) { try await turn(laneDriver) } } @@ -477,17 +551,20 @@ extension DatabaseManager { private func recordingTransaction( on driver: DatabaseDriver, for key: SessionTransactionTracker.Key, + readsOnly: Bool, _ work: () async throws -> R ) async throws -> R { guard driver.supportsTransactions else { return try await work() } + let turnHold = ManualCommitTurnHold() let outcome: Result do { - outcome = .success(try await work()) + outcome = .success(try await ManualCommit.$turnHold.withValue(turnHold) { try await work() }) } catch { outcome = .failure(error) } - let state = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) - sessionTransactions.record(state, on: driver, for: key) + let read = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) + let state = ManualCommit.recordedState(read: read, turnOpenedHold: turnHold.isHeld) + sessionTransactions.record(state, on: driver, for: key, readsOnly: readsOnly) return try outcome.get() } @@ -514,6 +591,12 @@ extension DatabaseManager { return scope.isServerScoped ? session.resolvedBrowseDatabase : scope.database } + // The key a scope's turn records its transaction under, which the session state ledger shares. + internal func sessionKey(for scope: DatabaseScope) -> SessionTransactionTracker.Key? { + guard let session = activeSessions[scope.connectionId] else { return nil } + return SessionTransactionTracker.Key(connectionId: scope.connectionId, lane: laneDatabase(for: scope, in: session)) + } + internal func sessionDriver(holding scope: DatabaseScope) -> DatabaseDriver? { guard let session = activeSessions[scope.connectionId] else { return nil } return sessionDriver(for: scope.connectionId, laneDatabase: laneDatabase(for: scope, in: session)) @@ -528,18 +611,17 @@ extension DatabaseManager { return sessionLanes.parkedDriver(for: connectionId, database: laneDatabase) } - /// A parked connection nobody watched may have died while it sat idle. One that no longer - /// answers is replaced before the work runs, and if it was holding a transaction the work is - /// refused once, because running it on the new connection would carry on as if that transaction - /// had not just been rolled back by the server. + // A parked connection nobody watched may have died while it sat idle. One that no longer answers is replaced + // before the work runs, and if it held a transaction or session state the work is refused once, because running it + // on the new connection would carry on as if the server had not just dropped them. private func verifiedParkedDriver( _ driver: DatabaseDriver, database: String, for connectionId: UUID ) async throws -> DatabaseDriver { if sessionLanes.isFresh(driver) { return driver } - /// Read before the ping: a ping that finds the socket closed leaves libpq reporting no - /// transaction state at all, which would read as nothing to lose. + // Read before the ping: a ping that finds the socket closed leaves libpq reporting no transaction state at + // all, which would read as nothing to lose. let held = await driver.heldSessionTransactionState() let wasOpened = sessionLanes.isOpened(database: database, for: connectionId) sessionLanes.beginVerifying(driver) @@ -549,12 +631,15 @@ extension DatabaseManager { sessionLanes.markVerified(driver) return driver } catch { + // Recorded before the close, which prunes the ledger, and before the reopen, which can fail or be + // superseded: the session is gone either way, and whatever runs on this database next has to hear of it. + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) - /// Recorded before the reopen, which can fail or be superseded: the transaction is gone - /// either way, and whatever runs on this database next has to hear about it. - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } let generation = sessionLanes.generation(for: connectionId) let reopened = try await sessionLanes.open( DatabaseScope(connectionId: connectionId, database: database, schema: nil) @@ -617,10 +702,9 @@ extension DatabaseManager { throw DatabaseError.notConnected } - /// Closes the connections opened for SQL on databases the user never browsed that nobody has - /// used for `idleFor`. One that is running work or holds a transaction stays. Whatever else the - /// session held goes with it unreported: on PostgreSQL its temp tables, `SET` values and - /// advisory locks. + // Closes the connections opened for SQL on databases the user never browsed that nobody has used for `idleFor`. + // One that is running work, holds a transaction or holds session state the user's statements set up stays until + // disconnect, as a browsed one does. What a routine set up is out of sight and goes with the close. internal func closeIdleOpenedLanes( for connectionId: UUID, idleFor: TimeInterval = SessionLanes.openedIdleTimeout, @@ -631,8 +715,8 @@ extension DatabaseManager { } } - /// Asked inside the lane's own turn, so no statement runs between the answer and the close. Only - /// a driver that says it is idle is closed: one that cannot say may be holding a transaction. + // Asked inside the lane's own turn, so no statement runs between the answer and the close. Only a driver that says + // it is idle is closed: one that cannot say may be holding a transaction. private func closeOpenedLaneIfIdle( _ database: String, driver: DatabaseDriver, @@ -640,10 +724,12 @@ extension DatabaseManager { ) async -> Bool { let key = SessionDriverGate.Key(connectionId: connectionId, database: database) guard !sessionDriverGate.isHeld(key) else { return false } + let ledgerKey = SessionTransactionTracker.Key(connectionId: connectionId, lane: database) let closed = try? await sessionDriverGate.withExclusiveAccess(key) { () async -> Bool in guard sessionLanes.isOpened(database: database, for: connectionId), sessionLanes.parkedDriver(for: connectionId, database: database) === driver, - !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database) + !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database), + sessionStateLedger.state(for: ledgerKey, heldBy: driver).isEmpty else { return false } let state = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) guard state == .idle, diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 2e0bfa525e..c3d63a5b56 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -20,7 +20,8 @@ extension DatabaseManager { _ requestedConnection: DatabaseConnection, passwordOverride incomingPasswordOverride: String? = nil, sshPasswordOverride: String? = nil, - trigger: ConnectTrigger = .request + trigger: ConnectTrigger = .request, + adoptsSSHProfileEdits: Bool = false ) async throws { let connection = resolvedConnectionDefinition(for: requestedConnection) @@ -49,24 +50,17 @@ extension DatabaseManager { let effectiveQueryTimeoutSeconds = prepareConnectingSession(for: connection) lastActiveSessionId = connection.id - let resolvedConnection: DatabaseConnection - do { - // Before anything signs in: the host resolves some credentials below, and a plugin the rest in `connect()`. - try await localCredentialConsent.check(connection, trigger: trigger) - resolvedConnection = try await envVarReferenceConsent.resolve(connection, trigger: trigger) - try await preConnectScriptConsent.authorize(connection, trigger: trigger) - // The answer can take a minute, long enough for the person to cancel the attempt. - guard !isAttemptCancelled(attempt, for: connection.id) else { throw CancellationError() } - try await runPreConnectScriptIfNeeded(connection, resolvedConnection: resolvedConnection, attempt: attempt) - } catch { - finalizeConnectionFailure( - for: connection.id, - cancelled: isAttemptCancelled(attempt, for: connection.id), - error: error, - attempt: attempt - ) - throw error - } + let (routedConnection, pinnedSSHRoute) = try pinSSHRoute( + for: connection, + adoptsSSHProfileEdits: adoptsSSHProfileEdits, + attempt: attempt + ) + + let (resolvedConnection, dialledConnection) = try await authorizeConnect( + connection, + trigger: trigger, + attempt: attempt + ) let (passwordOverride, promptsForPassword, promptCacheKey) = try await resolvePasswordOverride( for: connection, @@ -87,8 +81,9 @@ extension DatabaseManager { reportStage(.resolvingTunnel, attempt: attempt, for: connection.id) } effectiveConnection = try await buildEffectiveConnectionReportingStages( - for: resolvedConnection, + for: dialledConnection, sshPasswordOverride: sshPasswordOverride, + pinnedSSHRoute: pinnedSSHRoute, deadline: deadline, attempt: (attempt, connection.id) ) @@ -136,7 +131,7 @@ extension DatabaseManager { attempt: attempt, for: connection.id ) - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: effectiveQueryTimeoutSeconds, startupCommands: resolvedConnection.startupCommands, @@ -161,13 +156,15 @@ extension DatabaseManager { // Batch all session mutations into a single write to fire objectWillChange once. if var session = activeSessions[connection.id] { session.connection = successfulConnectionRecord( - from: connection, + from: routedConnection, preservingSessionFieldsFrom: session.connection ) + dialedConnections[connection.id] = routedConnection + session.commitMode = session.connection.commitMode session.driver = driver session.status = driver.status session.effectiveConnection = effectiveConnection - session.connectionWarning = driver.connectionWarning + session.connectionWarning = openingWarning(driver: driver, startupFailures: startupFailures, for: connection.id) // A connect answers whatever went wrong before it, including a reconnect that gave up on this entry. session.liveness = .live session.recovery = nil @@ -175,10 +172,7 @@ extension DatabaseManager { markSessionVerified(connection.id) if let passwordOverride, !connection.usesIAMToken { session.cachedPassword = passwordOverride - /// Only a password that actually authenticated is shared with the other - /// connections on this profile. Caching the prompt's answer before the connect - /// meant one mistyped password locked every one of them out until a relaunch, - /// because the prompt never came back. + // Only once it authenticated: a mistyped answer shared with the profile locked every connection out. if promptsForPassword { promptedPasswords[promptCacheKey] = passwordOverride } @@ -186,7 +180,7 @@ extension DatabaseManager { setSession(session, for: connection.id) } - /// Before `finish`, so the clear is still the current attempt's to make. + // Before `finish`, so the clear is still the current attempt's to make. clearConnectionStage(for: connection.id) connectionAttempts.finish(attempt, for: connection.id) @@ -229,7 +223,7 @@ extension DatabaseManager { } // A session's answer is reused only while the connection still dials where it was typed for. let session = activeSessions[connection.id] - let sessionPassword = session?.connection.destination.digest == connection.destination.digest + let sessionPassword = session?.connection.destination.digest == connection.dialedDestination.digest ? session?.cachedPassword : nil if let cached = promptedPasswords[promptCacheKey] ?? sessionPassword { @@ -258,8 +252,8 @@ extension DatabaseManager { static func promptCacheKey(for connection: DatabaseConnection) -> PromptedPasswordKey { let sharedByProfile = connection.credentialMode.profileId != nil return PromptedPasswordKey( - owner: ConnectionCredentialResolver.promptCacheKey(for: connection), - destination: sharedByProfile ? "" : connection.destination.digest + owner: ConnectionCredentialResolver.credentialOwner(for: connection), + destination: sharedByProfile ? "" : connection.dialedDestination.digest ) } @@ -358,6 +352,7 @@ extension DatabaseManager { guard let stored = connectionStorage.loadConnection(id: connection.id) else { return connection } var resolved = connection resolved.preferredSafeModeLevel = stored.preferredSafeModeLevel + resolved.commitMode = stored.commitMode return resolved } @@ -587,21 +582,21 @@ extension DatabaseManager { ) } - /// The connection parked for `database`, if it still answers. One that died is closed, and if it - /// was holding a transaction, the next work on that database is told the server rolled it back. - /// A turn running on it right now is answer enough, and asking would wait for that turn. + // The connection parked for `database`, if it still answers. One that died is closed, and if it held a transaction + // or session state, the next work on that database is told what the server dropped. A turn running on it right now + // is answer enough, and asking would wait for that turn. private func answeringParkedDriver(database: String, for connectionId: UUID) async -> DatabaseDriver? { let key = SessionDriverGate.Key(connectionId: connectionId, database: database) guard var parked = sessionLanes.parkedDriver(for: connectionId, database: database) else { return nil } - /// A check already running decides for this switch too: once it ends it has either confirmed - /// the driver or parked a replacement, and that is what gets promoted. + // A check already running decides for this switch too: once it ends it has either confirmed the driver or + // parked a replacement, and that is what gets promoted. if sessionLanes.isVerifying(parked) { await sessionLanes.waitForVerification(of: parked) guard let settled = sessionLanes.parkedDriver(for: connectionId, database: database) else { return nil } parked = settled } if sessionLanes.isFresh(parked) || sessionDriverGate.isHeld(key) { return parked } - /// Read before the ping, which on a closed socket leaves libpq reporting no state at all. + // Read before the ping, which on a closed socket leaves libpq reporting no state at all. let held = await parked.heldSessionTransactionState() do { try await sessionDriverGate.withExclusiveAccess(key) { @@ -611,10 +606,13 @@ extension DatabaseManager { return parked } catch { guard sessionLanes.parkedDriver(for: connectionId, database: database) === parked else { return nil } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: parked, for: connectionId), + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } return nil } } @@ -658,6 +656,8 @@ extension DatabaseManager { guard let previous = session(for: connectionId) else { throw DatabaseError.notConnected } + // Refused before anything is torn down, so the session stays on the server it dialed. + _ = try routedThroughLinkedProfile(previous.connection, pinnedSSHRoute: previous.connection.destinationTunnel) // The reconnect would end a live transaction; one on a connection that already stopped answering is gone anyway. if previous.liveness == .live, await browsedDriverHoldsTransaction(previous.driver, for: connectionId) { throw DatabaseError.connectionFailed( @@ -670,7 +670,7 @@ extension DatabaseManager { let previousSavedSchema = appSettingsStorage.loadLastSchema(for: connectionId) // Noted under the database being left, before the session names the target. - noteTransactionLost(with: previous.driver, for: connectionId) + noteSessionLost(with: previous.driver, for: connectionId, includingState: false) updateSession(connectionId) { session in session.connection.database = database session.browseDatabase = database @@ -785,6 +785,8 @@ extension DatabaseManager { /// first: `MainContentCoordinator.teardown()` clears them from memory and only the window-close /// path saves on its way out, which is how a disconnect used to take a window's tabs with it. func disconnectSession(_ sessionId: UUID, origin: SessionDisconnectOrigin = .appManaged) async { + // Before the lookup: closing the window of a connect that failed has no session to end, only the route. + dialedConnections[sessionId] = nil let lifecycleLogger = Logger(subsystem: "com.TablePro", category: "NativeTabLifecycle") guard let session = activeSessions[sessionId] else { lifecycleLogger.info( @@ -843,6 +845,7 @@ extension DatabaseManager { "[close] disconnectSession driver.disconnect done connId=\(sessionId, privacy: .public) elapsedMs=\(Int(Date().timeIntervalSince(driverStart) * 1_000))" ) removeSessionEntry(for: sessionId) + VaultDatabaseCredentials.shared.endUse(connection: sessionId, live: Set(activeSessions.keys)) await SchemaService.shared.invalidate(connectionId: sessionId) await DatabaseTreeMetadataService.shared.handleDisconnect(connectionId: sessionId) @@ -926,6 +929,7 @@ extension DatabaseManager { let stored = connectionStorage.loadConnection(id: id) else { continue } adoptDisplayFields(from: stored, into: session, for: id) setSafeModeLevel(stored.preferredSafeModeLevel, for: id) + setCommitMode(stored.commitMode, for: id) } } @@ -959,6 +963,7 @@ extension DatabaseManager { ) -> DatabaseConnection { var adopted = connection(attempted, adoptingDisplayFieldsFrom: current) adopted.preferredSafeModeLevel = current.preferredSafeModeLevel + adopted.commitMode = current.commitMode return adopted } @@ -1014,6 +1019,22 @@ extension DatabaseManager { _ = connectionStorage.updateSafeModeLevel(level, for: connectionId) } + // Written to the connection, as the Safe Mode level is, so the form and the Database menu are one setting. + func setCommitMode(_ mode: ConnectionCommitMode, for connectionId: UUID) { + guard var session = activeSessions[connectionId] else { return } + guard session.connection.commitMode != mode || session.commitMode != mode else { return } + session.connection.commitMode = mode + session.commitMode = mode + setSession(session, for: connectionId) + connectionStorage.mutateConnections(ids: [connectionId]) { $0.commitMode = mode } + } + + // What writes run under: a stored Manual Commit on an engine that can't hold runs as automatic. + func commitMode(for connectionId: UUID) -> ConnectionCommitMode { + guard let session = activeSessions[connectionId] else { return .automatic } + return ManualCommit.effectiveMode(session.commitMode, databaseType: session.connection.type) + } + internal func setSession(_ session: ConnectionSession, for connectionId: UUID) { /// A session created while a window is already in Agent mode takes the level stored on the /// connection, which is the user's own and may be Silent. Applying the floor only when the @@ -1027,6 +1048,7 @@ extension DatabaseManager { activeSessions[connectionId] = session connectionStatusVersions[connectionId, default: 0] &+= 1 sessionTransactions.prune(connectionId) + sessionStateLedger.prune(connectionId) AppEvents.shared.connectionStatusChanged.send( ConnectionStatusChange(connectionId: connectionId, status: session.reportedStatus) ) @@ -1083,10 +1105,79 @@ extension DatabaseManager { connectionStages.removeValue(forKey: connectionId) } + // What a connect settles before it dials, in order: approvals for this Mac's credentials, `${VAR}` references + // and the pre-connect script, the script itself, then the RDS endpoint a CNAME stands for, which can ask again. + // All of it runs before the connect's deadline starts, since each step can wait on a person. + private func authorizeConnect( + _ connection: DatabaseConnection, + trigger: ConnectTrigger, + attempt: Int + ) async throws -> (resolved: DatabaseConnection, dialled: DatabaseConnection) { + do { + // Before anything signs in: the host resolves some credentials below, and a plugin the rest in `connect()`. + try await localCredentialConsent.check(connection, trigger: trigger) + let resolved = try await envVarReferenceConsent.resolve(connection, trigger: trigger) + try await preConnectScriptConsent.authorize(connection, trigger: trigger) + // The answer can take a minute, long enough for the person to cancel the attempt. + guard !isAttemptCancelled(attempt, for: connection.id) else { throw CancellationError() } + try await runPreConnectScriptIfNeeded(connection, resolvedConnection: resolved, attempt: attempt) + // After the script, which can be what makes the name resolve, such as a VPN it brings up. + let dialled = try await resolvingRDSEndpointAlias(of: resolved, trigger: trigger) + guard !isAttemptCancelled(attempt, for: connection.id) else { throw CancellationError() } + return (resolved, dialled) + } catch { + finalizeConnectionFailure( + for: connection.id, + cancelled: isAttemptCancelled(attempt, for: connection.id), + error: error, + attempt: attempt + ) + throw error + } + } + + // Read once, before any approval is asked for, so the approvals and the dial name one route. A session that + // already dialed keeps its route under its window; only that window's Connect takes up a profile edit. + private func pinSSHRoute( + for connection: DatabaseConnection, + adoptsSSHProfileEdits: Bool, + attempt: Int + ) throws -> (routed: DatabaseConnection, pinned: ConnectionDestination.Tunnel) { + let routed = SSHProfileStorage.shared.refreshingLinkedProfile(connection) + let kept = adoptsSSHProfileEdits + ? nil + : activeSessions[connection.id].flatMap { $0.effectiveConnection == nil ? nil : $0.connection } + ?? dialedConnections[connection.id] + let pinned = kept?.destinationTunnel ?? routed.destinationTunnel + // A deleted profile leaves its last server inline, so a session dialed through one is checked as well. + guard Self.followsSSHProfile(routed) || kept.map(Self.followsSSHProfile) == true, + routed.destinationTunnel != pinned + else { return (routed, pinned) } + let error = ConnectionTunnelError.sshProfileMoved + if connectionAttempts.isCurrent(attempt, for: connection.id) { + clearConnectionStage(for: connection.id) + } + // A session that dialed stays, unreachable, under its window; an entry this attempt made goes. + if let driver = activeSessions[connection.id]?.driver { + let info = ConnectionFailureClassifier.info(for: error) + updateSession(connection.id) { $0.status = .error(info.message) } + markSessionUnreachable(connection.id, startedWith: driver, info: info) + } else { + finalizeConnectionFailure(for: connection.id, cancelled: false, error: error, attempt: attempt) + } + throw error + } + + private static func followsSSHProfile(_ connection: DatabaseConnection) -> Bool { + if case .profile = connection.sshTunnelMode { return true } + return false + } + // The proxy reports its download as a stage of this attempt. private func buildEffectiveConnectionReportingStages( for resolvedConnection: DatabaseConnection, sshPasswordOverride: String?, + pinnedSSHRoute: ConnectionDestination.Tunnel, deadline: ConnectionDeadline, attempt: (number: Int, connectionId: UUID) ) async throws -> DatabaseConnection { @@ -1096,6 +1187,7 @@ extension DatabaseManager { try await buildEffectiveConnection( for: resolvedConnection, sshPasswordOverride: sshPasswordOverride, + pinnedSSHRoute: pinnedSSHRoute, deadline: deadline, ownsAttempt: { !self.isAttemptCancelled(attempt.number, for: attempt.connectionId) } ) @@ -1172,6 +1264,7 @@ extension DatabaseManager { monitorPingedDrivers.removeValue(forKey: connectionId) sessionDriverGate.drain(connectionId: connectionId) connectionStatusVersions.removeValue(forKey: connectionId) + reportedStartupFailures.removeValue(forKey: connectionId) forgetVerification(for: connectionId) aiAccessApprovals.revoke(connectionId) AppEvents.shared.connectionStatusChanged.send( diff --git a/apple/TablePro/Core/Database/DatabaseManager+Startup.swift b/apple/TablePro/Core/Database/DatabaseManager+Startup.swift index 5f68e9c7a7..9f76d161dc 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Startup.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Startup.swift @@ -61,26 +61,119 @@ extension DatabaseManager { return statements } - @discardableResult nonisolated internal func executeStartupCommands( _ commands: String?, on driver: DatabaseDriver, connectionName: String - ) async -> [(statement: String, error: String)] { - var failures: [(statement: String, error: String)] = [] + ) async -> [StartupCommandFailure] { + var failures: [StartupCommandFailure] = [] let databaseType = driver.connection.type let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution for statement in Self.startupStatements(from: commands, databaseType: databaseType, grammar: grammar) { do { - _ = try await driver.execute(query: statement) + // Teradata ANSI keeps `DATABASE` pending and takes only COMMIT after it; a failed commit fails the statement. + _ = try await AppDefinitionCommit.run(on: driver, refusesOpenTransaction: false) { + try await driver.execute(query: statement) + } Self.startupLogger.info( - "Startup command succeeded for '\(connectionName)': \(statement)" + "Startup command succeeded for '\(connectionName, privacy: .private)': \(statement, privacy: .private)" ) } catch { Self.startupLogger.warning( - "Startup command failed for '\(connectionName)': \(statement) — \(error.localizedDescription)" + "Startup command failed for '\(connectionName, privacy: .private)': \(statement, privacy: .private): \(error.localizedDescription, privacy: .private)" ) - failures.append((statement: statement, error: error.localizedDescription)) + failures.append(StartupCommandFailure(statement: statement, error: error)) } } return failures } + + // A startup command can be a safety rail, `SET ROLE app_ro` or a read-only default, so a session that opened + // without one says so on the connection's warning rather than only in the log. + nonisolated internal static func startupWarning(for failures: [StartupCommandFailure]) -> String? { + guard let first = failures.first else { return nil } + guard failures.count > 1 else { + guard let code = first.code else { + return String( + format: String(localized: "The startup command %@ failed, so this session runs without it."), + shownStatement(first.statement) + ) + } + return String( + format: String(localized: "The startup command %1$@ failed (%2$@), so this session runs without it."), + shownStatement(first.statement), code + ) + } + let lines = failures.map { failure in + failure.code.map { "\(shownStatement(failure.statement)) (\($0))" } ?? shownStatement(failure.statement) + } + return String( + format: String(localized: "Startup commands failed, so this session runs without them:\n%@"), + lines.joined(separator: "\n") + ) + } + + // A startup command can carry a secret, a password after IDENTIFIED BY or a key in a literal, so the banner shows + // it only up to its first quote or credential clause. + nonisolated internal static func shownStatement(_ statement: String) -> String { + let quote = statement.firstIndex(where: { $0 == "'" || $0 == "\"" || $0 == "$" }) + let stops = [quote] + ["IDENTIFIED", "PASSWORD"].map { statement.range(of: $0, options: .caseInsensitive)?.lowerBound } + let cut = stops.compactMap { $0 }.min() ?? statement.endIndex + let kept = statement[.. limit else { return kept } + return String(kept.prefix(limit)) + "…" + } + + nonisolated internal static func joinedWarning(_ parts: [String?]) -> String? { + let present = parts.compactMap { $0 }.filter { !$0.isEmpty } + return present.isEmpty ? nil : present.joined(separator: "\n\n") + } + + // The warning a session opens with: the login's own, then what its setup could not do. It replaces the previous + // one, so a reconnect that ran every startup command drops a banner about the session before it, and the record + // of named failures starts over. + internal func openingWarning( + driver: DatabaseDriver, + startupFailures: [StartupCommandFailure], + restoreWarning: String? = nil, + for connectionId: UUID + ) -> String? { + reportedStartupFailures[connectionId] = Set(startupFailures.map { $0.statement }) + return Self.joinedWarning([driver.connectionWarning, Self.startupWarning(for: startupFailures), restoreWarning]) + } + + // A pool entry or a database lane opens a session of its own, which can fail a command the window's session ran. + // Each failing statement is named once per connect: the pool keeps opening entries while the window is open. + internal func reportStartupFailures(_ failures: [StartupCommandFailure], for connectionId: UUID) { + guard activeSessions[connectionId] != nil else { return } + var reported = reportedStartupFailures[connectionId] ?? [] + let unreported = failures.filter { reported.insert($0.statement).inserted } + reportedStartupFailures[connectionId] = reported + guard let warning = Self.startupWarning(for: unreported) else { return } + updateSession(connectionId) { session in + session.connectionWarning = Self.joinedWarning([session.connectionWarning, warning]) + } + } +} + +// What the banner may show of a failed startup command: the statement and the server's code, never its message, which +// can echo the values the statement carried. The message goes to the log, private. +struct StartupCommandFailure: Sendable, Equatable { + let statement: String + let code: String? + + init(statement: String, code: String?) { + self.statement = statement + self.code = code + } + + init(statement: String, error: Error) { + self.init(statement: statement, code: Self.code(of: error)) + } + + static func code(of error: Error) -> String? { + guard let driverError = error as? any PluginDriverError else { return nil } + if let state = driverError.pluginSqlState, !state.isEmpty { return state } + if let code = driverError.pluginErrorCode, code != 0 { return String(code) } + return nil + } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift b/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift index 93aead5136..7ab723f1bd 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift @@ -74,6 +74,7 @@ extension DatabaseManager { type: connection.type, sshConfig: SSHConfiguration(), sslConfig: tunnelSSL, + credentialMode: connection.credentialMode, passwordSource: connection.passwordSource, additionalFields: effectiveFields ) diff --git a/apple/TablePro/Core/Database/DatabaseManager.swift b/apple/TablePro/Core/Database/DatabaseManager.swift index 15072aca99..4b8501ee18 100644 --- a/apple/TablePro/Core/Database/DatabaseManager.swift +++ b/apple/TablePro/Core/Database/DatabaseManager.swift @@ -9,6 +9,7 @@ import Combine import Foundation import os import TableProPluginKit +import TableProSQLGrammar /// Manages database connections and active drivers @MainActor @@ -25,6 +26,7 @@ final class DatabaseManager: ObservableObject { internal let localCredentialConsent: LocalCredentialConsent internal let postDisconnectScripts: PostDisconnectScripts internal let localNetworkProbe: any LocalNetworkAccessProbing + internal let cnameResolver: any CNAMEResolving internal var historyRecorder: QueryHistoryRecording = QueryHistoryManager.shared // Keyed by the credential profile, or by the connection and where it dials: a linked-folder row keeps its id @@ -118,6 +120,10 @@ final class DatabaseManager: ObservableObject { /// before touching shared session state and discards its driver when it lost. internal var connectionAttempts = ConnectionAttemptRegistry() + // What each connection last dialed, kept through a failed connect so an unattended one cannot follow an SSH profile + // edit under the window; a disconnect ends it, and the window's own Connect replaces it. + internal var dialedConnections: [UUID: DatabaseConnection] = [:] + /// The step each in-flight connect last reported, so a window that joins one already running /// can seed itself. `AppEvents.connectionStageChanged` is a `PassthroughSubject`, so it holds /// nothing: an observer built after a step was sent could only report the generic fallback, @@ -126,6 +132,10 @@ final class DatabaseManager: ObservableObject { /// for the same reason every other shared write here is generation-checked. internal var connectionStages: [UUID: ConnectionStage] = [:] + // The startup statements a connection's warning has already named since its connect, so a pool entry or lane + // opened later names only a failure the window was not told about. + internal var reportedStartupFailures: [UUID: Set] = [:] + /// Orders operations that move the shared driver, so two windows cannot interleave /// their pins and each run against the other's database. internal let sessionDriverGate = SessionDriverGate() @@ -138,6 +148,11 @@ final class DatabaseManager: ObservableObject { self?.isLiveSessionDriver(driver, for: key) ?? false } + // What the user's statements set up on each session connection, read before a reconnect closes one. + internal lazy var sessionStateLedger = SessionStateLedger { [weak self] key, driver in + self?.isLiveSessionDriver(driver, for: key) ?? false + } + /// The driver each connection's health monitor last pinged, so the reconnect that follows a /// failed ping acts only on that driver and never on one a database switch put in its place. /// Held, not just identified: a closed driver that was freed would hand its address, and with it @@ -191,6 +206,7 @@ final class DatabaseManager: ObservableObject { localCredentialConsent: LocalCredentialConsent = LocalCredentialConsent(), postDisconnectScripts: PostDisconnectScripts = PostDisconnectScripts(), localNetworkProbe: any LocalNetworkAccessProbing = NetworkFrameworkLocalNetworkProbe(), + cnameResolver: any CNAMEResolving = SystemCNAMEResolver(), recoveryClock: any Clock = ContinuousClock() ) { self.connectionStorage = connectionStorage @@ -202,9 +218,14 @@ final class DatabaseManager: ObservableObject { self.localCredentialConsent = localCredentialConsent self.postDisconnectScripts = postDisconnectScripts self.localNetworkProbe = localNetworkProbe + self.cnameResolver = cnameResolver self.recoveryClock = recoveryClock sessionLanes.onClose = { [weak self] connectionId in self?.sessionTransactions.prune(connectionId) + self?.sessionStateLedger.prune(connectionId) + } + sessionLanes.recordedState = { [weak self] driver, connectionId in + self?.sessionStateLedger.state(heldBy: driver, for: connectionId) ?? [] } observeConnectionUpdates() observeHealthCheckSetting() diff --git a/apple/TablePro/Core/Database/EnvVarReferenceConsent.swift b/apple/TablePro/Core/Database/EnvVarReferenceConsent.swift index 49fe876501..96c2aca3b1 100644 --- a/apple/TablePro/Core/Database/EnvVarReferenceConsent.swift +++ b/apple/TablePro/Core/Database/EnvVarReferenceConsent.swift @@ -96,7 +96,7 @@ internal final class EnvVarReferenceConsent { EnvVarReferenceQuestion( connectionName: connection.name, names: EnvVarResolver.referencedNames(in: connection).sorted(), - destination: connection.destinationSummary, + destination: SSHProfileStorage.shared.refreshingLinkedProfile(connection).destinationSummary, origin: connection.origin ) } diff --git a/apple/TablePro/Core/Database/LocalCredentialConsent.swift b/apple/TablePro/Core/Database/LocalCredentialConsent.swift index a63b8a5d28..bddb50eca7 100644 --- a/apple/TablePro/Core/Database/LocalCredentialConsent.swift +++ b/apple/TablePro/Core/Database/LocalCredentialConsent.swift @@ -94,7 +94,7 @@ internal final class LocalCredentialConsent { uses: ConnectionFieldCatalog.localCredentialUses(in: connection.additionalFields).map { use in describe(use, fields: fields) }, - destination: connection.destinationSummary, + destination: SSHProfileStorage.shared.refreshingLinkedProfile(connection).destinationSummary, origin: connection.origin ) } diff --git a/apple/TablePro/Core/Database/NativeDumpDescriptor.swift b/apple/TablePro/Core/Database/NativeDumpDescriptor.swift index 2d93bf3683..c48f7f9e47 100644 --- a/apple/TablePro/Core/Database/NativeDumpDescriptor.swift +++ b/apple/TablePro/Core/Database/NativeDumpDescriptor.swift @@ -6,34 +6,26 @@ import Foundation import TableProPluginKit -/// Everything one engine's own dump and restore need, as data. -/// -/// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter -/// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` -/// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, `sqlite3` -/// needs no network arguments at all, and DuckDB runs no binary because its dump is a statement its -/// own in-process engine executes. So a descriptor names its mechanism and supplies its own -/// arguments or statements, rather than filling slots in one shared command shape. +// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter +// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` +// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, `sqlite3` +// needs no network arguments at all, and DuckDB runs no binary because its dump is a statement its +// own in-process engine executes. So a descriptor names its mechanism and supplies its own +// arguments or statements, rather than filling slots in one shared command shape. struct NativeDumpDescriptor: Sendable { - /// How the tool hands back what it produced. This is the difference that no shared argument - /// list can paper over. enum OutputDelivery: Sendable, Equatable { - /// The tool is told where to write, and writes there itself. case toolWritesFile - /// The tool writes to standard output and the caller redirects it to the file. case standardOutput } - /// What the caller offers as a file name, and what a restore will accept back. - /// - /// `id` exists because one engine can offer two: DuckDB writes either a single `.duckdb` file - /// or a directory of Parquet, and the sheet has to name which one it is about to write. + // `id` exists because one engine can offer two: DuckDB writes either a single `.duckdb` file + // or a directory of Parquet, and the sheet has to name which one it is about to write. struct ArchiveFormat: Sendable, Equatable, Identifiable { let id: String let fileExtension: String let contentDescription: String - /// A destination the user chooses with an open panel rather than a save panel, because the - /// engine writes a folder of files rather than one file. + // A destination the user chooses with an open panel rather than a save panel, because the + // engine writes a folder of files rather than one file. let producesDirectory: Bool init( @@ -56,25 +48,19 @@ struct NativeDumpDescriptor: Sendable { let password: String? let scope: NativeDumpScope - /// The engine's own name for the database in front of it, read from the live connection. - /// - /// Only DuckDB needs one, and it needs it badly: `COPY FROM DATABASE TO ` names - /// a catalog, and DuckDB derives that catalog from the file's basename. Measured, a file - /// `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` fails with - /// `Catalog "main" does not exist`. So neither the file name nor a constant will do. + // Only DuckDB needs one, and it needs it badly: `COPY FROM DATABASE TO ` names + // a catalog, and DuckDB derives that catalog from the file's basename. Measured, a file + // `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` fails with + // `Catalog "main" does not exist`. So neither the file name nor a constant will do. let currentCatalog: String? - /// The path a file-backed driver actually opens. - /// - /// SQLite keeps it in `database`, while DuckDB and libSQL keep it in a plugin-declared - /// additional field and leave `database` empty. Resolved by the caller through - /// `LocalFilePathField` rather than guessed here, because reading `connection.database` for - /// libSQL handed `sqlite3` an empty path, which exits 0 and writes a 52-byte file that the - /// result sheet then reports as a successful backup. + // SQLite keeps the file's path in `database`; DuckDB and libSQL keep it in a plugin-declared field and leave + // `database` empty. Resolved by the caller through `LocalFilePathField`: reading `connection.database` for libSQL + // handed `sqlite3` an empty path, which exits 0 and writes a 52-byte file the result sheet reports as a backup. let localFilePath: String? - /// What the live session's driver reports the server to be, so a tool can be given the - /// flags that server needs. `mysqldump` 8.0 reads a table no MariaDB server has. + // What the live session's driver reports the server to be, so a tool can be given the + // flags that server needs. `mysqldump` 8.0 reads a table no MariaDB server has. let serverVersion: String? // Whether the server's greeting offered TLS, for a tool whose SSL options depend on it. Nil when not read. @@ -83,6 +69,9 @@ struct NativeDumpDescriptor: Sendable { // Set by `NativeDumpService.buildCommand` before the arguments are built. var clientPluginFolder: MariaDBClientPluginFolder? + // Set by `NativeDumpService` when the tool's `needsSystemTrustAnchors` says so, and removed when the tool exits. + var systemTrustAnchorsFile: URL? + init( connection: DatabaseConnection, database: String, @@ -120,25 +109,24 @@ struct NativeDumpDescriptor: Sendable { let value: String } - /// A tool on the user's Mac that the app spawns. struct CommandLineTool: Sendable { - /// The candidate names, in the order they are tried. More than one because a tool can ship - /// under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old - /// name only as a symlink that some builds omit. + // The candidate names, in the order they are tried. More than one because a tool can ship + // under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old + // name only as a symlink that some builds omit. let backupBinaries: [String] let restoreBinaries: [String] - /// What to tell the user to install when neither name resolves. There is no portable - /// answer, so each engine names its own package. + // What to tell the user to install when neither name resolves. There is no portable + // answer, so each engine names its own package. let installHint: String let backupDelivery: OutputDelivery let restoreDelivery: OutputDelivery - /// True when the tool has no channel for a password but its own argument list, which every - /// process on the machine can read through `ps`. Only `sqlpackage` is in that position; - /// every other engine here takes one from the environment or a file. The flows that touch - /// such a tool say so rather than leaving the user to find out. + // True when the tool has no channel for a password but its own argument list, which every + // process on the machine can read through `ps`. Only `sqlpackage` is in that position; + // every other engine here takes one from the environment or a file. The flows that touch + // such a tool say so rather than leaving the user to find out. let exposesPasswordInArguments: Bool let configurationFileEntries: @Sendable (Request) -> [ConfigurationEntry] @@ -147,14 +135,14 @@ struct NativeDumpDescriptor: Sendable { internal let requiresUntranslatedMessages: Bool - /// Picks the binary to run for a known server version, when the engine's tools refuse - /// some servers. Nil leaves the plain PATH lookup in place. + // Picks the binary to run for a known server version, when the engine's tools refuse + // some servers. Nil leaves the plain PATH lookup in place. let toolForServer: (@Sendable (_ binary: String, _ serverVersion: String?) -> NativeDumpToolSelection)? - /// Says what the resolved binary actually is, for an engine whose tools forked their - /// options. It runs a process, so it runs once per resolution rather than once per - /// argument list. Nil leaves the tool unidentified, which is what every engine but MySQL - /// and MariaDB wants. + // Says what the resolved binary actually is, for an engine whose tools forked their + // options. It runs a process, so it runs once per resolution rather than once per + // argument list. Nil leaves the tool unidentified, which is what every engine but MySQL + // and MariaDB wants. let identifyExecutable: (@Sendable (_ name: String, _ path: String) -> NativeDumpResolvedTool)? let backupArguments: @Sendable (Request, NativeDumpResolvedTool) throws -> [String] @@ -166,6 +154,9 @@ struct NativeDumpDescriptor: Sendable { let clientPluginFolder: @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? + // True when the tool has no trust store of its own to check the server with, so it gets a file of this Mac's. + let needsSystemTrustAnchors: @Sendable (Request) -> Bool + init( backupBinaries: [String], restoreBinaries: [String], @@ -182,7 +173,8 @@ struct NativeDumpDescriptor: Sendable { environment: @escaping @Sendable (Request) -> [String: String] = { _ in [:] }, configurationFileEntries: @escaping @Sendable (Request) -> [ConfigurationEntry] = { _ in [] }, prepare: @escaping @Sendable (Request, NativeDumpResolvedTool) async throws -> Request = { request, _ in request }, - clientPluginFolder: @escaping @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? = { _, _ in nil } + clientPluginFolder: @escaping @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? = { _, _ in nil }, + needsSystemTrustAnchors: @escaping @Sendable (Request) -> Bool = { _ in false } ) { self.backupBinaries = backupBinaries self.restoreBinaries = restoreBinaries @@ -200,6 +192,7 @@ struct NativeDumpDescriptor: Sendable { self.configurationFileEntries = configurationFileEntries self.prepare = prepare self.clientPluginFolder = clientPluginFolder + self.needsSystemTrustAnchors = needsSystemTrustAnchors } func binaries(for kind: NativeDumpKind) -> [String] { @@ -214,8 +207,8 @@ struct NativeDumpDescriptor: Sendable { try kind == .backup ? backupArguments(request, resolved) : restoreArguments(request, resolved) } - /// What the app knows about the binary it resolved. An engine that declares no - /// `identifyExecutable` gets the plain answer, which is what its arguments already assume. + // What the app knows about the binary it resolved. An engine that declares no + // `identifyExecutable` gets the plain answer, which is what its arguments already assume. func identify(name: String, path: String) -> NativeDumpResolvedTool { identifyExecutable?(name, path) ?? NativeDumpResolvedTool(name: name, path: path) } @@ -229,19 +222,17 @@ struct NativeDumpDescriptor: Sendable { } } - /// Statements the engine already in front of the app runs for itself. - /// - /// There is no binary to find, no password to hand over and nothing to reap, which is why this - /// cannot be a `CommandLineTool` with an empty argument list. An embedded engine also has no - /// server to write on the user's behalf, so the file lands where the panel picked. + // There is no binary to find, no password to hand over and nothing to reap, which is why this + // cannot be a `CommandLineTool` with an empty argument list. An embedded engine also has no + // server to write on the user's behalf, so the file lands where the panel picked. struct EngineStatements: Sendable { - /// The statements for one direction, in order. `alias` is a name for the attached backup, - /// unique per run so a `DETACH` that could not run leaves nothing blocking the next one. + // The statements for one direction, in order. `alias` is a name for the attached backup, + // unique per run so a `DETACH` that could not run leaves nothing blocking the next one. let backupStatements: @Sendable (Request, String) -> [String] let restoreStatements: @Sendable (Request, String) -> [String] - /// Run after a failure and ignored. A `DETACH` inside an aborted transaction fails too, and - /// the caller has the engine's real error to report rather than this one. + // Run after a failure and ignored. A `DETACH` inside an aborted transaction fails too, and + // the caller has the engine's real error to report rather than this one. let cleanupStatements: @Sendable (String) -> [String] // Set when the archive's own SQL must not run on the session: the driver imports it into a new file of this @@ -271,9 +262,9 @@ struct NativeDumpDescriptor: Sendable { let objectScope: NativeDumpObjectScope let restoreSemantics: RestoreSemantics - /// True when the tool opens the database file itself and so cannot reach a connection that has - /// no local file. libSQL claims the SQLite descriptor and reaches either a file or a Turso URL, - /// and only the file can be handed to `sqlite3`. + // True when the tool opens the database file itself and so cannot reach a connection that has + // no local file. libSQL claims the SQLite descriptor and reaches either a file or a Turso URL, + // and only the file can be handed to `sqlite3`. let requiresLocalFile: Bool init( diff --git a/apple/TablePro/Core/Database/NativeDumpRegistry.swift b/apple/TablePro/Core/Database/NativeDumpRegistry.swift index d1ec365aa5..88421a5b11 100644 --- a/apple/TablePro/Core/Database/NativeDumpRegistry.swift +++ b/apple/TablePro/Core/Database/NativeDumpRegistry.swift @@ -6,21 +6,19 @@ import Foundation import TableProPluginKit import TableProSQLQuoting - -/// Which engines can dump and restore, and how to drive them. -/// -/// Curated here rather than declared by each driver plugin, because the facts are about tools on -/// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static -/// for them would mean an ABI change and a re-release of every registry plugin for data no plugin -/// produces. -/// -/// A password never reaches the argument list. Anything in `argv` is readable by every process on -/// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` -/// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, -/// whose tools read neither. +import TableProTLSClientIdentity + +// Curated here rather than declared by each driver plugin, because the facts are about tools on +// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static +// for them would mean an ABI change and a re-release of every registry plugin for data no plugin +// produces. +// A password never reaches the argument list. Anything in `argv` is readable by every process on +// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` +// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, +// whose tools read neither. enum NativeDumpRegistry { - /// The formats one engine can write, in the order the sheet offers them. The first is the - /// default. Only DuckDB has more than one. + // The formats one engine can write, in the order the sheet offers them. The first is the + // default. Only DuckDB has more than one. static func formats(for type: DatabaseType) -> [NativeDumpDescriptor.ArchiveFormat] { switch type { case .duckdb: @@ -58,12 +56,11 @@ enum NativeDumpRegistry { descriptor(for: type) != nil } - /// Whether this particular connection can be dumped, which is not always the same question as - /// whether its engine can. - /// - /// libSQL reaches either a local file or a Turso URL, and only the file can be handed to - /// `sqlite3`. Answering by type alone is how a remote libSQL connection produced a 52-byte - /// backup the result sheet reported as a success. + // Whether this particular connection can be dumped, which is not always the same question as + // whether its engine can. + // libSQL reaches either a local file or a Turso URL, and only the file can be handed to + // `sqlite3`. Answering by type alone is how a remote libSQL connection produced a 52-byte + // backup the result sheet reported as a success. static func supports(_ connection: DatabaseConnection, localFilePath: String?) -> Bool { guard let descriptor = descriptor(for: connection.type) else { return false } guard descriptor.requiresLocalFile else { return true } @@ -73,8 +70,8 @@ enum NativeDumpRegistry { // MARK: - PostgreSQL - /// Redshift reports 8.0.2, which no `pg_dump` release claims to support, so only PostgreSQL - /// itself is matched to the server. + // Redshift reports 8.0.2, which no `pg_dump` release claims to support, so only PostgreSQL + // itself is matched to the server. static let postgresToolForServer: @Sendable (String, String?) -> NativeDumpToolSelection = { binary, serverVersion in PostgreSQLDumpToolLocator.select(binary: binary, serverVersion: serverVersion) } @@ -121,10 +118,14 @@ enum NativeDumpRegistry { environment["PGHOSTADDR"] = hostaddr } environment.merge( - postgresSSLEnvironment(request.connection.transportSSLConfiguration) + postgresSSLEnvironment( + request.connection.transportSSLConfiguration, + systemTrustAnchorsFile: request.systemTrustAnchorsFile + ) ) { _, new in new } return environment - } + }, + needsSystemTrustAnchors: { request in postgresNeedsSystemTrustAnchors(request) } ) ), archiveFormat: NativeDumpDescriptor.ArchiveFormat( @@ -186,18 +187,23 @@ enum NativeDumpRegistry { } } - /// The whole of the connection's SSL configuration, not just its mode. - /// - /// libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, and measured - /// with pg_dump 17.11 a `verify-ca` connection with no such file fails before connecting with - /// `root certificate file "..." does not exist`. So a Verify CA connection that opens in the - /// app could never be dumped. The rule matches what `LibPQConnectionString` already sends on - /// the live connection, down to the CA being tied to the modes that verify. - static func postgresSSLEnvironment(_ ssl: SSLConfiguration) -> [String: String] { + // What `LibPQConnectionString` sends on the live connection. Measured with pg_dump 17.11, a verifying mode with no + // root file named and no `~/.postgresql/root.crt` fails before connecting with `root certificate file "..." does not + // exist`. + static func postgresSSLEnvironment( + _ ssl: SSLConfiguration, + systemTrustAnchorsFile: URL? = nil, + inherited: [String: String] = ProcessInfo.processInfo.environment + ) -> [String: String] { guard ssl.isEnabled, let mode = postgresSSLMode(ssl.mode) else { return [:] } var environment = ["PGSSLMODE": mode] if ssl.verifiesCertificate, !ssl.caCertificatePath.isEmpty { environment["PGSSLROOTCERT"] = ssl.caCertificatePath + } else if ssl.verifiesHostname, let systemTrustAnchorsFile { + environment["PGSSLROOTCERT"] = systemTrustAnchorsFile.path + } else if ssl.verifiesCertificate, let named = inherited["PGSSLROOTCERT"], !named.isEmpty { + // The tool starts from a minimal environment, so a root file the user named for libpq is carried to it. + environment["PGSSLROOTCERT"] = named } if !ssl.clientCertificatePath.isEmpty { environment["PGSSLCERT"] = ssl.clientCertificatePath @@ -208,10 +214,32 @@ enum NativeDumpRegistry { return environment } + // The driver's rule, `LibPQConnectionString.rootCertificateSource`: Verify Identity with no CA file checks this Mac's + // trusted roots, unless the server is a socket or libpq's own `~/.postgresql/root.crt` is there. The tool inherits + // HOME, so it looks for that file where this does. + static func postgresNeedsSystemTrustAnchors( + _ request: NativeDumpDescriptor.Request, + environment: [String: String] = ProcessInfo.processInfo.environment, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> Bool { + let ssl = request.connection.transportSSLConfiguration + return LibPQSystemTrustRule.usesSystemTrustAnchors( + verifiesHostname: ssl.verifiesHostname, + caCertificatePath: ssl.caCertificatePath, + host: postgresEndpoint(request).host, + environment: environment, + fileExists: fileExists + ) + } + + static func postgresDefaultRootCertificatePath(environment: [String: String]) -> String { + LibPQSystemTrustRule.defaultRootCertificatePath(environment: environment) + } + // MARK: - MySQL and MariaDB - /// MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some - /// builds leave out, so both spellings are tried. + // MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some + // builds leave out, so both spellings are tried. private static var mysql: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -269,12 +297,12 @@ enum NativeDumpRegistry { ) } - /// `--` before the database, because `my_getopt` does not stop parsing options at the first - /// positional argument and every name after it is the user's. Measured with mysqldump 8.4.11 - /// and 12.3.2, a database named `--no-data` with the terminator behind it dumped a *different* - /// database with no rows and exited 0, which the result sheet reports as a successful backup; - /// a table named `--ssl-mode=DISABLED` in the same slot sends the whole dump in cleartext. With - /// `--` in front of the database, both are names again. + // `--` before the database, because `my_getopt` does not stop parsing options at the first + // positional argument and every name after it is the user's. Measured with mysqldump 8.4.11 + // and 12.3.2, a database named `--no-data` with the terminator behind it dumped a *different* + // database with no rows and exited 0, which the result sheet reports as a successful backup; + // a table named `--ssl-mode=DISABLED` in the same slot sends the whole dump in cleartext. With + // `--` in front of the database, both are names again. private static func mysqlObjectArguments(_ request: NativeDumpDescriptor.Request) -> [String] { ["--", request.database] + request.scope.objects.map(\.name) } @@ -390,9 +418,9 @@ enum NativeDumpRegistry { // MARK: - MongoDB - /// `mongodump` reads no password from the environment, and a password in `argv` is readable by - /// every process on the machine. Its `--config` file is the only channel left, so the caller - /// writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. + // `mongodump` reads no password from the environment, and a password in `argv` is readable by + // every process on the machine. Its `--config` file is the only channel left, so the caller + // writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. private static var mongodb: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -432,8 +460,8 @@ enum NativeDumpRegistry { // The tools check the name in the URI, 127.0.0.1 through a tunnel, and no option names another. Verify CA skips it. private static let mongoModesThroughTunnel: [SSLMode] = [.disabled, .preferred, .required, .verifyCa] - /// A whole-database dump names the database; a narrowed one names each collection, because - /// `--db` plus `--collection` takes only one collection at a time while `--nsInclude` repeats. + // A whole-database dump names the database; a narrowed one names each collection, because + // `--db` plus `--collection` takes only one collection at a time while `--nsInclude` repeats. private static func mongoNamespaceFlags(_ request: NativeDumpDescriptor.Request) -> [String] { guard !request.scope.isWholeDatabase else { return ["--db=\(request.database)"] } return request.scope.objects.map { object in @@ -458,13 +486,12 @@ enum NativeDumpRegistry { // MARK: - SQL Server - /// `sqlpackage` is Microsoft's own client-side export, and the only one that writes a whole - /// database to a local file. `bcp` moves one table at a time and produces no schema, so it - /// cannot stand in for a dump. - /// - /// Not verified against a live server here: the arguments come from Microsoft's documented CLI - /// surface rather than from a run, so a mistake in them surfaces as the tool's own usage error - /// rather than a wrong file. + // `sqlpackage` is Microsoft's own client-side export, and the only one that writes a whole + // database to a local file. `bcp` moves one table at a time and produces no schema, so it + // cannot stand in for a dump. + // Not verified against a live server here: the arguments come from Microsoft's documented CLI + // surface rather than from a run, so a mistake in them surfaces as the tool's own usage error + // rather than a wrong file. private static var sqlServer: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -522,10 +549,10 @@ enum NativeDumpRegistry { return "\(schema).\(object.name)" } - /// `sqlpackage` takes no password flag and no password environment variable: the connection - /// string is the only channel it has. That puts the password in `argv`, where `ps` can read it, - /// so the string is built here and `NativeDumpService` keeps SQL Server out of the flows that - /// promise otherwise. Anything better needs Entra or integrated auth, which have no password. + // `sqlpackage` takes no password flag and no password environment variable: the connection + // string is the only channel it has. That puts the password in `argv`, where `ps` can read it, + // so the string is built here and `NativeDumpService` keeps SQL Server out of the flows that + // promise otherwise. Anything better needs Entra or integrated auth, which have no password. private static func sqlServerConnectionString(_ request: NativeDumpDescriptor.Request) throws -> String { let quoted = NativeDumpArgumentQuoting.sqlServerConnectionStringValue var parts = try [ @@ -559,12 +586,11 @@ enum NativeDumpRegistry { // MARK: - SQLite - /// The database is a file the tool opens directly, so there is no host, port or password, and - /// `.dump` writes SQL to standard output. - /// - /// The path comes from `localFilePath` rather than from `connection.database`. libSQL claims - /// this descriptor too and keeps its path in a plugin-declared additional field, so reading - /// `database` handed `sqlite3` an empty string, which exits 0 after writing a 52-byte file. + // The database is a file the tool opens directly, so there is no host, port or password, and + // `.dump` writes SQL to standard output. + // The path comes from `localFilePath` rather than from `connection.database`. libSQL claims + // this descriptor too and keeps its path in a plugin-declared additional field, so reading + // `database` handed `sqlite3` an empty string, which exits 0 after writing a 52-byte file. private static var sqlite: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( diff --git a/apple/TablePro/Core/Database/NativeDumpService.swift b/apple/TablePro/Core/Database/NativeDumpService.swift index 33b825cf0a..577d3e9b85 100644 --- a/apple/TablePro/Core/Database/NativeDumpService.swift +++ b/apple/TablePro/Core/Database/NativeDumpService.swift @@ -14,25 +14,24 @@ import os import TableProConnectionLibrary import TableProPluginKit import TableProProcess +import TableProTLSClientIdentity import Yams // MARK: - Public Types -/// What the service is doing: dump (back up) a database or restore a dump file. enum NativeDumpKind: Equatable, Sendable { case backup case restore } -/// Observable state of a backup or restore. enum NativeDumpState: Equatable { case idle case running(database: String, fileURL: URL, bytesProcessed: Int64, totalBytes: Int64?) case cancelling case finished(database: String, fileURL: URL, bytesProcessed: Int64, skippedSettings: [String] = []) - /// A restore that fails part way through has already replayed some of the dump, and the - /// target is left in whatever state that reached. A backup writes only to its own file, which - /// is removed, so nothing of the user's is touched. + // A restore that fails part way through has already replayed some of the dump, and the + // target is left in whatever state that reached. A backup writes only to its own file, which + // is removed, so nothing of the user's is touched. case failed(message: String, targetMayBeModified: Bool) case cancelled } @@ -48,6 +47,7 @@ enum NativeDumpError: LocalizedError, Equatable { case unreachableServer(message: String) case nulInArgument case uncheckedThroughTunnel(tool: String, mode: SSLMode, supportedModes: [SSLMode]) + case systemTrustUnavailable(message: String) var errorDescription: String? { switch self { @@ -67,7 +67,7 @@ enum NativeDumpError: LocalizedError, Equatable { return String(localized: "The selected backup file is not readable.") case .engineStatementUnavailable: return String(localized: "TablePro could not read the database's own name from the connection.") - case .incompatibleTool(let message), .unreachableServer(let message): + case .incompatibleTool(let message), .unreachableServer(let message), .systemTrustUnavailable(let message): return message case .nulInArgument: return String(localized: "The database name or a sign-in value holds a NUL character, which a command line cannot carry.") @@ -98,29 +98,28 @@ internal enum NativeDumpExitPolicy: Equatable, Sendable { } } -/// Parameters for a single backup or restore subprocess. struct NativeDumpCommand: Equatable { let executable: URL let arguments: [String] let environment: [String: String] let stderrByteCap: Int - /// Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while - /// `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. + // Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while + // `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. let delivery: NativeDumpDescriptor.OutputDelivery - /// The file standard output is redirected to, or read from on a restore. Nil when the tool - /// handles the file itself. + // The file standard output is redirected to, or read from on a restore. Nil when the tool + // handles the file itself. let redirectedFileURL: URL? - /// A credentials file written for tools that read a password from neither the environment nor - /// standard input. Removed once the process exits, however it exits. + // A credentials file written for tools that read a password from neither the environment nor + // standard input. Removed once the process exits, however it exits. let temporaryCredentialsFileURL: URL? let clientPluginFolder: MariaDBClientPluginFolder? - /// Which direction a redirected file goes: a restore feeds it in as standard input, a backup - /// takes standard output and writes it out. + // Which direction a redirected file goes: a restore feeds it in as standard input, a backup + // takes standard output and writes it out. let isRestore: Bool internal let exitPolicy: NativeDumpExitPolicy @@ -150,11 +149,10 @@ struct NativeDumpCommand: Equatable { } } -/// Parameters for a dump the engine runs for itself. struct NativeDumpStatementJob: Sendable, Equatable { let statements: [String] - /// Run after a failure and ignored. A `DETACH` inside a transaction the engine already aborted - /// fails too, and the caller has the real error to report instead of this one. + // Run after a failure and ignored. A `DETACH` inside a transaction the engine already aborted + // fails too, and the caller has the real error to report instead of this one. let cleanupStatements: [String] let scope: DatabaseScope var isolatedImport: NativeDumpIsolatedImport? @@ -168,8 +166,8 @@ struct NativeDumpIsolatedImport: Sendable, Equatable { let file: URL } -/// What a run is, so the service can pick a runner without either runner accepting a shape it -/// cannot execute. +// What a run is, so the service can pick a runner without either runner accepting a shape it +// cannot execute. enum NativeDumpJob { case process(NativeDumpCommand) case statements(NativeDumpStatementJob) @@ -201,7 +199,6 @@ internal extension NativeDumpJob { } } -/// Captured terminal state of a finished/cancelled run. struct NativeDumpRunResult: Equatable { let exitCode: Int32 let stderr: String @@ -209,14 +206,12 @@ struct NativeDumpRunResult: Equatable { var targetUntouched = false } -/// Runs one backup or restore and reports how it ended. Abstracted so the state machine can be -/// tested without launching a process or reaching a database. +// Runs one backup or restore and reports how it ended. Abstracted so the state machine can be +// tested without launching a process or reaching a database. protocol NativeDumpRunner: AnyObject { - /// Begins the work. Throws synchronously if it cannot be started at all. func start() throws - /// Asks for cancellation. Safe to call more than once. + // Safe to call more than once. func cancel() - /// Resolves once the work has ended, however it ended. var result: NativeDumpRunResult { get async } // How far a restore has read its file, or nil when the runner cannot tell. var bytesRead: Int64? { get } @@ -243,7 +238,10 @@ final class NativeDumpService: ObservableObject { private var cancelRequested = false private var preparation: Task? // The Vault lease the tool signs in with, held until the tool exits. - private var credentialLease: VaultLease? + private var credentialLease: VaultLeaseHold? + private var endsCredentialLease = false + // The roots file the tool checks the server with, removed when the tool exits. + private var systemTrustAnchors: SystemTrustAnchorFile? private let runningWork: RunningWorkRegistry private let reportCompletion: @MainActor (OperationCompletion) -> Void private var trackedRestore: TrackedRestore? @@ -273,13 +271,13 @@ final class NativeDumpService: ObservableObject { state = newState switch newState { case .finished: - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.succeeded(OperationSummary())) case .failed(let message, _): - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.failed(reason: message)) case .cancelled: - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.cancelled) case .idle, .running, .cancelling: break @@ -400,11 +398,15 @@ final class NativeDumpService: ObservableObject { var effective = session?.effectiveConnection ?? connection let resolvedFields = DatabaseDriverFactory.resolvedAdditionalFields(for: effective) ?? effective.additionalFields + // A password on the tool's command line is readable while it runs, so that Vault user is the tool's alone and + // is revoked when it exits. + let exposesPassword = descriptor.exposesPasswordInArguments // The connect's own resolution, errors included: a Keychain read misses every other source. let signIn = try await ConnectionCredentialResolver.resolveSignIn( for: effective, fields: resolvedFields, - override: session?.cachedPassword + override: session?.cachedPassword, + leaseOwner: exposesPassword ? UUID() : nil ) // A tool signs in with the Keychain secrets the driver gets beside the password. if let token = resolvedFields[ConnectionSecretOptions.mongoSessionTokenFieldId], !token.isEmpty { @@ -417,12 +419,13 @@ final class NativeDumpService: ObservableObject { effective.username = username } let password = signIn.password - releaseCredentialLease() + releaseRunResources() credentialLease = signIn.lease + endsCredentialLease = exposesPassword signIn.lease?.start { _ in } var jobStarted = false defer { - if !jobStarted { releaseCredentialLease() } + if !jobStarted { releaseRunResources() } } let localFilePath = Self.localFilePath(for: effective) let catalog = descriptor.engineStatements == nil @@ -453,6 +456,7 @@ final class NativeDumpService: ObservableObject { return } request = prepared + request.systemTrustAnchorsFile = try await holdSystemTrustAnchors(for: request, tool: tool) let command = try Self.buildCommand( kind: kind, tool: tool, @@ -480,9 +484,33 @@ final class NativeDumpService: ObservableObject { Self.logger.info("\(self.toolName, privacy: .public) started db=\(database, privacy: .private(mask: .hash))") } - private func releaseCredentialLease() { - credentialLease?.release() + private func releaseRunResources() { + if endsCredentialLease { + credentialLease?.end() + } else { + credentialLease?.release() + } credentialLease = nil + systemTrustAnchors?.remove() + systemTrustAnchors = nil + } + + // Off the main actor: the first build in a minute reads every trust setting, 150 to 275 ms measured. + private func holdSystemTrustAnchors( + for request: NativeDumpDescriptor.Request, + tool: NativeDumpDescriptor.CommandLineTool + ) async throws -> URL? { + let file = try await Task.detached { () throws -> SystemTrustAnchorFile? in + guard tool.needsSystemTrustAnchors(request) else { return nil } + do { + return try SystemTrustAnchors.shared.writeFile() + } catch { + throw NativeDumpError.systemTrustUnavailable(message: error.localizedDescription) + } + }.value + systemTrustAnchors?.remove() + systemTrustAnchors = file + return file?.url } // Nil when a cancel arrived while it ran: the checks it was making stop, and the run reads as cancelled rather than @@ -561,12 +589,11 @@ final class NativeDumpService: ObservableObject { } } - /// A cancel that arrives before the tool is running is latched rather than dropped. - /// - /// `start` finds the binary and asks it what it is before there is anything to cancel, and both - /// steps spawn a process, so the window is real: a Cancel taken during it used to be discarded - /// and the tool launched afterwards, which on a restore means writing to the target database - /// the user just said to leave alone. + // A cancel that arrives before the tool is running is latched rather than dropped. + // `start` finds the binary and asks it what it is before there is anything to cancel, and both + // steps spawn a process, so the window is real: a Cancel taken during it used to be discarded + // and the tool launched afterwards, which on a restore means writing to the target database + // the user just said to leave alone. func cancel() { cancelRequested = true preparation?.cancel() @@ -634,21 +661,18 @@ final class NativeDumpService: ObservableObject { return refusal.map { .incompatible($0) } ?? .missing } - /// The path a file-backed driver actually opens, read from wherever that driver keeps it. - /// SQLite uses `database`; DuckDB and libSQL use a plugin-declared additional field. + // SQLite uses `database`; DuckDB and libSQL use a plugin-declared additional field. static func localFilePath(for connection: DatabaseConnection) -> String? { guard let field = PluginManager.shared.localFilePathField(for: connection.type) else { return nil } let path = connection.localFilePath(in: field) return path.isEmpty ? nil : path } - /// The engine's own name for the database being dumped. - /// - /// `COPY FROM DATABASE` names a catalog, and DuckDB derives a catalog name from the file's - /// basename: measured, `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` - /// fails with `Catalog "main" does not exist`. DuckDB also reaches more than one catalog at once - /// through `ATTACH`, so the answer is the picked database when the engine lists it, and only - /// otherwise the one the session is on. + // `COPY FROM DATABASE` names a catalog, and DuckDB derives a catalog name from the file's + // basename: measured, `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` + // fails with `Catalog "main" does not exist`. DuckDB also reaches more than one catalog at once + // through `ATTACH`, so the answer is the picked database when the engine lists it, and only + // otherwise the one the session is on. static func resolvedCatalog(connectionId: UUID, database: String) async -> String? { guard let scope = DatabaseManager.shared.resolvedScope( database: database, schema: nil, for: connectionId @@ -664,9 +688,9 @@ final class NativeDumpService: ObservableObject { } } - /// Pure so the choice is testable without an engine. The picked name wins when the engine says - /// it exists, because a connection with two catalogs attached has to dump the one that was - /// ticked rather than whichever the session is parked on. + // Pure so the choice is testable without an engine. The picked name wins when the engine says + // it exists, because a connection with two catalogs attached has to dump the one that was + // ticked rather than whichever the session is parked on. nonisolated static func catalogName( picked: String, reported: [String], @@ -716,18 +740,18 @@ final class NativeDumpService: ObservableObject { ) } - /// Unique per run, so a `DETACH` that could not execute leaves nothing behind that blocks the - /// next attempt with "database with name already exists". + // Unique per run, so a `DETACH` that could not execute leaves nothing behind that blocks the + // next attempt with "database with name already exists". static func backupAlias() -> String { "tablepro_backup_\(UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(12))" } // MARK: - Command Construction - /// The descriptor supplies the arguments and the environment; this adds the parts every tool - /// shares. A password never joins the argument list, because `argv` is world readable through - /// `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it - /// does not. + // The descriptor supplies the arguments and the environment; this adds the parts every tool + // shares. A password never joins the argument list, because `argv` is world readable through + // `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it + // does not. nonisolated static func buildCommand( kind: NativeDumpKind, tool: NativeDumpDescriptor.CommandLineTool, @@ -773,9 +797,9 @@ final class NativeDumpService: ObservableObject { ) } - /// `mongodump` and `mongorestore` read a password from neither the environment nor standard - /// input, and one in `argv` is readable by every process on the machine. Their `--config` file - /// is the remaining channel, so it is written owner-only and removed when the process exits. + // `mongodump` and `mongorestore` read a password from neither the environment nor standard + // input, and one in `argv` is readable by every process on the machine. Their `--config` file + // is the remaining channel, so it is written owner-only and removed when the process exits. nonisolated static func writeMongoConfigurationFile( _ entries: [NativeDumpDescriptor.ConfigurationEntry] ) throws -> URL { @@ -799,10 +823,10 @@ final class NativeDumpService: ObservableObject { return try Yams.serialize(node: .mapping(Node.Mapping(pairs)), width: -1, allowUnicode: true) } - /// The locale variables a spawned tool reads, named rather than spelled at each use site. The - /// preference-key guard scans this target's sources for a quoted string passed to a `forKey:` - /// label and reports anything outside the `com.TablePro` namespace, so a locale variable - /// written that way reads as a stray `UserDefaults` key. Comments are scanned too. + // The locale variables a spawned tool reads, named rather than spelled at each use site. The + // preference-key guard scans this target's sources for a quoted string passed to a `forKey:` + // label and reports anything outside the `com.TablePro` namespace, so a locale variable + // written that way reads as a stray `UserDefaults` key. Comments are scanned too. nonisolated internal enum LocaleEnvironmentKey { static let everyCategory = "LC_ALL" static let characterHandling = "LC_CTYPE" @@ -875,9 +899,9 @@ final class NativeDumpService: ObservableObject { Self.logger.error("\(self.toolName, privacy: .public) failed code=\(result.exitCode) db=\(database, privacy: .private(mask: .hash)) stderr=\(result.stderr)") } - /// A DuckDB Parquet backup is a folder, so its size is the sum of what is inside it and its - /// removal is recursive. `removeItem` already handles a directory; `attributesOfItem` does not - /// sum one. + // A DuckDB Parquet backup is a folder, so its size is the sum of what is inside it and its + // removal is recursive. `removeItem` already handles a directory; `attributesOfItem` does not + // sum one. nonisolated static func sizeOfDestination(_ url: URL) -> Int64 { let manager = FileManager.default var isDirectory: ObjCBool = false @@ -937,13 +961,12 @@ final class NativeDumpService: ObservableObject { // MARK: - Database Size Helper extension NativeDumpService { - /// Best-effort estimate of the database's on-disk size. Used as an upper bound for the backup - /// progress bar; the dump file is typically much smaller because of compression, so the bar - /// tops out at the size and then jumps when the tool exits. Returns nil if the query fails. - /// - /// Routed through `withMetadataDriver` rather than taken straight off `driver(for:)`. On the - /// session driver this query queues behind whatever the user is running and joins whatever - /// transaction a query tab left open, which is the hazard `metadataRoute` exists to avoid. + // Best-effort estimate of the database's on-disk size. Used as an upper bound for the backup + // progress bar; the dump file is typically much smaller because of compression, so the bar + // tops out at the size and then jumps when the tool exits. + // Routed through `withMetadataDriver` rather than taken straight off `driver(for:)`. On the + // session driver this query queues behind whatever the user is running and joins whatever + // transaction a query tab left open, which is the hazard `metadataRoute` exists to avoid. static func estimatedDatabaseSize( connection: DatabaseConnection, database: String @@ -959,8 +982,8 @@ extension NativeDumpService { return Int64(text) } - /// An engine with no cheap size answer returns nil, which leaves the progress bar - /// indeterminate rather than showing a percentage of a number nobody measured. + // An engine with no cheap size answer returns nil, which leaves the progress bar + // indeterminate rather than showing a percentage of a number nobody measured. nonisolated static func sizeQuery(for type: DatabaseType) -> String? { switch type { case .postgresql, .redshift: diff --git a/apple/TablePro/Core/Database/ObjectCommentEditing.swift b/apple/TablePro/Core/Database/ObjectCommentEditing.swift index 15b025159d..3502c381ad 100644 --- a/apple/TablePro/Core/Database/ObjectCommentEditing.swift +++ b/apple/TablePro/Core/Database/ObjectCommentEditing.swift @@ -101,12 +101,16 @@ enum DatabaseObjectCommandRunner { if let denial = decision.denialError { throw denial } let startedAt = Date() + let route = DatabaseManager.shared.schemaChangeRoute(for: scope) + let refusesOpenTransaction = !route.isPooled try await DatabaseManager.shared.withScopedDriver( scope: scope, - route: DatabaseManager.shared.schemaChangeRoute(for: scope), + route: route, cancellation: .protectedWrite ) { driver in - _ = try await driver.execute(query: sql) + _ = try await AppDefinitionCommit.run(on: driver, refusesOpenTransaction: refusesOpenTransaction) { + try await driver.execute(query: sql) + } } await DatabaseManager.shared.historyRecorder.record( diff --git a/apple/TablePro/Core/Database/SessionLanes.swift b/apple/TablePro/Core/Database/SessionLanes.swift index ebcd95ee09..b1c291d62b 100644 --- a/apple/TablePro/Core/Database/SessionLanes.swift +++ b/apple/TablePro/Core/Database/SessionLanes.swift @@ -5,6 +5,18 @@ import Foundation import os +import TableProSQLGrammar + +// What a connection replaced or closed without the user asking took with it: an open transaction, what the user's +// own statements set up, or both. +internal struct SessionLoss: Equatable, Sendable { + var transaction: Bool + var state: SQLSessionStateKinds + + func merging(_ other: SessionLoss) -> SessionLoss { + SessionLoss(transaction: transaction || other.transaction, state: state.union(other.state)) + } +} /// The user connections a session keeps open on databases other than the one it is browsing. /// @@ -33,9 +45,9 @@ internal final class SessionLanes { /// Drivers being checked before use. A turn held on one of these is not a query proving the /// driver answers, so a switch back waits for the check instead of promoting it on trust. private var verifying: [ObjectIdentifier: [CheckedContinuation]] = [:] - /// Databases whose connection died holding a transaction and was replaced. The next turn there - /// is refused once, so work meant to continue that transaction does not carry on without it. - private var lostTransactions: [UUID: Set] = [:] + // Databases whose connection died holding a transaction or session state and was replaced. The next turn there is + // refused once, so work meant for that session does not carry on without it. + private var losses: [UUID: [String: SessionLoss]] = [:] private struct Opened { weak var driver: AnyObject? @@ -46,8 +58,8 @@ internal final class SessionLanes { /// switched away from, nothing on screen closes one, so it is closed once idle. Browsing its /// database takes it off this list. private var opened: [UUID: [String: Opened]] = [:] - /// Databases whose opened connection was closed for being idle. Work queued for one reopens it: - /// no transaction was lost. The session's temp tables, `SET` values and advisory locks were. + // Databases whose opened connection was closed for being idle. Work queued for one reopens it. One holding a + // transaction or session state the user's statements set up is never closed idle. private var closedIdle: [UUID: Set] = [:] internal static let openedIdleTimeout: TimeInterval = MetadataConnectionPool.idleTimeout @@ -59,6 +71,9 @@ internal final class SessionLanes { /// Told after a parked connection closes, which ends any transaction it held. internal var onClose: (@MainActor (UUID) -> Void)? + // What the user's statements set up on a driver, from the ledger the session owns. + internal var recordedState: (@MainActor (DatabaseDriver, UUID) -> SQLSessionStateKinds)? + internal init(opener: Opener? = nil) { self.opener = opener ?? { scope in try await MetadataConnectionPool.openServerDriver(for: scope, purpose: .session) @@ -122,13 +137,13 @@ internal final class SessionLanes { /// Closes every parked connection of a session. A disconnect ends them, and so does a rebuilt /// transport: they were dialed through the tunnel port that is going away. Opens still in flight /// lose their generation, so one that lands afterwards closes its own driver. - internal func closeAll(for connectionId: UUID, keepingLostTransactions: Bool = false) { + internal func closeAll(for connectionId: UUID, keepingLosses: Bool = false) { generations[connectionId, default: 0] &+= 1 let drivers = parked.removeValue(forKey: connectionId) ?? [:] opened.removeValue(forKey: connectionId) closedIdle.removeValue(forKey: connectionId) - if !keepingLostTransactions { - lostTransactions.removeValue(forKey: connectionId) + if !keepingLosses { + losses.removeValue(forKey: connectionId) } for driver in drivers.values { verifiedAt.removeValue(forKey: ObjectIdentifier(driver)) @@ -196,35 +211,54 @@ internal final class SessionLanes { } } - internal func markTransactionLost(database: String, for connectionId: UUID) { - lostTransactions[connectionId, default: []].insert(database) + internal func markLost( + database: String, + transaction: Bool = false, + state: SQLSessionStateKinds = [], + for connectionId: UUID + ) { + guard transaction || !state.isEmpty else { return } + let loss = SessionLoss(transaction: transaction, state: state) + losses[connectionId, default: [:]][database] = losses[connectionId]?[database]?.merging(loss) ?? loss + Self.logger.info( + "noted a lost session connId=\(connectionId, privacy: .public) transaction=\(transaction) kinds=\(state.rawValue)" + ) } - internal func hasTransactionLoss(database: String, for connectionId: UUID) -> Bool { - lostTransactions[connectionId]?.contains(database) ?? false + internal func hasLoss(database: String, for connectionId: UUID) -> Bool { + losses[connectionId]?[database] != nil } - /// Closes every parked connection when the transport under them is rebuilt, first noting each - /// one that was holding a transaction, so the next work on that database is told the server - /// rolled it back instead of quietly starting over on a new connection. - internal func closeAllNotingLostTransactions(for connectionId: UUID) async { + // Closes every parked connection when the transport under them is rebuilt, first noting each one that held a + // transaction or session state, so the next work on that database is told what the server dropped instead of + // quietly starting over on a new connection. + internal func closeAllNotingLosses(for connectionId: UUID) async { for (database, driver) in parked[connectionId] ?? [:] { let held = await driver.heldSessionTransactionState() - if !held.permitsAppTransaction { - lostTransactions[connectionId, default: []].insert(database) - } + markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: recordedState?(driver, connectionId) ?? [], + for: connectionId + ) } - closeAll(for: connectionId, keepingLostTransactions: true) + closeAll(for: connectionId, keepingLosses: true) } - /// Reports a lost transaction once, clearing it. - internal func takeTransactionLoss(database: String, for connectionId: UUID) -> Bool { - lostTransactions[connectionId]?.remove(database) != nil + // Reported once, whole, to the first turn that owns session state, as a lost transaction always was. + internal func takeLoss(database: String, for connectionId: UUID) -> SessionLoss? { + let loss = losses[connectionId]?.removeValue(forKey: database) + if losses[connectionId]?.isEmpty == true { + losses.removeValue(forKey: connectionId) + } + return loss } - // A connection with one driver for every database lost the transaction whichever name it was noted under. - internal func takeAnyTransactionLoss(for connectionId: UUID) -> String? { - lostTransactions.removeValue(forKey: connectionId)?.min() + // A connection with one driver for every database lost the session whichever name it was noted under. + internal func takeAnyLoss(for connectionId: UUID) -> (database: String, loss: SessionLoss)? { + guard let noted = losses.removeValue(forKey: connectionId), let database = noted.keys.min() else { return nil } + let loss = noted.values.reduce(SessionLoss(transaction: false, state: [])) { $0.merging($1) } + return (database: database, loss: loss) } internal func markOpened(_ driver: DatabaseDriver, database: String, for connectionId: UUID, at date: Date = Date()) { diff --git a/apple/TablePro/Core/Database/SessionStateLedger.swift b/apple/TablePro/Core/Database/SessionStateLedger.swift new file mode 100644 index 0000000000..8ed93187c8 --- /dev/null +++ b/apple/TablePro/Core/Database/SessionStateLedger.swift @@ -0,0 +1,171 @@ +// +// SessionStateLedger.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +// What the user's own statements set up on each session connection, as kinds only: a value a `SET` carried is kept +// nowhere. A reconnect reads it for the driver it is about to close, since the server cannot be asked once it is gone. +@MainActor +internal final class SessionStateLedger { + private struct Entry { + // Weak, and compared by identity: a closed driver that was freed hands its address to the next one allocated. + weak var driver: AnyObject? + var state: SQLSessionStateKinds + } + + private var entries: [SessionTransactionTracker.Key: Entry] = [:] + private let isLive: @MainActor (SessionTransactionTracker.Key, DatabaseDriver) -> Bool + + internal init(isLive: @escaping @MainActor (SessionTransactionTracker.Key, DatabaseDriver) -> Bool) { + self.isLive = isLive + } + + internal func record(_ effect: SQLSessionStateEffect, on driver: DatabaseDriver, for key: SessionTransactionTracker.Key) { + let current = entries[key].flatMap { $0.driver === driver ? $0.state : nil } ?? [] + // A turn can outlive its session's teardown; its late answer must not land on the replacement. + guard isLive(key, driver) else { + if entries[key]?.driver === driver { entries.removeValue(forKey: key) } + return + } + let state = effect.applied(to: current) + entries[key] = state.isEmpty ? nil : Entry(driver: driver, state: state) + } + + internal func state(for key: SessionTransactionTracker.Key, heldBy driver: DatabaseDriver) -> SQLSessionStateKinds { + guard let entry = entries[key], entry.driver === driver else { return [] } + return entry.state + } + + internal func state(heldBy driver: DatabaseDriver, for connectionId: UUID) -> SQLSessionStateKinds { + entries.reduce(into: SQLSessionStateKinds()) { state, item in + if item.key.connectionId == connectionId, item.value.driver === driver { + state.formUnion(item.value.state) + } + } + } + + // Drops what a closed, replaced or disconnected driver held, as the transaction tracker drops its holds. + internal func prune(_ connectionId: UUID) { + for (key, entry) in entries where key.connectionId == connectionId { + if let driver = entry.driver as? DatabaseDriver, isLive(key, driver) { continue } + entries.removeValue(forKey: key) + } + } +} + +private struct SessionStateReading: Sendable { + let key: SessionTransactionTracker.Key + let dialect: SQLSessionDialect + let grammar: SQLLexicalGrammar + let startup: [String] +} + +extension DatabaseManager { + // Called inside the user's own lease, where the driver is the one the text ran on. A statement that failed counts: + // it can have created a temporary table before its error. + internal func noteSessionStatements(_ statements: [String], ranOn driver: DatabaseDriver, scope: DatabaseScope) async { + guard !statements.isEmpty, + let session = activeSessions[scope.connectionId], + let dialect = SQLSessionDialect(TransactionEngineFamily.of(session.connection.type)), + let key = sessionKey(for: scope), + isLiveSessionDriver(driver, for: key) + else { return } + let databaseType = session.connection.type + let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution + // A reconnect runs the startup commands again, so what they set up is never lost. + let startup = Self.startupStatements( + from: session.connection.startupCommands, databaseType: databaseType, grammar: grammar + ) + let reading = SessionStateReading(key: key, dialect: dialect, grammar: grammar, startup: startup) + let effect = await Self.sessionStateEffect(of: statements, reading: reading) + guard !effect.isEmpty else { return } + sessionStateLedger.record(effect, on: driver, for: key) + } + + // A Run All of a long script reads every statement, which would hold the main actor. + @concurrent + nonisolated private static func sessionStateEffect( + of statements: [String], + reading: SessionStateReading + ) async -> SQLSessionStateEffect { + SQLSessionStateClassifier.effect( + ofStatements: statements, dialect: reading.dialect, grammar: reading.grammar, skipping: reading.startup + ) + } + + // Kinds only, never a statement, a name or a value: a `SET` can carry a secret. + static func sessionLostMessage(database: String, loss: SessionLoss) -> String { + guard !loss.state.isEmpty else { return parkedTransactionLostMessage(database: database) } + let kinds = ListFormatter.localizedString(byJoining: loss.state.localizedNames) + guard loss.transaction else { + return String( + format: String( + // swiftlint:disable:next line_length + localized: "The connection to %1$@ was lost while it waited and has been reconnected. The new session does not have what the old one set up: %2$@. Set it up again, then run the statements." + ), + database, kinds + ) + } + return String( + format: String( + // swiftlint:disable:next line_length + localized: "The connection to %1$@ was lost while it waited, and the server rolled back its open transaction. The new session does not have what the old one set up either: %2$@. Set it up again, then run the statements." + ), + database, kinds + ) + } +} + +extension SQLSessionDialect { + // Every engine left out records nothing, as before the ledger existed. + init?(_ family: TransactionEngineFamily) { + switch family { + case .postgres, .redshift, .cockroach: + self = .postgres + case .mysql: + self = .mySQL + case .sqlServer: + self = .sqlServer + case .oracle: + self = .oracle + case .sqlite: + self = .sqlite + case .duckdb: + self = .duckDB + case .redis, .teradata, .other: + return nil + } + } +} + +extension SQLSessionStateKinds { + // In bit order, so a message names the kinds the same way every time. + var localizedNames: [String] { + var names: [String] = [] + if contains(.temporaryObjects) { names.append(String(localized: "temporary tables")) } + if contains(.sessionSettings) { names.append(String(localized: "session settings")) } + if contains(.userVariables) { names.append(String(localized: "user variables")) } + if contains(.preparedStatements) { names.append(String(localized: "prepared statements")) } + if contains(.heldCursors) { names.append(String(localized: "cursors")) } + if contains(.sessionLocks) { names.append(String(localized: "session locks")) } + if contains(.listeners) { names.append(String(localized: "LISTEN registrations")) } + return names + } +} + +extension BatchStatementOutcome { + // The statement that failed ran too, as `CatalogEvent.statementsRan` counts it. A Stop ran only what it returned. + func ranCount(of totalCount: Int) -> Int { + switch self { + case .completed: + return totalCount + case .cancelled(let results): + return results.count + case .failed(let results, let failure, _): + return failure.ranStatementCount(executedCount: results.count, totalCount: totalCount) + } + } +} diff --git a/apple/TablePro/Core/Database/SessionTransactionTracker.swift b/apple/TablePro/Core/Database/SessionTransactionTracker.swift index 1cd91819ea..1471a74411 100644 --- a/apple/TablePro/Core/Database/SessionTransactionTracker.swift +++ b/apple/TablePro/Core/Database/SessionTransactionTracker.swift @@ -28,8 +28,7 @@ internal final class SessionTransactionTracker: ObservableObject { internal struct Hold: Identifiable, Equatable { let key: Key let state: State - /// Moves with every turn recorded on the lane, so an action confirmed after other work ran - /// there is refused rather than ending a transaction the user has not seen. + // Moves with every turn that could have changed the transaction, so a confirmation made before it is refused. let revision: Int var id: Key { key } @@ -43,10 +42,17 @@ internal final class SessionTransactionTracker: ObservableObject { var state: State } + // Whether a lane's driver ever answered with a definite state; a timed-out read keeps an earlier yes. + private struct Reader { + weak var driver: AnyObject? + var answers: Bool + } + @Published private(set) var holds: [Key: Hold] = [:] private var drivers: [Key: Entry] = [:] private var revisions: [Key: Int] = [:] + private var readers: [Key: Reader] = [:] private let isLive: @MainActor (Key, DatabaseDriver) -> Bool internal init(isLive: @escaping @MainActor (Key, DatabaseDriver) -> Bool) { @@ -64,6 +70,10 @@ internal final class SessionTransactionTracker: ObservableObject { transactionState(connectionId: connectionId, container: container) != nil } + internal func hold(connectionId: UUID, container: String) -> Hold? { + holds(for: connectionId).first { $0.key.lane == nil || $0.key.lane == container } + } + // An aborted hold wins, because it is the one that blocks every statement until a rollback. internal func transactionState(connectionId: UUID, container: String) -> State? { let states = holds.values @@ -73,7 +83,7 @@ internal final class SessionTransactionTracker: ObservableObject { return states.isEmpty ? nil : .open } - // Moves with every turn recorded on any of the connection's lanes. + // Moves with every turn on any of the connection's lanes, except a read-only one that changed nothing. internal func turnRevision(for connectionId: UUID) -> Int { revisions.reduce(0) { total, entry in entry.key.connectionId == connectionId ? total &+ entry.value : total @@ -89,33 +99,70 @@ internal final class SessionTransactionTracker: ObservableObject { holds[hold.key] == hold } - /// An engine that cannot say keeps what an earlier answer from the same driver said, since a - /// refresh never clears what it refreshes; from another driver it says nothing at all. - internal func record(_ state: PluginSessionTransactionState, on driver: DatabaseDriver, for key: Key) { + // An engine that cannot say keeps what an earlier answer from the same driver said. A read-only turn + // moves the revision only when the state it read differs, so a background read never voids a confirmation. + internal func record( + _ state: PluginSessionTransactionState, + on driver: DatabaseDriver, + for key: Key, + readsOnly: Bool = false + ) { // A turn can outlive its session's teardown; its late answer must not touch the replacement's hold. guard isLive(key, driver) else { if drivers[key]?.driver === driver { clear(key) } return } - revisions[key, default: 0] &+= 1 + noteReader(driver, answered: state != .unknown, for: key) + let next = heldState(after: state, from: driver, for: key) + let unchanged = next == drivers[key]?.state && (next == nil || drivers[key]?.driver === driver) + if !(readsOnly && unchanged) { + revisions[key, default: 0] &+= 1 + } + if let next { + set(next, on: driver, for: key) + } else { + clear(key) + } + } + + // True once the live driver of a lane the container runs on has only ever answered `.unknown`. + internal func cannotReportState(connectionId: UUID, container: String) -> Bool { + readers.contains { key, reader in + guard key.connectionId == connectionId, key.lane == nil || key.lane == container, + let driver = reader.driver as? DatabaseDriver, isLive(key, driver) + else { return false } + return !reader.answers + } + } + + private func heldState( + after state: PluginSessionTransactionState, + from driver: DatabaseDriver, + for key: Key + ) -> State? { switch state { case .inTransaction: - set(.open, on: driver, for: key) + return .open case .abortedTransaction: - set(.aborted, on: driver, for: key) + return .aborted case .idle, .holdsSessionLocks: - clear(key) + return nil case .unknown: - if drivers[key]?.driver !== driver { - clear(key) - } else if let entry = drivers[key] { - set(entry.state, on: driver, for: key) - } + guard let entry = drivers[key], entry.driver === driver else { return nil } + return entry.state @unknown default: - clear(key) + return nil } } + private func noteReader(_ driver: DatabaseDriver, answered: Bool, for key: Key) { + if let reader = readers[key], reader.driver === driver { + if answered { readers[key]?.answers = true } + return + } + readers[key] = Reader(driver: driver, answers: answered) + } + /// Drops every hold whose connection is not the lane's live one any more: a disconnect, a /// reconnect onto a new driver, a closed entry or a lost parked connection all end the /// transaction on the server. @@ -124,6 +171,10 @@ internal final class SessionTransactionTracker: ObservableObject { if let driver = entry.driver as? DatabaseDriver, isLive(key, driver) { continue } clear(key) } + for (key, reader) in readers where key.connectionId == connectionId { + if let driver = reader.driver as? DatabaseDriver, isLive(key, driver) { continue } + readers.removeValue(forKey: key) + } } private func set(_ state: State, on driver: DatabaseDriver, for key: Key) { diff --git a/apple/TablePro/Core/Database/TriggerEditing.swift b/apple/TablePro/Core/Database/TriggerEditing.swift index c3744ad7e4..69942fbfcd 100644 --- a/apple/TablePro/Core/Database/TriggerEditing.swift +++ b/apple/TablePro/Core/Database/TriggerEditing.swift @@ -14,7 +14,6 @@ import TableProSQLGrammar enum TriggerEditingError: LocalizedError { case notConnected - case denied(String) case dropUnavailable case notCreateTrigger case definitionUnavailable @@ -23,7 +22,6 @@ enum TriggerEditingError: LocalizedError { var errorDescription: String? { switch self { case .notConnected: String(localized: "Not connected to database") - case let .denied(reason): reason case .dropUnavailable: String(localized: "This database cannot drop triggers") case .notCreateTrigger: String( @@ -121,9 +119,7 @@ enum TriggerApplyStrategy: Equatable { enum TriggerEditing { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "TriggerEditing") - /// Runs on the schema change route, never on the session driver: the trigger's own BEGIN - /// would join a transaction a query tab left open, and its COMMIT or ROLLBACK would take - /// that tab's uncommitted work with it. + // On the schema change route: a BEGIN on the session driver would join a query tab's open transaction. static func apply( scope: DatabaseScope, connection: DatabaseConnection, @@ -132,7 +128,8 @@ enum TriggerEditing { isEdit: Bool, originalName: String?, originalDefinition: String?, - gate: any ExecutionGate = ExecutionGateProvider.shared + gate: any ExecutionGate = ExecutionGateProvider.shared, + holdConfirmation: SchemaChangeHoldConfirmation = DatabaseManager.confirmSchemaChangeBesideHold ) async throws { let plan = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in TriggerApplyPlan( @@ -162,9 +159,12 @@ enum TriggerEditing { statements: plan.gatedStatements(for: sql) ) ) - guard case .authorized = decision else { - throw TriggerEditingError.denied(decision.deniedReason ?? String(localized: "Operation not permitted")) - } + if let denial = decision.denialError { throw denial } + let route = DatabaseManager.shared.schemaChangeRoute(for: scope) + try await DatabaseManager.shared.confirmSchemaChange( + on: route, connectionId: connection.id, confirmation: holdConfirmation + ) + let isPooled = route.isPooled let startedAt = Date() defer { @@ -173,10 +173,15 @@ enum TriggerEditing { ) } do { - try await withSchemaChangeDriver(scope: scope) { driver in + try await withSchemaChangeDriver(scope: scope, route: route) { driver in switch plan.strategy { case let .transactional(dropFirst): - try await runInTransaction(driver: driver, dropSQL: dropFirst ? plan.dropSQL : nil, sql: sql) + let wraps = await DatabaseManager.wrapsSchemaChange( + on: driver, isPooled: isPooled, checksTransactionalDDL: false + ) + try await runInTransaction( + driver: driver, dropSQL: dropFirst ? plan.dropSQL : nil, sql: sql, wraps: wraps + ) case .dropThenCreate: guard let dropSQL = plan.dropSQL, let restore else { throw TriggerEditingError.dropUnavailable } try await runDropThenCreate(driver: driver, dropSQL: dropSQL, sql: sql, rollback: restore) @@ -230,10 +235,7 @@ enum TriggerEditing { name: String, gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { - /// Built on a driver in the scope that will run it, the way `apply` builds its own inside - /// the lease. The session driver is wherever the sidebar last went, and an engine that - /// qualifies its DDL with the connection's current database writes that name into a - /// statement the pooled connection then runs somewhere else. + // Built in the scope that runs it: the session driver may qualify the DDL with another database. let generated = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in driver.generateDropTriggerSQL(name: name, table: tableName) } @@ -252,9 +254,7 @@ enum TriggerEditing { operationDescription: String(localized: "Drop Trigger") ) ) - guard case .authorized = decision else { - throw TriggerEditingError.denied(decision.deniedReason ?? String(localized: "Operation not permitted")) - } + if let denial = decision.denialError { throw denial } let startedAt = Date() try await withSchemaChangeDriver(scope: scope) { driver in @@ -271,17 +271,24 @@ enum TriggerEditing { private static func withSchemaChangeDriver( scope: DatabaseScope, + route: ScopedDriverRoute? = nil, _ body: @Sendable @escaping (DatabaseDriver) async throws -> Void ) async throws { try await DatabaseManager.shared.withScopedDriver( scope: scope, - route: DatabaseManager.shared.schemaChangeRoute(for: scope), + route: route ?? DatabaseManager.shared.schemaChangeRoute(for: scope), cancellation: .protectedWrite, body ) } - static func runInTransaction(driver: DatabaseDriver, dropSQL: String?, sql: String) async throws { + // Unwrapped, the edit joins the transaction the session holds and is taken back with it. + static func runInTransaction(driver: DatabaseDriver, dropSQL: String?, sql: String, wraps: Bool = true) async throws { + guard wraps else { + if let dropSQL { _ = try await driver.execute(query: dropSQL) } + _ = try await driver.execute(query: sql) + return + } try await driver.beginTransaction(mode: .readWrite) do { if let dropSQL { _ = try await driver.execute(query: dropSQL) } diff --git a/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift b/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift index 3f4391e661..1108b7eb85 100644 --- a/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift +++ b/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift @@ -48,7 +48,7 @@ actor VaultAuthenticator { private let readFile: @Sendable (String) -> String? private let now: @Sendable () -> ContinuousClock.Instant private let openURL: @Sendable (URL) async -> Void - private let makeCallbackReceiver: @Sendable () -> any VaultOIDCCallbackReceiving + private let makeCallbackReceiver: @Sendable (_ expectedState: String) -> any VaultOIDCCallbackReceiving private var minted: [LoginKey: VaultToken] = [:] private var oidcSignIns: [LoginKey: Task] = [:] @@ -61,7 +61,9 @@ actor VaultAuthenticator { openURL: @escaping @Sendable (URL) async -> Void = { url in await MainActor.run { _ = WebPageOpener.open(url, scope: .httpsOrLoopbackHTTP) } }, - makeCallbackReceiver: @escaping @Sendable () -> any VaultOIDCCallbackReceiving = { VaultOIDCLoopback() } + makeCallbackReceiver: @escaping @Sendable (_ expectedState: String) -> any VaultOIDCCallbackReceiving = { + VaultOIDCLoopback(expectedState: $0) + } ) { self.transport = transport self.environment = environment @@ -147,7 +149,7 @@ actor VaultAuthenticator { do { grant = try await VaultClient(endpoint: endpoint, transport: transport).lookupSelf(token: value) } catch let error as VaultRequestError { - throw VaultError.tokenRejected(source, error.message) + throw VaultError.signInFailure(error, at: endpoint, otherwise: .tokenRejected(source, error.message)) } // The user owns this token, so it is never renewed here. return VaultToken( @@ -174,16 +176,12 @@ actor VaultAuthenticator { grant = try await VaultClient(endpoint: endpoint, transport: transport) .appRoleLogin(mount: mount, roleId: roleId, secretId: secretId) } catch let error as VaultRequestError { - throw VaultError.tokenRejected(.appRole, error.message) + throw VaultError.signInFailure(error, at: endpoint, otherwise: .tokenRejected(.appRole, error.message)) } return token(from: grant, source: .appRole) } private func performOIDCSignIn(_ login: VaultOIDCLogin) async throws -> VaultToken { - let receiver = makeCallbackReceiver() - try await receiver.start() - defer { receiver.stop() } - let client = VaultClient(endpoint: login.endpoint, transport: transport) let nonce = UUID().uuidString.replacingOccurrences(of: "-", with: "") + UUID().uuidString.replacingOccurrences(of: "-", with: "") @@ -198,13 +196,23 @@ actor VaultAuthenticator { } catch let error as VaultRequestError { throw VaultError.oidcFailed(error.message) } + // Vault checks this state on the callback; the loopback lets no other callback end the wait. + guard let state = Self.singleQueryValue(named: "state", in: authURL), !state.isEmpty else { + throw VaultError.oidcFailed(String( + localized: "Vault returned a sign-in page without a state, so TablePro could not check the browser's answer." + )) + } + + let receiver = makeCallbackReceiver(state) + try await receiver.start() + defer { receiver.stop() } await openURL(authURL) let parameters = try await receiver.callback(timeout: Self.oidcCallbackTimeout) if let error = parameters["error"] { throw VaultError.oidcFailed(parameters["error_description"] ?? error) } - guard let state = parameters["state"], let code = parameters["code"], !code.isEmpty else { + guard let code = parameters["code"], !code.isEmpty else { throw VaultError.oidcFailed(String(localized: "The browser returned no authorization code.")) } do { @@ -216,6 +224,11 @@ actor VaultAuthenticator { } } + private static func singleQueryValue(named name: String, in url: URL) -> String? { + let items = URLComponents(url: url, resolvingAgainstBaseURL: false)?.queryItems ?? [] + return LoopbackRedirectRequest.singleValue(named: name, in: items) + } + // MARK: - Minted tokens private func reusable(_ key: LoginKey, validFor: Duration?) async -> VaultToken? { diff --git a/apple/TablePro/Core/Database/Vault/VaultClient.swift b/apple/TablePro/Core/Database/Vault/VaultClient.swift index cbc60ae066..8e8ac88fcc 100644 --- a/apple/TablePro/Core/Database/Vault/VaultClient.swift +++ b/apple/TablePro/Core/Database/Vault/VaultClient.swift @@ -74,6 +74,9 @@ struct VaultRequestError: Error, Equatable { var isPermissionDenied: Bool { status == 403 } + // Rate limited, sealed, or a proxy in front of it that cannot reach it: the same request can work in a moment. + var isTransient: Bool { [429, 502, 503, 504].contains(status) } + // Vault wraps several causes as "2 errors occurred:\n\t* permission denied\n\t* invalid token". var message: String { let lines = messages.flatMap { $0.components(separatedBy: .newlines) } diff --git a/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift b/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift index bdc0d04403..66ffd7b1cb 100644 --- a/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift +++ b/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift @@ -6,31 +6,41 @@ import Foundation import os -// Every call reads a new lease. The credential lives only in the returned value and the driver it configures. +// The credential lives only in memory: in the cached lease and the drivers that sign in with it. struct VaultDatabaseCredentials: Sendable { static let shared = VaultDatabaseCredentials() // A lease dies with the token that created it, so a minted token close to its end is renewed or replaced first. static let minimumTokenLife: Duration = .seconds(300) + // Quit waits this long for Vault to revoke the users still leased. + static let quitRevokeLimit: Duration = .seconds(3) private let authenticator: VaultAuthenticator private let transport: any VaultTransport private let clock: any Clock + private let leases: VaultLeaseCache - init( + init>( authenticator: VaultAuthenticator = .shared, transport: any VaultTransport = URLSessionVaultTransport.shared, - clock: any Clock = ContinuousClock() + clock: C = ContinuousClock() ) { self.authenticator = authenticator self.transport = transport self.clock = clock + self.leases = VaultLeaseCache(clock: clock) } - func signIn(_ role: VaultDatabaseRole, interactive: Bool) async throws -> ResolvedSignIn { + // `owner` is the credential profile, or the connection for a role of its own. One owner signs in as one user + // while its lease lives. `connection` is the one signing in, nil for a test of the source alone. + func signIn( + _ role: VaultDatabaseRole, + owner: UUID, + for connection: UUID?, + interactive: Bool + ) async throws -> ResolvedSignIn { let endpoint = try VaultEndpoint.resolve(address: role.address, environment: authenticator.currentEnvironment) let client = VaultClient(endpoint: endpoint, transport: transport) - let path = role.credentialsPath var token = try await authenticator.token( for: role.auth, @@ -42,6 +52,31 @@ struct VaultDatabaseCredentials: Sendable { await authenticator.forget(role.auth, endpoint: endpoint) token = try await authenticator.token(for: role.auth, endpoint: endpoint, interactive: interactive) } + let key = VaultLeaseCache.Key(owner: owner, role: role, endpoint: endpoint, token: token.value) + let issuing = token + let hold = try await leases.acquire(key, for: connection) { + try await readLease(role, endpoint: endpoint, client: client, token: issuing, interactive: interactive) + } + return ResolvedSignIn(username: hold.username, password: hold.password, lease: hold) + } + + func endUse(connection: UUID, live: Set) { + leases.endUse(connection: connection, live: live) + } + + func revokeAllLeases() async { + await leases.revokeAll(within: Self.quitRevokeLimit) + } + + private func readLease( + _ role: VaultDatabaseRole, + endpoint: VaultEndpoint, + client: VaultClient, + token initialToken: VaultToken, + interactive: Bool + ) async throws -> VaultLeaseCache.Issued { + let path = role.credentialsPath + var token = initialToken let credential: VaultLeasedCredential let issuedAt: ContinuousClock.Instant do { @@ -54,7 +89,7 @@ struct VaultDatabaseCredentials: Sendable { credential = try await Self.read(path: path, token: token.value, client: client) issuedAt = authenticator.currentInstant } catch let error as VaultRequestError { - throw VaultError.credentialsFailed(path: path, message: error.message) + throw Self.failure(error, path: path, client: client) } if token.source.isMinted, @@ -88,7 +123,12 @@ struct VaultDatabaseCredentials: Sendable { renewer: renewer, clock: clock ) - return ResolvedSignIn(username: credential.username, password: credential.password, lease: lease) + return VaultLeaseCache.Issued( + username: credential.username, + password: credential.password, + lease: lease, + token: token.value + ) } // Only a token TablePro minted bounds the lease's schedule. The user's own token is theirs to renew; rotating @@ -101,9 +141,13 @@ struct VaultDatabaseCredentials: Sendable { do { return try await client.readCredentials(path: path, token: token) } catch let error as VaultRequestError { - throw VaultError.credentialsFailed(path: path, message: error.message) + throw failure(error, path: path, client: client) } } + + private static func failure(_ error: VaultRequestError, path: String, client: VaultClient) -> VaultError { + .signInFailure(error, at: client.endpoint, otherwise: .credentialsFailed(path: path, message: error.message)) + } } // Renews and revokes with the token that issued the lease: Vault revokes a lease with that token, whatever token diff --git a/apple/TablePro/Core/Database/Vault/VaultError.swift b/apple/TablePro/Core/Database/Vault/VaultError.swift index b19321fa91..123b39465e 100644 --- a/apple/TablePro/Core/Database/Vault/VaultError.swift +++ b/apple/TablePro/Core/Database/Vault/VaultError.swift @@ -45,8 +45,21 @@ enum VaultError: LocalizedError, Equatable { case oidcCallbackPortInUse case credentialsFailed(path: String, message: String) case unreachable(String, String) + // Vault's own failure, such as a standby with no active node: worth another try. + case serverFailed(String) case malformedResponse + // A sign-in's Vault HTTP failure: a busy, sealed or failing Vault is worth another try, anything else is `refusal`. + static func signInFailure( + _ error: VaultRequestError, + at endpoint: VaultEndpoint, + otherwise refusal: VaultError + ) -> VaultError { + if error.isTransient { return .unreachable(endpoint.address.absoluteString, error.message) } + if error.status >= 500 { return .serverFailed(error.message) } + return refusal + } + var errorDescription: String? { switch self { case .noAddress: @@ -94,6 +107,8 @@ enum VaultError: LocalizedError, Equatable { return String(format: String(localized: "Vault could not issue credentials from %@: %@"), path, message) case let .unreachable(address, message): return String(format: String(localized: "Could not reach Vault at %@: %@"), address, message) + case let .serverFailed(message): + return String(format: String(localized: "Vault could not complete the request: %@"), message) case .malformedResponse: return String(localized: "Vault returned a response TablePro could not read.") } diff --git a/apple/TablePro/Core/Database/Vault/VaultLease.swift b/apple/TablePro/Core/Database/Vault/VaultLease.swift index 5547b63fdb..c27991a8af 100644 --- a/apple/TablePro/Core/Database/Vault/VaultLease.swift +++ b/apple/TablePro/Core/Database/Vault/VaultLease.swift @@ -58,8 +58,7 @@ struct VaultLeaseSchedule: Equatable, Sendable { } } -// Owned by whatever signed in with it, a driver or a backup job. Dropping the last reference revokes it too, so a -// lease a timed-out connect never adopted does not outlive it. +// Owned by `VaultLeaseCache`, which hands out holds. Dropping the last reference revokes it too. final class VaultLease: Sendable { let leaseId: String @@ -98,6 +97,9 @@ final class VaultLease: Sendable { var hasEnded: Bool { control.hasEnded } + // What `onEnding` gets when a renewal finds the lease gone: the time a running statement has to finish in. + var endingLead: Duration { schedule.lead } + // `onEnding` gets how long the lease has left, which is the time to sign in again in. func start(onEnding: @escaping @Sendable (Duration) -> Void) { let leaseId = leaseId @@ -130,6 +132,23 @@ final class VaultLease: Sendable { } } + func revoke() async { + guard control.release() else { return } + await renewer.revoke(leaseId: leaseId) + } + + // Vault Agent drops a cached lease once renewing it fails; this asks the same question on demand. Nil when the + // lease cannot be renewed or Vault gave no definite answer. + func stillExists(timeout: Duration) async -> Bool? { + guard schedule.renewable, !hasEnded else { return nil } + do { + _ = try await renewer.renew(leaseId: leaseId, increment: schedule.initialDuration, timeout: timeout) + return true + } catch { + return Self.isDefinitive(error) ? false : nil + } + } + // Returns when the lease is about to end and can no longer be renewed. private static func keepAlive( leaseId: String, @@ -227,8 +246,10 @@ final class VaultLease: Sendable { if let request = error as? VaultRequestError { return (400 ..< 500).contains(request.status) && request.status != 429 } - if case .unreachable = error as? VaultError { return false } - return error is VaultError + switch error as? VaultError { + case .unreachable, .serverFailed: return false + default: return error is VaultError + } } private final class Control: Sendable { diff --git a/apple/TablePro/Core/Database/Vault/VaultLeaseCache.swift b/apple/TablePro/Core/Database/Vault/VaultLeaseCache.swift new file mode 100644 index 0000000000..f38a31ad29 --- /dev/null +++ b/apple/TablePro/Core/Database/Vault/VaultLeaseCache.swift @@ -0,0 +1,499 @@ +// +// VaultLeaseCache.swift +// TablePro +// + +import CryptoKit +import Foundation +import os +import TableProPluginKit +import TableProUtilities + +// Every sign-in of one owner and role shares a lease, so a database still loading a new user meets the same user +// on the next attempt instead of yet another new one. +final class VaultLeaseCache: Sendable { + struct Key: Hashable, Sendable { + let owner: UUID + let role: VaultDatabaseRole + let endpoint: VaultEndpoint + // Vault Agent reuses a lease only for the token that read it; so does this. + let tokenDigest: Data + + init(owner: UUID, role: VaultDatabaseRole, endpoint: VaultEndpoint, token: String) { + self.owner = owner + self.role = role + self.endpoint = endpoint + self.tokenDigest = Data(SHA256.hash(data: Data(token.utf8))) + } + + func reading(with token: String) -> Key { + Key(owner: owner, role: role, endpoint: endpoint, token: token) + } + } + + struct Issued: Sendable { + let username: String + let password: String + let lease: VaultLease + // A re-login while reading leaves the lease with another token than the one the request was keyed by. + let token: String + } + + // An unused lease goes when the pool lets an unused connection go. + static let idleRetention: Duration = .seconds(MetadataConnectionPool.idleTimeout) + // Vitess, ProxySQL and replicas refuse a user Vault just created until they load it. HashiCorp's Terraform + // provider retries a new credential for the same 30 seconds, then gives up on it. + static let acceptanceWindow: Duration = .seconds(30) + // Vault's own default request timeout. + static let vaultRequestTimeout: Duration = .seconds(30) + + let clock: any Clock + // Starts timing a lease's age on `clock`. + private let stopwatch: @Sendable () -> @Sendable () -> Duration + private let state = OSAllocatedUnfairLock(initialState: State()) + + init>(clock: C = ContinuousClock()) { + self.clock = clock + stopwatch = { + let start = clock.now + return { start.duration(to: clock.now) } + } + } + + // `connection` is the connection that signs in, nil for a test of the source alone. + func acquire( + _ key: Key, + for connection: UUID?, + issue: @escaping @Sendable () async throws -> Issued + ) async throws -> VaultLeaseHold { + let lookup = state.withLock { state -> Lookup in + if let id = state.listed[key], let hold = attach(to: id, for: connection, joinedRead: false, in: &state) { + return .held(hold) + } + if var running = state.minting[key] { + running.waiters += 1 + state.minting[key] = running + return .reading(running.task) + } + let task = read(key, issue: issue) + state.minting[key] = Minting(task: task, waiters: 1) + return .reading(task) + } + switch lookup { + case .held(let hold): + return hold + case .reading(let task): + let id = try await task.value + // Only a quit that revoked everything takes a slot from under a sign-in that joined its read. + guard let hold = state.withLock({ attach(to: id, for: connection, joinedRead: true, in: &$0) }) else { + throw CancellationError() + } + return hold + } + } + + // The user ended `connection`. A lease no connection in `live` signed in with is no longer handed out, and goes + // once its last driver lets go, which a pooled read or a backup can do after this returns. + func endUse(connection: UUID, live: Set) { + let leases = state.withLock { state -> [VaultLease] in + let ended = state.slots.compactMap { id, slot -> UInt64? in + slot.users.contains(connection) ? id : nil + } + return ended.compactMap { id in + state.slots[id]?.users.remove(connection) + guard state.slots[id]?.users.isDisjoint(with: live) == true else { return nil } + unlist(id, in: &state) + return settle(id, in: &state) + } + } + leases.forEach { $0.release() } + } + + // A revoke sent from a detached task at quit never leaves the process, so these are awaited, within `limit`. + func revokeAll(within limit: Duration) async { + let leases = state.withLock { state -> [VaultLease] in + state.closed = true + return Array(state.slots.keys).compactMap { drop($0, in: &state) } + } + guard !leases.isEmpty else { return } + let clock = clock + await withTaskGroup(of: Void.self) { group in + group.addTask { + await withTaskGroup(of: Void.self) { revokes in + for lease in leases { + revokes.addTask { await lease.revoke() } + } + } + } + group.addTask { + try? await clock.sleep(for: limit) + } + await group.next() + group.cancelAll() + } + } + + // MARK: - Holds + + fileprivate func subscribe(slot id: UInt64, hold: UInt64, onEnding: @escaping @Sendable (Duration) -> Void) { + let ended = state.withLock { state -> Duration? in + guard state.slots[id]?.holds[hold] != nil else { return nil } + state.slots[id]?.holds[hold]?.onEnding = onEnding + return state.slots[id]?.ending + } + if let ended { onEnding(ended) } + } + + // `ending` revokes the lease too, unless another hold still uses it. + fileprivate func release(slot id: UInt64, hold: UInt64, ending: Bool) { + let lease = state.withLock { state -> VaultLease? in + guard state.slots[id]?.holds.removeValue(forKey: hold) != nil else { return nil } + if ending, state.slots[id]?.isIdle == true { + unlist(id, in: &state) + } + return settle(id, in: &state) + } + lease?.release() + } + + // A refusal alone cannot say whether Vault revoked the user or the database turned it away for its own reasons + // (another database, a privilege, a user it has not loaded), so Vault is asked. Only a lease it no longer has ends. + fileprivate func signInAccepted(slot id: UInt64) { + state.withLock { $0.slots[id]?.accepted = true } + } + + fileprivate func signInRefused(slot id: UInt64, credentials: Bool) { + let lease = state.withLock { state -> VaultLease? in + guard let slot = state.slots[id], slot.ending == nil else { return nil } + if credentials { + state.slots[id]?.credentialsRefused = true + } + guard !slot.asking else { return nil } + state.slots[id]?.asking = true + return slot.issued.lease + } + guard let lease else { return } + Task { [weak self] in + let exists = await lease.stillExists(timeout: Self.vaultRequestTimeout) + self?.answered(id, exists: exists) + } + } + + fileprivate func hasEnded(slot id: UInt64) -> Bool { + state.withLock { $0.slots[id]?.ending != nil } + } + + fileprivate func redialDelay(slot id: UInt64, afterFailures failures: Int) -> Duration? { + state.withLock { state -> Duration? in + guard let slot = state.slots[id], slot.ending == nil, !slot.issued.lease.hasEnded else { return nil } + let left = Self.acceptanceWindow - slot.age() + guard left > .zero else { return nil } + return min(.seconds(ExponentialBackoff.delay(for: failures)), left) + } + } + + // MARK: - State + + private struct Holder: Sendable { + var onEnding: (@Sendable (Duration) -> Void)? + } + + private struct Slot: Sendable { + let key: Key + let issued: Issued + let age: @Sendable () -> Duration + // Sign-ins that joined the read and have not taken their hold yet. + var joinedReads: Int + var holds: [UInt64: Holder] = [:] + // Connections that signed in with it and were not ended since. + var users: Set = [] + // Vault is being asked whether it still has the lease. + var asking = false + var accepted = false + var credentialsRefused = false + var ending: Duration? + var idleRelease: Task? + + var isIdle: Bool { holds.isEmpty && joinedReads == 0 } + } + + private struct Minting: Sendable { + let task: Task + var waiters: Int + } + + private enum Lookup: Sendable { + case held(VaultLeaseHold) + case reading(Task) + } + + private struct State: Sendable { + var slots: [UInt64: Slot] = [:] + var listed: [Key: UInt64] = [:] + var minting: [Key: Minting] = [:] + var nextId: UInt64 = 0 + // Set at quit: a lease read after the revoke would outlive the app. + var closed = false + } + + private func read(_ key: Key, issue: @escaping @Sendable () async throws -> Issued) -> Task { + Task { + let issued: Issued + do { + issued = try await issue() + } catch { + state.withLock { _ = $0.minting.removeValue(forKey: key) } + throw error + } + guard let (id, replaced) = register(issued, requestedAs: key) else { + issued.lease.release() + throw CancellationError() + } + replaced?.release() + hearEnding(of: issued.lease, slot: id) + return id + } + } + + private func hearEnding(of lease: VaultLease, slot id: UInt64) { + lease.start { [weak self] remaining in + self?.leaseEnding(id, remaining: remaining) + } + } + + private func register(_ issued: Issued, requestedAs requested: Key) -> (UInt64, replaced: VaultLease?)? { + let age = stopwatch() + return state.withLock { state in + let waiters = state.minting.removeValue(forKey: requested)?.waiters ?? 0 + guard !state.closed else { return nil } + let key = requested.reading(with: issued.token) + var replaced: VaultLease? + if let previous = state.listed[key] { + unlist(previous, in: &state) + replaced = settle(previous, in: &state) + } + state.nextId &+= 1 + let id = state.nextId + state.slots[id] = Slot(key: key, issued: issued, age: age, joinedReads: waiters) + state.listed[key] = id + return (id, replaced) + } + } + + // Nil for a slot that is gone, or, outside a joined read, for a lease that ended before its ending was handled. + private func attach(to id: UInt64, for connection: UUID?, joinedRead: Bool, in state: inout State) -> VaultLeaseHold? { + guard var slot = state.slots[id] else { return nil } + if joinedRead { + slot.joinedReads -= 1 + } else if slot.issued.lease.hasEnded { + return nil + } + slot.idleRelease?.cancel() + slot.idleRelease = nil + state.nextId &+= 1 + let hold = state.nextId + slot.holds[hold] = Holder() + if let connection { + slot.users.insert(connection) + } + state.slots[id] = slot + return VaultLeaseHold(cache: self, slot: id, hold: hold, issued: slot.issued) + } + + private func answered(_ id: UInt64, exists: Bool?) { + let (lead, dropped) = state.withLock { state -> (Duration?, VaultLease?) in + guard let slot = state.slots[id] else { return (nil, nil) } + state.slots[id]?.asking = false + state.slots[id]?.credentialsRefused = false + // Vault renews most engines' users without asking the database, so a user dropped there still exists to + // Vault. Once a server took it and then refused its password, new sign-ins read another; sessions keep it. + if exists != false, slot.accepted, slot.credentialsRefused { + unlist(id, in: &state) + return (nil, settle(id, in: &state)) + } + return (slot.issued.lease.endingLead, nil) + } + dropped?.release() + if exists == false, let lead { + leaseEnding(id, remaining: lead) + } + } + + private func leaseEnding(_ id: UInt64, remaining: Duration) { + let (handlers, lease) = state.withLock { state -> ([@Sendable (Duration) -> Void], VaultLease?) in + guard let slot = state.slots[id], slot.ending == nil else { return ([], nil) } + state.slots[id]?.ending = remaining + unlist(id, in: &state) + let handlers = state.slots[id]?.holds.values.compactMap(\.onEnding) ?? [] + return (handlers, settle(id, in: &state)) + } + handlers.forEach { $0(remaining) } + lease?.release() + } + + private func expireIdle(_ id: UInt64) { + let lease = state.withLock { state -> VaultLease? in + guard state.slots[id]?.isIdle == true else { return nil } + return drop(id, in: &state) + } + lease?.release() + } + + // For a slot that may have lost its last user: an unlisted one goes now, a listed one after the retention. + private func settle(_ id: UInt64, in state: inout State) -> VaultLease? { + guard let slot = state.slots[id], slot.isIdle else { return nil } + guard state.listed[slot.key] == id else { return drop(id, in: &state) } + slot.idleRelease?.cancel() + let clock = clock + state.slots[id]?.idleRelease = Task { [weak self] in + do { + try await clock.sleep(for: Self.idleRetention) + } catch { + return + } + self?.expireIdle(id) + } + return nil + } + + private func unlist(_ id: UInt64, in state: inout State) { + guard let key = state.slots[id]?.key, state.listed[key] == id else { return } + state.listed[key] = nil + } + + private func drop(_ id: UInt64, in state: inout State) -> VaultLease? { + guard let slot = state.slots.removeValue(forKey: id) else { return nil } + slot.idleRelease?.cancel() + if state.listed[slot.key] == id { + state.listed[slot.key] = nil + } + return slot.issued.lease + } +} + +// One user of a cached lease. Releasing it, or dropping the last reference to it, hands the lease back. +final class VaultLeaseHold: Sendable { + let username: String + let password: String + let leaseId: String + + private let lease: VaultLease + private let cache: VaultLeaseCache + private let slot: UInt64 + private let hold: UInt64 + private let released = OSAllocatedUnfairLock(initialState: false) + + fileprivate init(cache: VaultLeaseCache, slot: UInt64, hold: UInt64, issued: VaultLeaseCache.Issued) { + self.cache = cache + self.slot = slot + self.hold = hold + self.username = issued.username + self.password = issued.password + self.leaseId = issued.lease.leaseId + self.lease = issued.lease + } + + deinit { + release() + } + + var hasEnded: Bool { lease.hasEnded || cache.hasEnded(slot: slot) } + + var clock: any Clock { cache.clock } + + // `onEnding` gets how long the lease has left, which is the time to sign in again in. + func start(onEnding: @escaping @Sendable (Duration) -> Void) { + cache.subscribe(slot: slot, hold: hold, onEnding: onEnding) + } + + func release() { + release(ending: false) + } + + // For a user whose password left the process, such as a backup tool's arguments: revoked as soon as nothing + // else holds it, not kept for the next sign-in. + func end() { + release(ending: true) + } + + private func release(ending: Bool) { + let first = released.withLock { released -> Bool in + defer { released = true } + return !released + } + guard first else { return } + cache.release(slot: slot, hold: hold, ending: ending) + } + + func signInAccepted() { + cache.signInAccepted(slot: slot) + } + + // `credentials`: the server named the password or user as the reason, as MySQL's 1045 does. + func signInRefused(credentials: Bool) { + cache.signInRefused(slot: slot, credentials: credentials) + } + + // Nil once the lease is past the acceptance window or has ended: a refusal then is final. + func redialDelay(afterFailures failures: Int) -> Duration? { + cache.redialDelay(slot: slot, afterFailures: failures) + } + + // Dials until the database takes this user, while the lease is young. `waiting` hears each wait before it + // starts; `isCurrent` says whether the connect still wants a dial once it ends. + func dial( + _ connect: () async throws -> Void, + waiting: (Duration) -> Void, + isCurrent: () -> Bool + ) async throws { + let startedYoung = redialDelay(afterFailures: 1) != nil + var failures = 0 + while true { + do { + try await connect() + return + } catch { + guard let refusal = error as? any PluginDriverError, Self.isRefusedSignIn(refusal) else { throw error } + failures += 1 + guard let delay = redialDelay(afterFailures: failures) else { + throw startedYoung && !hasEnded ? VaultUserNotLoaded(refusal: refusal) : error + } + waiting(delay) + try await clock.sleep(for: delay) + guard isCurrent() else { throw CancellationError() } + guard !hasEnded else { throw error } + } + } + } + + // MySQL answers 1045 for a user it has not loaded yet, the same as for a wrong password. + static func isRefusedSignIn(_ error: Error) -> Bool { + guard let error = error as? any PluginDriverError else { return false } + return error.pluginErrorCode == 1_045 && error.pluginSqlState == "28000" + } +} + +extension VaultLeaseHold: Equatable { + static func == (lhs: VaultLeaseHold, rhs: VaultLeaseHold) -> Bool { lhs === rhs } +} + +// A new Vault user the server refused for the whole acceptance window. The lease is kept, so the next attempt meets +// the same user, which the server may have loaded by then. +struct VaultUserNotLoaded: PluginDriverError { + let refusal: any PluginDriverError + + var pluginErrorMessage: String { refusal.pluginErrorMessage } + var pluginErrorCode: Int? { refusal.pluginErrorCode } + var pluginSqlState: String? { refusal.pluginSqlState } + var pluginErrorDetail: String? { refusal.pluginErrorDetail } + + var recoverySuggestion: String? { + String( + format: String(localized: """ + The database refused the new user Vault created for it for %lld seconds. Some databases load new \ + users on a timer, so try again in a moment. + """), + Int64(VaultLeaseCache.acceptanceWindow.components.seconds) + ) + } +} diff --git a/apple/TablePro/Core/Database/Vault/VaultOIDCLoopback.swift b/apple/TablePro/Core/Database/Vault/VaultOIDCLoopback.swift index 3cb80e148f..70a5e695d5 100644 --- a/apple/TablePro/Core/Database/Vault/VaultOIDCLoopback.swift +++ b/apple/TablePro/Core/Database/Vault/VaultOIDCLoopback.swift @@ -4,8 +4,7 @@ // import Foundation -import Network -import os +import TableProHTTP protocol VaultOIDCCallbackReceiving: Sendable { func start() async throws @@ -14,202 +13,71 @@ protocol VaultOIDCCallbackReceiving: Sendable { } // Listens where the vault CLI does, http://localhost:8250/oidc/callback, the redirect URI an OIDC role already allows. -final class VaultOIDCLoopback: VaultOIDCCallbackReceiving, @unchecked Sendable { +final class VaultOIDCLoopback: VaultOIDCCallbackReceiving, Sendable { static let port: UInt16 = 8_250 static let callbackPath = "/oidc/callback" - private static let logger = Logger(subsystem: "com.TablePro", category: "VaultOIDCLoopback") - private static let maximumRequestHeadBytes = 8_192 + private let listener: LoopbackRedirectListener<[String: String]> - private let queue = DispatchQueue(label: "com.TablePro.VaultOIDC.loopback") - private let lock = NSLock() - private var listener: NWListener? - private var connections: [ObjectIdentifier: NWConnection] = [:] - private var readyContinuation: CheckedContinuation? - private var callbackContinuation: CheckedContinuation<[String: String], Error>? - private var outcome: Result<[String: String], Error>? - private var timeoutItem: DispatchWorkItem? - - func start() async throws { - let parameters = NWParameters.tcp - parameters.requiredLocalEndpoint = .hostPort(host: .ipv4(.loopback), port: NWEndpoint.Port(rawValue: Self.port) ?? .any) - parameters.allowLocalEndpointReuse = true - let listener: NWListener - do { - listener = try NWListener(using: parameters) - } catch { - throw VaultError.oidcCallbackPortInUse - } - listener.stateUpdateHandler = { [weak self] state in self?.listenerChanged(state) } - listener.newConnectionHandler = { [weak self] connection in - guard let self else { - connection.cancel() - return - } - self.accept(connection) - } - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - lock.withLock { - self.listener = listener - readyContinuation = continuation - } - listener.start(queue: queue) - } - } - - func callback(timeout: Duration) async throws -> [String: String] { - try await withTaskCancellationHandler { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<[String: String], Error>) in - let settled = lock.withLock { () -> Result<[String: String], Error>? in - if let outcome { return outcome } - callbackContinuation = continuation - let item = DispatchWorkItem { [weak self] in - self?.finish(.failure(VaultError.oidcFailed(String(localized: "The browser sign-in timed out.")))) - } - timeoutItem = item - let seconds = Double(timeout.components.seconds) + Double(timeout.components.attoseconds) / 1e18 - queue.asyncAfter(deadline: .now() + seconds, execute: item) - return nil - } - if let settled { continuation.resume(with: settled) } + // Only a callback carrying the `state` from Vault's sign-in page ends the wait. + init(expectedState: String, port: UInt16 = VaultOIDCLoopback.port, headTimeout: TimeInterval = 5) { + listener = LoopbackRedirectListener(port: port, headTimeout: headTimeout) { head in + guard let parameters = Self.callbackParameters(in: head, expectedState: expectedState) else { + return .ignore(LoopbackRedirectPage.badRequest) } - } onCancel: { - finish(.failure(CancellationError())) + return .settle(Self.successPage(), .success(parameters)) } } - func stop() { - finish(.failure(CancellationError())) + func start() async throws { + try await startListening() } - private func listenerChanged(_ state: NWListener.State) { - switch state { - case .ready: - let continuation = lock.withLock { () -> CheckedContinuation? in - defer { readyContinuation = nil } - return readyContinuation - } - continuation?.resume() - case .failed(let error), .waiting(let error): - Self.logger.error("Vault OIDC callback listener failed: \(error.publicLogShape, privacy: .public)") - let continuation = lock.withLock { () -> CheckedContinuation? in - defer { readyContinuation = nil } - return readyContinuation - } - if let continuation { - continuation.resume(throwing: VaultError.oidcCallbackPortInUse) - } - finish(.failure(VaultError.oidcCallbackPortInUse)) - case .cancelled: - let continuation = lock.withLock { () -> CheckedContinuation? in - defer { readyContinuation = nil } - return readyContinuation - } - continuation?.resume(throwing: CancellationError()) - default: - break + @discardableResult + func startListening() async throws -> UInt16 { + do { + return try await listener.start() + } catch is LoopbackRedirectListenerError { + throw VaultError.oidcCallbackPortInUse } } - private func accept(_ connection: NWConnection) { - let admitted = lock.withLock { () -> Bool in - guard outcome == nil, connections.count < 16 else { return false } - connections[ObjectIdentifier(connection)] = connection - return true - } - guard admitted else { - connection.cancel() - return - } - connection.start(queue: queue) - receive(on: connection, head: Data()) + func callback(timeout: Duration) async throws -> [String: String] { + let seconds = Double(timeout.components.seconds) + Double(timeout.components.attoseconds) / 1e18 + return try await listener.outcome( + timeout: seconds, + timeoutError: VaultError.oidcFailed(String(localized: "The browser sign-in timed out.")) + ) } - private func receive(on connection: NWConnection, head: Data) { - let remaining = Self.maximumRequestHeadBytes - head.count - connection.receive(minimumIncompleteLength: 1, maximumLength: remaining) { [weak self] content, _, isComplete, error in - guard let self else { - connection.cancel() - return - } - var received = head - if let content { received.append(content) } - if let parameters = Self.callbackParameters(in: received) { - self.respond(on: connection, status: "200 OK", body: Self.successPage()) - self.finish(.success(parameters)) - return - } - if Self.headIsComplete(received) || received.count >= Self.maximumRequestHeadBytes || isComplete || error != nil { - self.respond(on: connection, status: "404 Not Found", body: "") - return - } - self.receive(on: connection, head: received) - } + func stop() { + listener.stop() } - static func callbackParameters(in head: Data) -> [String: String]? { - guard headIsComplete(head), let text = String(data: head, encoding: .utf8), - let requestLine = text.components(separatedBy: .newlines).first - else { return nil } - let parts = requestLine.split(separator: " ") - guard parts.count >= 2, parts[0] == "GET", - let components = URLComponents(string: "http://localhost" + parts[1]), + static func callbackParameters(in head: Data, expectedState: String) -> [String: String]? { + guard !expectedState.isEmpty, + LoopbackRedirectRequest.isComplete(head), + let target = LoopbackRedirectRequest.getTarget(LoopbackRedirectRequest.text(of: head)), + let components = URLComponents(string: target), + components.host == nil, components.path == callbackPath else { return nil } + let items = LoopbackRedirectRequest.formDecodedItems(in: components) + guard let state = LoopbackRedirectRequest.singleValue(named: "state", in: items), + LoopbackRedirectRequest.constantTimeEquals(state, expectedState) + else { return nil } var parameters: [String: String] = [:] - for item in components.queryItems ?? [] { + for item in items { if let value = item.value { parameters[item.name] = value } } return parameters } - private static func headIsComplete(_ head: Data) -> Bool { - head.range(of: Data("\r\n\r\n".utf8)) != nil || head.range(of: Data("\n\n".utf8)) != nil - } - - private func respond(on connection: NWConnection, status: String, body: String) { - let payload = Data(body.utf8) - let head = "HTTP/1.1 \(status)\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: \(payload.count)\r\n" - + "Connection: close\r\nCache-Control: no-store\r\n\r\n" - connection.send(content: Data(head.utf8) + payload, completion: .contentProcessed { [weak self] _ in - connection.cancel() - self?.release(connection) - }) - } - - private func release(_ connection: NWConnection) { - lock.withLock { connections[ObjectIdentifier(connection)] = nil } - } - - private func finish(_ result: Result<[String: String], Error>) { - let (continuation, listener, settled, abandoned) = lock.withLock { - () -> (CheckedContinuation<[String: String], Error>?, NWListener?, Result<[String: String], Error>, [NWConnection]) in - let firstOutcome = outcome == nil - if firstOutcome { outcome = result } - timeoutItem?.cancel() - timeoutItem = nil - let continuation = callbackContinuation - callbackContinuation = nil - let listener = self.listener - self.listener = nil - // A connection still sending the success page closes itself once it is sent. - var abandoned: [NWConnection] = [] - if firstOutcome, case .failure = result { - abandoned = Array(connections.values) - } - return (continuation, listener, outcome ?? result, abandoned) - } - listener?.cancel() - abandoned.forEach { $0.cancel() } - continuation?.resume(with: settled) - } - - private static func successPage() -> String { - let message = String(localized: "Signed in to Vault. You can close this window and return to TablePro.") - return """ - TablePro - -

\(message)

- """ + private static func successPage() -> Data { + LoopbackRedirectPage.response( + status: "200 OK", + title: "TablePro", + message: String(localized: "Signed in to Vault. You can close this window and return to TablePro.") + ) } } diff --git a/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift b/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift index d5ae1c5575..c5f72e9acf 100644 --- a/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift +++ b/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift @@ -1,35 +1,12 @@ -// -// TableLoadHistoryStore.swift -// TablePro -// - import Foundation import os -/// Where a finished trace goes. The one requirement is that the producer is a synchronous -/// `@MainActor` method, so this cannot be `async`: the call has to return before any disk work -/// happens or the measurement pays for its own recording. +// Not async: the producer is a synchronous main-actor method that must return before any disk work, or the measurement pays for its own recording. internal protocol TableLoadSummarySink: Sendable { func record(_ record: TableLoadPerformanceRecord) } -/// What `TableLoadTracer.shared` gets under XCTest. Dozens of coordinator suites drive -/// `openTableTab`, which begins real traces, and `AppStorageEnvironment` resolves to the production -/// directory under a unit-test host, so a live sink would append to the developer's own history on -/// every test run. Discarding at the sink means no file is opened and nothing needs cleaning up. -internal struct DiscardingTableLoadSummarySink: TableLoadSummarySink { - internal func record(_ record: TableLoadPerformanceRecord) {} -} - -/// A bounded, device-local history of how long table loads took, kept so an intermittent slowdown -/// can be looked at after it happened rather than only while `log stream` is running. -/// -/// Newline-delimited JSON rather than the single re-encoded array `ExecutionAuditLog` uses. Two -/// reasons, both about the caps this store has and that one does not: rewriting the whole array costs -/// the size of the store on every append, which at the ten thousand records allowed here is megabytes -/// written per table click; and a write torn by a crash takes the entire array with it, where a torn -/// final line costs one record. Nothing is held in memory between appends either, because ten -/// thousand decoded records is a lot of residency for a file the running app never reads. +// Newline-delimited JSON: the retention pass rewrites the file, and a write torn by a crash costs one line, not the file. internal actor TableLoadHistoryStore: TableLoadSummarySink { internal static let shared = TableLoadHistoryStore() @@ -41,8 +18,7 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { private static let newline = UInt8(ascii: "\n") private static let logger = Logger(subsystem: "com.TablePro", category: "TableLoadHistory") - /// Fractional seconds, because a store that measures milliseconds and stamps whole seconds - /// cannot order a burst of navigation inside one, and the retention pass sorts on this field. + // Fractional seconds: the retention pass sorts on this field, and whole seconds cannot order a burst of navigation. private static let timestamps = Date.ISO8601FormatStyle(includingFractionalSeconds: true) private let fileURL: URL @@ -55,8 +31,6 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { self.fileURL = fileURL } - /// Resolved through `AppStorageEnvironment` like every other store, so a UI test writes into its - /// own sandbox rather than the history of the person running it. internal static func defaultFileURL() -> URL { let directory = AppStorageEnvironment.shared.supportDirectory try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -75,8 +49,7 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { pruneIfNeeded() } - /// The first touch in a process reconciles the file against the retention rules, so a history - /// left behind by a build that quit before pruning cannot grow past its caps. + // A history left by a build that quit before pruning must not grow past the caps. private func loadIfNeeded() { guard !loaded else { return } loaded = true @@ -90,14 +63,7 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { prune() } - /// An undecodable line is dropped rather than refused. A history whose tail was torn by a crash - /// has to keep working, because it is diagnostic data and refusing to read it would make the - /// store itself the outage. The count is logged rather than swallowed, so a schema change that - /// makes an older record unreadable shows up as a number instead of a file that quietly shrank. - /// - /// The counts are set from the file as read before the rewrite, so a write that fails leaves the - /// caps reflecting what is still on disk. Left at zero they would read as under capacity and the - /// only retry would be the hourly one, which is an hour of appending with no bound enforced. + // A torn line is dropped and counted; the caps come from the file as read, so a failed rewrite leaves them true. private func prune() { lastPrunedAt = Date() diff --git a/apple/TablePro/Core/Diagnostics/TableLoadTracer.swift b/apple/TablePro/Core/Diagnostics/TableLoadTracer.swift index 1524bc11e2..a9ddee61b4 100644 --- a/apple/TablePro/Core/Diagnostics/TableLoadTracer.swift +++ b/apple/TablePro/Core/Diagnostics/TableLoadTracer.swift @@ -1,8 +1,3 @@ -// -// TableLoadTracer.swift -// TablePro -// - import Foundation import os @@ -18,7 +13,7 @@ import os /// keeps a bounded local history an intermittent slowdown can be looked at in after the fact. @MainActor internal final class TableLoadTracer { - internal static let shared = TableLoadTracer(sink: TableLoadTracer.defaultSink()) + internal static let shared = TableLoadTracer(sink: TableLoadHistoryStore.shared) private struct Interval { let id: OSSignpostID @@ -40,15 +35,6 @@ internal final class TableLoadTracer { self.sink = sink } - /// A unit test that drives a coordinator opens real table tabs and so begins real traces, and - /// `AppStorageEnvironment` resolves to the production directory under a unit-test host. Choosing - /// the sink here keeps those traces off the developer's own history without the store having to - /// know it is being tested. - private static func defaultSink() -> any TableLoadSummarySink { - guard NSClassFromString("XCTestCase") == nil else { return DiscardingTableLoadSummarySink() } - return TableLoadHistoryStore.shared - } - /// A sidebar click that opens a new native window tab finishes in a different coordinator, so the /// click instant is parked here and picked up by whichever window ends up doing the load. Without /// it the trace starts after the window has already mounted and hides that cost. diff --git a/apple/TablePro/Core/Events/AppEvents.swift b/apple/TablePro/Core/Events/AppEvents.swift index eb2894539b..04c5dfa6a5 100644 --- a/apple/TablePro/Core/Events/AppEvents.swift +++ b/apple/TablePro/Core/Events/AppEvents.swift @@ -13,56 +13,49 @@ final class AppEvents { // MARK: - Theme & Accessibility - /// Carries the palette revision and the effective appearance. A consumer that snapshots - /// colours or fonts needs both: a forced Light/Dark switch never changes the theme, and a - /// theme switch never changes the appearance. Subscribing reaches a view that is alive but - /// detached from every window, which a walk of the window tree would miss. + // Carries both because a Light/Dark switch keeps the theme and a theme switch keeps the appearance; it also reaches views detached from every window. let themeChanged = PassthroughSubject() let accessibilityTextSizeChanged = PassthroughSubject() // MARK: - System Environment - /// The Mac's time zone moved. Formatted text that was derived in the old one is now wrong, and - /// nothing else re-derives it: a grid keeps its cached strings until the result is re-fetched. + // A grid keeps strings formatted in the old time zone until a re-fetch, and nothing else re-derives them. let systemTimeZoneChanged = PassthroughSubject() // MARK: - Settings let editorSettingsChanged = PassthroughSubject() - /// A live session's health monitor is started once, at connect, so a change to how often - /// TablePro checks its connections reaches nothing already open without this. + // A session's health monitor starts once at connect, so an open session sees a new interval only through this. let connectionHealthCheckChanged = PassthroughSubject() let dataGridSettingsChanged = PassthroughSubject() - /// The menu bar re-syncs itself through `MainMenuBuilder`, but a window's toolbar advertises - /// the same shortcuts in its tooltips and overflow menu and AppKit never revisits either. + let valueColorSettingsChanged = PassthroughSubject() + + // The menu bar re-syncs itself, but AppKit never revisits the shortcuts in a toolbar's tooltips or overflow menu. let keyboardSettingsChanged = PassthroughSubject() let currentSchemaChanged = PassthroughSubject() let aiSettingsChanged = PassthroughSubject() + // A new API key lives in the Keychain, so the save that brings it can leave `AISettings` unchanged. + let aiProviderSaved = PassthroughSubject() // MARK: - Connections let connectionStatusChanged = PassthroughSubject() - /// A workspace's window phase changed. Carries the connection id; the phase lives on the workspace. + // The phase lives on the workspace, so the payload is only the connection id. let connectionPhaseChanged = PassthroughSubject() // A driver's Vault lease can no longer be renewed, so the driver has to sign in again. let credentialLeaseEnding = PassthroughSubject() - /// The step a connection attempt is currently on. Presentational detail inside the - /// connecting phase, never a second source of truth for which pane a window shows. + // Presentational only: which pane a window shows comes from the phase, never from the stage. let connectionStageChanged = PassthroughSubject() - /// Connection metadata changed (name, color, group, type, etc.). - /// Payload is the affected connection's id, or `nil` for bulk updates - /// (sync pull, multi-import) where the sender doesn't track individual ids. - /// Subscribers scoped to a single connection should filter `payload == id`; - /// list-level subscribers refresh on every event regardless. + // nil for a bulk update (sync pull, multi-import) whose sender does not track ids. let connectionUpdated = PassthroughSubject() let databaseDidConnect = PassthroughSubject() @@ -73,66 +66,45 @@ final class AppEvents { let mainWindowWillClose = PassthroughSubject() - /// A connection window was registered, closed, or became key. - /// Subscribers that present the set of open connections, or which window - /// stands for each one, refresh on every event. + // Also fires when a connection window becomes key, since that can change which window stands for a connection. let connectionWindowsChanged = PassthroughSubject() - /// The user reordered the workspace rail. Every open rail shares one - /// arrangement, so all of them reload. + // Every open rail shares one arrangement, so all of them reload. let workspaceRailOrderChanged = PassthroughSubject() - /// The workspace rail was shown or hidden. Every window carries its own - /// rail, so all of them follow the one setting. + // Every window carries its own rail, and all of them follow the one setting. let workspaceRailVisibilityChanged = PassthroughSubject() - /// A window's tabs changed in a way that changes which containers they hold - /// open. Fired only when the set of held containers can differ, not on every - /// keystroke, so the rail reloads when a workspace appears or goes away. + // Fired only when the set of held containers can differ, not on every keystroke. let workspaceTabsChanged = PassthroughSubject() - /// A connection's browse cursor moved to another database or schema. - /// Payload is the connection's id. let browseContainerChanged = PassthroughSubject() - /// A window put another connections-strip entry in front, so the strip's highlight moves. let visibleContainerChanged = PassthroughSubject() - /// A session connection started or stopped holding a transaction. Payload is the connection's id. let sessionTransactionsChanged = PassthroughSubject() // MARK: - Data Sources - /// Query history changed (entry added, deleted, or cleared). - /// Payload is the affected connection's id, or `nil` for cross-connection - /// operations (delete-by-id without connection lookup, clear-all). - /// Per-connection subscribers should refresh on `payload == nil || payload == self.connectionId`. + // nil when the sender cannot name one connection (delete by id, clear all), so a per-connection subscriber refreshes on nil too. let queryHistoryDidUpdate = PassthroughSubject() - /// SQL favorites or favorite folders changed. - /// Payload is the connection whose list changed, or `nil` when every connection's did. - /// Per-connection subscribers should refresh on `payload == nil || payload == self.connectionId`. - /// - /// A global favorite is part of every connection's list, so a record moving into or out of - /// global scope changes them all and is sent as `nil`, as are bulk operations and any delete. - /// Only the sender can know this: the scope it moved from is no longer in the record by the - /// time a subscriber sees the event. `SQLFavoriteManager.scopeToAnnounce(for:newConnectionId:)` - /// is where that is decided. + // SQLFavoriteManager.scopeToAnnounce picks the payload, since a subscriber never sees the scope a record moved from. let sqlFavoritesDidUpdate = PassthroughSubject() let linkedFoldersDidUpdate = PassthroughSubject() let teamLibraryDidUpdate = PassthroughSubject() - /// Linked SQL folder rescan completed; cached file index changed. - /// Senders are bulk rescans across all enabled folders, so payload is always `nil`. - /// The shape is kept consistent with `sqlFavoritesDidUpdate` so subscribers can - /// uniformly handle "this update may affect me" via `payload == nil || payload == self.connectionId`. + // Always nil, since every rescan is bulk; typed like sqlFavoritesDidUpdate so subscribers share one filter. let linkedSQLFoldersDidUpdate = PassthroughSubject() // Refresh on a tab that loads its own content rather than through the coordinator, by tab id. let tabRefreshRequested = PassthroughSubject() + // A query tab's edit history was written. The tab's id, or nil after a purge or Clear History. + let queryTabVersionsDidUpdate = PassthroughSubject() + // MARK: - License & Sync let licenseStatusDidChange = PassthroughSubject() @@ -147,8 +119,7 @@ final class AppEvents { let pluginsRejected = PassthroughSubject<[RejectedPlugin], Never>() - /// Not private so a test can hand an isolated bus to the object under test. App code uses - /// `shared`, which is the only instance anything observes. + // Not private so a test can hand an isolated bus to the object under test; app code uses only shared. init() {} } diff --git a/apple/TablePro/Core/Events/SucceededStatements.swift b/apple/TablePro/Core/Events/SucceededStatements.swift index 156a74b96e..8f8cdd8f0b 100644 --- a/apple/TablePro/Core/Events/SucceededStatements.swift +++ b/apple/TablePro/Core/Events/SucceededStatements.swift @@ -35,8 +35,15 @@ enum StatementCommitEvidence: Sendable, Equatable { plan: BatchTransactionPlan, completed: Bool ) -> StatementCommitEvidence { - guard plan.opensTransaction else { return .runStartedIn(start, endedIn: end, appTransaction: .none) } - return .runStartedIn(start, endedIn: end, appTransaction: completed ? .committed : .rolledBack) + switch plan { + case .appTransaction: + return .runStartedIn(start, endedIn: end, appTransaction: completed ? .committed : .rolledBack) + // A held run sent no COMMIT, so what it ran is pending until the user ends the transaction. + case .heldTransaction: + return .runStartedIn(start, endedIn: end, appTransaction: completed ? .none : .rolledBack) + case .scriptTransaction, .autocommit, .sessionTransaction: + return .runStartedIn(start, endedIn: end, appTransaction: .none) + } } } diff --git a/apple/TablePro/Core/Execution/TabExecutionRegistry.swift b/apple/TablePro/Core/Execution/TabExecutionRegistry.swift index dce0de9b95..2e022ef07e 100644 --- a/apple/TablePro/Core/Execution/TabExecutionRegistry.swift +++ b/apple/TablePro/Core/Execution/TabExecutionRegistry.swift @@ -68,6 +68,7 @@ internal struct TabExecutionRegistry { private var entries: [UUID: Entry] = [:] private var contentEpochs: [UUID: Int] = [:] private var unclaimedWork: [UUID: Set] = [:] + private var stoppedEpochs: [UUID: Int] = [:] private var lastEpoch: Int = 0 internal init() {} @@ -75,6 +76,7 @@ internal struct TabExecutionRegistry { internal mutating func claim(_ tabId: UUID, startedAt: ContinuousClock.Instant = .now) -> TabExecutionClaim { lastEpoch += 1 entries[tabId] = Entry(epoch: lastEpoch, startedAt: startedAt) + stoppedEpochs.removeValue(forKey: tabId) contentEpochs[tabId] = lastEpoch return TabExecutionClaim(tabId: tabId, epoch: lastEpoch, startedAt: startedAt) } @@ -113,6 +115,7 @@ internal struct TabExecutionRegistry { EndedExecution(tabId: tabId, startedAt: $0.startedAt, reason: reason) } unclaimedWork.removeValue(forKey: tabId) + stoppedEpochs.removeValue(forKey: tabId) lastEpoch += 1 contentEpochs[tabId] = lastEpoch return ended @@ -125,6 +128,7 @@ internal struct TabExecutionRegistry { } entries.removeAll() unclaimedWork.removeAll() + stoppedEpochs.removeAll() for tabId in tabIds { lastEpoch += 1 contentEpochs[tabId] = lastEpoch @@ -179,7 +183,12 @@ internal struct TabExecutionRegistry { guard entries[tabId]?.isUninterruptible != true else { return StopOutcome(ended: [], keptUninterruptibleClaim: true) } - let ended = entries.removeValue(forKey: tabId).map { + let removed = entries.removeValue(forKey: tabId) + if let removed { + // Statements that ran before the Stop can stay on the server, and the run still has to keep them. + stoppedEpochs[tabId] = removed.epoch + } + let ended = removed.map { [EndedExecution(tabId: tabId, startedAt: $0.startedAt, reason: .cancelledByUser)] } ?? [] lastEpoch += 1 @@ -187,6 +196,17 @@ internal struct TabExecutionRegistry { return StopOutcome(ended: ended, keptUninterruptibleClaim: false) } + // Replacing content mints a new epoch, so work started on the old result after the Stop cannot write over the kept one. + internal mutating func settleStopped(_ claim: TabExecutionClaim, replacingContent: Bool) -> Bool { + guard stoppedEpochs[claim.tabId] == claim.epoch else { return false } + stoppedEpochs.removeValue(forKey: claim.tabId) + if replacingContent { + lastEpoch += 1 + contentEpochs[claim.tabId] = lastEpoch + } + return true + } + /// Work that runs against a tab without owning its result. /// /// Fetch All is why this exists. `claim` mints a new content epoch, which is the very value the diff --git a/apple/TablePro/Core/LSP/LSPDocumentManager.swift b/apple/TablePro/Core/LSP/LSPDocumentManager.swift index 787d684eb0..cd56839cd9 100644 --- a/apple/TablePro/Core/LSP/LSPDocumentManager.swift +++ b/apple/TablePro/Core/LSP/LSPDocumentManager.swift @@ -7,47 +7,28 @@ import Foundation @MainActor final class LSPDocumentManager { - struct DocumentState { - var uri: String - var version: Int - var languageId: String - } - - private var documents: [String: DocumentState] = [:] + private var versions: [String: Int] = [:] func openDocument(uri: String, languageId: String, text: String) -> LSPTextDocumentItem { - let state = DocumentState(uri: uri, version: 0, languageId: languageId) - documents[uri] = state + versions[uri] = 0 return LSPTextDocumentItem(uri: uri, languageId: languageId, version: 0, text: text) } - func changeDocument( - uri: String, - newText: String - ) -> (versioned: LSPVersionedTextDocumentIdentifier, changes: [LSPTextDocumentContentChangeEvent])? { - guard var state = documents[uri] else { return nil } - state.version += 1 - documents[uri] = state - return ( - versioned: LSPVersionedTextDocumentIdentifier(uri: uri, version: state.version), - changes: [LSPTextDocumentContentChangeEvent(text: newText)] + func changeDocument(uri: String, changes: [LSPTextDocumentContentChangeEvent]) -> LSPDidChangeParams? { + guard let version = versions[uri] else { return nil } + versions[uri] = version + 1 + return LSPDidChangeParams( + textDocument: LSPVersionedTextDocumentIdentifier(uri: uri, version: version + 1), + contentChanges: changes ) } func closeDocument(uri: String) -> LSPTextDocumentIdentifier? { - guard documents.removeValue(forKey: uri) != nil else { return nil } + guard versions.removeValue(forKey: uri) != nil else { return nil } return LSPTextDocumentIdentifier(uri: uri) } func version(for uri: String) -> Int? { - documents[uri]?.version - } - - func isOpen(_ uri: String) -> Bool { - documents[uri] != nil - } - - func resetAll() { - documents.removeAll() + versions[uri] } } diff --git a/apple/TablePro/Core/LSP/LSPTransport.swift b/apple/TablePro/Core/LSP/LSPTransport.swift index 47e274cc6a..9ee56f2959 100644 --- a/apple/TablePro/Core/LSP/LSPTransport.swift +++ b/apple/TablePro/Core/LSP/LSPTransport.swift @@ -32,6 +32,45 @@ enum LSPTransportError: Error, LocalizedError { } } +// MARK: - LSPMessageWriter + +// Writes on its own queue, so a server that stops reading blocks the queue and never the transport. +final class LSPMessageWriter: @unchecked Sendable { + private let handle: FileHandle + private let queue = DispatchQueue(label: "com.TablePro.LSPTransport.writer") + private var isClosed = false + + init(_ handle: FileHandle) throws { + // Without it, a write to a server that has exited raises SIGPIPE and ends the app. + guard fcntl(handle.fileDescriptor, F_SETNOSIGPIPE, 1) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + self.handle = handle + } + + func write(_ frame: Data, onFailure: @escaping @Sendable (any Error) -> Void) { + queue.async { [self] in + guard !isClosed else { + onFailure(LSPTransportError.processNotRunning) + return + } + do { + try handle.write(contentsOf: frame) + } catch { + onFailure(error) + } + } + } + + func close() { + queue.async { [self] in + guard !isClosed else { return } + isClosed = true + try? handle.close() + } + } +} + // MARK: - LSPTransport actor LSPTransport { @@ -39,8 +78,10 @@ actor LSPTransport { private var process: Process? private var stdinPipe: Pipe? + private var writer: LSPMessageWriter? private var stdoutPipe: Pipe? private var stderrPipe: Pipe? + private var stdoutReader: PipeReader? private var stderrReader: PipeReader? private var nextRequestID: Int = 1 private var pendingRequests: [Int: CheckedContinuation] = [:] @@ -66,6 +107,7 @@ actor LSPTransport { proc.standardOutput = stdout proc.standardError = stderr + self.writer = try LSPMessageWriter(stdin.fileHandleForWriting) self.stdinPipe = stdin self.stdoutPipe = stdout self.stderrPipe = stderr @@ -87,9 +129,15 @@ actor LSPTransport { try proc.run() - let handle = stdout.fileHandleForReading + // Not FileHandle.bytes: its reads share one queue, so an idle server would stall every other AsyncBytes read. + let (chunks, chunkSink) = AsyncStream.makeStream(of: Data.self) + let stdoutReader = PipeReader(stdout.fileHandleForReading) + stdoutReader.start { chunkSink.yield($0) } atEnd: { _ in + chunkSink.finish() + } + self.stdoutReader = stdoutReader readerTask = Task { [weak self] in - await self?.runReadLoop(handle: handle) + await self?.runReadLoop(chunks) } Self.logger.info("LSP transport started: \(executablePath)") @@ -105,9 +153,10 @@ actor LSPTransport { readerTask?.cancel() readerTask = nil - if let stdinHandle = stdinPipe?.fileHandleForWriting { - try? stdinHandle.close() - } + writer?.close() + writer = nil + stdoutReader?.stop() + stdoutReader = nil if let stdoutHandle = stdoutPipe?.fileHandleForReading { try? stdoutHandle.close() } @@ -144,7 +193,7 @@ actor LSPTransport { return try await withCheckedThrowingContinuation { continuation in pendingRequests[requestID] = continuation do { - try writeMessage(data) + try writeMessage(data, failingRequest: requestID) } catch { pendingRequests.removeValue(forKey: requestID) continuation.resume(throwing: error) @@ -212,40 +261,37 @@ actor LSPTransport { // MARK: - Private - private func writeMessage(_ data: Data) throws { - guard let stdinPipe else { + private func writeMessage(_ data: Data, failingRequest requestID: Int? = nil) throws { + guard let writer else { throw LSPTransportError.processNotRunning } - - let header = "Content-Length: \(data.count)\r\n\r\n" - guard let headerData = header.data(using: .utf8) else { - throw LSPTransportError.invalidResponse + let frame = Data("Content-Length: \(data.count)\r\n\r\n".utf8) + data + writer.write(frame) { [weak self] error in + Task { [weak self] in + await self?.writeFailed(error, requestID: requestID) + } } + } - let handle = stdinPipe.fileHandleForWriting - handle.write(headerData) - handle.write(data) + private func writeFailed(_ error: any Error, requestID: Int?) { + Self.logger.warning("LSP write failed: \(error.localizedDescription)") + guard let requestID, let continuation = pendingRequests.removeValue(forKey: requestID) else { return } + continuation.resume(throwing: error) } - private func runReadLoop(handle: FileHandle) async { + private func runReadLoop(_ chunks: AsyncStream) async { var buffer = Data() - do { - for try await byte in handle.bytes { - if Task.isCancelled { return } - buffer.append(byte) - while let (messageData, _) = Self.parseMessageFromBuffer(&buffer) { - dispatchMessage(messageData) - } - } - } catch { - if !Task.isCancelled { - Self.logger.debug("LSP read loop ended: \(error.localizedDescription)") + for await chunk in chunks { + // A stream ended by cancellation still hands over the chunks it buffered. + if Task.isCancelled { return } + buffer.append(chunk) + while let (messageData, _) = Self.parseMessageFromBuffer(&buffer) { + dispatchMessage(messageData) } } } - /// Parse a single LSP message from the buffer. - /// Returns (messageBody, totalBytesConsumed) or nil if buffer is incomplete. + // Nil until the buffer holds a whole message; the Int is the bytes it consumed. private static func parseMessageFromBuffer(_ buffer: inout Data) -> (Data, Int)? { let separator: [UInt8] = [0x0D, 0x0A, 0x0D, 0x0A] guard let separatorRange = buffer.range(of: Data(separator)) else { @@ -320,7 +366,7 @@ actor LSPTransport { if let handler = notificationHandlers[method] { handler(data) } - // Server-initiated request (has both id and method) — reply with handler result or null + // Server-initiated request (has both id and method): reply with the handler's result or null if let id { if let deferred = deferredRequestHandlers[method] { deferred(data, id) @@ -341,6 +387,9 @@ actor LSPTransport { private func handleProcessExit(code: Int32) { Self.logger.info("LSP process exited with code \(code)") + writer?.close() + writer = nil + stdinPipe = nil let pending = pendingRequests pendingRequests.removeAll() for (_, continuation) in pending { diff --git a/apple/TablePro/Core/LSP/LSPTypes.swift b/apple/TablePro/Core/LSP/LSPTypes.swift index 0df7dd49b7..f6888062ad 100644 --- a/apple/TablePro/Core/LSP/LSPTypes.swift +++ b/apple/TablePro/Core/LSP/LSPTypes.swift @@ -35,7 +35,18 @@ struct LSPVersionedTextDocumentIdentifier: Codable, Sendable, Equatable { } struct LSPTextDocumentContentChangeEvent: Codable, Sendable, Equatable { + let range: LSPRange? let text: String + + init(text: String) { + self.range = nil + self.text = text + } + + init(range: LSPRange, text: String) { + self.range = range + self.text = text + } } struct LSPInlineCompletionItem: Codable, Sendable, Equatable { diff --git a/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift b/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift index c93f2427a7..c6a30934fb 100644 --- a/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift +++ b/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift @@ -13,6 +13,7 @@ public struct MCPValidatedToken: Sendable, Equatable { public let expiresAt: Date? public let isBridgeCredential: Bool public let isClaudeAgentCredential: Bool + public let toolAllowlist: Set? public init( tokenId: UUID, @@ -22,7 +23,8 @@ public struct MCPValidatedToken: Sendable, Equatable { issuedAt: Date, expiresAt: Date?, isBridgeCredential: Bool = false, - isClaudeAgentCredential: Bool = false + isClaudeAgentCredential: Bool = false, + toolAllowlist: Set? = nil ) { self.tokenId = tokenId self.label = label @@ -32,6 +34,7 @@ public struct MCPValidatedToken: Sendable, Equatable { self.isClaudeAgentCredential = isClaudeAgentCredential self.issuedAt = issuedAt self.expiresAt = expiresAt + self.toolAllowlist = toolAllowlist } } @@ -58,7 +61,8 @@ internal extension MCPTokenStore { issuedAt: authToken.createdAt, expiresAt: authToken.expiresAt, isBridgeCredential: authToken.isBridgeCredential, - isClaudeAgentCredential: authToken.isClaudeAgentCredential + isClaudeAgentCredential: authToken.isClaudeAgentCredential, + toolAllowlist: authToken.toolAllowlist ) } } @@ -133,7 +137,8 @@ public actor MCPBearerTokenAuthenticator: MCPAuthenticator { expiresAt: validated.expiresAt, isBridgeCredential: validated.isBridgeCredential, isClaudeAgentCredential: validated.isClaudeAgentCredential - ) + ), + toolAllowlist: validated.toolAllowlist ) MCPAuditLogger.logAuthSuccess( tokenId: validated.tokenId, diff --git a/apple/TablePro/Core/MCP/Clients/MCPClientCatalogBundle.swift b/apple/TablePro/Core/MCP/Clients/MCPClientCatalogBundle.swift new file mode 100644 index 0000000000..e6539dd60f --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPClientCatalogBundle.swift @@ -0,0 +1,91 @@ +import Foundation +import os +import TableProMCPClients + +enum MCPClientCatalogBundle { + private static let logger = Logger(subsystem: "com.TablePro", category: "MCPClientCatalog") + + static let shared: MCPClientCatalog? = load(from: .main) + + static func load(from bundle: Bundle) -> MCPClientCatalog? { + guard let url = bundle.url(forResource: "clients", withExtension: "json") else { + logger.error("The app bundle has no clients.json") + return nil + } + do { + return try MCPClientCatalog.decode(Data(contentsOf: url)) + } catch { + logger.error("clients.json could not be read: \(error.publicLogShape, privacy: .public)") + return nil + } + } +} + +struct MCPClientSessionSighting: Equatable, Sendable { + let id: String + let clientInfoName: String + // The client's clientInfo title when it sent one, such as "Codex", else its name. + let displayName: String + let tokenName: String? + + init(id: String, clientInfoName: String, displayName: String? = nil, tokenName: String?) { + self.id = id + self.clientInfoName = clientInfoName + self.displayName = displayName ?? clientInfoName + self.tokenName = tokenName + } +} + +extension MCPClientSessionSighting { + init(_ snapshot: MCPServerManager.SessionSnapshot) { + self.init( + id: snapshot.id, + clientInfoName: snapshot.clientInfoName ?? snapshot.clientName, + displayName: snapshot.clientName, + tokenName: snapshot.tokenName + ) + } +} + +extension MCPClientCatalog { + // The in-app Claude Agent runs the claude CLI, which names itself Claude Code; that session is TablePro's own. + func clients(forSessionNamed clientInfoName: String, tokenName: String?) -> [MCPClientDefinition] { + guard tokenName != MCPTokenStore.claudeAgentTokenName else { return [] } + return clients(matchingClientInfoName: clientInfoName) + } + + // ChatGPT and Codex CLI send one clientInfo name: the session is the one of them found on this Mac, else neither. + func client(forSessionNamed clientInfoName: String, tokenName: String?, detected: Set) -> MCPClientDefinition? { + guard tokenName != MCPTokenStore.claudeAgentTokenName else { return nil } + return client(matchingClientInfoName: clientInfoName, detected: detected) + } + + // The rows a session lights: its own client, or every row it could be when the Mac cannot tell them apart. + func clientIDs(forSessionNamed clientInfoName: String, tokenName: String?, detected: Set) -> Set { + if let client = client(forSessionNamed: clientInfoName, tokenName: tokenName, detected: detected) { + return [client.id] + } + return Set(clients(forSessionNamed: clientInfoName, tokenName: tokenName).map(\.id)) + } + + // An icon is still right when every client the session could be shares it, as ChatGPT and Codex CLI do. + func iconSlug(forSessionNamed clientInfoName: String, tokenName: String?, detected: Set) -> String? { + if let client = client(forSessionNamed: clientInfoName, tokenName: tokenName, detected: detected) { + return client.icon + } + let icons = Set(clients(forSessionNamed: clientInfoName, tokenName: tokenName).map(\.icon)) + return icons.count == 1 ? icons.first : nil + } + + // The clients whose sessions only detection can tell apart. + func ambiguousClientIDs(forSessionsNamed sessions: [(clientInfoName: String, tokenName: String?)]) -> Set { + var ids: Set = [] + for session in sessions { + let candidates = clients(forSessionNamed: session.clientInfoName, tokenName: session.tokenName) + if candidates.count > 1 { + ids.formUnion(candidates.map(\.id)) + } + } + return ids + } +} diff --git a/apple/TablePro/Core/MCP/Clients/MCPClientRow.swift b/apple/TablePro/Core/MCP/Clients/MCPClientRow.swift new file mode 100644 index 0000000000..df84a8ac57 --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPClientRow.swift @@ -0,0 +1,304 @@ +import Foundation +import TableProMCPClients + +enum MCPClientConfigState: Equatable, Sendable { + case notConfigured + case current + case moved + case otherCopy + case outdated + // `project` is the folder whose own setup wins there, as the detail text shows it. + case shadowed(project: String) + case otherScope + case disabled + case usesToken + case unreadable + case unknown + + init(_ status: MCPClientConfigStatus, displayPath: (String) -> String = { $0 }) { + switch status { + case .notConfigured: + self = .notConfigured + case .current: + self = .current + case .stale(let reason): + switch reason { + case .commandMissing: + self = .moved + case .otherCommand: + self = .otherCopy + case .otherURL, .otherToken: + self = .outdated + case .shadowed(let scope): + self = .shadowed(project: displayPath(scope)) + } + case .otherScope: + self = .otherScope + case .disabled: + self = .disabled + case .usesToken: + self = .usesToken + case .unreadable: + self = .unreadable + case .unknown: + self = .unknown + } + } + + var hasEntry: Bool { + switch self { + case .current, .moved, .otherCopy, .outdated, .shadowed, .disabled, .usesToken: true + case .notConfigured, .otherScope, .unreadable, .unknown: false + } + } + + var needsUpdate: Bool { + switch self { + case .moved, .otherCopy, .outdated: true + case .current, .shadowed, .notConfigured, .otherScope, .disabled, .usesToken, .unreadable, .unknown: false + } + } + + // The user's own choice: a token they scoped, or TablePro turned off in the client. Install or Update would undo it. + var isUserChoice: Bool { + self == .usesToken || self == .disabled + } + + var isShadowed: Bool { + if case .shadowed = self { return true } + return false + } +} + +enum MCPClientRoute: Equatable, Sendable { + case deeplink(scheme: String) + case cli(binary: String) + case file + case copy + + var handsOff: Bool { + if case .deeplink = self { return true } + return false + } +} + +enum MCPClientUnsupported: Equatable, Sendable { + case remoteOnly + case stdioOnly +} + +enum MCPClientApplyMoment: Equatable, Sendable { + case live + case newSession + case restartApp + case reload +} + +struct MCPClientRow: Identifiable, Equatable, Sendable { + let id: String + let name: String + let iconSlug: String + let isDetected: Bool + let isInstalled: Bool + let config: MCPClientConfigState + let installRoute: MCPClientRoute + let updateRoute: MCPClientRoute + let removeRoute: MCPClientRoute? + let unsupported: MCPClientUnsupported? + let applies: MCPClientApplyMoment? + let configPath: String? + let displayConfigPath: String? + let configFileExists: Bool + let isConfigWritable: Bool + let snippet: String? + let commandLine: String? + + // A file only the client may write (Claude Code's .claude.json) is set up through its command, not pasted into. + var copyText: String? { + if let commandLine, !isConfigWritable { return commandLine } + return snippet ?? commandLine + } + + var configFileName: String? { + configPath.map { ($0 as NSString).lastPathComponent } + } +} + +struct MCPClientScanner: Sendable { + let catalog: MCPClientCatalog + let context: MCPClientSetupContext + + func scan() -> [MCPClientRow] { + catalog.clients.compactMap(row(for:)) + } + + func row(for client: MCPClientDefinition) -> MCPClientRow? { + guard let spec = client.spec(for: .macos) else { return nil } + let location = catalog.location(for: client, platform: .macos) + let configPath = location.map { context.expand($0.path) } + let fileExists = configPath.map { FileManager.default.fileExists(atPath: $0) } ?? false + let config = configState(location: location, path: configPath, fileExists: fileExists) + let detected = spec.detect.contains(where: isDetected) + let isInstalled = detected || config.hasEntry || config == .otherScope + let isWritable = location.map { $0.isWritable && $0.format != .yaml } ?? false + + let renderer = MCPTemplateRenderer(serverName: catalog.serverName, target: .stdio(command: context.bridgePath)) + let unsupported = spec.unsupported.map(Self.unsupported) + let entry = unsupported == nil ? try? renderer.entry(spec) : nil + + let available = unsupported == nil ? availableRoutes(spec, isWritable: isWritable) : [] + let installRoute = isInstalled ? (available.first ?? .copy) : .copy + // Update merges into the file and keeps the user's own keys; a client's add command replaces the whole entry. + let updateRoute = isInstalled + ? (available.first { $0 == .file } ?? available.first { !$0.handsOff } ?? available.first ?? .copy) + : .copy + // A shadowed row's user entry works; Remove would take it out and leave the project's old one in charge. + let removeRoute = config.hasEntry && !config.isShadowed && unsupported == nil + ? removalRoute(spec, isWritable: isWritable, fileExists: fileExists) + : nil + + return MCPClientRow( + id: client.id, + name: client.name, + iconSlug: client.icon, + isDetected: detected, + isInstalled: isInstalled, + config: config, + installRoute: installRoute, + updateRoute: updateRoute, + removeRoute: removeRoute, + unsupported: unsupported, + applies: spec.applies.map(Self.applyMoment), + configPath: configPath, + displayConfigPath: configPath.map(context.displayPath), + configFileExists: fileExists, + isConfigWritable: isWritable, + snippet: entry.map { renderer.snippet(for: location, entry: $0) }, + commandLine: entry.flatMap { commandLine(spec, renderer: renderer, entry: $0) } + ) + } + + func executable(named binary: String) -> URL? { + context.executable(named: binary, catalogPaths: catalogPaths(forBinary: binary)) + } + + func detectedIDs(among ids: Set) -> Set { + let found = catalog.clients.filter { client in + ids.contains(client.id) && client.spec(for: .macos)?.detect.contains(where: isDetected) == true + } + return Set(found.map(\.id)) + } + + private func configState(location: MCPConfigLocation?, path: String?, fileExists: Bool) -> MCPClientConfigState { + guard let location, let path else { return .unknown } + var text: String? + if fileExists { + guard let read = try? String(contentsOfFile: path, encoding: .utf8) else { return .unreadable } + text = read + } + let status = MCPClientStatusEvaluator.status( + configText: text, + location: location, + serverName: catalog.serverName, + expected: .stdio(command: context.bridgePath), + fileExists: { FileManager.default.fileExists(atPath: $0) } + ) + return MCPClientConfigState(status, displayPath: context.displayPath) + } + + private func isDetected(_ rule: MCPClientDetectRule) -> Bool { + switch rule { + case .bundleID(let identifier): + context.workspace.applicationURL(forBundleIdentifier: identifier) != nil + case .binary(let name, _): + executable(named: name) != nil + case .path(let path): + FileManager.default.fileExists(atPath: context.expand(path)) + } + } + + private func availableRoutes(_ spec: MCPClientPlatformSpec, isWritable: Bool) -> [MCPClientRoute] { + spec.install.compactMap { method -> MCPClientRoute? in + switch method { + case .deeplink(let scheme, _): + hasHandler(forScheme: scheme) ? .deeplink(scheme: scheme) : nil + case .cli(let cli): + executable(named: cli.binary) != nil ? .cli(binary: cli.binary) : nil + case .file: + isWritable ? .file : nil + } + } + } + + private func removalRoute(_ spec: MCPClientPlatformSpec, isWritable: Bool, fileExists: Bool) -> MCPClientRoute? { + for method in spec.install { + switch method { + case .cli(let cli) where cli.remove != nil && executable(named: cli.binary) != nil: + return .cli(binary: cli.binary) + case .file where isWritable && fileExists: + return .file + default: + continue + } + } + return nil + } + + private func hasHandler(forScheme scheme: String) -> Bool { + guard let probe = URL(string: "\(scheme)://") else { return false } + return context.workspace.applicationURL(toOpen: probe) != nil + } + + private func commandLine(_ spec: MCPClientPlatformSpec, renderer: MCPTemplateRenderer, entry: JSONValue) -> String? { + guard let cli = spec.cliInstall, let arguments = try? renderer.arguments(cli.add, entry: entry) else { return nil } + return renderer.commandLine(binary: cli.binary, arguments: arguments) + } + + private func catalogPaths(forBinary binary: String) -> [String] { + catalog.clients + .flatMap { $0.spec(for: .macos)?.detect ?? [] } + .flatMap { rule -> [String] in + if case let .binary(name, paths) = rule, name == binary { return paths } + return [] + } + } + + private static func unsupported(_ reason: MCPClientUnsupportedReason) -> MCPClientUnsupported { + switch reason { + case .remoteOnly: .remoteOnly + case .stdioOnly: .stdioOnly + } + } + + private static func applyMoment(_ applies: MCPClientApplies) -> MCPClientApplyMoment { + switch applies { + case .live: .live + case .newSession: .newSession + case .restartApp: .restartApp + case .reload: .reload + } + } +} + +extension MCPClientPlatformSpec { + var cliInstall: MCPCLIInstall? { + for method in install { + if case .cli(let cli) = method { return cli } + } + return nil + } + + func cliInstall(binary: String) -> MCPCLIInstall? { + for method in install { + if case .cli(let cli) = method, cli.binary == binary { return cli } + } + return nil + } + + func deeplinkTemplate(scheme: String) -> String? { + for method in install { + if case let .deeplink(candidate, template) = method, candidate == scheme { return template } + } + return nil + } +} diff --git a/apple/TablePro/Core/MCP/Clients/MCPClientRowPresentation.swift b/apple/TablePro/Core/MCP/Clients/MCPClientRowPresentation.swift new file mode 100644 index 0000000000..21a1f29426 --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPClientRowPresentation.swift @@ -0,0 +1,244 @@ +import Foundation + +enum MCPClientActionKind: Equatable, Sendable { + case install + case update + case remove +} + +enum MCPClientActivity: Equatable, Sendable { + case running(MCPClientActionKind) + case handedOff + case failed(MCPClientActionKind, message: String, output: String?) +} + +enum MCPClientConnection: Equatable, Sendable { + case notConnected + case connected + case justConnected +} + +enum MCPClientRowAction: Equatable, Sendable { + case install(handsOff: Bool) + case update(handsOff: Bool) + case copy + + var title: String { + switch self { + case .install(let handsOff): + if handsOff { String(localized: "Install…") } else { String(localized: "Install") } + case .update(let handsOff): + if handsOff { String(localized: "Update…") } else { String(localized: "Update") } + case .copy: + String(localized: "Copy") + } + } + + // Every row's button reads the same verb, so VoiceOver needs the client's name to tell them apart. + func accessibilityLabel(clientName: String) -> String { + switch self { + case .install: + String(format: String(localized: "Install in %@"), clientName) + case .update: + String(format: String(localized: "Update in %@"), clientName) + case .copy: + String(format: String(localized: "Copy Setup for %@"), clientName) + } + } +} + +struct MCPClientRowPresentation: Equatable, Sendable { + let title: String + let subtitle: String + let status: String? + let action: MCPClientRowAction? + let isBusy: Bool + + init(row: MCPClientRow, activity: MCPClientActivity?, connection: MCPClientConnection) { + title = row.name + if case .running = activity { + isBusy = true + } else { + isBusy = false + } + if let unsupported = row.unsupported { + subtitle = Self.unsupportedSubtitle(unsupported) + status = nil + action = nil + return + } + var route = row.config.needsUpdate ? row.updateRoute : row.installRoute + if row.config == .unreadable, route == .file { + route = .copy + } + subtitle = Self.subtitle(route: route, row: row) + status = Self.status(row: row, activity: activity, connection: connection) + action = isBusy ? nil : Self.action(row: row) + } + + static func action(row: MCPClientRow) -> MCPClientRowAction? { + guard row.unsupported == nil else { return nil } + switch row.config { + case .current, .shadowed, .disabled, .usesToken: + return nil + case .moved, .otherCopy, .outdated: + return row.updateRoute == .copy ? .copy : .update(handsOff: row.updateRoute.handsOff) + case .notConfigured, .otherScope, .unreadable, .unknown: + if row.installRoute == .copy || (row.config == .unreadable && row.installRoute == .file) { + return row.copyText == nil ? nil : .copy + } + return .install(handsOff: row.installRoute.handsOff) + } + } + + private static func status( + row: MCPClientRow, + activity: MCPClientActivity?, + connection: MCPClientConnection + ) -> String? { + switch activity { + case .running(let kind): + return runningStatus(kind) + case .failed(let kind, _, _): + return failedStatus(kind) + case .handedOff where !row.config.hasEntry && connection == .notConnected: + return String(format: String(localized: "Finish in %@"), row.name) + case .handedOff, .none: + break + } + switch connection { + case .justConnected: + return String(localized: "Connected just now") + case .connected: + return String(localized: "Connected") + case .notConnected: + break + } + switch row.config { + case .current: + return String(localized: "Set up") + case .moved: + return String(localized: "TablePro moved") + case .otherCopy: + return String(localized: "Uses another copy") + case .outdated: + return String(localized: "Out of date") + case .shadowed: + return String(localized: "Old setup in a project") + case .otherScope: + return String(localized: "One project only") + case .disabled: + return String(localized: "Turned off") + case .usesToken: + return String(localized: "Set up with a token") + case .unreadable: + return String(localized: "Can't read settings") + case .notConfigured, .unknown: + return row.isInstalled ? nil : String(localized: "Not found") + } + } + + private static func runningStatus(_ kind: MCPClientActionKind) -> String { + switch kind { + case .install: String(localized: "Installing…") + case .update: String(localized: "Updating…") + case .remove: String(localized: "Removing…") + } + } + + private static func failedStatus(_ kind: MCPClientActionKind) -> String { + switch kind { + case .install: String(localized: "Couldn't install") + case .update: String(localized: "Couldn't update") + case .remove: String(localized: "Couldn't remove") + } + } + + private static func subtitle(route: MCPClientRoute, row: MCPClientRow) -> String { + switch route { + case .deeplink: + return String(format: String(localized: "Opens %@ to confirm"), row.name) + case .cli(let binary): + return String(format: String(localized: "Adds through the %@ command"), binary) + case .file: + return String(format: String(localized: "Edits %@"), row.displayConfigPath ?? row.name) + case .copy: + if let path = row.displayConfigPath, row.isConfigWritable { + return String(format: String(localized: "Copy the setup into %@"), path) + } + if row.commandLine != nil { + return String(localized: "Copy the command and run it in Terminal") + } + return String(format: String(localized: "Add it in %@'s settings"), row.name) + } + } + + private static func unsupportedSubtitle(_ reason: MCPClientUnsupported) -> String { + switch reason { + case .remoteOnly: + String(localized: "Reaches only servers on the internet") + case .stdioOnly: + String(localized: "Starts only local command servers") + } + } +} + +enum MCPClientDetailText { + static func applies(_ moment: MCPClientApplyMoment, name: String) -> String { + switch moment { + case .live: + String(format: String(localized: "%@ picks up the change right away."), name) + case .newSession: + String(format: String(localized: "Start a new %@ session to load it."), name) + case .restartApp: + String(format: String(localized: "Quit and reopen %@ to load it."), name) + case .reload: + String(format: String(localized: "Reload %@ to load it."), name) + } + } + + static func unsupported(_ reason: MCPClientUnsupported, name: String) -> String { + switch reason { + case .remoteOnly: + String( + format: String(localized: "%@ reaches MCP servers only from its maker's cloud, so it cannot reach TablePro on this Mac."), + name + ) + case .stdioOnly: + String( + format: String(localized: "%@ starts MCP servers only as local commands, so it cannot reach TablePro's HTTP server."), + name + ) + } + } + + static func configState(_ state: MCPClientConfigState, name: String) -> String? { + switch state { + case .moved: + String(format: String(localized: "%@ is set up for a copy of TablePro that is no longer there."), name) + case .otherCopy: + String(format: String(localized: "%@ is set up for a different copy of TablePro. Update points it to this one."), name) + case .outdated: + String(format: String(localized: "%@ is set up with an address or token that no longer works."), name) + case .shadowed(let project): + String( + format: String(localized: "%1$@ is set up, but the project %2$@ has its own older TablePro setup that wins there. Remove TablePro from that project's settings."), + name, + project + ) + case .otherScope: + String(format: String(localized: "%@ has TablePro in one project only. Install adds it for every project."), name) + case .disabled: + String(format: String(localized: "TablePro is turned off in %1$@. Turn it on in %1$@'s MCP settings to use it."), name) + case .usesToken: + String( + format: String(localized: "%@ connects with a token from Settings > MCP > Authentication, which can limit what it reaches. TablePro leaves it as is."), + name + ) + case .unreadable: + String(format: String(localized: "TablePro could not read %@'s settings file, so it leaves the file alone. Copy the setup and add it by hand."), name) + case .current, .notConfigured, .unknown: + nil + } + } +} diff --git a/apple/TablePro/Core/MCP/Clients/MCPClientSetupContext.swift b/apple/TablePro/Core/MCP/Clients/MCPClientSetupContext.swift new file mode 100644 index 0000000000..4367ae3228 --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPClientSetupContext.swift @@ -0,0 +1,83 @@ +import Foundation +import TableProMCPClients + +struct MCPClientSetupContext: Sendable { + let home: String + let environment: [String: String] + let bridgePath: String + let workspace: any MCPClientWorkspace + let isExecutable: @Sendable (String) -> Bool + + init( + home: String, + environment: [String: String], + bridgePath: String, + workspace: any MCPClientWorkspace = SystemMCPClientWorkspace(), + isExecutable: @escaping @Sendable (String) -> Bool = { FileManager.default.isExecutableFile(atPath: $0) } + ) { + self.home = home + self.environment = environment + self.bridgePath = bridgePath + self.workspace = workspace + self.isExecutable = isExecutable + } + + // A sandboxed run reads and writes client settings in a home inside the sandbox, never the real one. + static func live(storage: AppStorageEnvironment = .shared, bundle: Bundle = .main) -> MCPClientSetupContext { + let processEnvironment = ProcessInfo.processInfo.environment + let bridgePath = bundle.bundleURL.appendingPathComponent("Contents/MacOS/tablepro-mcp").path + guard storage.isIsolated else { + return MCPClientSetupContext( + home: NSHomeDirectory(), + environment: processEnvironment, + bridgePath: bridgePath + ) + } + return MCPClientSetupContext( + home: storage.supportDirectory.appendingPathComponent("ClientHome", isDirectory: true).path, + environment: processEnvironment["PATH"].map { ["PATH": $0] } ?? [:], + bridgePath: bridgePath + ) + } + + var searchDirectories: [String] { + [ + "\(home)/.local/bin", + "\(home)/.claude/local", + "/opt/homebrew/bin", + "/usr/local/bin", + "\(home)/.npm-global/bin", + "\(home)/.bun/bin", + "\(home)/.volta/bin", + "\(home)/.opencode/bin", + "/usr/bin", + "/bin" + ] + AgentCLIDiscovery.nodeVersionManagerDirectories(home: home) + } + + func expand(_ path: String) -> String { + MCPPathExpander.expand(path, home: home, environment: environment) + } + + func displayPath(_ path: String) -> String { + guard path.hasPrefix(home + "/") else { return path } + return "~" + path.dropFirst(home.count) + } + + // A GUI app gets launchd's PATH, and a node-based CLI needs node on it. + func processEnvironment() -> [String: String] { + AgentCLIDiscovery(candidatePaths: [], preferredPathDirectories: searchDirectories) + .makeEnvironment(base: environment, adding: ["HOME": home]) + } + + func executable(named name: String, catalogPaths: [String]) -> URL? { + var seen = Set() + let candidates = (catalogPaths.map(expand) + searchDirectories.map { "\($0)/\(name)" }) + .filter { seen.insert($0).inserted } + return AgentCLIDiscovery( + candidatePaths: candidates, + preferredPathDirectories: searchDirectories, + isExecutable: isExecutable + ).resolveExecutable(override: nil) + } +} diff --git a/apple/TablePro/Core/MCP/Clients/MCPClientSetupService.swift b/apple/TablePro/Core/MCP/Clients/MCPClientSetupService.swift new file mode 100644 index 0000000000..5798fbc180 --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPClientSetupService.swift @@ -0,0 +1,405 @@ +import AppKit +import Combine +import Foundation +import os +import TableProMCPClients + +enum MCPClientActionOutcome: Equatable, Sendable { + case done(backupPath: String?) + case handedOff + case refusedReadOnlyVolume + case failed(message: String, output: String?) + case unavailable +} + +private struct MCPClientActionFailure: Error { + let message: String + let output: String? +} + +@MainActor +final class MCPClientSetupService: ObservableObject { + private static let logger = Logger(subsystem: "com.TablePro", category: "MCPClientSetup") + private static let justConnectedDuration: Duration = .seconds(60) + + @Published private(set) var rows: [MCPClientRow] = [] + @Published private(set) var hasLoaded = false + @Published private(set) var activities: [String: MCPClientActivity] = [:] + @Published private(set) var connectedClientIDs: Set = [] + @Published private(set) var justConnectedClientIDs: Set = [] + + let isAppOnReadOnlyVolume: Bool + var isCatalogAvailable: Bool { catalog != nil } + + private let catalog: MCPClientCatalog? + private let context: MCPClientSetupContext + private let runner: any MCPClientCommandRunning + private let announce: @MainActor (String) -> Void + private var knownSessionIDs: Set? + private var sightings: [MCPClientSessionSighting] = [] + private var refreshGeneration = 0 + private var settleTasks: [String: Task] = [:] + private var subscriptions: Set = [] + + init( + catalog: MCPClientCatalog? = MCPClientCatalogBundle.shared, + context: MCPClientSetupContext = .live(), + runner: any MCPClientCommandRunning = AgentCLICommandRunner(), + isAppOnReadOnlyVolume: Bool = CommandLineShim.isOnReadOnlyVolume(Bundle.main.bundleURL), + sessions: AnyPublisher<[MCPClientSessionSighting], Never> = MCPClientSetupService.liveSessions(), + activations: AnyPublisher = MCPClientSetupService.liveActivations(), + announce: @escaping @MainActor (String) -> Void = { AccessibilityAnnouncement.post($0) } + ) { + self.catalog = catalog + self.context = context + self.runner = runner + self.isAppOnReadOnlyVolume = isAppOnReadOnlyVolume + self.announce = announce + + sessions + .sink { [weak self] sightings in self?.sessionsChanged(sightings) } + .store(in: &subscriptions) + activations + .sink { [weak self] in + guard let self else { return } + Task { await self.returnedToTablePro() } + } + .store(in: &subscriptions) + } + + deinit { + for task in settleTasks.values { + task.cancel() + } + } + + static func liveSessions() -> AnyPublisher<[MCPClientSessionSighting], Never> { + MCPServerManager.shared.$connectedClients + .map { snapshots in snapshots.map(MCPClientSessionSighting.init) } + .eraseToAnyPublisher() + } + + static func liveActivations() -> AnyPublisher { + NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification) + .map { _ in () } + .eraseToAnyPublisher() + } + + func row(id: String) -> MCPClientRow? { + rows.first { $0.id == id } + } + + func presentation(for row: MCPClientRow) -> MCPClientRowPresentation { + MCPClientRowPresentation(row: row, activity: activities[row.id], connection: connection(for: row.id)) + } + + func connection(for id: String) -> MCPClientConnection { + if justConnectedClientIDs.contains(id) { return .justConnected } + if connectedClientIDs.contains(id) { return .connected } + return .notConnected + } + + func refresh() async { + guard let catalog else { + hasLoaded = true + return + } + refreshGeneration += 1 + let generation = refreshGeneration + let scanned = await Self.scan(MCPClientScanner(catalog: catalog, context: context)) + guard generation == refreshGeneration else { return } + rows = scanned + hasLoaded = true + // Which of two rows sharing a session is on this Mac is known only once the rows are scanned. + connectedClientIDs = connectedIDs(for: sightings, catalog: catalog) + justConnectedClientIDs.formIntersection(connectedClientIDs) + for row in scanned where row.config.hasEntry && activities[row.id] == .handedOff { + activities[row.id] = nil + } + } + + // A client with no settings file to read gives no sign the hand-off finished; coming back is the best one. + func returnedToTablePro() async { + await refresh() + for row in rows where row.config == .unknown && activities[row.id] == .handedOff { + activities[row.id] = nil + } + } + + @concurrent + nonisolated private static func scan(_ scanner: MCPClientScanner) async -> [MCPClientRow] { + scanner.scan() + } + + @discardableResult + func install(_ id: String) async -> MCPClientActionOutcome { + await perform(.install, on: id) + } + + @discardableResult + func update(_ id: String) async -> MCPClientActionOutcome { + await perform(.update, on: id) + } + + @discardableResult + func remove(_ id: String) async -> MCPClientActionOutcome { + await perform(.remove, on: id) + } + + func sessionsChanged(_ sightings: [MCPClientSessionSighting]) { + guard let catalog else { return } + self.sightings = sightings + let previouslyConnected = connectedClientIDs + let connected = connectedIDs(for: sightings, catalog: catalog) + connectedClientIDs = connected + justConnectedClientIDs.formIntersection(connected) + + let ids = Set(sightings.map(\.id)) + guard let known = knownSessionIDs else { + // Sessions already open when the sheet appears read Connected, not just connected. + knownSessionIDs = ids + return + } + knownSessionIDs = ids + let arrivals = sightings.filter { !known.contains($0.id) } + guard !arrivals.isEmpty else { return } + + let detected = detectedClientIDs + for sighting in arrivals { + let clientIDs = catalog.clientIDs(forSessionNamed: sighting.clientInfoName, tokenName: sighting.tokenName, detected: detected) + let fresh = clientIDs.subtracting(previouslyConnected).subtracting(justConnectedClientIDs) + guard !fresh.isEmpty else { continue } + for id in fresh { + markJustConnected(id) + } + let client = catalog.client(forSessionNamed: sighting.clientInfoName, tokenName: sighting.tokenName, detected: detected) + announce(String(format: String(localized: "%@ connected"), client?.name ?? sighting.displayName)) + } + Task { await self.refresh() } + } + + private var detectedClientIDs: Set { + Set(rows.filter { $0.isDetected }.map(\.id)) + } + + private func connectedIDs(for sightings: [MCPClientSessionSighting], catalog: MCPClientCatalog) -> Set { + let detected = detectedClientIDs + return sightings.reduce(into: Set()) { ids, sighting in + ids.formUnion(catalog.clientIDs(forSessionNamed: sighting.clientInfoName, tokenName: sighting.tokenName, detected: detected)) + } + } + + private func markJustConnected(_ id: String) { + justConnectedClientIDs.insert(id) + settleTasks[id]?.cancel() + settleTasks[id] = Task { [weak self] in + try? await Task.sleep(for: Self.justConnectedDuration) + guard !Task.isCancelled else { return } + self?.justConnectedClientIDs.remove(id) + self?.settleTasks[id] = nil + } + } + + private func perform(_ kind: MCPClientActionKind, on id: String) async -> MCPClientActionOutcome { + guard let catalog, + let row = row(id: id), + let client = catalog.client(id: id), + let spec = client.spec(for: .macos), + spec.unsupported == nil + else { return .unavailable } + if case .running? = activities[id] { return .unavailable } + if kind != .remove, row.config.isUserChoice { return .unavailable } + // A disk image or a translocated copy is gone after eject or relaunch, so a client set up for it breaks. + if kind != .remove, isAppOnReadOnlyVolume { return .refusedReadOnlyVolume } + + let route: MCPClientRoute? = switch kind { + case .install: row.installRoute + case .update: row.updateRoute + case .remove: row.removeRoute + } + guard let route, route != .copy else { return .unavailable } + + activities[id] = .running(kind) + let outcome: MCPClientActionOutcome + do { + outcome = try await run(kind, route: route, client: client, spec: spec, catalog: catalog) + } catch let failure as MCPClientActionFailure { + outcome = .failed(message: failure.message, output: failure.output) + } catch { + outcome = .failed( + message: Self.message(for: error, fileName: row.configFileName ?? row.name), + output: Self.technicalDetail(for: error) + ) + } + + switch outcome { + case let .failed(message, output): + activities[id] = .failed(kind, message: message, output: output) + Self.logger.error("A client setup action failed for \(id, privacy: .public)") + case .handedOff: + activities[id] = .handedOff + case .done, .refusedReadOnlyVolume, .unavailable: + activities[id] = nil + } + await refresh() + return outcome + } + + private func run( + _ kind: MCPClientActionKind, + route: MCPClientRoute, + client: MCPClientDefinition, + spec: MCPClientPlatformSpec, + catalog: MCPClientCatalog + ) async throws -> MCPClientActionOutcome { + let renderer = MCPTemplateRenderer(serverName: catalog.serverName, target: .stdio(command: context.bridgePath)) + switch route { + case .deeplink(let scheme): + guard let template = spec.deeplinkTemplate(scheme: scheme) else { return .unavailable } + let url = try renderer.url(template, entry: try renderer.entry(spec)) + guard url.scheme?.caseInsensitiveCompare(scheme) == .orderedSame else { return .unavailable } + try await context.workspace.open(url) + return .handedOff + + case .cli(let binary): + guard let cli = spec.cliInstall(binary: binary) else { return .unavailable } + let scanner = MCPClientScanner(catalog: catalog, context: context) + guard let executable = scanner.executable(named: binary) else { + throw MCPClientActionFailure( + message: String(format: String(localized: "The %@ command is no longer installed."), binary), + output: nil + ) + } + let entry = try renderer.entry(spec) + switch kind { + case .install: + try await runCommand(executable, try renderer.arguments(cli.add, entry: entry), isRemoval: false) + case .update: + if cli.update == .removeThenAdd, let remove = cli.remove { + try await runCommand(executable, try renderer.arguments(remove, entry: entry), isRemoval: true) + } + try await runCommand(executable, try renderer.arguments(cli.add, entry: entry), isRemoval: false) + case .remove: + guard let remove = cli.remove else { return .unavailable } + try await runCommand(executable, try renderer.arguments(remove, entry: entry), isRemoval: true) + } + return .done(backupPath: nil) + + case .file: + guard let location = catalog.location(for: client, platform: .macos), + location.isWritable, + location.format != .yaml + else { return .unavailable } + let url = URL(fileURLWithPath: context.expand(location.path)) + let name = catalog.serverName + let writer = MCPConfigFileWriter() + switch kind { + case .install, .update: + guard let template = spec.entry else { return .unavailable } + let entry = try renderer.entry(spec) + let result = try writer.update(url) { text in + try Self.updating(name, entry: entry, template: template, at: location, in: text ?? "") + } + return .done(backupPath: result.backupURL?.path) + case .remove: + guard FileManager.default.fileExists(atPath: url.path) else { return .done(backupPath: nil) } + let result = try writer.update(url) { text in + try Self.removing(name, at: location, in: text ?? "") + } + return .done(backupPath: result.backupURL?.path) + } + + case .copy: + return .unavailable + } + } + + private func runCommand(_ executable: URL, _ arguments: [String], isRemoval: Bool) async throws { + let result = try await runner.run( + executable: executable, + arguments: arguments, + environment: context.processEnvironment() + ) + guard result.exitCode != 0 else { return } + // claude exits 1 for a server it does not have; for a removal that is the goal already reached. + if isRemoval, Self.reportsMissingEntry(result.output) { return } + let output = result.output.trimmingCharacters(in: .whitespacesAndNewlines) + throw MCPClientActionFailure( + message: String( + format: String(localized: "The %@ command stopped with exit code %d."), + executable.lastPathComponent, + Int(result.exitCode) + ), + output: output.isEmpty ? nil : output + ) + } + + nonisolated static func reportsMissingEntry(_ output: String) -> Bool { + let text = output.lowercased() + return ["no mcp server", "not found", "does not exist", "no such server"].contains { text.contains($0) } + } + + nonisolated private static func updating( + _ name: String, + entry: JSONValue, + template: JSONValue, + at location: MCPConfigLocation, + in text: String + ) throws -> String { + switch location.format { + case .toml: + try TOMLTableEditor.updating(name, entry: entry, template: template, at: location.keyPath, in: text) + case .json, .jsonc, .yaml: + try JSONCMemberEditor.updating(name, entry: entry, template: template, at: location.keyPath, in: text) + } + } + + nonisolated private static func removing(_ name: String, at location: MCPConfigLocation, in text: String) throws -> String { + switch location.format { + case .toml: + try TOMLTableEditor.removing(name, at: location.keyPath, in: text) + case .json, .jsonc, .yaml: + try JSONCMemberEditor.removing(name, at: location.keyPath, in: text) + } + } + + nonisolated static func message(for error: Error, fileName: String) -> String { + if let edit = error as? MCPConfigEditError { + switch edit { + case .invalidSyntax: + return String(format: String(localized: "%@ could not be read as valid settings, so TablePro left it unchanged."), fileName) + case .duplicateKey: + return String(format: String(localized: "%@ names the same key twice, so TablePro left it unchanged."), fileName) + case .notAnObject: + return String(format: String(localized: "%@ has an unexpected shape where TablePro would add itself, so TablePro left it unchanged."), fileName) + case .unsupportedShape: + return String(format: String(localized: "%@ defines TablePro in a form TablePro cannot edit safely, so it left the file unchanged."), fileName) + case .verificationFailed: + return verificationFailedMessage(fileName) + } + } + if let write = error as? MCPConfigWriteError { + switch write { + case .changedDuringWrite: + return String(format: String(localized: "%@ kept changing while TablePro wrote it. Try again."), fileName) + case .verificationFailed: + return verificationFailedMessage(fileName) + case .io: + return String(format: String(localized: "TablePro could not write %@."), fileName) + } + } + if error is MCPTemplateError { + return String(localized: "TablePro's client list has a setup for this client that it cannot fill in.") + } + return error.localizedDescription + } + + nonisolated static func technicalDetail(for error: Error) -> String? { + guard case .io(let detail)? = error as? MCPConfigWriteError else { return nil } + return detail + } + + nonisolated private static func verificationFailedMessage(_ fileName: String) -> String { + String(format: String(localized: "TablePro could not confirm its change to %@. Check the file before you try again."), fileName) + } +} diff --git a/apple/TablePro/Core/MCP/Clients/MCPClientWorkspace.swift b/apple/TablePro/Core/MCP/Clients/MCPClientWorkspace.swift new file mode 100644 index 0000000000..b573eca560 --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPClientWorkspace.swift @@ -0,0 +1,76 @@ +import AppKit +import Foundation + +protocol MCPClientWorkspace: Sendable { + func applicationURL(forBundleIdentifier identifier: String) -> URL? + func applicationURL(toOpen url: URL) -> URL? + func open(_ url: URL) async throws +} + +struct SystemMCPClientWorkspace: MCPClientWorkspace { + func applicationURL(forBundleIdentifier identifier: String) -> URL? { + NSWorkspace.shared.urlForApplication(withBundleIdentifier: identifier) + } + + func applicationURL(toOpen url: URL) -> URL? { + NSWorkspace.shared.urlForApplication(toOpen: url) + } + + // Only a catalog deeplink reaches here, and only after a handler app was found for its scheme. + func open(_ url: URL) async throws { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + NSWorkspace.shared.open(url, configuration: NSWorkspace.OpenConfiguration()) { _, error in + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume() + } + } + } + } +} + +struct MCPClientCommandResult: Equatable, Sendable { + let exitCode: Int32 + let output: String +} + +protocol MCPClientCommandRunning: Sendable { + func run(executable: URL, arguments: [String], environment: [String: String]) async throws -> MCPClientCommandResult +} + +enum MCPClientCommandError: Error, LocalizedError, Equatable { + case timedOut(command: String, seconds: Int) + + var errorDescription: String? { + switch self { + case let .timedOut(command, seconds): + String(format: String(localized: "The %@ command did not finish within %d seconds."), command, seconds) + } + } +} + +struct AgentCLICommandRunner: MCPClientCommandRunning { + var timeoutSeconds = 60 + + func run(executable: URL, arguments: [String], environment: [String: String]) async throws -> MCPClientCommandResult { + let process = AgentCLIProcess(label: executable.lastPathComponent) + let launch = AgentCLILaunch(executable: executable, arguments: arguments, environment: environment) + let limit = timeoutSeconds + return try await withThrowingTaskGroup(of: MCPClientCommandResult?.self) { group in + group.addTask { + let finished = try await process.run(launch) + return MCPClientCommandResult(exitCode: finished.code, output: finished.output) + } + group.addTask { + try await Task.sleep(for: .seconds(limit)) + return nil + } + defer { group.cancelAll() } + guard let first = try await group.next(), let result = first else { + throw MCPClientCommandError.timedOut(command: executable.lastPathComponent, seconds: limit) + } + return result + } + } +} diff --git a/apple/TablePro/Core/MCP/Clients/MCPHTTPClientSetup.swift b/apple/TablePro/Core/MCP/Clients/MCPHTTPClientSetup.swift new file mode 100644 index 0000000000..5cb0252fee --- /dev/null +++ b/apple/TablePro/Core/MCP/Clients/MCPHTTPClientSetup.swift @@ -0,0 +1,69 @@ +import Foundation +import TableProMCPClients + +struct MCPHTTPClientChoice: Identifiable, Hashable, Sendable { + let id: String + let name: String +} + +struct MCPHTTPClientSnippet: Equatable, Sendable { + let displayConfigPath: String? + let commandLine: String? + let snippet: String? +} + +struct MCPHTTPClientSetup: Sendable { + static let otherClientID = "__other__" + + let catalog: MCPClientCatalog? + let context: MCPClientSetupContext + let url: String + + init(catalog: MCPClientCatalog? = MCPClientCatalogBundle.shared, context: MCPClientSetupContext = .live(), port: Int) { + self.catalog = catalog + self.context = context + self.url = "http://127.0.0.1:\(port)/mcp" + } + + var choices: [MCPHTTPClientChoice] { + let catalogChoices = (catalog?.clients ?? []) + .filter { $0.spec(for: .macos) != nil && catalog?.httpEntry(for: $0) != nil } + .map { MCPHTTPClientChoice(id: $0.id, name: $0.name) } + return catalogChoices + [MCPHTTPClientChoice(id: Self.otherClientID, name: String(localized: "Other Client"))] + } + + static func authorization(token: String) -> String { + "Bearer \(token)" + } + + func render(clientID: String, token: String) -> MCPHTTPClientSnippet? { + guard let catalog, + let client = catalog.client(id: clientID), + let template = catalog.httpEntry(for: client), + let spec = Self.platformSpec(of: client, holding: template) + else { return nil } + let renderer = MCPTemplateRenderer( + serverName: catalog.serverName, + target: .http(url: url, authorization: Self.authorization(token: token)) + ) + guard let entry = try? renderer.entry(spec) else { return nil } + let location = catalog.location(for: client, platform: .macos) ?? catalog.location(for: client, platform: .linux) + let commandLine = spec.cliInstall.flatMap { cli in + (try? renderer.arguments(cli.add, entry: entry)).map { renderer.commandLine(binary: cli.binary, arguments: $0) } + } + let isWritable = location?.isWritable ?? true + return MCPHTTPClientSnippet( + displayConfigPath: location.map { context.displayPath(context.expand($0.path)) }, + commandLine: commandLine, + snippet: commandLine != nil && !isWritable ? nil : renderer.snippet(for: location, entry: entry) + ) + } + + // The HTTP entry is written for whichever platform serves TablePro over HTTP; its spec carries the matching CLI. + private static func platformSpec(of client: MCPClientDefinition, holding entry: JSONValue) -> MCPClientPlatformSpec? { + [MCPClientPlatform.macos, .linux] + .lazy + .compactMap { client.spec(for: $0) } + .first { $0.entry == entry } + } +} diff --git a/apple/TablePro/Core/MCP/Identity/MCPApprovalLedger.swift b/apple/TablePro/Core/MCP/Identity/MCPApprovalLedger.swift index c602f06ceb..3de9a1253e 100644 --- a/apple/TablePro/Core/MCP/Identity/MCPApprovalLedger.swift +++ b/apple/TablePro/Core/MCP/Identity/MCPApprovalLedger.swift @@ -57,9 +57,7 @@ public actor MCPApprovalLedger { return await store.denial(subject: principal.grantSubject.storageKey, connectionId: connectionId, now: now) } - /// Drops the approvals a revoked token was carrying. Rotating the bridge credential is not a - /// revocation: the bridge re-mints itself roughly every 45 minutes and deletes the token it - /// replaces, so treating that as the user withdrawing consent re-asks them all day. + // A bridge credential rotation never reaches here: it drops the old token without a revocation. func clear(subject: MCPGrantSubject) async { await store.revokeAll(subject: subject.storageKey) } diff --git a/apple/TablePro/Core/MCP/Legacy/Handlers/InitializeHandler.swift b/apple/TablePro/Core/MCP/Legacy/Handlers/InitializeHandler.swift index 8317f1f683..94757be96a 100644 --- a/apple/TablePro/Core/MCP/Legacy/Handlers/InitializeHandler.swift +++ b/apple/TablePro/Core/MCP/Legacy/Handlers/InitializeHandler.swift @@ -11,7 +11,7 @@ public struct InitializeHandler: MCPMethodHandler { public func handle(params: JsonValue?, context: MCPRequestContext) async throws -> MCPResult { let negotiated = try Self.negotiate(requestedVersion: params?["protocolVersion"]?.stringValue) - return Self.result(protocolVersion: negotiated) + return Self.result(protocolVersion: negotiated, scopes: context.principal.scopes) } public static func negotiate(requestedVersion: String?) throws -> MCPProtocolVersion { @@ -25,12 +25,25 @@ public struct InitializeHandler: MCPMethodHandler { return requested } - public static func result(protocolVersion: MCPProtocolVersion) -> MCPResult { + public static func result(protocolVersion: MCPProtocolVersion, scopes: Set) -> MCPResult { MCPResult.complete([ "protocolVersion": .string(protocolVersion.rawValue), - "capabilities": MCPMethodRegistry.capabilities().asJsonValue(era: .legacy), + "capabilities": capabilities(for: scopes).asJsonValue(era: .legacy), "serverInfo": MCPMethodRegistry.serverInfo.asJsonValue, "instructions": .string(MCPMethodRegistry.instructions) ]) } + + // A client told it has resources or prompts lists them at connect, and a 403 there can end the session. + static func capabilities(for scopes: Set) -> MCPServerCapabilities { + let all = MCPMethodRegistry.capabilities() + let readsResources = scopes.contains(.resourcesRead) + return MCPServerCapabilities( + tools: scopes.contains(.toolsRead) ? all.tools : nil, + resources: readsResources ? all.resources : nil, + prompts: readsResources ? all.prompts : nil, + completions: readsResources && all.completions, + extensions: all.extensions + ) + } } diff --git a/apple/TablePro/Core/MCP/Legacy/MCPLegacyEraAdapter.swift b/apple/TablePro/Core/MCP/Legacy/MCPLegacyEraAdapter.swift index 2ded825edf..9af96c8329 100644 --- a/apple/TablePro/Core/MCP/Legacy/MCPLegacyEraAdapter.swift +++ b/apple/TablePro/Core/MCP/Legacy/MCPLegacyEraAdapter.swift @@ -96,7 +96,7 @@ public actor MCPLegacyEraAdapter { Self.logger.info( "Legacy initialize: client=\(clientInfo?.name ?? "unknown", privacy: .private(mask: .hash)) version=\(negotiated.rawValue, privacy: .public)" ) - return (InitializeHandler.result(protocolVersion: negotiated), sessionId) + return (InitializeHandler.result(protocolVersion: negotiated, scopes: principal.scopes), sessionId) } public func terminate(sessionId: MCPLegacySessionId) async { diff --git a/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift b/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift index cc9c7020ee..ea8ab78310 100644 --- a/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift +++ b/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift @@ -31,14 +31,24 @@ internal enum MCPServerStartError: Error, LocalizedError, Equatable { @MainActor internal final class MCPServerManager: ObservableObject { - internal struct SessionSnapshot: Sendable, Identifiable { + internal struct SessionSnapshot: Sendable, Identifiable, Equatable { + internal enum Transport: Sendable, Equatable { + // Shares the bridge credential with every other stdio client, so Disconnect cannot lock it out. + case stdioBridge + // Holds a token of its own, which Disconnect revokes. + case http + } + internal let id: String internal let clientName: String + // What the client sent as clientInfo.name; clientName prefers its title. + internal let clientInfoName: String? internal let clientVersion: String? internal let connectedSince: Date internal let lastActivityAt: Date internal let tokenName: String? internal let remoteAddress: String? + internal let transport: Transport } private struct BridgeCredential { @@ -62,14 +72,13 @@ internal final class MCPServerManager: ObservableObject { private let handshake: MCPHandshakeFile private let isEnabled: @MainActor () -> Bool - @Published private var composition: MCPServerComposition? + @Published private(set) var composition: MCPServerComposition? @Published private var bridgeCredential: BridgeCredential? @Published private var instanceId = "" @Published private var generation = 0 @Published private var revocationObserverId: UUID? @Published private var dispatchTask: Task? @Published private var stateTask: Task? - @Published private var clientRefreshTask: Task? @Published private var tokenRenewalTask: Task? internal var isRunning: Bool { @@ -150,32 +159,40 @@ internal final class MCPServerManager: ObservableObject { composition?.authPolicy ?? MCPAuthPolicy() } - // False when the Keychain write failed: the token works again after a restart. + // False when nothing was revoked: no server is running, or the Keychain write failed and the token + // works again after a restart. @discardableResult internal func revokeToken(id: UUID) async -> Bool { - guard let tokenStore else { return true } + guard let tokenStore else { return false } return await tokenStore.revoke(tokenId: id) } + // False when the token revocation did not reach the Keychain. A stdio row revokes nothing: every + // stdio client shares the bridge credential, and a revoked one would lock all of them out. @discardableResult internal func disconnectClient(_ clientId: String) async -> Bool { guard let composition else { return true } let entries = await composition.activityLedger.snapshot(now: Date()) - - guard let tokenId = entries.first(where: { $0.id == clientId })?.tokenId else { + guard let entry = entries.first(where: { $0.id == clientId }) else { await composition.activityLedger.forget(id: clientId) await refreshClients() return true } - var withdrawn: MCPGrantSubject? = .token(tokenId) - if let ended = await tokenStore?.token(id: tokenId) { - withdrawn = ended.subjectWithdrawnWhenEnded + var stored: MCPAuthToken? + if let tokenId = entry.tokenId { + stored = await tokenStore?.token(id: tokenId) } - let persisted = await revokeToken(id: tokenId) - if let withdrawn { - await composition.authPolicy.clearApprovals(subject: withdrawn) + guard entry.transport == .http, stored?.isBridgeCredential != true, let tokenId = entry.tokenId else { + await composition.dispatcher.cancelInflight(matchingClientId: clientId) + await composition.activityLedger.forget(id: clientId) + await refreshClients() + return true } + + let withdrawn = stored?.subjectWithdrawnWhenEnded ?? .token(tokenId) + let persisted = await revokeToken(id: tokenId) + await composition.authPolicy.clearApprovals(subject: withdrawn) await composition.activityLedger.forget(tokenId: tokenId) await refreshClients() return persisted @@ -186,9 +203,7 @@ internal final class MCPServerManager: ObservableObject { private func performStart(port: UInt16) async -> MCPServerStartError? { guard isEnabled() else { await performStop() - handshake.writeRefusal( - MCPStartRefusal(pid: ProcessInfo.processInfo.processIdentifier, refusedAt: Date()) - ) + writeRefusal(.turnedOff) Self.logger.info("Refused to start the MCP server: it is turned off") return .turnedOff } @@ -212,8 +227,7 @@ internal final class MCPServerManager: ObservableObject { socketPath = try handshake.prepareSocketPath() } catch { Self.logger.error("MCP socket folder is not usable: \(error.publicLogShape, privacy: .public)") - state = .failed(.socketFolderUnusable) - tokenStore = nil + failStart(.socketFolderUnusable) return nil } @@ -237,23 +251,33 @@ internal final class MCPServerManager: ObservableObject { } Self.logger.error("MCP server failed to start: \(lastFailure, privacy: .private)") - state = .failed(.listenerUnavailable(detail: lastFailure)) - tokenStore = nil + failStart(.listenerUnavailable(detail: lastFailure)) return nil } + // The bridge that launched TablePro reads the refusal and names the failure instead of timing out. + private func failStart(_ failure: MCPServerFailure) { + state = .failed(failure) + tokenStore = nil + writeRefusal(.failedToStart) + } + + private func writeRefusal(_ reason: MCPStartRefusal.Reason) { + handshake.writeRefusal( + MCPStartRefusal(pid: ProcessInfo.processInfo.processIdentifier, refusedAt: Date(), reason: reason) + ) + } + private func activate(_ built: MCPServerComposition, generation: Int, store: MCPTokenStore) async { guard let boundPort = await built.transport.listeningPort else { await built.transport.stop() - state = .failed(.noBoundPort) - tokenStore = nil + failStart(.noBoundPort) return } guard let credential = await mintBridgeCredential(store: store) else { await built.transport.stop() - state = .failed(.credentialUnavailable) - tokenStore = nil + failStart(.credentialUnavailable) return } @@ -264,7 +288,6 @@ internal final class MCPServerManager: ObservableObject { await built.start() startDispatchLoop(built, generation: generation) startStateLoop(built, generation: generation) - startClientRefresh(built, generation: generation) startBridgeCredentialRenewal(generation: generation) await registerRevocationObserver(built, store: store, generation: generation) @@ -292,8 +315,6 @@ internal final class MCPServerManager: ObservableObject { dispatchTask = nil stateTask?.cancel() stateTask = nil - clientRefreshTask?.cancel() - clientRefreshTask = nil tokenRenewalTask?.cancel() tokenRenewalTask = nil @@ -310,7 +331,7 @@ internal final class MCPServerManager: ObservableObject { await releaseBridgeCredential() tokenStore = nil - connectedClients = [] + publishClients([]) state = .stopped MCPAuditLogger.logServerStopped() } @@ -358,7 +379,7 @@ internal final class MCPServerManager: ObservableObject { private func renewBridgeCredential(generation: Int) async { guard self.generation == generation, let store = tokenStore, - let socketPath = composition?.socketPath, + let composition, case .running = state else { return } let previous = bridgeCredential @@ -370,10 +391,13 @@ internal final class MCPServerManager: ObservableObject { } bridgeCredential = credential - writeHandshake(socketPath: socketPath, credential: credential) + writeHandshake(socketPath: composition.socketPath, credential: credential) if let previous { - await store.delete(tokenId: previous.tokenId) + // Not a revocation: the old credential opens nothing new, but what it started may finish until it expires. + await store.dropSupersededBridgeCredential(tokenId: previous.tokenId) + guard self.generation == generation else { return } + await composition.dispatcher.cancelInflight(matchingTokenId: previous.tokenId, at: previous.expiresAt) } Self.logger.info("Rotated the MCP bridge credential") } @@ -410,13 +434,11 @@ internal final class MCPServerManager: ObservableObject { } } - private func startClientRefresh(_ built: MCPServerComposition, generation: Int) { - clientRefreshTask = Task { @MainActor [weak self] in - while !Task.isCancelled { - guard let self, self.isCurrent(generation) else { return } - await self.refreshClients() - try? await Task.sleep(for: Self.clientRefreshInterval) - } + // Only a view showing the list reads it between changes, so the poll lives as long as that view's task. + internal func refreshClientsWhileShown() async { + while !Task.isCancelled { + await refreshClients() + try? await Task.sleep(for: Self.clientRefreshInterval) } } @@ -475,24 +497,32 @@ internal final class MCPServerManager: ObservableObject { self.generation == generation } - private func refreshClients() async { + internal func refreshClients() async { guard let composition else { - connectedClients = [] + publishClients([]) return } let now = Date() await composition.activityLedger.prune(now: now) let entries = await composition.activityLedger.snapshot(now: now) - connectedClients = entries.map { entry in + publishClients(entries.map { entry in SessionSnapshot( id: entry.id, clientName: entry.clientName, + clientInfoName: entry.clientInfoName, clientVersion: entry.clientVersion, connectedSince: entry.firstSeenAt, lastActivityAt: entry.lastSeenAt, tokenName: entry.tokenName, - remoteAddress: entry.address + remoteAddress: entry.address, + transport: entry.transport == .stdioBridge ? .stdioBridge : .http ) - } + }) + } + + // Every observer of the manager re-renders on an assignment, equal or not. + private func publishClients(_ clients: [SessionSnapshot]) { + guard clients != connectedClients else { return } + connectedClients = clients } } diff --git a/apple/TablePro/Core/MCP/MCPAuditLogStorage.swift b/apple/TablePro/Core/MCP/MCPAuditLogStorage.swift index ab6a3ec121..6ded8af32a 100644 --- a/apple/TablePro/Core/MCP/MCPAuditLogStorage.swift +++ b/apple/TablePro/Core/MCP/MCPAuditLogStorage.swift @@ -28,25 +28,26 @@ struct MCPAuditChainLink: Sendable, Equatable { entry.connectionId?.uuidString ?? "", entry.action, entry.outcome, - entry.details ?? "", - previousHash + entry.details ?? "" ] - let joined = fields.joined(separator: "\u{1F}") + // Only a row with a client hashes it, so every row written before the column keeps its digest. + let client = entry.clientName.map { [$0] } ?? [] + let joined = (fields + client + [previousHash]).joined(separator: "\u{1F}") return SHA256.hash(data: Data(joined.utf8)).hexEncoded } } actor MCPAuditLogStorage { - static let shared = MCPAuditLogStorage() + static let shared = MCPAuditLogStorage(directory: AppStorageEnvironment.shared.supportDirectory) + static let databaseFileName = "mcp-audit.db" private static let logger = Logger(subsystem: "com.TablePro", category: "MCPAuditLogStorage") private static let retentionDays: Int = 90 private static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) - private static var isRunningTests: Bool { - NSClassFromString("XCTestCase") != nil - } + private static let entryColumns = + "id, timestamp, category, token_id, token_name, connection_id, action, outcome, details, client_name" private struct DatabaseHandle: @unchecked Sendable { var pointer: OpaquePointer? @@ -57,60 +58,38 @@ actor MCPAuditLogStorage { private var db: OpaquePointer? { if !isPrepared { isPrepared = true - setupDatabase() + let setAside = setupDatabase() prune(olderThan: Self.retentionDays) loadChainHead() + if let setAside { + recordSetAside(setAside) + } } return dbHandle.pointer } private var dbPath: String? - private let testDatabaseSuffix: String? + private let directory: URL private var nextSequence: Int = 0 private var lastHash: String = MCPAuditChainLink.genesisHash private var isPrepared = false - init() { - self.testDatabaseSuffix = nil - } - - #if DEBUG - init(isolatedForTesting: Bool) { - self.testDatabaseSuffix = isolatedForTesting ? "_\(UUID().uuidString)" : nil + init(directory: URL) { + self.directory = directory } - var databaseFilePaths: [String] { - guard db != nil, let dbPath else { return [] } - return [dbPath, dbPath + "-wal", dbPath + "-shm"] - } - #endif - deinit { if let pointer = dbHandle.pointer { sqlite3_close(pointer) } - if Self.isRunningTests, let dbPath { - try? FileManager.default.removeItem(atPath: dbPath) - for suffix in ["-wal", "-shm"] { - try? FileManager.default.removeItem(atPath: dbPath + suffix) - } - } } - private func setupDatabase() { + private func setupDatabase() -> SetAsideDamagedStore? { let fileManager = FileManager.default - let directory = AppStorageEnvironment.shared.applicationSupportRoot.appendingPathComponent("TablePro") try? fileManager.createDirectory(at: directory, withIntermediateDirectories: true) Self.restrict(path: directory.path, to: 0o700) - if Self.isRunningTests { - Self.purgeAbandonedTestDatabases(in: directory, fileManager: fileManager) - } - - let suffix = testDatabaseSuffix ?? "" - let fileName = Self.isRunningTests - ? "mcp-audit-test_\(ProcessInfo.processInfo.processIdentifier)\(suffix).db" - : "mcp-audit.db" - let path = directory.appendingPathComponent(fileName).path(percentEncoded: false) + let url = directory.appendingPathComponent(Self.databaseFileName) + let path = url.path(percentEncoded: false) self.dbPath = path if !fileManager.fileExists(atPath: path) { @@ -122,9 +101,16 @@ actor MCPAuditLogStorage { } Self.restrict(path: path, to: 0o600) - if sqlite3_open(path, &dbHandle.pointer) != SQLITE_OK { + // The damaged log is evidence, so it is set aside for review, never deleted. + guard case .opened(let pointer, let setAside) = LocalSQLiteStoreOpener.open(at: url) else { Self.logger.error("Error opening MCP audit database") - return + return nil + } + dbHandle.pointer = pointer + if setAside != nil { + Self.logger.fault("The MCP audit log was damaged and set aside; a new chain starts") + // SQLite created the fresh file with the default mode; its -wal and -shm copy this one. + Self.restrict(path: path, to: 0o600) } execute("PRAGMA journal_mode=WAL;") @@ -133,6 +119,16 @@ actor MCPAuditLogStorage { createTables() backfillChainIfNeeded() restrictDatabaseFiles() + return setAside + } + + private func recordSetAside(_ setAside: SetAsideDamagedStore) { + addEntry(AuditEntry( + category: .admin, + action: "auditLog.setAside", + outcome: AuditOutcome.error, + details: "setAside=\(setAside.setAsideURL.lastPathComponent)" + )) } private func restrictDatabaseFiles() { @@ -143,14 +139,6 @@ actor MCPAuditLogStorage { } } - private static func purgeAbandonedTestDatabases(in directory: URL, fileManager: FileManager) { - let ownMarker = "mcp-audit-test_\(ProcessInfo.processInfo.processIdentifier)" - guard let names = try? fileManager.contentsOfDirectory(atPath: directory.path) else { return } - for name in names where name.hasPrefix("mcp-audit-test_") && !name.hasPrefix(ownMarker) { - try? fileManager.removeItem(atPath: directory.appendingPathComponent(name).path) - } - } - private static func restrict(path: String, to permissions: Int) { guard FileManager.default.fileExists(atPath: path) else { return } do { @@ -176,21 +164,24 @@ actor MCPAuditLogStorage { details TEXT, sequence INTEGER, previous_hash TEXT, - entry_hash TEXT + entry_hash TEXT, + client_name TEXT ); """) - addMissingChainColumns() + addMissingColumns() execute("CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_entries(timestamp DESC);") execute("CREATE INDEX IF NOT EXISTS idx_audit_token ON audit_entries(token_id, timestamp DESC);") execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_sequence ON audit_entries(sequence);") } - private func addMissingChainColumns() { + // Older builds read and write only the columns they name, so an added column keeps the log theirs too. + private func addMissingColumns() { let existing = columnNames(ofTable: "audit_entries") let required: [(String, String)] = [ ("sequence", "INTEGER"), ("previous_hash", "TEXT"), - ("entry_hash", "TEXT") + ("entry_hash", "TEXT"), + ("client_name", "TEXT") ] for (name, type) in required where !existing.contains(name) { execute("ALTER TABLE audit_entries ADD COLUMN \(name) \(type);") @@ -217,7 +208,7 @@ actor MCPAuditLogStorage { guard unchainedRowCount() > 0 else { return } let sql = """ - SELECT id, timestamp, category, token_id, token_name, connection_id, action, outcome, details + SELECT \(Self.entryColumns) FROM audit_entries ORDER BY timestamp ASC, id ASC; """ @@ -287,6 +278,8 @@ actor MCPAuditLogStorage { @discardableResult func addEntry(_ entry: AuditEntry) -> Bool { + // Opening the log loads the chain head, so it has to happen before the head is read. + let handle = db let sequence = nextSequence let previousHash = lastHash let hash = MCPAuditChainLink.digest(entry: entry, sequence: sequence, previousHash: previousHash) @@ -294,12 +287,12 @@ actor MCPAuditLogStorage { let sql = """ INSERT INTO audit_entries (id, timestamp, category, token_id, token_name, connection_id, action, outcome, details, - sequence, previous_hash, entry_hash) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + sequence, previous_hash, entry_hash, client_name) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ var statement: OpaquePointer? - guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { + guard sqlite3_prepare_v2(handle, sql, -1, &statement, nil) == SQLITE_OK else { Self.logger.warning("Failed to prepare audit insert statement") return false } @@ -317,6 +310,7 @@ actor MCPAuditLogStorage { sqlite3_bind_int64(statement, 10, Int64(sequence)) sqlite3_bind_text(statement, 11, previousHash, -1, Self.SQLITE_TRANSIENT) sqlite3_bind_text(statement, 12, hash, -1, Self.SQLITE_TRANSIENT) + bindOptionalText(statement, index: 13, value: entry.clientName) guard sqlite3_step(statement) == SQLITE_DONE else { Self.logger.warning("Failed to append audit entry \(entry.action, privacy: .public)") @@ -342,16 +336,20 @@ actor MCPAuditLogStorage { func query( category: AuditCategory? = nil, tokenId: UUID? = nil, + tokenName: String? = nil, + clientName: String? = nil, since: Date? = nil, limit: Int = 500 ) -> [AuditEntry] { var conditions: [String] = [] if category != nil { conditions.append("category = ?") } if tokenId != nil { conditions.append("token_id = ?") } + if tokenName != nil { conditions.append("token_name = ?") } + if clientName != nil { conditions.append("client_name = ?") } if since != nil { conditions.append("timestamp >= ?") } var sql = """ - SELECT id, timestamp, category, token_id, token_name, connection_id, action, outcome, details + SELECT \(Self.entryColumns) FROM audit_entries """ if !conditions.isEmpty { @@ -375,6 +373,10 @@ actor MCPAuditLogStorage { sqlite3_bind_text(statement, bindIndex, tokenId.uuidString, -1, Self.SQLITE_TRANSIENT) bindIndex += 1 } + for value in [tokenName, clientName].compactMap({ $0 }) { + sqlite3_bind_text(statement, bindIndex, value, -1, Self.SQLITE_TRANSIENT) + bindIndex += 1 + } if let since { sqlite3_bind_double(statement, bindIndex, since.timeIntervalSince1970) bindIndex += 1 @@ -390,10 +392,28 @@ actor MCPAuditLogStorage { return entries } + func clientNames() -> [String] { + let sql = """ + SELECT DISTINCT client_name FROM audit_entries + WHERE client_name IS NOT NULL + ORDER BY client_name COLLATE NOCASE; + """ + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return [] } + defer { sqlite3_finalize(statement) } + + var names: [String] = [] + while sqlite3_step(statement) == SQLITE_ROW { + if let raw = sqlite3_column_text(statement, 0) { + names.append(String(cString: raw)) + } + } + return names + } + func verify() -> MCPAuditVerification { let sql = """ - SELECT id, timestamp, category, token_id, token_name, connection_id, action, outcome, details, - sequence, previous_hash, entry_hash + SELECT \(Self.entryColumns), sequence, previous_hash, entry_hash FROM audit_entries ORDER BY sequence ASC; """ @@ -411,11 +431,11 @@ actor MCPAuditLogStorage { while sqlite3_step(statement) == SQLITE_ROW { guard let entry = parseEntry(statement) else { - return .broken(atSequence: Int(sqlite3_column_int64(statement, 9))) + return .broken(atSequence: Int(sqlite3_column_int64(statement, 10))) } - let sequence = Int(sqlite3_column_int64(statement, 9)) - let previousHash = sqlite3_column_text(statement, 10).map { String(cString: $0) } ?? "" - let storedHash = sqlite3_column_text(statement, 11).map { String(cString: $0) } ?? "" + let sequence = Int(sqlite3_column_int64(statement, 10)) + let previousHash = sqlite3_column_text(statement, 11).map { String(cString: $0) } ?? "" + let storedHash = sqlite3_column_text(statement, 12).map { String(cString: $0) } ?? "" let recomputed = MCPAuditChainLink.digest( entry: entry, @@ -499,6 +519,7 @@ actor MCPAuditLogStorage { let action = String(cString: actionCString) let outcome = String(cString: outcomeCString) let details = sqlite3_column_text(statement, 8).map { String(cString: $0) } + let clientName = sqlite3_column_text(statement, 9).map { String(cString: $0) } return AuditEntry( id: id, @@ -509,7 +530,8 @@ actor MCPAuditLogStorage { connectionId: connectionId, action: action, outcome: outcome, - details: details + details: details, + clientName: clientName ) } } diff --git a/apple/TablePro/Core/MCP/MCPAuditLogger.swift b/apple/TablePro/Core/MCP/MCPAuditLogger.swift index 56c75a47aa..e5e8dfbf96 100644 --- a/apple/TablePro/Core/MCP/MCPAuditLogger.swift +++ b/apple/TablePro/Core/MCP/MCPAuditLogger.swift @@ -34,6 +34,7 @@ enum MCPAuditLogger { private static let serverResource = Logger(subsystem: "com.TablePro", category: "MCPResource") private static let messageExcerptLimit = 256 + private static let clientNameLimit = 128 static func flush() async { await MCPAuditWriteQueue.shared.flush() @@ -166,6 +167,7 @@ enum MCPAuditLogger { static func logQueryExecuted( principal: MCPPrincipal, + client: String? = nil, connectionId: UUID, sql: String, durationMs: Int, @@ -201,12 +203,14 @@ enum MCPAuditLogger { connectionId: connectionId, action: "query.executed", outcome: outcome, - details: detailParts.joined(separator: " ") + details: detailParts.joined(separator: " "), + client: client ) } static func logToolCalled( principal: MCPPrincipal, + client: String? = nil, toolName: String, connectionId: UUID? = nil, outcome: AuditOutcome, @@ -233,22 +237,13 @@ enum MCPAuditLogger { connectionId: connectionId, action: "tool.\(toolName)", outcome: outcome, - details: detailParts.joined(separator: " ") + details: detailParts.joined(separator: " "), + client: client ) } - /// A call TablePro made to an MCP server somebody else runs. - /// - /// Every other entry in this log records something an outside client asked of TablePro. This one - /// records the opposite direction, which is the direction that can take the user's schema and - /// their query results off the machine, so it is the one worth keeping. It is written before the - /// request leaves rather than when it returns: a server that never answers has still been sent - /// the arguments. - /// - /// The payload is recorded as its size and its SHA-256, not its text. The arguments of a tool - /// call carry table names, filter values and anything the model copied out of a result, and an - /// audit log that stored them would be a second copy of the data the log exists to watch. The - /// digest still answers "was this the same call as that one". + // A call TablePro made to someone else's MCP server, written before it leaves: a server that never + // answers was still sent the arguments. The payload is kept as size and digest, never as text. static func logOutboundToolCall( serverId: UUID, serverName: String, @@ -258,9 +253,7 @@ enum MCPAuditLogger { payload: Data ) { let digest = SHA256.hash(data: payload).compactMap { String(format: "%02x", $0) }.joined() - /// Named for what it digests. In this surface a fingerprint means a pairing token's, which - /// no log line may carry, and the `mcp_log_token_material` SwiftLint rule reads that - /// vocabulary rather than the value. + // Not "fingerprint": the mcp_log_token_material SwiftLint rule reserves that word for token material. let payloadDigest = String(digest.prefix(16)) serverTool.info( """ @@ -289,6 +282,7 @@ enum MCPAuditLogger { static func logResourceRead( principal: MCPPrincipal, + client: String? = nil, uri: String, outcome: AuditOutcome, errorMessage: String? = nil @@ -312,7 +306,8 @@ enum MCPAuditLogger { tokenName: principal.auditLabel, action: "resource.read", outcome: outcome, - details: detailParts.joined(separator: " ") + details: detailParts.joined(separator: " "), + client: client ) } @@ -320,6 +315,18 @@ enum MCPAuditLogger { SHA256.hash(data: Data(sql.utf8)).hexEncoded } + // The name Connected Clients lists, before any catalog lookup, so a row reads the same in every build. + // English like the rest of the row; nil when the client sent no clientInfo. + static func clientName(for meta: MCPRequestMeta) -> String? { + guard let info = meta.clientInfo else { return nil } + if info.name == CLIClientIdentity.name { + return CLIClientIdentity.pairingName + } + return [info.title, info.name] + .compactMap { $0?.trimmingCharacters(in: .whitespacesAndNewlines) } + .first { !$0.isEmpty } + } + private static func composeDetails(ip: String, extra: String?) -> String { guard let extra, !extra.isEmpty else { return "ip=\(ip)" } return "ip=\(ip) \(extra)" @@ -332,7 +339,8 @@ enum MCPAuditLogger { connectionId: UUID? = nil, action: String, outcome: AuditOutcome, - details: String? = nil + details: String? = nil, + client: String? = nil ) { let entry = AuditEntry( category: category, @@ -341,7 +349,8 @@ enum MCPAuditLogger { connectionId: connectionId, action: action, outcome: outcome, - details: details + details: details, + clientName: client.map { truncate($0, to: clientNameLimit) } ) MCPAuditWriteQueue.shared.enqueue(entry, into: MCPAuditLogStorage.shared) } diff --git a/apple/TablePro/Core/MCP/MCPAuthPolicy.swift b/apple/TablePro/Core/MCP/MCPAuthPolicy.swift index 1fc13c4522..13743bf717 100644 --- a/apple/TablePro/Core/MCP/MCPAuthPolicy.swift +++ b/apple/TablePro/Core/MCP/MCPAuthPolicy.swift @@ -143,10 +143,7 @@ public actor MCPAuthPolicy { ) } - /// An aggregate read spans connections, so it cannot prompt for one that asks each time. It - /// answers the narrower question every per-connection check already asks: may this principal - /// see this connection at all. Anything it excludes is a connection the client could not have - /// named directly either. + // May this principal see the connection at all; a call that names one is asked about it on top of this. func readableConnectionIds(principal: MCPPrincipal) async -> Set { let candidates = await connectionIdsProvider() return await readableConnectionIds(principal: principal, from: candidates) @@ -172,6 +169,16 @@ public actor MCPAuthPolicy { return true } + // A read spanning connections cannot ask about each one, so it keeps only those already allowed. + func connectionIdsReadableWithoutPrompt(principal: MCPPrincipal) async -> Set { + var allowed: Set = [] + for candidate in await readableConnectionIds(principal: principal) { + guard await isAllowedWithoutPrompt(principal: principal, connectionId: candidate) else { continue } + allowed.insert(candidate) + } + return allowed + } + func resolveAndAuthorize( principal: MCPPrincipal, tool: MCPToolName, @@ -268,10 +275,7 @@ public actor MCPAuthPolicy { await approvalLedger.record(principal: principal, connectionId: connectionId, approved: true) } - /// Called when a token is revoked. The bundled bridge re-mints its credential roughly every - /// 45 minutes and deletes the one it replaces, which reaches here as a revocation; treating that - /// as the user withdrawing consent is what made the prompt come back all day, so the caller - /// passes the token's name and the bridge's own rotation is skipped. + // The bridge's approvals belong to the bridge, not to one of its rotating credentials. func clearApprovals(tokenId: UUID, wasBridgeCredential: Bool = false) async { guard !wasBridgeCredential else { return } await clearApprovals(subject: .token(tokenId)) diff --git a/apple/TablePro/Core/MCP/MCPConnectionApprovalReading.swift b/apple/TablePro/Core/MCP/MCPConnectionApprovalReading.swift index 9609610e4f..f9f083d6d6 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionApprovalReading.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionApprovalReading.swift @@ -1,9 +1,6 @@ import Foundation -/// Reads the user's MCP approval setting. -/// -/// A seam rather than a direct `AppSettingsManager` read, because the auth policy is an actor and -/// its tests must be able to state the policy without standing up main-actor settings. +// A seam so the auth policy actor and its tests can state the setting without main-actor settings. protocol MCPConnectionApprovalReading: Sendable { func connectionApproval() async -> MCPConnectionApproval } @@ -13,15 +10,3 @@ struct MCPSettingsApprovalReader: MCPConnectionApprovalReading { await MainActor.run { AppSettingsManager.shared.mcp.connectionApproval } } } - -struct MCPFixedApprovalReader: MCPConnectionApprovalReading { - let approval: MCPConnectionApproval - - init(_ approval: MCPConnectionApproval) { - self.approval = approval - } - - func connectionApproval() async -> MCPConnectionApproval { - approval - } -} diff --git a/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift b/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift index 6912d91cf7..4034f8353c 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift @@ -290,11 +290,13 @@ struct MCPExplainRequest: Equatable { } extension MCPConnectionBridge { + // The router reads the whole plan, so the budget holds only what goes out: the plan text, the parsed plan, the rows. func explainQuery( _ operation: ApprovedOperation, request: MCPExplainRequest, timeoutSeconds: Int, - cancellation: MCPCancellationToken? + cancellation: MCPCancellationToken?, + budget: ExternalAnswerBudget = .mcpAnswer ) async throws -> JsonValue { let databaseType = operation.databaseType let masking = ExternalRowMasking.plan(in: request.format) @@ -316,13 +318,15 @@ extension MCPConnectionBridge { declaredVariants: databaseType.explainVariants ) } else { - outcome = try await runStatement( + let statement = try await runStatement( operation, maxRows: Self.explainRowLimit, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - masking: masking + masking: masking, + budget: nil ) + outcome = (statement.result, statement.executionTimeMs) routed = ExplainResultRouter.route( sql: request.statement, columns: outcome.result.columns, @@ -332,24 +336,63 @@ extension MCPConnectionBridge { ) } + var spent = budget + let planText = routed.map { spent.keep(text: $0.rawText) } + // The tree repeats every node's text, so it goes out only when it fits whole; plan_text already carries it. + // No `nil` literal near a JsonValue here: it is ExpressibleByNilLiteral, so `nil` can become `.null`. + var plan: JsonValue? + var planLeftOut = false + if let encodedPlan = routed?.plan.map(Self.encode(plan:)) { + if spent.keepWhole(bytes: Self.encodedBytes(of: encodedPlan)) { + plan = encodedPlan + } else { + planLeftOut = true + } + } + let kept = spent.keep(outcome.result) var payload: [String: JsonValue] = [ "statement": .string(request.statement), "execution_time_ms": .double(outcome.executionTimeMs), "columns": .array(outcome.result.columns.map { .string($0) }), - "rows": .array(outcome.result.rows.map { row in .array(row.map(Self.cellValue)) }) + "rows": .array(kept.rows.map { row in .array(row.map(Self.cellValue)) }) ] if let statusMessage = outcome.result.statusMessage { payload["status_message"] = .string(statusMessage) } - if let routed { - payload["plan_text"] = .string(routed.rawText) - if let plan = routed.plan { - payload["plan"] = Self.encode(plan: plan) - } + if let planText { + payload["plan_text"] = .string(planText) + } + if let plan { + payload["plan"] = plan + } + if spent.sizeLimited || planLeftOut { + payload["is_truncated"] = .bool(true) + } + var notes = spent.notes + if planLeftOut { + notes.append(String( + format: String(localized: "The parsed plan did not fit in TablePro's %@ limit for one answer, so it is left out. Read plan_text instead."), + ByteCountFormatter.string(fromByteCount: Int64(spent.limitBytes), countStyle: .memory) + )) + } + if !notes.isEmpty { + payload["notes"] = .array(notes.map(JsonValue.string)) } return .object(payload) } + // What `value` encodes to, measured without rendering it; a number counts as the core counts a scalar. + static func encodedBytes(of value: JsonValue) -> Int { + switch value { + case .null: return 4 + case .bool: return 5 + case .int, .double: return 40 + case .string(let text): return text.utf8.count + 2 + case .array(let items): return items.reduce(2) { $0 + encodedBytes(of: $1) + 1 } + case .object(let fields): return fields.reduce(2) { $0 + $1.key.utf8.count + 3 + encodedBytes(of: $1.value) + 1 } + } + } + static let explainRowLimit = 5_000 static func explainStatement( diff --git a/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift b/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift index ed5436db9e..75d4538ac6 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift @@ -481,7 +481,35 @@ extension MCPConnectionBridge { ) let entries = page.entries.filter { Self.isSearchable($0, by: search, masksValues: masksValues) } - return .object(["history": .array(entries.map { Self.encode(historyEntry: $0, masksValues: masksValues) })]) + let answer = Self.withinAnswerBudget(entries.map { Self.encode(historyEntry: $0, masksValues: masksValues) }) + return .object([ + "history": .array(answer.entries), + "is_truncated": .bool(answer.isTruncated) + ]) + } + + // Entries keep their order; the first one that would pass the budget ends the answer. + static func withinAnswerBudget(_ entries: [JsonValue]) -> (entries: [JsonValue], isTruncated: Bool) { + var kept: [JsonValue] = [] + var usedBytes = 0 + for entry in entries { + usedBytes += entry.jsonString().utf8.count + guard usedBytes <= ExternalAnswerBudget.answerBytes else { + return (kept, true) + } + kept.append(entry) + } + return (kept, false) + } + + // The chat reaches only its own connection, so it lists only that one. + func listConnection(id connectionId: UUID) async -> JsonValue { + let listings = await MainActor.run { ExternalConnectionDirectory.listings() } + return Self.connectionListing(listings, id: connectionId) + } + + static func connectionListing(_ listings: [ExternalConnectionListing], id connectionId: UUID) -> JsonValue { + MCPConnectionListEncoder.encode(listings, access: .limited([connectionId]), purpose: .agent) } static func scopePath(_ scope: PluginPrivilegeScope) -> String { @@ -528,7 +556,9 @@ extension MCPConnectionBridge { // A statement the app wrote from the data itself carries stored values: a document edit's `replaceOne` holds every // field of the original and the edit, an import the file's values. What a person or a client typed is theirs. static func externalQueryText(of entry: QueryHistoryEntry, masksValues: Bool) -> String { - withholdsQuery(of: entry, masksValues: masksValues) ? ServerErrorText.withheldStatementText : entry.query + withholdsQuery(of: entry, masksValues: masksValues) + ? ServerErrorText.withheldStatementText + : ExternalAnswerBudget.cut(entry.query) } // The search matches the stored statement, so an entry whose statement is withheld would answer for its text. @@ -550,7 +580,7 @@ extension MCPConnectionBridge { // A failure the editor or a grid save recorded keeps the server's words, which can repeat a value it read. static func externalErrorText(_ error: String, masksValues: Bool) -> String { - masksValues ? ServerErrorText.withheldMessage : MCPErrorRedactor.redact(error) + masksValues ? ServerErrorText.withheldMessage : ExternalAnswerBudget.cut(MCPErrorRedactor.redact(error)) } } diff --git a/apple/TablePro/Core/MCP/MCPConnectionBridge.swift b/apple/TablePro/Core/MCP/MCPConnectionBridge.swift index 8a3f3dcbd2..019a44a119 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionBridge.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionBridge.swift @@ -19,10 +19,6 @@ public actor MCPConnectionBridge { await listConnections(access: principal.connectionAccess, purpose: purpose) } - func listConnections() async -> JsonValue { - await listConnections(access: .all, purpose: .agent) - } - private func listConnections(access: ConnectionAccess, purpose: MCPConnectionListPurpose) async -> JsonValue { let listings = await MainActor.run { ExternalConnectionDirectory.listings() } return MCPConnectionListEncoder.encode(listings, access: access, purpose: purpose) @@ -125,19 +121,22 @@ public actor MCPConnectionBridge { maxRows: Int, timeoutSeconds: Int, cancellation: MCPCancellationToken?, - masking: ExternalRowMasking = .byOrigin + masking: ExternalRowMasking = .byOrigin, + budget: ExternalAnswerBudget = .mcpAnswer ) async throws -> JsonValue { let outcome = try await runStatement( operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - masking: masking + masking: masking, + budget: budget ) return Self.encode( result: outcome.result, scope: operation.scope, - executionTimeMs: outcome.executionTimeMs + executionTimeMs: outcome.executionTimeMs, + notes: outcome.budget?.notes ?? [] ) } @@ -145,32 +144,36 @@ public actor MCPConnectionBridge { _ operation: ApprovedOperation, maxRows: Int, timeoutSeconds: Int, - cancellation: MCPCancellationToken? + cancellation: MCPCancellationToken?, + budget: ExternalAnswerBudget = .mcpAnswer ) async throws -> (payload: JsonValue, rowsReturned: Int) { let outcome = try await access.runScript( operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + budget: budget ) return (Self.encode(script: outcome, scope: operation.scope), outcome.rowsReturned) } + // A nil budget sends the rows whole, for a caller that holds them to a budget of its own. func runStatement( _ operation: ApprovedOperation, maxRows: Int, timeoutSeconds: Int, cancellation: MCPCancellationToken?, - masking: ExternalRowMasking = .byOrigin - ) async throws -> (result: QueryResult, executionTimeMs: Double) { - let outcome = try await access.runStatement( + masking: ExternalRowMasking = .byOrigin, + budget: ExternalAnswerBudget? = .mcpAnswer + ) async throws -> DatabaseAccessBridge.StatementOutcome { + try await access.runStatement( operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - masking: masking + masking: masking, + budget: budget ) - return (outcome.result, outcome.executionTimeMs) } func runExplainPlan( @@ -215,11 +218,15 @@ public actor MCPConnectionBridge { func insertRows(_ operation: ApprovedOperation, table: String) async throws -> JsonValue { let inserted = try await access.runInserts(operation) - return .object([ + var reply: [String: JsonValue] = [ "table": .string(table), "rows_submitted": .int(operation.statements.count), - "rows_affected": .int(inserted) - ]) + "rows_affected": .int(inserted.affected) + ] + if let notice = inserted.notice { + reply["status_message"] = .string(notice) + } + return .object(reply) } func transactionControl(_ operation: ApprovedOperation, action: String) async throws -> JsonValue { @@ -245,14 +252,24 @@ public actor MCPConnectionBridge { return .object(["status": .string("dropped"), "schema": .string(name)]) } - static func encode(result: QueryResult, scope: DatabaseScope, executionTimeMs: Double) -> JsonValue { - .object(fields(of: result, scope: scope, executionTimeMs: executionTimeMs)) + static func encode( + result: QueryResult, + scope: DatabaseScope, + executionTimeMs: Double, + notes: [String] = [] + ) -> JsonValue { + .object(fields(of: result, scope: scope, executionTimeMs: executionTimeMs, notes: notes)) } /// A script's answer in the shape a statement's has, so a client that reads one result reads the first, plus /// `result_sets` with every one of them once there is more than one to read. static func encode(script: DatabaseAccessBridge.ScriptOutcome, scope: DatabaseScope) -> JsonValue { - var response = fields(of: script.primary, scope: scope, executionTimeMs: script.executionTimeMs) + var response = fields( + of: script.primary, + scope: scope, + executionTimeMs: script.executionTimeMs, + notes: script.budget?.notes ?? [] + ) if script.resultSets.count > 1 { response["result_sets"] = .array(script.resultSets.map(encode(resultSet:))) } @@ -272,7 +289,12 @@ public actor MCPConnectionBridge { return .object(entry) } - private static func fields(of result: QueryResult, scope: DatabaseScope, executionTimeMs: Double) -> [String: JsonValue] { + private static func fields( + of result: QueryResult, + scope: DatabaseScope, + executionTimeMs: Double, + notes: [String] + ) -> [String: JsonValue] { var response: [String: JsonValue] = [ "columns": .array(result.columns.map { .string($0) }), "rows": .array(result.rows.map { row in .array(row.map(cellValue)) }), @@ -288,6 +310,9 @@ public actor MCPConnectionBridge { if let statusMessage = result.statusMessage { response["status_message"] = .string(statusMessage) } + if !notes.isEmpty { + response["notes"] = .array(notes.map(JsonValue.string)) + } return response } diff --git a/apple/TablePro/Core/MCP/MCPPairingService.swift b/apple/TablePro/Core/MCP/MCPPairingService.swift index 23a1e369fe..2f234aa67f 100644 --- a/apple/TablePro/Core/MCP/MCPPairingService.swift +++ b/apple/TablePro/Core/MCP/MCPPairingService.swift @@ -12,9 +12,7 @@ struct PairingExchangeRecord: Sendable, Equatable { let expiresAt: Date } -/// A failed verification burns the code. PKCE protects the exchange only while each code allows one -/// attempt; leaving a code alive after a mismatch turns the five-minute window into an offline -/// guessing game against a value the caller can retry without limit. +// A failed verification burns the code: PKCE holds only while each code allows one attempt. actor PairingExchangeStore { static let exchangeWindow: TimeInterval = 300 static let maxPendingCodes = 50 @@ -96,6 +94,30 @@ actor PairingExchangeStore { } } +// The approval sheet has closed by the time a token is minted, so this is what the user reads. +enum MCPPairingError: Error, LocalizedError, Equatable { + case tokenNotSaved + case tokenNotCreated + + init(issuing error: Error) { + switch error as? MCPTokenStoreError { + case .notSaved?, .keychainUnavailable?: + self = .tokenNotSaved + case .storedTokensUnreadable?, .randomGenerationFailed?, nil: + self = .tokenNotCreated + } + } + + var errorDescription: String? { + switch self { + case .tokenNotSaved: + String(localized: "TablePro couldn't save a token for this client to the Keychain, so the client wasn't paired. Try pairing again.") + case .tokenNotCreated: + String(localized: "TablePro couldn't create a token for this client, so the client wasn't paired. Try pairing again.") + } + } +} + @MainActor final class MCPPairingService { static let shared = MCPPairingService() @@ -168,20 +190,57 @@ final class MCPPairingService { throw error } + let redirect: URL + do { + redirect = try await issueCode(for: request, target: target, approval: approval, tokenStore: tokenStore) + } catch { + // The requesting app waits on its redirect, so a failure after approval still answers it. + if let failed = PairingRedirectBuilder.failed( + base: target.url, + state: request.state, + mode: request.responseMode + ) { + NSWorkspace.shared.open(failed) + } + throw error + } + + Self.logger.info( + """ + Pairing approved for client '\(request.clientName, privacy: .private(mask: .hash))' \ + redirect=\(target.displayValue, privacy: .private(mask: .hash)) + """ + ) + NSWorkspace.shared.open(redirect) + } + + // Mints the token and parks it behind a one-time code; nothing is left behind when it throws. + func issueCode( + for request: PairingRequest, + target: PairingRedirectTarget, + approval: PairingApproval, + tokenStore: MCPTokenStore + ) async throws -> URL { let connectionAccess: ConnectionAccess = approval.allowedConnectionIds.map { .limited($0) } ?? .all let extraScopes = request.scopeRequest.grantedOptionalScopes( approved: approval.extraScopes, allowsHiddenConnectionListing: AppSettingsManager.shared.mcp.allowsHiddenConnectionListing ) - let result = try await tokenStore.generate( - name: request.clientName, - permissions: approval.grantedPermissions, - connectionAccess: connectionAccess, - expiresAt: approval.expiresAt, - isBridgeCredential: false, - isClaudeAgentCredential: false, - extraScopes: extraScopes - ) + let result: (token: MCPAuthToken, plaintext: String) + do { + result = try await tokenStore.generate( + name: request.clientName, + permissions: approval.grantedPermissions, + connectionAccess: connectionAccess, + expiresAt: approval.expiresAt, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: extraScopes + ) + } catch { + Self.logger.error("Pairing token was not issued: \(error.publicLogShape, privacy: .public)") + throw MCPPairingError(issuing: error) + } let code = UUID().uuidString do { @@ -211,14 +270,7 @@ final class MCPPairingService { await tokenStore.delete(tokenId: result.token.id) throw DatabaseAccessError.invalidArgument("redirect URL") } - - Self.logger.info( - """ - Pairing approved for client '\(request.clientName, privacy: .private(mask: .hash))' \ - redirect=\(target.displayValue, privacy: .private(mask: .hash)) - """ - ) - NSWorkspace.shared.open(redirect) + return redirect } func exchange( diff --git a/apple/TablePro/Core/MCP/MCPTokenStore.swift b/apple/TablePro/Core/MCP/MCPTokenStore.swift index 0e01d81646..f0e1da94c2 100644 --- a/apple/TablePro/Core/MCP/MCPTokenStore.swift +++ b/apple/TablePro/Core/MCP/MCPTokenStore.swift @@ -9,6 +9,7 @@ enum MCPTokenStoreError: Error, Sendable, Equatable { case randomGenerationFailed(OSStatus) case keychainUnavailable(OSStatus) case storedTokensUnreadable + case notSaved var message: String { switch self { @@ -18,6 +19,8 @@ enum MCPTokenStoreError: Error, Sendable, Equatable { return "Keychain unavailable (OSStatus \(status))" case .storedTokensUnreadable: return "The saved MCP tokens could not be read, so a new token cannot be saved" + case .notSaved: + return "The new MCP token could not be saved to the Keychain" } } } @@ -39,9 +42,17 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { // Set only by the Claude Agent chat's own mint, on the same terms. let isClaudeAgentCredential: Bool let extraScopes: Set + // Set only by the Claude Agent chat's own mint; no pairing input reaches it. + let toolProfile: MCPToolProfile? var scopes: Set { - permissions.scopes.union(extraScopes) + let granted = permissions.scopes.union(extraScopes) + guard let toolProfile else { return granted } + return granted.intersection(toolProfile.scopes) + } + + var toolAllowlist: Set? { + toolProfile?.toolNames } var isExpired: Bool { @@ -51,6 +62,25 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { var isEffectivelyActive: Bool { isActive && !isExpired } + func renamed(_ name: String) -> MCPAuthToken { + MCPAuthToken( + id: id, + name: name, + prefix: prefix, + tokenHash: tokenHash, + salt: salt, + permissions: permissions, + connectionAccess: connectionAccess, + createdAt: createdAt, + lastUsedAt: lastUsedAt, + expiresAt: expiresAt, + isActive: isActive, + isBridgeCredential: isBridgeCredential, + isClaudeAgentCredential: isClaudeAgentCredential, + extraScopes: extraScopes + ) + } + var grantSubject: MCPGrantSubject { MCPGrantSubject(tokenId: id, isBridgeCredential: isBridgeCredential, isClaudeAgentCredential: isClaudeAgentCredential) } @@ -74,7 +104,8 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { isActive: Bool, isBridgeCredential: Bool = false, isClaudeAgentCredential: Bool = false, - extraScopes: Set = [] + extraScopes: Set = [], + toolProfile: MCPToolProfile? = nil ) { self.id = id self.name = name @@ -90,6 +121,7 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { self.isBridgeCredential = isBridgeCredential self.isClaudeAgentCredential = isClaudeAgentCredential self.extraScopes = extraScopes.intersection(MCPScope.optionalGrants) + self.toolProfile = toolProfile } private enum CodingKeys: String, CodingKey { @@ -107,6 +139,7 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { case isBridgeCredential case isClaudeAgentCredential case extraScopes + case toolProfile } init(from decoder: Decoder) throws { @@ -128,6 +161,12 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { let storedExtraScopes = (try? container.decodeIfPresent([String].self, forKey: .extraScopes)) ?? [] self.extraScopes = Set(storedExtraScopes.compactMap(MCPScope.init(rawValue:))) .intersection(MCPScope.optionalGrants) + // Strict, unlike extraScopes: a profile only narrows, so a damaged value read as absent would widen the token. + if container.contains(.toolProfile) { + self.toolProfile = try container.decode(MCPToolProfile.self, forKey: .toolProfile) + } else { + self.toolProfile = nil + } } func encode(to encoder: Encoder) throws { @@ -150,6 +189,34 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { if !extraScopes.isEmpty { try container.encode(extraScopes.map(\.rawValue).sorted(), forKey: .extraScopes) } + try container.encodeIfPresent(toolProfile, forKey: .toolProfile) + } +} + +enum MCPToolProfile: String, Codable, Sendable { + // The chat's Ask mode: the schema lookups, no rows, no history. + case schemaLookup + + var scopes: Set { + switch self { + case .schemaLookup: + [.toolsRead] + } + } + + var toolNames: Set { + switch self { + case .schemaLookup: + [ + ListConnectionsTool.name, + GetConnectionStatusTool.name, + ListDatabasesTool.name, + ListSchemasTool.name, + ListTablesTool.name, + DescribeTableTool.name, + GetTableDdlTool.name + ] + } } } @@ -185,6 +252,7 @@ enum TokenPermissions: String, Codable, Sendable, CaseIterable, Identifiable { actor MCPTokenStore { static let stdioBridgeTokenName = "__stdio_bridge__" + static let claudeAgentTokenName = "__claude_agent__" static let bridgeTokenPermissions: TokenPermissions = .readWrite static let defaultTokenLifetime: TimeInterval = 90 * 24 * 60 * 60 @@ -220,7 +288,7 @@ actor MCPTokenStore { revocationObservers.removeValue(forKey: id) } - // Only the bridge's own mint may take its name: a pairing client could otherwise claim the bridge's grants. + // Pairing and Settings mint without a profile; only the Claude Agent mint passes one. func generate( name: String, permissions: TokenPermissions, @@ -230,9 +298,34 @@ actor MCPTokenStore { isClaudeAgentCredential: Bool, extraScopes: Set ) throws -> (token: MCPAuthToken, plaintext: String) { - let name = (!isBridgeCredential && name == Self.stdioBridgeTokenName) - ? name + " (client)" - : name + try generate( + name: name, + permissions: permissions, + connectionAccess: connectionAccess, + expiresAt: expiresAt, + isBridgeCredential: isBridgeCredential, + isClaudeAgentCredential: isClaudeAgentCredential, + extraScopes: extraScopes, + toolProfile: nil + ) + } + + // Only the bridge's and the agent's own mints may take their names: a pairing client picks its name, + // and the window labels both names as TablePro's own. loadFromDisk enforces the same for older tokens. + func generate( + name: String, + permissions: TokenPermissions, + connectionAccess: ConnectionAccess, + expiresAt: Date?, + isBridgeCredential: Bool, + isClaudeAgentCredential: Bool, + extraScopes: Set, + toolProfile: MCPToolProfile? + ) throws -> (token: MCPAuthToken, plaintext: String) { + let isAgentMint = isClaudeAgentCredential && !isBridgeCredential + let claimsReservedName = (!isBridgeCredential && name == Self.stdioBridgeTokenName) + || (!isAgentMint && name == Self.claudeAgentTokenName) + let name = claimsReservedName ? Self.clientName(claiming: name) : name // The bridge credential is dropped at every load, so it alone may live in memory only. if !isBridgeCredential, !adoptStoredTokensIfUnread() { Self.logger.error("Refused to mint an MCP token: the stored list is still unreadable") @@ -255,12 +348,18 @@ actor MCPTokenStore { expiresAt: expiresAt, isActive: true, isBridgeCredential: isBridgeCredential, - isClaudeAgentCredential: isClaudeAgentCredential && !isBridgeCredential, - extraScopes: isBridgeCredential ? [] : extraScopes + isClaudeAgentCredential: isAgentMint, + extraScopes: isBridgeCredential ? [] : extraScopes, + toolProfile: toolProfile ) tokens.append(token) - save() + // A token the Keychain did not take would vanish at the next launch while the client still holds it. + if !save(), !isBridgeCredential { + tokens.removeAll { $0.id == token.id } + Self.logger.error("Refused to hand out an MCP token the Keychain did not save") + throw MCPTokenStoreError.notSaved + } Self.logger.info("Generated MCP token '\(name, privacy: .private(mask: .hash))'") MCPAuditLogger.logTokenCreated(tokenId: token.id, tokenName: name) @@ -312,28 +411,45 @@ actor MCPTokenStore { return revoke(tokenId: token.id) ? .revoked : .notPersisted } - func delete(tokenId: UUID) { + // False when the Keychain write failed: the token stops working now, but until a later save lands a + // restart brings it back. + @discardableResult + func delete(tokenId: UUID) -> Bool { remove(tokenId: tokenId, withdrawsApprovals: true) } // A Claude Agent turn ends its own token this way: the approvals belong to the agent, not to one turn. - func retire(tokenId: UUID) { + @discardableResult + func retire(tokenId: UUID) -> Bool { remove(tokenId: tokenId, withdrawsApprovals: false) } - private func remove(tokenId: UUID, withdrawsApprovals: Bool) { + // A rotation revokes nothing, so no observer hears of it; any other token goes through delete. + func dropSupersededBridgeCredential(tokenId: UUID) { + guard let index = tokens.firstIndex(where: { $0.id == tokenId }), tokens[index].isBridgeCredential else { + delete(tokenId: tokenId) + return + } + tokens.remove(at: index) + save() + Self.logger.info("Dropped the superseded bridge credential") + } + + @discardableResult + private func remove(tokenId: UUID, withdrawsApprovals: Bool) -> Bool { guard let index = tokens.firstIndex(where: { $0.id == tokenId }) else { Self.logger.warning("Attempted to delete non-existent token \(tokenId.uuidString, privacy: .public)") - return + return true } let name = tokens[index].name let withdrawn = withdrawsApprovals ? tokens[index].subjectWithdrawnWhenEnded : nil tokens.remove(at: index) - save() + let persisted = save() notifyRevocationObservers(tokenId: tokenId, withdrawing: withdrawn) Self.logger.info("Deleted MCP token '\(name, privacy: .private(mask: .hash))'") + return persisted } private func notifyRevocationObservers(tokenId: UUID, withdrawing subject: MCPGrantSubject?) { @@ -383,16 +499,31 @@ actor MCPTokenStore { } let staleCount = tokens.filter(Self.isStaleBridge).count - guard staleCount > 0 else { return } tokens.removeAll(where: Self.isStaleBridge) + let renamedCount = tokens.filter(Self.claimsAgentName).count + tokens = tokens.map(Self.releasingAgentName) + guard staleCount > 0 || renamedCount > 0 else { return } save() - Self.logger.info("Cleaned up \(staleCount) stale bridge token(s)") + Self.logger.info("Cleaned up \(staleCount) stale bridge token(s), renamed \(renamedCount) client token(s)") } private static func isStaleBridge(_ token: MCPAuthToken) -> Bool { token.isBridgeCredential || token.name == stdioBridgeTokenName } + // Builds before the agent's name was reserved let a pairing client take it. + private static func claimsAgentName(_ token: MCPAuthToken) -> Bool { + token.name == claudeAgentTokenName && !token.isClaudeAgentCredential + } + + private static func releasingAgentName(_ token: MCPAuthToken) -> MCPAuthToken { + claimsAgentName(token) ? token.renamed(clientName(claiming: token.name)) : token + } + + private static func clientName(claiming reservedName: String) -> String { + reservedName + " (client)" + } + // One entry this build cannot decode costs that entry only, and it is written back untouched. static func decodeStored(_ data: Data) -> (tokens: [MCPAuthToken], undecoded: [Data]) { guard let entries = try? JSONSerialization.jsonObject(with: data) as? [Any] else { @@ -443,7 +574,9 @@ actor MCPTokenStore { } storedTokensUnread = false let minted = Set(tokens.map(\.id)) - tokens = stored.tokens.filter { !minted.contains($0.id) && !Self.isStaleBridge($0) } + tokens + tokens = stored.tokens + .filter { !minted.contains($0.id) && !Self.isStaleBridge($0) } + .map(Self.releasingAgentName) + tokens undecodedEntries = stored.undecoded Self.logger.info("Read the stored MCP tokens after an earlier failure") return true diff --git a/apple/TablePro/Core/MCP/PairingTypes.swift b/apple/TablePro/Core/MCP/PairingTypes.swift index 55afb9969b..92ec3e64a8 100644 --- a/apple/TablePro/Core/MCP/PairingTypes.swift +++ b/apple/TablePro/Core/MCP/PairingTypes.swift @@ -43,8 +43,7 @@ struct PairingRequest: Sendable, Equatable { } } -/// `scopes` names one permission level plus optional grants, separated by commas or spaces. -/// Unknown entries are skipped (RFC 6749 §3.3), and when several levels are named the lowest wins. +// One permission level plus optional grants; unknown entries are skipped (RFC 6749 3.3) and the lowest level wins. struct PairingScopeRequest: Sendable, Equatable { let permissions: TokenPermissions let optionalGrants: Set @@ -59,8 +58,7 @@ struct PairingScopeRequest: Sendable, Equatable { ) } - /// A grant needs all three: the client asked for it, the user ticked it on the sheet, and the - /// setting allows it. + // Granted only when the client asked, the user ticked it on the sheet, and the setting allows it. func grantedOptionalScopes(approved: Set, allowsHiddenConnectionListing: Bool) -> Set { guard allowsHiddenConnectionListing else { return [] } return optionalGrants.intersection(approved) @@ -80,8 +78,7 @@ struct PairingScopeRequest: Sendable, Equatable { } } -/// `legacy` is a link without `response_mode`: deprecated, and kept byte for byte for the clients -/// that shipped before the parameter existed. +// legacy is a link without response_mode, kept byte for byte for clients that shipped before it. enum PairingResponseMode: Sendable, Equatable { case legacy case query @@ -155,10 +152,8 @@ enum PairingValidationError: Error, Sendable, Equatable { } } -/// A pairing code is a bearer credential for the whole grant, so the address it is delivered to -/// decides who ends up holding it. Only two shapes can reach the machine the user is sitting at: a -/// loopback HTTP listener (RFC 8252's native-app redirect) and a private-use scheme registered by an -/// installed app. Any other origin is a web page, and handing it the code hands it the token. +// Whoever receives the code holds the grant, so only a loopback listener (RFC 8252) or an installed +// app's private-use scheme may: any other origin is a web page. enum PairingRedirectValidator { static let loopbackHosts: Set = [ "127.0.0.1", @@ -167,8 +162,7 @@ enum PairingRedirectValidator { "0:0:0:0:0:0:0:1" ] - /// `tablepro` would hand the code back to this app and open another pairing sheet. The rest run - /// or read something locally, or make their handler fetch from the host the URL names. + // tablepro would reopen a pairing sheet; the rest run something locally or fetch from the named host. static let deniedSchemes: Set = [ "about", "afp", "blob", "cifs", "data", "dav", "davs", "feed", "feeds", "file", "ftp", "ftps", "git", "gopher", "irc", "ircs", "itpc", "javascript", "ldap", "ldaps", "news", "nfs", "nntp", @@ -177,8 +171,7 @@ enum PairingRedirectValidator { "wss", "xmpp" ] - /// Every scheme this app handles, database URLs included: opening one of them with the code - /// appended would start a connection to whatever host the redirect names. + // Opening one of these with the code appended would connect to whatever host the redirect names. static let appRegisteredSchemes: Set = { let types = Bundle.main.object(forInfoDictionaryKey: "CFBundleURLTypes") as? [[String: Any]] ?? [] let schemes = types.flatMap { $0["CFBundleURLSchemes"] as? [String] ?? [] } @@ -211,7 +204,7 @@ enum PairingRedirectValidator { isWellFormedScheme(scheme) else { throw PairingValidationError.redirectSchemeNotAllowed(scheme) } - /// An opaque URL such as `myapp:https://example.com/` keeps its whole address in the path. + // An opaque URL such as myapp:https://example.com/ keeps its whole address in the path. let components = URLComponents(url: url, resolvingAgainstBaseURL: false) let authority = components?.host.map { "//\($0)" } ?? "" let port = components?.port.map { ":\($0)" } ?? "" @@ -222,8 +215,7 @@ enum PairingRedirectValidator { ) } - /// A browser that owns a custom scheme can unwrap it into a web page: Chrome opens - /// `google-chrome:https://…` and Safari opens `x-safari-https://…`. + // A browser that owns a custom scheme unwraps it into a web page (google-chrome:https://, x-safari-https://). static func validateHandler( of target: PairingRedirectTarget, handlerBundleIdentifier: String?, @@ -247,7 +239,7 @@ enum PairingRedirectValidator { } enum PairingRedirectBuilder { - /// Form decoders split on `&` and `=`, read `+` as a space, and some still split on `;`. + // Form decoders split on & and =, read + as a space, and some still split on ;. private static let valueCharacters = CharacterSet.urlQueryAllowed .subtracting(CharacterSet(charactersIn: "&=+;")) @@ -265,6 +257,16 @@ enum PairingRedirectBuilder { ) } + // The user approved, then TablePro could not issue the token (RFC 6749 4.1.2.1 server_error). + static func failed(base: URL, state: String?, mode: PairingResponseMode) -> URL? { + redirect( + base: base, + parameters: [("error", "server_error"), ("error_description", "token_not_issued")], + state: state, + mode: mode + ) + } + private static func redirect( base: URL, parameters: [(name: String, value: String)], @@ -303,7 +305,7 @@ enum PairingRedirectBuilder { guard var components = URLComponents(url: base, resolvingAgainstBaseURL: false) else { return nil } - /// Decoding and re-encoding the client's own items would turn its `%2B` into `+`. + // Decoding and re-encoding the client's own items would turn its %2B into +. var items = components.percentEncodedQueryItems ?? [] for parameter in parameters { guard let value = parameter.value.addingPercentEncoding(withAllowedCharacters: valueCharacters) else { @@ -316,9 +318,7 @@ enum PairingRedirectBuilder { } } -/// RFC 7636 fixes both halves of PKCE: the verifier is 43 to 128 characters of the unreserved set, -/// and the challenge is its base64url SHA-256, which is always 43 characters. Anything else is not a -/// PKCE exchange and is rejected before it can reach a comparison. +// RFC 7636: a verifier of 43 to 128 unreserved characters, a challenge of 43; anything else never reaches a comparison. enum PairingPkceValidator { static let minimumVerifierLength = 43 static let maximumVerifierLength = 128 diff --git a/apple/TablePro/Core/MCP/Prompts/MCPPromptSchemaReader.swift b/apple/TablePro/Core/MCP/Prompts/MCPPromptSchemaReader.swift index 663a939832..90ab1da09b 100644 --- a/apple/TablePro/Core/MCP/Prompts/MCPPromptSchemaReader.swift +++ b/apple/TablePro/Core/MCP/Prompts/MCPPromptSchemaReader.swift @@ -316,7 +316,7 @@ struct MCPPromptSchemaReader: Sendable { return MCPProtocolError(code: JsonRpcErrorCode.expired, message: detail, httpStatus: .ok) case .timeout(let detail, _): return MCPProtocolError.requestTimeout(detail: detail) - case .userCancelled: + case .userCancelled, .stoppedWhileRunning: return MCPProtocolError.requestCancelled() case .dataSourceError(let detail): return MCPProtocolError.internalError(detail: detail) diff --git a/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Notifications.swift b/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Notifications.swift index 02a4ac25e3..23d1f13908 100644 --- a/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Notifications.swift +++ b/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Notifications.swift @@ -21,13 +21,45 @@ extension MCPProtocolDispatcher { Self.logger.debug("Dropping cancellation without a usable requestId") return } - let key = MCPInflightKey(principal: principal, requestId: requestId) + guard let key = await cancellationKey( + requestId: requestId, + params: notification.params, + principal: principal, + exchange: exchange + ) else { + Self.logger.debug("Dropping a cancellation that does not say which stdio client sent it") + return + } let reason = notification.params?["reason"]?.stringValue let cancelled = await inflight.cancel(key: key, reason: .clientRequested(reason)) guard !cancelled else { return } Self.logger.debug("Cancellation referenced a request that is no longer in flight") } + // The sender is resolved the way its request was, so a cancellation reaches only that client's request. + private func cancellationKey( + requestId: JsonRpcId, + params: JsonValue?, + principal: MCPPrincipal, + exchange: MCPInboundExchange + ) async -> MCPInflightKey? { + guard principal.metadata.isBridgeCredential else { + return MCPInflightKey(principal: principal, requestId: requestId) + } + if let bridgeInstance = MCPRequestMeta.bridgeInstance(in: params) { + return MCPInflightKey(bridgeInstance: bridgeInstance, requestId: requestId) + } + let meta: MCPRequestMeta + switch try? await legacyAdapter.resolve(message: exchange.message, inbound: exchange.context) { + case .modern(let resolved)?, .legacy(let resolved, _)?: + meta = resolved + case .legacyInitialize?, nil: + return nil + } + let client = MCPClientIdentity(meta: meta, principal: principal, address: exchange.context.clientAddress) + return MCPInflightKey(client: client, requestId: requestId) + } + internal static func cancellationRequestId(in params: JsonValue?) -> JsonRpcId? { switch params?["requestId"] { case .string(let value): diff --git a/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Requests.swift b/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Requests.swift index 3e34ad8fdb..bd3281c6a4 100644 --- a/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Requests.swift +++ b/apple/TablePro/Core/MCP/Protocol/Extensions/MCPProtocolDispatcher+Requests.swift @@ -118,19 +118,22 @@ extension MCPProtocolDispatcher { return } - await activityLedger.record( - meta: meta, - principal: principal, - address: exchange.context.clientAddress, - at: exchange.context.receivedAt - ) + let client = MCPClientIdentity(meta: meta, principal: principal, address: exchange.context.clientAddress) + if !Self.activityExemptMethods.contains(request.method) { + await activityLedger.record(identity: client, at: exchange.context.receivedAt) + } - let key = MCPInflightKey(principal: principal, requestId: request.id) + let key = MCPInflightKey( + client: client, + bridgeInstance: MCPRequestMeta.bridgeInstance(in: request.params), + requestId: request.id + ) let cancellation = MCPCancellationToken() let registered = await inflight.register( key: key, token: cancellation, tokenId: principal.tokenId, + clientId: client.id, method: request.method, startedAt: exchange.context.receivedAt ) diff --git a/apple/TablePro/Core/MCP/Protocol/Handlers/PromptsGetHandler.swift b/apple/TablePro/Core/MCP/Protocol/Handlers/PromptsGetHandler.swift index bace8167ee..4450e19380 100644 --- a/apple/TablePro/Core/MCP/Protocol/Handlers/PromptsGetHandler.swift +++ b/apple/TablePro/Core/MCP/Protocol/Handlers/PromptsGetHandler.swift @@ -42,6 +42,7 @@ public struct PromptsGetHandler: MCPMethodHandler { ) MCPAuditLogger.logResourceRead( principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), uri: Self.auditUri(name: name), outcome: .success ) @@ -50,6 +51,7 @@ public struct PromptsGetHandler: MCPMethodHandler { let protocolError = error as? MCPProtocolError MCPAuditLogger.logResourceRead( principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), uri: Self.auditUri(name: name), outcome: protocolError?.code == JsonRpcErrorCode.forbidden ? .denied : .error, errorMessage: protocolError?.message ?? error.localizedDescription diff --git a/apple/TablePro/Core/MCP/Protocol/Handlers/ResourcesReadHandler.swift b/apple/TablePro/Core/MCP/Protocol/Handlers/ResourcesReadHandler.swift index 73ebc728f1..a84cc5b945 100644 --- a/apple/TablePro/Core/MCP/Protocol/Handlers/ResourcesReadHandler.swift +++ b/apple/TablePro/Core/MCP/Protocol/Handlers/ResourcesReadHandler.swift @@ -27,6 +27,7 @@ public struct ResourcesReadHandler: MCPMethodHandler { Self.logger.debug("resources/read uri=\(uri, privacy: .private(mask: .hash))") MCPAuditLogger.logResourceRead( principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), uri: uri, outcome: .success ) @@ -38,6 +39,7 @@ public struct ResourcesReadHandler: MCPMethodHandler { } catch { MCPAuditLogger.logResourceRead( principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), uri: uri, outcome: .error, errorMessage: (error as? MCPProtocolError)?.message ?? error.localizedDescription @@ -130,7 +132,7 @@ public struct ResourcesReadHandler: MCPMethodHandler { return MCPProtocolError(code: JsonRpcErrorCode.expired, message: detail, httpStatus: .ok) case .timeout(let detail, _): return .requestTimeout(detail: detail) - case .userCancelled: + case .userCancelled, .stoppedWhileRunning: return .requestCancelled() case .dataSourceError(let detail): return .internalError(detail: detail) diff --git a/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsCallHandler.swift b/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsCallHandler.swift index df4650c1b0..d9031f02bc 100644 --- a/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsCallHandler.swift +++ b/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsCallHandler.swift @@ -28,6 +28,18 @@ public struct ToolsCallHandler: MCPMethodHandler { } let connectionId = Self.connectionId(in: arguments) + // Before the scope check: no scope step-up opens a tool the token's profile leaves out. + guard context.principal.allowsTool(toolName) else { + Self.audit( + context: context, + tool: toolName, + connectionId: connectionId, + outcome: .denied, + error: "tool_not_in_profile" + ) + throw MCPProtocolError.forbidden(reason: "Tool '\(toolName)' is not available to this token") + } + let requiredScopes = type(of: tool).requiredScopes guard requiredScopes.isSubset(of: context.principal.scopes) else { Self.audit( @@ -120,6 +132,7 @@ public struct ToolsCallHandler: MCPMethodHandler { ) { MCPAuditLogger.logToolCalled( principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), toolName: tool, connectionId: connectionId, outcome: outcome, diff --git a/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsListHandler.swift b/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsListHandler.swift index 68eae9bf53..96995a16b6 100644 --- a/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsListHandler.swift +++ b/apple/TablePro/Core/MCP/Protocol/Handlers/ToolsListHandler.swift @@ -34,7 +34,10 @@ public struct ToolsListHandler: MCPMethodHandler { private static func visibleTools(for principal: MCPPrincipal) -> [any MCPToolImplementation] { MCPToolRegistry.allTools - .filter { type(of: $0).requiredScopes.isSubset(of: principal.scopes) } + .filter { tool in + let toolType = type(of: tool) + return toolType.requiredScopes.isSubset(of: principal.scopes) && principal.allowsTool(toolType.name) + } .sorted { type(of: $0).name < type(of: $1).name } } diff --git a/apple/TablePro/Core/MCP/Protocol/MCPProtocolDispatcher.swift b/apple/TablePro/Core/MCP/Protocol/MCPProtocolDispatcher.swift index 10a4d2c6d5..701de80598 100644 --- a/apple/TablePro/Core/MCP/Protocol/MCPProtocolDispatcher.swift +++ b/apple/TablePro/Core/MCP/Protocol/MCPProtocolDispatcher.swift @@ -21,6 +21,10 @@ public actor MCPProtocolDispatcher { public static let deadlineExemptMethods: Set = ["subscriptions/listen"] + // Needs no scope and opens no connection. The bridge and the tablepro command probe with it before + // every session, so recording it listed them as clients of their own. + public static let activityExemptMethods: Set = ["server/discover"] + internal static let legacyInitializeMethod = "initialize" internal static let legacySessionHeader = "Mcp-Session-Id" internal static let cancelledNotificationMethod = "notifications/cancelled" @@ -99,6 +103,34 @@ public actor MCPProtocolDispatcher { return count } + // Ends one client's requests without touching the credential it shares with other stdio clients. + @discardableResult + public func cancelInflight(matchingClientId clientId: String) async -> Int { + let count = await inflight.cancelAll(matchingClientId: clientId, reason: .clientDisconnected) + if count > 0 { + Self.logger.info("Cancelled \(count, privacy: .public) in-flight request(s) for a disconnected client") + } + return count + } + + // For a superseded credential: its requests may finish, but none outlives the credential's own expiry. + public func cancelInflight(matchingTokenId tokenId: UUID, at deadline: Date) async { + let clock = self.clock + let remaining = deadline.timeIntervalSince(await clock.now()) + guard remaining > 0 else { + await cancelInflight(matchingTokenId: tokenId) + return + } + Task { [weak self] in + do { + try await clock.sleep(for: .seconds(remaining)) + } catch { + return + } + _ = await self?.cancelInflight(matchingTokenId: tokenId) + } + } + @discardableResult public func cancelAllInflight(reason: MCPCancellationReason = .serverShuttingDown) async -> Int { await inflight.cancelAll(reason: reason) diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ConfirmDestructiveOperationTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ConfirmDestructiveOperationTool.swift index cf0418a0da..1e675ad88c 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ConfirmDestructiveOperationTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ConfirmDestructiveOperationTool.swift @@ -11,7 +11,8 @@ public struct ConfirmDestructiveOperationTool: MCPToolImplementation { with no WHERE: one statement, or on SQL Server a whole script. Anything else is refused. The user \ approves it before it runs: through your elicitation prompt when your client supports elicitation, \ otherwise through TablePro's own confirmation dialog on the user's Mac. Needs tools:write and a \ - connection the user left writable for external clients. + connection the user left writable for external clients. The transaction rules of execute_query apply, \ + Manual Commit's among them. """ ) public static let requiredScopes: Set = [.toolsWrite] diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/DataTools.swift b/apple/TablePro/Core/MCP/Protocol/Tools/DataTools.swift index 771dea9de4..989b5526b3 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/DataTools.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/DataTools.swift @@ -311,7 +311,8 @@ public struct InsertRowsTool: MCPToolImplementation { public static let description = String( localized: """ Insert rows into a table using bound parameters, so values are never spliced into SQL text. \ - Needs tools:write and goes through the connection's Safe Mode approval. + Needs tools:write and goes through the connection's Safe Mode approval. With Manual Commit on, the rows \ + stay pending until the user commits in TablePro. """ ) public static let requiredScopes: Set = [.toolsWrite] @@ -347,7 +348,8 @@ public struct InsertRowsTool: MCPToolImplementation { properties: [ "table": MCPToolSchema.string(String(localized: "Table that received the rows")), "rows_submitted": MCPToolSchema.integer(String(localized: "Rows sent")), - "rows_affected": MCPToolSchema.integer(String(localized: "Rows the engine reported inserting")) + "rows_affected": MCPToolSchema.integer(String(localized: "Rows the engine reported inserting")), + "status_message": MCPToolSchema.string(String(localized: "What happened to the open transaction, when it matters")) ], required: ["table", "rows_submitted", "rows_affected"] ) diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ExecuteQueryTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ExecuteQueryTool.swift index 87522b7b7a..98dee6bb14 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ExecuteQueryTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ExecuteQueryTool.swift @@ -13,7 +13,11 @@ public struct ExecuteQueryTool: MCPToolImplementation { Destructive statements, such as DROP, TRUNCATE, ALTER...DROP or a DELETE with no WHERE, go through \ confirm_destructive_operation instead. Statements that read or write files, or run server-side code, \ are refused. Send one statement per call, except on SQL Server, where a script runs batch by batch at \ - its GO lines and result_sets lists every result set when there is more than one. + its GO lines and result_sets lists every result set when there is more than one. A statement that starts \ + a transaction while one is open is refused. With Manual Commit on, a write stays pending until the user \ + commits it in TablePro, and statements that begin, commit or roll back a transaction are refused. While a \ + transaction is open on PostgreSQL or CockroachDB a statement runs inside a savepoint, so its failure leaves \ + the transaction usable; on Redshift and DuckDB it is refused. """ ) public static let requiredScopes: Set = [.toolsRead] diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift index 057a66292c..69d3515294 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift @@ -54,6 +54,13 @@ public struct ExplainQueryTool: MCPToolImplementation { ), "plan_text": MCPToolSchema.string(String(localized: "Flattened plan text")), "status_message": MCPToolSchema.string(String(localized: "Why values are masked, when they are")), + "is_truncated": MCPToolSchema.boolean( + String(localized: "Present and true when the answer's size limit cut plan_text or left the parsed plan or rows out") + ), + "notes": MCPToolSchema.array( + String(localized: "Why rows were left out or values cut, when they were"), + of: MCPToolSchema.stringItem + ), "plan": MCPToolSchema.object( properties: [ "root": MCPToolSchema.anyValue, @@ -170,20 +177,18 @@ public struct ExplainQueryTool: MCPToolImplementation { ) } - do { - var payload = try await services.connectionBridge.explainQuery( - operation, - request: request, - timeoutSeconds: timeoutSeconds, - cancellation: context.cancellation - ) - if case .object(var fields) = payload { - fields["available_variants"] = MCPConnectionBridge.explainVariants(for: meta.databaseType) - payload = .object(fields) - } - return .structured(payload) - } catch { - throw ToolQueryExecutor.translate(error, secrets: meta.redactionSecrets) + var payload = try await ToolQueryExecutor.explainAndLog( + services: services, + operation: operation, + request: request, + timeoutSeconds: timeoutSeconds, + context: context, + secrets: meta.redactionSecrets + ) + if case .object(var fields) = payload { + fields["available_variants"] = MCPConnectionBridge.explainVariants(for: meta.databaseType) + payload = .object(fields) } + return .structured(payload) } } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift index 14e1f492ab..5f7f4dec82 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift @@ -56,7 +56,13 @@ public struct ExportDataTool: MCPToolImplementation { properties: [ "format": MCPToolSchema.string(String(localized: "Format that was produced")), "rows_exported": MCPToolSchema.integer(String(localized: "Total rows written")), - "is_truncated": MCPToolSchema.boolean(String(localized: "Whether the row limit clipped any export")), + "is_truncated": MCPToolSchema.boolean( + String(localized: "Whether the row limit or the answer's size limit left rows out of any export") + ), + "notes": MCPToolSchema.array( + String(localized: "Why rows were left out, when the size limit left them out"), + of: MCPToolSchema.stringItem + ), "path": MCPToolSchema.string(String(localized: "File that was written, when output_path was given")), "exports": MCPToolSchema.array( String(localized: "One entry per query or table"), @@ -64,7 +70,9 @@ public struct ExportDataTool: MCPToolImplementation { properties: [ "label": MCPToolSchema.string(String(localized: "Table name, or 'query'")), "row_count": MCPToolSchema.integer(String(localized: "Rows in this export")), - "is_truncated": MCPToolSchema.boolean(String(localized: "Whether this export was clipped")), + "is_truncated": MCPToolSchema.boolean( + String(localized: "Whether the row limit or the answer's size limit left rows out") + ), "data": MCPToolSchema.string( String(localized: "Exported text. Omitted when the export was written to a file.") ) @@ -168,6 +176,8 @@ public struct ExportDataTool: MCPToolImplementation { var documents: [String] = [] var totalRows = 0 var anyTruncated = false + // Every table of the call shares one answer, so each gets what the ones before it left. + var budget = ExternalAnswerBudget.export for (statement, classification) in zip(statements, classifications) { let planned: OperationStatement @@ -191,15 +201,18 @@ public struct ExportDataTool: MCPToolImplementation { services: services ) - let result = try await ToolQueryExecutor.executeResultAndLog( + let read = try await ToolQueryExecutor.executeResultAndLog( services: services, operation: operation, maxRows: maxRows + 1, timeoutSeconds: timeoutSeconds, context: context, secrets: meta.redactionSecrets, - masking: statement.masking + masking: statement.masking, + budget: budget ) + let result = read.result + budget = read.budget let rowCount = min(result.rows.count, maxRows) let isTruncated = result.rows.count > maxRows || result.isTruncated @@ -233,6 +246,10 @@ public struct ExportDataTool: MCPToolImplementation { "is_truncated": .bool(anyTruncated), "exports": .array(exports) ] + let notes = budget.notes + if !notes.isEmpty { + payload["notes"] = .array(notes.map(JsonValue.string)) + } guard let destination else { return .structured(.object(payload)) diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/FocusQueryTabTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/FocusQueryTabTool.swift index c964c2f0f8..11b534ac74 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/FocusQueryTabTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/FocusQueryTabTool.swift @@ -6,7 +6,10 @@ public struct FocusQueryTabTool: MCPToolImplementation { public static let name = "focus_query_tab" public static let title: String? = String(localized: "Focus Tab") public static let description = String( - localized: "Bring an open tab to the front, using a tab id from list_recent_tabs." + localized: """ + Bring an open tab to the front, using a tab id from list_recent_tabs. TablePro asks the user about \ + the tab's connection first, as it does for any call that names a connection. + """ ) public static let requiredScopes: Set = [.toolsWrite] public static let annotations = MCPToolAnnotations( @@ -32,7 +35,15 @@ public struct FocusQueryTabTool: MCPToolImplementation { required: ["status", "tab_id", "connection_id"] ) - public init() {} + private let collectSnapshots: MCPTabSnapshotCollector + + public init() { + self.init(collectSnapshots: MCPTabSnapshotProvider.liveSnapshots) + } + + init(collectSnapshots: @escaping MCPTabSnapshotCollector) { + self.collectSnapshots = collectSnapshots + } public func perform( arguments: JsonValue, @@ -44,7 +55,8 @@ public struct FocusQueryTabTool: MCPToolImplementation { let snapshots = await MCPTabSnapshotProvider.readableSnapshots( context: context, - services: services + services: services, + collect: collectSnapshots ) guard let snapshot = snapshots.first(where: { $0.tabId == tabId }) else { throw MCPToolExecutionError.notFound( @@ -52,6 +64,13 @@ public struct FocusQueryTabTool: MCPToolImplementation { ) } + // The call names a tab, not a connection, so ToolsCallHandler had nothing to ask about. + try await services.authPolicy.resolveAndAuthorize( + principal: context.principal, + tool: Self.name, + connectionId: snapshot.connectionId + ) + let raised = await MainActor.run { () -> Bool in guard Self.focus(tabId: tabId, among: WindowManager.shared.coordinators(for: snapshot.connectionId)) else { return false diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ListRecentTabsTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ListRecentTabsTool.swift index 7125a08ed9..5ed6de9d97 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ListRecentTabsTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ListRecentTabsTool.swift @@ -6,8 +6,9 @@ public struct ListRecentTabsTool: MCPToolImplementation { public static let title: String? = String(localized: "List Open Tabs") public static let description = String( localized: """ - List the tabs open in TablePro for connections this client may reach. Each entry carries the tab \ - id that focus_query_tab takes and the window id that open_table_tab returns. + List the tabs open in TablePro. With connection_id, TablePro asks the user about that connection first. \ + Without it, the list covers only connections this client may already reach without asking. Each entry \ + carries the tab id that focus_query_tab takes and the window id that open_table_tab returns. """ ) public static let requiredScopes: Set = [.toolsRead] @@ -22,7 +23,7 @@ public struct ListRecentTabsTool: MCPToolImplementation { public static let inputSchema = MCPToolSchema.object( properties: [ "connection_id": MCPToolSchema.string( - String(localized: "Restrict to one connection. Omit for every readable connection.") + String(localized: "Restrict to one connection. Omit for every connection already approved.") ), "limit": MCPToolSchema.integer( String(localized: "Maximum tabs to return (1-500, default 20)"), @@ -42,7 +43,15 @@ public struct ListRecentTabsTool: MCPToolImplementation { required: ["tabs"] ) - public init() {} + private let collectSnapshots: MCPTabSnapshotCollector + + public init() { + self.init(collectSnapshots: MCPTabSnapshotProvider.liveSnapshots) + } + + init(collectSnapshots: @escaping MCPTabSnapshotCollector) { + self.collectSnapshots = collectSnapshots + } public func perform( arguments: JsonValue, @@ -53,17 +62,25 @@ public struct ListRecentTabsTool: MCPToolImplementation { let connectionId = try MCPArgumentDecoder.optionalUuid(arguments, key: "connection_id") let limit = try MCPArgumentDecoder.optionalInt(arguments, key: "limit", range: 1...500) ?? 20 - var snapshots = await MCPTabSnapshotProvider.readableSnapshots( - context: context, - services: services - ) + let snapshots: [MCPTabSnapshot] if let connectionId { + // ToolsCallHandler already asked about the named connection. try await MCPToolAuthorization.requireReadAccess( connectionId, context: context, services: services ) - snapshots = snapshots.filter { $0.connectionId == connectionId } + snapshots = await MCPTabSnapshotProvider.readableSnapshots( + context: context, + services: services, + collect: collectSnapshots + ).filter { $0.connectionId == connectionId } + } else { + snapshots = await MCPTabSnapshotProvider.snapshotsReadableWithoutPrompt( + context: context, + services: services, + collect: collectSnapshots + ) } let payload = snapshots.prefix(limit).map(MCPTabSnapshotProvider.encode) diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/MCPErrorRedactor.swift b/apple/TablePro/Core/MCP/Protocol/Tools/MCPErrorRedactor.swift index b5f466803f..8e37e6c8e0 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/MCPErrorRedactor.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/MCPErrorRedactor.swift @@ -3,6 +3,9 @@ import Foundation enum MCPErrorRedactor { static let placeholder = "[redacted]" static let maximumLength = 600 + // The URL pattern still costs more than linear time on a long run of scheme:// with no `@`, and only the first + // maximumLength characters are sent, so the patterns read no further than this. + static let scannedLength = 8_192 private static let credentialKeys: [String] = [ "host", "hostaddr", "hostname", "server", "port", "user", "username", "uid", @@ -11,10 +14,12 @@ enum MCPErrorRedactor { "service_account", "connection_string", "dsn" ] + // The DSN and host:port patterns start only where a run of their characters starts, and never give characters + // back: unanchored, each position inside a long run rescanned the rest of it, and a 200 KB error took minutes. private static let patterns: [String] = [ - "[A-Za-z][A-Za-z0-9+.-]*://[^\\s]*@[^\\s]*", + "(?= 3 { output = output.replacingOccurrences(of: secret, with: placeholder, options: [.caseInsensitive]) } + output = scannedPrefix(output) for expression in compiled { let range = NSRange(output.startIndex..., in: output) output = expression.stringByReplacingMatches( @@ -71,4 +77,14 @@ enum MCPErrorRedactor { guard (output as NSString).length > maximumLength else { return output } return (output as NSString).substring(to: maximumLength) + "…" } + + // Cut at a blank, so no piece of a token is left without the rest a pattern needs to see it. + private static func scannedPrefix(_ text: String) -> String { + let source = text as NSString + guard source.length > scannedLength else { return text } + let head = source.substring(to: scannedLength) as NSString + let blank = head.rangeOfCharacter(from: .whitespacesAndNewlines, options: .backwards) + guard blank.location != NSNotFound else { return placeholder } + return head.substring(to: blank.location) + } } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/MCPTabSnapshotProvider.swift b/apple/TablePro/Core/MCP/Protocol/Tools/MCPTabSnapshotProvider.swift index 4415afdd91..381395f52b 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/MCPTabSnapshotProvider.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/MCPTabSnapshotProvider.swift @@ -16,6 +16,8 @@ struct MCPTabSnapshot { weak var window: NSWindow? } +typealias MCPTabSnapshotCollector = @MainActor @Sendable () -> [MCPTabSnapshot] + enum MCPTabSnapshotProvider { static let openPollInterval: Duration = .milliseconds(50) static let openTimeout: Duration = .seconds(8) @@ -58,13 +60,31 @@ enum MCPTabSnapshotProvider { } } + static let liveSnapshots: MCPTabSnapshotCollector = { MCPTabSnapshotProvider.collectTabSnapshots() } + + // Tabs of every connection the token may see; the caller asks about one before acting on it. static func readableSnapshots( context: MCPRequestContext, - services: MCPToolServices + services: MCPToolServices, + collect: MCPTabSnapshotCollector = MCPTabSnapshotProvider.liveSnapshots ) async -> [MCPTabSnapshot] { let readable = await MCPToolAuthorization.readableConnectionIds(context: context, services: services) - let snapshots = await MainActor.run { collectTabSnapshots() } - return snapshots.filter { readable.contains($0.connectionId) } + let collected = await collect() + return snapshots(collected, in: readable) + } + + static func snapshotsReadableWithoutPrompt( + context: MCPRequestContext, + services: MCPToolServices, + collect: MCPTabSnapshotCollector = MCPTabSnapshotProvider.liveSnapshots + ) async -> [MCPTabSnapshot] { + let allowed = await services.authPolicy.connectionIdsReadableWithoutPrompt(principal: context.principal) + let collected = await collect() + return snapshots(collected, in: allowed) + } + + static func snapshots(_ snapshots: [MCPTabSnapshot], in connectionIds: Set) -> [MCPTabSnapshot] { + snapshots.filter { connectionIds.contains($0.connectionId) } } /// A database or schema the caller left out means the one being browsed, which is where the diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolExecutionError.swift b/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolExecutionError.swift index 30fdfebc5d..8b63947854 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolExecutionError.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolExecutionError.swift @@ -12,7 +12,8 @@ public enum MCPToolErrorCode: String, Sendable, Equatable { case internalFailure = "internal_failure" } -public struct MCPToolExecutionError: Error, Sendable, Equatable { +// LocalizedError: the assistant's transcript shows `localizedDescription`, which was Foundation's generic text. +public struct MCPToolExecutionError: LocalizedError, Sendable, Equatable { public let code: MCPToolErrorCode public let message: String @@ -21,6 +22,10 @@ public struct MCPToolExecutionError: Error, Sendable, Equatable { self.message = message } + public var errorDescription: String? { + "\(code.rawValue): \(message)" + } + public var asToolResult: MCPToolCallResult { MCPToolCallResult(content: [.text("\(code.rawValue): \(message)")], isError: true) } @@ -73,6 +78,9 @@ public struct MCPToolExecutionError: Error, Sendable, Equatable { ) case .dataSourceError(let detail): return MCPToolExecutionError(code: .queryFailed, message: MCPErrorRedactor.redact(detail, secrets: secrets)) + case .stoppedWhileRunning(let detail): + // The code a write past its deadline gets, the other answer that means "may have been applied". + return MCPToolExecutionError(code: .timeout, message: detail) } } } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolImplementation.swift b/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolImplementation.swift index 6c409723c5..e282074c06 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolImplementation.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/MCPToolImplementation.swift @@ -50,6 +50,16 @@ public extension MCPToolImplementation { } catch let error as MCPToolExecutionError { return error.asToolResult } catch let error as DatabaseAccessError { + // A cancelled request keeps its documented -33002, and says the write it sent may have been applied. + if case .stoppedWhileRunning = error, Task.isCancelled { + let cancelled = MCPProtocolError.requestCancelled() + throw MCPProtocolError( + code: cancelled.code, + message: cancelled.message, + httpStatus: cancelled.httpStatus, + data: .object(["outcome": .string("unknown")]) + ) + } return MCPToolExecutionError.from(error).asToolResult } catch is CancellationError { throw MCPProtocolError.requestCancelled() diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift b/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift index 832833c5bb..9e022fc2c8 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift @@ -236,7 +236,9 @@ public struct TransactionControlTool: MCPToolImplementation { localized: """ Begin, commit, or roll back a transaction on the connection's shared session. The transaction \ stays open across calls until committed or rolled back, and it is the same session the user's \ - own tabs run on, so leave nothing open. Needs tools:write. + own tabs run on, so leave nothing open. Begin is refused while a transaction is already open, \ + and every action is refused on a connection with Manual Commit on, where the user commits in \ + TablePro. Needs tools:write. """ ) public static let requiredScopes: Set = [.toolsWrite] diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift index 46bac19014..7d8903f65d 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift @@ -6,8 +6,9 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { public static let title: String? = String(localized: "Search Query History") public static let description = String( localized: """ - Search the query history TablePro keeps on this Mac. Results cover only connections this client \ - may reach, whether or not connection_id is given. + Search the query history TablePro keeps on this Mac. With connection_id, TablePro asks the user about \ + that connection first. Without it, results cover only connections this client may already reach \ + without asking. """ ) public static let requiredScopes: Set = [.toolsRead] @@ -22,7 +23,9 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { public static let inputSchema = MCPToolSchema.object( properties: [ "query": MCPToolSchema.string(String(localized: "Text matched against the recorded statement")), - "connection_id": MCPToolSchema.string(String(localized: "Restrict to one connection")), + "connection_id": MCPToolSchema.string( + String(localized: "Restrict to one connection. Omit for every connection already approved.") + ), "limit": MCPToolSchema.integer( String(localized: "Maximum entries to return (1-500, default 50)"), minimum: 1, @@ -60,9 +63,12 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { "row_count", "was_successful" ] ) + ), + "is_truncated": MCPToolSchema.boolean( + String(localized: "Whether the size limit left out matching entries") ) ], - required: ["entries"] + required: ["entries", "is_truncated"] ) public init() {} @@ -90,21 +96,16 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { ) } + let allowlist: Set if let connectionId { - try await services.authPolicy.resolveAndAuthorize( - principal: context.principal, - tool: Self.name, - connectionId: connectionId, - sql: nil - ) - } - - var allowlist = await services.authPolicy.readableConnectionIds(principal: context.principal) - if let connectionId { - allowlist = allowlist.intersection([connectionId]) + // ToolsCallHandler already asked about this connection; Ask Every Time records nothing, so asking here asked twice. + try await MCPToolAuthorization.requireReadAccess(connectionId, context: context, services: services) + allowlist = [connectionId] + } else { + allowlist = await services.authPolicy.connectionIdsReadableWithoutPrompt(principal: context.principal) } guard !allowlist.isEmpty else { - return .structured(.object(["entries": .array([])])) + return .structured(.object(["entries": .array([]), "is_truncated": .bool(false)])) } var anyMasksValues = false @@ -161,6 +162,10 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { return .object(fields) } - return .structured(.object(["entries": .array(payload)])) + let answer = MCPConnectionBridge.withinAnswerBudget(payload) + return .structured(.object([ + "entries": .array(answer.entries), + "is_truncated": .bool(answer.isTruncated) + ])) } } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift index 81d490792e..7341213ccd 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift @@ -18,10 +18,12 @@ enum ToolQueryExecutor { maxRows: maxRows, timeoutSeconds: timeoutSeconds, principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), cancellation: context.cancellation, secrets: secrets, source: historySource(for: context), - masking: masking + masking: masking, + budget: .mcpAnswer ) } @@ -36,15 +38,18 @@ enum ToolQueryExecutor { maxRows: Int, timeoutSeconds: Int, principal: MCPPrincipal, + client: String? = nil, cancellation: MCPCancellationToken? = nil, secrets: [String] = [], source: QueryHistorySource = .mcp, - masking: ExternalRowMasking = .byOrigin + masking: ExternalRowMasking = .byOrigin, + budget: ExternalAnswerBudget = .mcpAnswer ) async throws -> JsonValue { try await logged( services: services, approved: operation, principal: principal, + client: client, secrets: secrets, source: source ) { @@ -54,11 +59,13 @@ enum ToolQueryExecutor { maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - masking: masking + masking: masking, + budget: budget ) } } + // The rows held to `budget`, and what is left of it for the next table of the same export. static func executeResultAndLog( services: MCPToolServices, operation: ApprovedOperation, @@ -66,13 +73,15 @@ enum ToolQueryExecutor { timeoutSeconds: Int, context: MCPRequestContext, secrets: [String], - masking: ExternalRowMasking = .byOrigin - ) async throws -> QueryResult { + masking: ExternalRowMasking = .byOrigin, + budget: ExternalAnswerBudget + ) async throws -> (result: QueryResult, budget: ExternalAnswerBudget) { try await context.cancellation.throwIfCancelled() return try await logged( services: services, approved: operation, principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), secrets: secrets, source: historySource(for: context) ) { @@ -81,9 +90,44 @@ enum ToolQueryExecutor { maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: context.cancellation, - masking: masking + masking: masking, + budget: budget ) - return (outcome.result, outcome.result.rows.count) + return ((result: outcome.result, budget: outcome.budget ?? budget), outcome.result.rows.count) + } + } + + // An analyzed write ran, so it is recorded the way execute_query records one; a plan of a read is not. + static func explainAndLog( + services: MCPToolServices, + operation: ApprovedOperation, + request: MCPExplainRequest, + timeoutSeconds: Int, + context: MCPRequestContext, + secrets: [String] + ) async throws -> JsonValue { + guard operation.kind.declaresWrite else { + do { + return try await services.connectionBridge.explainQuery( + operation, request: request, timeoutSeconds: timeoutSeconds, cancellation: context.cancellation + ) + } catch { + throw translate(error, secrets: secrets) + } + } + try await context.cancellation.throwIfCancelled() + return try await logged( + services: services, + approved: operation, + principal: context.principal, + client: MCPAuditLogger.clientName(for: context.meta), + secrets: secrets, + source: historySource(for: context) + ) { + let payload = try await services.connectionBridge.explainQuery( + operation, request: request, timeoutSeconds: timeoutSeconds, cancellation: context.cancellation + ) + return (payload, payload["rows"]?.arrayValue?.count ?? 0) } } @@ -91,6 +135,7 @@ enum ToolQueryExecutor { services: MCPToolServices, approved: ApprovedOperation, principal: MCPPrincipal, + client: String?, secrets: [String], source: QueryHistorySource, operation: () async throws -> (Output, Int) @@ -122,6 +167,7 @@ enum ToolQueryExecutor { ) MCPAuditLogger.logQueryExecuted( principal: principal, + client: client, connectionId: connectionId, sql: query, durationMs: Int(elapsed * 1_000), @@ -146,6 +192,7 @@ enum ToolQueryExecutor { ) MCPAuditLogger.logQueryExecuted( principal: principal, + client: client, connectionId: connectionId, sql: query, durationMs: Int(elapsed * 1_000), @@ -170,7 +217,8 @@ enum ToolQueryExecutor { maxRows: Int, timeoutSeconds: Int, cancellation: MCPCancellationToken?, - masking: ExternalRowMasking + masking: ExternalRowMasking, + budget: ExternalAnswerBudget ) async throws -> (payload: JsonValue, rowsReturned: Int) { switch operation.unit { case .statement: @@ -179,7 +227,8 @@ enum ToolQueryExecutor { maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - masking: masking + masking: masking, + budget: budget ) return (payload, payload["row_count"]?.intValue ?? 0) case .script: @@ -187,13 +236,16 @@ enum ToolQueryExecutor { operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + budget: budget ) } } static func translate(_ error: Error, secrets: [String]) -> Error { if let dataError = error as? DatabaseAccessError { + // Kept as it is, so an MCP call answers it as the cancel it was and the assistant shows its own words. + if case .stoppedWhileRunning = dataError { return dataError } return MCPToolExecutionError.from(dataError, secrets: secrets) } if let toolError = error as? MCPToolExecutionError { @@ -208,9 +260,7 @@ enum ToolQueryExecutor { return MCPToolExecutionError.queryFailed(MCPErrorRedactor.message(for: error, secrets: secrets)) } - /// An MCP query has no tab and no window of its own, so its completion is owned by the - /// connection. Clicking the notification brings the connection's most recent window forward - /// rather than focusing a tab that never existed. + // An MCP query has no tab, so its notification brings the connection's most recent window forward. @MainActor private static func reportMcpQueryFinished( _ outcome: OperationOutcome, diff --git a/apple/TablePro/Core/Menu/DatabaseMenuBuilder.swift b/apple/TablePro/Core/Menu/DatabaseMenuBuilder.swift index f1e617b865..167aff86f2 100644 --- a/apple/TablePro/Core/Menu/DatabaseMenuBuilder.swift +++ b/apple/TablePro/Core/Menu/DatabaseMenuBuilder.swift @@ -10,6 +10,10 @@ import AppKit enum DatabaseMenuBuilder { static let identifier = NSUserInterfaceItemIdentifier("menu.database") static let liveMessagesIdentifier = NSUserInterfaceItemIdentifier("menu.database.liveMessages") + static let manualCommitFootnoteIdentifier = NSUserInterfaceItemIdentifier("menu.database.manualCommitFootnote") + static let manualCommitUnavailableFootnoteIdentifier = NSUserInterfaceItemIdentifier( + "menu.database.manualCommitUnavailableFootnote" + ) static func build(keyboard: KeyboardSettings, modules: Set = []) -> NSMenuItem { let item = MenuItemFactory.menu(String(localized: "Database"), items: [ @@ -38,6 +42,7 @@ enum DatabaseMenuBuilder { shortcut: .refresh, keyboard: keyboard ), + autoRefreshSubmenu(), MenuItemFactory.separator, MenuItemFactory.item( String(localized: "New Database…"), @@ -109,7 +114,8 @@ enum DatabaseMenuBuilder { ), schemaSubmenu(), sessionContextSubmenu(), - safeModeSubmenu(), + safeModeSubmenu() + ] + transactionItems(keyboard: keyboard) + [ favoriteDatabaseSubmenu(), maintenanceSubmenu(), MenuItemFactory.item( @@ -278,6 +284,49 @@ enum DatabaseMenuBuilder { return container } + private static func transactionItems(keyboard: KeyboardSettings) -> [NSMenuItem] { + [ + MenuItemFactory.item( + String(localized: "Manual Commit"), + action: #selector(MainSplitViewController.toggleManualCommit(_:)) + ), + manualCommitFootnote( + String(localized: "This database can't report an open transaction, so it commits each change."), + identifier: manualCommitFootnoteIdentifier + ), + manualCommitFootnote(ManualCommit.unavailableExplanation, identifier: manualCommitUnavailableFootnoteIdentifier), + MenuItemFactory.item( + String(localized: "Commit Transaction"), + action: #selector(MainSplitViewController.commitOpenTransaction(_:)), + shortcut: .commitTransaction, + keyboard: keyboard + ), + MenuItemFactory.item( + String(localized: "Roll Back Transaction"), + action: #selector(MainSplitViewController.rollBackOpenTransaction(_:)), + shortcut: .rollbackTransaction, + keyboard: keyboard + ) + ] + } + + // Validation shows it under Manual Commit, which it shares an action with so it always validates as disabled. + private static func manualCommitFootnote(_ text: String, identifier: NSUserInterfaceItemIdentifier) -> NSMenuItem { + let item = MenuFootnote.item(text) + item.action = #selector(MainSplitViewController.toggleManualCommit(_:)) + item.identifier = identifier + item.isHidden = true + return item + } + + private static let autoRefreshDelegate = TableAutoRefreshMenuDelegate() + + private static func autoRefreshSubmenu() -> NSMenuItem { + let container = MenuItemFactory.submenu(String(localized: "Auto Refresh"), items: []) + container.submenu?.delegate = autoRefreshDelegate + return container + } + private static let safeModeDelegate = SafeModeMenuDelegate() /// The same list the toolbar's Safe Mode control opens, so every toolbar item keeps a menu-bar diff --git a/apple/TablePro/Core/Menu/FileMenuBuilder.swift b/apple/TablePro/Core/Menu/FileMenuBuilder.swift index 888331b772..2ce0f5d02b 100644 --- a/apple/TablePro/Core/Menu/FileMenuBuilder.swift +++ b/apple/TablePro/Core/Menu/FileMenuBuilder.swift @@ -7,11 +7,12 @@ import AppKit @MainActor enum FileMenuBuilder { - /// Retained for the menu's lifetime, which is the app's: `NSMenu.delegate` is unowned. + // Retained for the app's lifetime: `NSMenu.delegate` is unowned. private static let closeTitleDelegate = CloseCommandMenuDelegate() private static let importFormatDelegate = ImportFormatMenuDelegate() private static let agentSessionDelegate = AgentSessionMenuDelegate() private static let conversationHistoryDelegate = ConversationHistoryMenuDelegate() + private static let revertToDelegate = QueryTabRevertMenuDelegate() static func build(keyboard: KeyboardSettings) -> NSMenuItem { let file = MenuItemFactory.menu(String(localized: "File"), items: [ @@ -84,14 +85,9 @@ enum FileMenuBuilder { shortcut: .saveAs, keyboard: keyboard ), + revertToSubmenu(), MenuItemFactory.separator, - /// `performClose:` is the close command every `NSWindow` implements and validates, so - /// Command W reaches Settings, the integrations activity window and every viewer the - /// app opens. An app-specific selector here reached only the editor window, and the - /// windows that missed out had to answer it by hand. `EditorWindow` overrides it to - /// close the front tab, which is where a window that draws its own tabs says so. - /// Built with the title the resolver gives a window with no tabs; the File menu's - /// delegate resolves it from the key window from then on. + // `performClose:` reaches every window, not only the editor; `EditorWindow` overrides it to close the front tab. MenuItemFactory.item( CloseCommandTitleResolver.windowTitle, action: #selector(NSWindow.performClose(_:)), @@ -143,14 +139,8 @@ enum FileMenuBuilder { return file } - /// Agent mode's four session commands and the assistant's three conversation commands, which - /// between them had no menu-bar home at all: the rail's buttons and the pane header's menu were - /// the only routes, so none of them could be found by search, rebound, or reached by a user who - /// had the rail collapsed. They sit in File because a session and a conversation are things this - /// window opens, closes and throws away, which is what the rest of this menu is about. - /// - /// Each command has exactly one item here, and each item leaves `target` nil, so the window - /// validates it through the responder chain and dims what the mode cannot run. + // A session and a conversation are things this window opens and closes. Each item leaves `target` nil, so the + // window validates it through the responder chain. private static func sessionSubmenu(keyboard: KeyboardSettings) -> NSMenuItem { MenuItemFactory.submenu(String(localized: "Session"), items: [ MenuItemFactory.item( @@ -159,11 +149,7 @@ enum FileMenuBuilder { shortcut: .newAgentSession, keyboard: keyboard ), - /// Opens the session the rail has highlighted, which is what Return on the rail does. - /// A separate leaf rather than the list's own row: AppKit ignores a key equivalent on an - /// item that owns a submenu, so making this the list would hand Settings a binding that - /// records, reads back and never fires. That is the same trap Import Data… is split - /// around, and it is why the list is a row of its own below. + // A leaf beside the list: AppKit ignores a key equivalent on an item that owns a submenu. MenuItemFactory.item( String(localized: "Open Session"), action: #selector(MainSplitViewController.openAgentSession(_:)), @@ -198,23 +184,27 @@ enum FileMenuBuilder { ]) } - /// Every session the connection on screen owns, latest first, filled when it opens. A session - /// list built at menu-build time would be one window's sessions frozen at launch. + // Filled when it opens; a list built with the menu bar would be frozen at launch. private static func recentSessionsSubmenu() -> NSMenuItem { let container = MenuItemFactory.submenu(String(localized: "Recent Sessions"), items: []) container.submenu?.delegate = agentSessionDelegate return container } - /// The assistant's stored conversations, filled when it opens for the same reason: the set - /// changes with every reply. The pane header's own menu offers the same list, and both put the - /// choice through the window so neither can act on a connection the other is showing. + // Filled when it opens, since the set changes with every reply. private static func conversationHistorySubmenu() -> NSMenuItem { let container = MenuItemFactory.submenu(String(localized: "Conversation History"), items: []) container.submenu?.delegate = conversationHistoryDelegate return container } + // The selected query tab's recent versions, filled when it opens. No shortcut: AppKit ignores one on this row. + private static func revertToSubmenu() -> NSMenuItem { + let container = MenuItemFactory.submenu(String(localized: "Revert To"), items: []) + container.submenu?.delegate = revertToDelegate + return container + } + private static func importSubmenu(keyboard: KeyboardSettings) -> NSMenuItem { let container = MenuItemFactory.submenu(String(localized: "Import"), items: [ MenuItemFactory.item( @@ -253,9 +243,7 @@ enum FileMenuBuilder { return container } - /// Every format the connection imports from, for naming one a file's extension does not. - /// Import Data… above it reads the format off the file instead. The Actions pull-down offers - /// the same list under the same title, and the two are filled by the same class when they open. + // For a format the file's extension does not name; the Actions pull-down fills the same list with the same class. private static func importFormatsSubmenu() -> NSMenuItem { let container = MenuItemFactory.submenu(String(localized: "Import Data From"), items: []) container.submenu?.delegate = importFormatDelegate diff --git a/apple/TablePro/Core/Menu/QueryTabRevertMenuDelegate.swift b/apple/TablePro/Core/Menu/QueryTabRevertMenuDelegate.swift new file mode 100644 index 0000000000..0dc068c075 --- /dev/null +++ b/apple/TablePro/Core/Menu/QueryTabRevertMenuDelegate.swift @@ -0,0 +1,67 @@ +// +// QueryTabRevertMenuDelegate.swift +// TablePro +// + +import AppKit + +internal struct QueryTabRevertChoice: Equatable { + let tabId: UUID + let versionId: Int64 +} + +// File > Revert To, filled for the selected query tab when it opens. Items carry no target, so the window validates them. +@MainActor +internal final class QueryTabRevertMenuDelegate: NSObject, NSMenuDelegate { + static let revertAction = #selector(MainSplitViewController.revertToQueryTabVersion(_:)) + static let browseAction = #selector(MainSplitViewController.browseQueryTabVersions(_:)) + static let recentVersionLimit = 10 + + // Resolved when the menu opens, so building the menu bar touches no store. + private let storage: QueryTabVersionStorage? + + init(storage: QueryTabVersionStorage? = nil) { + self.storage = storage + } + + func menuNeedsUpdate(_ menu: NSMenu) { + let controller = NSApp.target(forAction: Self.browseAction, to: nil, from: nil) as? MainSplitViewController + let tabId = controller?.selectedQueryTabId + let store = storage ?? .shared + let summaries = tabId.map { store.summaries(tabId: $0, limit: Self.recentVersionLimit) } ?? [] + Self.fill(menu, tabId: tabId, summaries: summaries) + } + + static func fill(_ menu: NSMenu, tabId: UUID?, summaries: [QueryTabVersionSummary]) { + menu.removeAllItems() + if let tabId, !summaries.isEmpty { + for summary in summaries { + menu.addItem(item(for: summary, tabId: tabId)) + } + } else { + menu.addItem(MenuPlaceholder.item()) + } + menu.addItem(.separator()) + let browse = NSMenuItem(title: String(localized: "Browse All Versions…"), action: browseAction, keyEquivalent: "") + browse.target = nil + menu.addItem(browse) + } + + static func item(for summary: QueryTabVersionSummary, tabId: UUID) -> NSMenuItem { + let entry = QueryTabVersionHistoryProvider.entry(for: summary) + let item = NSMenuItem(title: VersionHistoryFormatting.menuTitle(for: entry), action: revertAction, keyEquivalent: "") + item.target = nil + item.representedObject = QueryTabRevertChoice(tabId: tabId, versionId: summary.id) + return item + } + + // Keeps AppKit's key-equivalent search from reading the store on every modified keystroke. + func menuHasKeyEquivalent( + _ menu: NSMenu, + for event: NSEvent, + target: AutoreleasingUnsafeMutablePointer, + action: UnsafeMutablePointer + ) -> Bool { + false + } +} diff --git a/apple/TablePro/Core/Menu/TableAutoRefreshMenuDelegate.swift b/apple/TablePro/Core/Menu/TableAutoRefreshMenuDelegate.swift new file mode 100644 index 0000000000..f7df6b2695 --- /dev/null +++ b/apple/TablePro/Core/Menu/TableAutoRefreshMenuDelegate.swift @@ -0,0 +1,38 @@ +// +// TableAutoRefreshMenuDelegate.swift +// TablePro +// + +import AppKit + +// Filled when it opens, so the checkmark names the selected tab's interval; validation dims the items off a table tab. +@MainActor +final class TableAutoRefreshMenuDelegate: NSObject, NSMenuDelegate { + static let action = #selector(MainSplitViewController.setTableAutoRefreshInterval(_:)) + + func menuNeedsUpdate(_ menu: NSMenu) { + let controller = NSApp.target(forAction: Self.action, to: nil, from: nil) as? MainSplitViewController + Self.populate(menu, current: controller?.tableAutoRefreshInterval) + } + + internal static func populate(_ menu: NSMenu, current: TableAutoRefreshInterval?) { + menu.removeAllItems() + for interval in TableAutoRefreshInterval.allCases { + let item = NSMenuItem(title: interval.title, action: action, keyEquivalent: "") + item.target = nil + item.representedObject = interval.rawValue + item.state = interval == current ? .on : .off + menu.addItem(item) + } + } + + // Keeps AppKit's key-equivalent search from rebuilding the menu on every modified keystroke. + func menuHasKeyEquivalent( + _ menu: NSMenu, + for event: NSEvent, + target: AutoreleasingUnsafeMutablePointer, + action: UnsafeMutablePointer + ) -> Bool { + false + } +} diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 164f6c3613..5380089732 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -12,11 +12,11 @@ // // The read side is only as incremental as the driver is. `streamRows` has a // protocol default that calls `execute` and yields the whole result at once, -// which Dameng and Teradata still inherit, so on those two a large table is -// materialised before the first batch is written and Stop cannot land until -// it is. Every other SQL driver implements it for real. Export has the same -// property through the same call; a capability that lets this refuse rather -// than inherit would have to be declared by all 32 plugins. +// which Dameng still inherits, so there a large table is materialised before +// the first batch is written and Stop cannot land until it is. Every other SQL +// driver implements it for real. Export has the same property through the same +// call; a capability that lets this refuse rather than inherit would have to be +// declared by every plugin. // // This is the write path CSV and JSON import already use, through the same // `SQLStatementGenerator` and its per-engine bind-parameter ceiling. The one @@ -38,6 +38,8 @@ internal struct ObjectCopyRowCopier: Sendable { internal struct Outcome: Sendable { internal let inserted: Int internal let cancelled: Bool + // The limit the source's stream stopped at, so the table holds only its leading rows. + internal var rowCut: Int? } /// A table that failed after some of its batches were written. Outside a transaction those stay @@ -53,14 +55,18 @@ internal struct ObjectCopyRowCopier: Sendable { internal let sourceDatabaseType: DatabaseType internal let targetDatabaseType: DatabaseType + // `readContext` is Read-Only's, for a source whose connection runs at it: a view as the source runs its functions. internal func copy( from sourceDriver: any PluginDatabaseDriver, to targetDriver: any PluginDatabaseDriver, + readContext: PluginStatementContext? = nil, onProgress: @Sendable (Int) -> Void ) async throws -> Outcome { let scopes = await openSessionScopes(source: sourceDriver, target: targetDriver) do { - let outcome = try await transfer(from: sourceDriver, to: targetDriver, onProgress: onProgress) + let outcome = try await transfer( + from: sourceDriver, to: targetDriver, readContext: readContext, onProgress: onProgress + ) await scopes?.close() return outcome } catch { @@ -86,6 +92,7 @@ internal struct ObjectCopyRowCopier: Sendable { private func transfer( from sourceDriver: any PluginDatabaseDriver, to targetDriver: any PluginDatabaseDriver, + readContext: PluginStatementContext?, onProgress: @Sendable (Int) -> Void ) async throws -> Outcome { if let statement = step.serverSideInsert { @@ -94,7 +101,16 @@ internal struct ObjectCopyRowCopier: Sendable { let generator = try makeGenerator(targetDriver: targetDriver) let budget = SQLWriteBatchBudget(columnCount: step.columns.count, generator: generator) var filler = SQLWriteBatchFiller<[PluginCellValue]>(budget: budget) - var stream = sourceDriver.streamRows(query: step.sourceQuery).makeAsyncIterator() + let cut = OSAllocatedUnfairLock(initialState: nil) + // The copy's SELECT states no engine ceiling, so only a limit the driver declares applies, as for its own export query. + let rows = readContext.map { context in + var context = context + context.readFunctionCalls(in: step.sourceQuery, databaseType: sourceDatabaseType) + return sourceDriver.streamRows(query: step.sourceQuery, context: context) + } ?? sourceDriver.streamRows(query: step.sourceQuery) + var stream = ExportRowCut.reporting(rows, engineCeiling: nil) { limit in + cut.withLock { $0 = limit } + }.makeAsyncIterator() var inserted = 0 @@ -121,7 +137,7 @@ internal struct ObjectCopyRowCopier: Sendable { inserted += try await write(batch, generator: generator, driver: targetDriver) onProgress(inserted) } - return Outcome(inserted: inserted, cancelled: Task.isCancelled) + return Outcome(inserted: inserted, cancelled: Task.isCancelled, rowCut: cut.withLock { $0 }) } catch is CancellationError { return Outcome(inserted: inserted, cancelled: true) } catch where inserted > 0 { @@ -180,13 +196,13 @@ internal struct ObjectCopyRowCopier: Sendable { format: String(localized: "Could not build an INSERT for %@."), step.qualifiedTargetName )) } - /// The generator's own parameter array rather than the batch flattened again: the two - /// agree today, and taking the statement's own leaves no second place for the placeholder - /// order to be decided. The round trip is lossless because `PluginCellValue` is text, - /// bytes or null and `asAny` maps each to itself. + // The generator's own parameter array rather than the batch flattened again, so the placeholder order is + // decided in one place. `PluginCellValue` is text, bytes or null and `asAny` maps each to itself. The context + // carries each value's target column type: Oracle wrote `00123` into a VARCHAR2 as 123 without it. _ = try await driver.executeParameterized( query: statement.sql, - parameters: statement.parameters.map(PluginDriverAdapter.cellValue(for:)) + parameters: statement.parameters.map(PluginDriverAdapter.cellValue(for:)), + context: statement.context ) return batch.count } diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift index 6cef58ed51..78076942f4 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift @@ -22,6 +22,7 @@ internal struct ObjectCopyObjectOutcome: Identifiable, Sendable { internal let selection: ObjectCopySelection internal let rowsCopied: Int internal let error: String? + internal var warning: String? internal var id: String { selection.id } internal var succeeded: Bool { error == nil } @@ -67,6 +68,8 @@ internal struct ObjectCopyRunResult: Sendable { internal var firstError: String? { outcomes.compactMap(\.error).first } + internal var warnings: [String] { outcomes.compactMap(\.warning) } + /// Every failure, each with an identity of its own. /// /// One object fails once per phase it reaches, so a table whose `CREATE` failed under Skip and @@ -112,9 +115,11 @@ internal struct ObjectCopyRunner { /// pending work. `DataWriteExecutor`, `DatabaseManager+Principals` and /// `StructureRebuildPlanRunner` ask the same question for the same reason. nonisolated private static func transactionOwner(of driver: DatabaseDriver) async -> WriteTransactionOwner { + // `.automatic`: a pooled lease is closed while idle, which would roll a held copy back after it reported success. WriteTransactionOwner.resolve( supportsTransactions: driver.supportsTransactions, - sessionState: await driver.heldSessionTransactionState() + sessionState: await driver.heldSessionTransactionState(), + commitMode: .automatic ) } @@ -128,6 +133,16 @@ internal struct ObjectCopyRunner { ) defer { ProcessInfo.processInfo.endActivity(activity) } + // A row filter is the user's text, read from a source that may be at Read-Only: it meets the gate as a Run would. + try await ReadOnlyReadGate.authorize( + plan.dataSteps.filter(\.hasRowFilter).map(\.sourceQuery), + connectionId: request.source.connectionId, + databaseType: request.source.databaseType, + gate: gate, + operationDescription: String(localized: "Execute Query"), + runsIn: request.source.scope + ) + return try await gate.authorizing(authorizationRequest(for: plan)) { try await self.execute(plan, request: request, progress: progress) } @@ -261,6 +276,7 @@ internal struct ObjectCopyRunner { let pinsSession = plan.isStructureReadPinned && VerbatimTransferScopes.isSameEngine( source: request.source.databaseType, target: request.target.databaseType ) + let family = TransactionEngineFamily.of(request.target.databaseType) return try await manager.withMetadataDriver(scope: scope) { driver in guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { @@ -269,8 +285,9 @@ internal struct ObjectCopyRunner { let owner = await Self.transactionOwner(of: driver) // The other two settings keep what ran, and PostgreSQL commits an aborted transaction as a ROLLBACK. let usesTransaction = hasCleanup && errorHandling == .stopAndRollback - && plugin.supportsTransactionalDDL && owner.opensTransaction + && plugin.supportsTransactionalDDL && owner.sendsBegin && owner.commitsAtEnd if usesTransaction { try await plugin.beginTransaction(mode: .readWrite) } + let committingEach = usesTransaction ? nil : Self.standaloneFamily(family, owner: owner) var result: DDLResult do { @@ -283,6 +300,7 @@ internal struct ObjectCopyRunner { on: plugin, pinsSession: pinsSession, errorHandling: errorHandling, + committingEach: committingEach, progress: progress ) } @@ -295,7 +313,8 @@ internal struct ObjectCopyRunner { /// `sql_mode`: it gets its own creation settings, or else the target's. if !result.stopped { let definitions = await Self.execute( - definitionGroups, on: plugin, errorHandling: errorHandling, progress: progress + definitionGroups, on: plugin, errorHandling: errorHandling, committingEach: committingEach, + progress: progress ) result.outcomes += definitions.outcomes result.cancelled = definitions.cancelled @@ -338,6 +357,7 @@ internal struct ObjectCopyRunner { on driver: any PluginDatabaseDriver, pinsSession: Bool, errorHandling: ImportErrorHandling, + committingEach family: TransactionEngineFamily?, progress: ObjectCopyProgress ) async throws -> DDLResult { let writesAnything = !schemaStatements.isEmpty || groups.contains { !$0.statements.isEmpty } @@ -349,9 +369,11 @@ internal struct ObjectCopyRunner { /// table, view or routine can be created if the schema naming it is not there, so /// carrying on would report every one of them as its own failure for one cause. for statement in schemaStatements { - _ = try await driver.execute(query: statement.sql) + try await executeStatement(statement.sql, on: driver, committingEach: family) } - let result = await execute(groups, on: driver, errorHandling: errorHandling, progress: progress) + let result = await execute( + groups, on: driver, errorHandling: errorHandling, committingEach: family, progress: progress + ) await scopes?.close() return result } catch { @@ -360,10 +382,34 @@ internal struct ObjectCopyRunner { } } + // nil inside the phase's own transaction; otherwise a statement the session keeps pending is committed on its own. + nonisolated private static func executeStatement( + _ sql: String, + on driver: any PluginDatabaseDriver, + committingEach family: TransactionEngineFamily? + ) async throws { + guard let family else { + _ = try await driver.execute(query: sql) + return + } + _ = try await AppDefinitionCommit.run(on: driver, family: family, refusesOpenTransaction: false) { + try await driver.execute(query: sql) + } + } + + // A phase that joined a transaction the session already held leaves it to its owner, as its result reports. + nonisolated private static func standaloneFamily( + _ family: TransactionEngineFamily, + owner: WriteTransactionOwner + ) -> TransactionEngineFamily? { + owner == .session ? nil : family + } + nonisolated private static func execute( _ groups: [ObjectCopyStatementGroup], on driver: any PluginDatabaseDriver, errorHandling: ImportErrorHandling, + committingEach family: TransactionEngineFamily?, progress: ObjectCopyProgress ) async -> DDLResult { var result = DDLResult() @@ -377,7 +423,7 @@ internal struct ObjectCopyRunner { let opened = await openSettings(of: group, on: driver) do { for statement in group.statements { - _ = try await driver.execute(query: statement.sql) + try await executeStatement(statement.sql, on: driver, committingEach: family) } if opened { await closeSettings(of: group, on: driver) } result.outcomes.append(ObjectCopyObjectOutcome( @@ -481,13 +527,15 @@ internal struct ObjectCopyRunner { guard !runnable.isEmpty else { return DDLResult() } let errorHandling = request.errorHandling + let family = TransactionEngineFamily.of(request.target.databaseType) return try await manager.withMetadataDriver(scope: request.target.scope) { driver in guard let plugin = CompareMetadataService.pluginDriver(from: driver) else { throw ObjectCopyError.refused(Self.noTargetDriver) } let owner = await Self.transactionOwner(of: driver) var result = await Self.execute( - runnable, on: plugin, errorHandling: errorHandling, progress: progress + runnable, on: plugin, errorHandling: errorHandling, + committingEach: Self.standaloneFamily(family, owner: owner), progress: progress ) result.joinedSessionTransaction = owner == .session return result @@ -507,9 +555,14 @@ internal struct ObjectCopyRunner { /// would send a retry over them. var rowsCopied = 0 - mutating func recordCopy(of step: ObjectCopyTableStep, inserted: Int) { + mutating func recordCopy(of step: ObjectCopyTableStep, inserted: Int, rowCut: Int? = nil) { rowsCopied += inserted - outcomes.append(ObjectCopyObjectOutcome(selection: step.selection, rowsCopied: inserted, error: nil)) + outcomes.append(ObjectCopyObjectOutcome( + selection: step.selection, + rowsCopied: inserted, + error: nil, + warning: rowCut.map { StreamRowCutWarning.text(table: step.selection.qualifiedName, limit: $0) } + )) } /// Stop reaching the driver mid-stream is the user stopping, not the copy failing, and the @@ -579,6 +632,7 @@ internal struct ObjectCopyRunner { let sourceType = request.source.databaseType let targetType = request.target.databaseType let sourceScope = request.source.scope + let sourceIsReadOnly = ExecutionGateProvider.runsAtReadOnly(request.source.connectionId) let errorHandling = request.errorHandling let relaxesRows = plan.keysInside.rows @@ -592,15 +646,16 @@ internal struct ObjectCopyRunner { throw ObjectCopyError.refused(Self.noTargetDriver) } let owner = await Self.transactionOwner(of: targetDriver) - let usesTransaction = owner.opensTransaction + let usesTransaction = owner.sendsBegin && owner.commitsAtEnd if usesTransaction { try await targetPlugin.beginTransaction(mode: .readWrite) } var result = DataResult() result.joinedSessionTransaction = owner == .session // Checks stay on for the clears: a table outside the copy that references a cleared one // keeps its own ON DELETE rule, and MySQL cannot prove no other database holds one. + // In this phase's own transaction or the session's, so a clear never commits on its own. let cleared = await Self.execute( - clearGroups, on: targetPlugin, errorHandling: errorHandling, progress: progress + clearGroups, on: targetPlugin, errorHandling: errorHandling, committingEach: nil, progress: progress ) result.outcomes += cleared.outcomes.filter { $0.error != nil } result.cancelled = cleared.cancelled @@ -620,6 +675,7 @@ internal struct ObjectCopyRunner { manager: manager, sourceType: sourceType, targetType: targetType, + sourceIsReadOnly: sourceIsReadOnly, errorHandling: errorHandling, progress: progress ) @@ -672,6 +728,7 @@ internal struct ObjectCopyRunner { manager: any ScopedMetadataProviding, sourceType: DatabaseType, targetType: DatabaseType, + sourceIsReadOnly: Bool, errorHandling: ImportErrorHandling, progress: ObjectCopyProgress ) async -> DataResult { @@ -694,15 +751,18 @@ internal struct ObjectCopyRunner { guard let sourcePlugin = CompareMetadataService.pluginDriver(from: sourceDriver) else { throw ObjectCopyError.refused(noSourceDriver) } - return try await copier.copy(from: sourcePlugin, to: targetPlugin) { rows in - progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + let readContext = sourceDriver.readOnlyRunContext(safeModeIsReadOnly: sourceIsReadOnly) + return try await BulkReadTimeout.lifted(on: [sourceDriver]) { + try await copier.copy(from: sourcePlugin, to: targetPlugin, readContext: readContext) { rows in + progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + } } } guard !outcome.cancelled else { result.recordStop(of: step, inserted: outcome.inserted, progress: progress) break } - result.recordCopy(of: step, inserted: outcome.inserted) + result.recordCopy(of: step, inserted: outcome.inserted, rowCut: outcome.rowCut) } catch is CancellationError { result.recordStop(of: step, inserted: 0, progress: progress) break @@ -748,7 +808,7 @@ internal struct ObjectCopyRunner { result.recordStop(of: step, inserted: outcome.inserted, progress: progress) break } - result.recordCopy(of: step, inserted: outcome.inserted) + result.recordCopy(of: step, inserted: outcome.inserted, rowCut: outcome.rowCut) } catch is CancellationError { result.recordStop(of: step, inserted: 0, progress: progress) break @@ -776,9 +836,11 @@ internal struct ObjectCopyRunner { ) async throws -> CopiedRows { let sourceScope = request.source.scope let targetScope = request.target.scope + let sourceIsReadOnly = ExecutionGateProvider.runsAtReadOnly(request.source.connectionId) let copier = ObjectCopyRowCopier( step: step, sourceDatabaseType: request.source.databaseType, targetDatabaseType: targetType ) + let family = TransactionEngineFamily.of(targetType) /// The injected manager on both sides. Reaching for the singleton on the target bypassed /// the caller's own connections and routing, which is the whole point of injecting one. @@ -790,17 +852,48 @@ internal struct ObjectCopyRunner { throw ObjectCopyError.refused(Self.noTargetDriver) } let owner = await Self.transactionOwner(of: targetDriver) - return try await manager.withMetadataDriver( - scope: sourceScope, workload: .bulk - ) { sourceDriver in - guard let sourcePlugin = CompareMetadataService.pluginDriver(from: sourceDriver) else { - throw ObjectCopyError.refused(Self.noSourceDriver) + let committingEach = Self.standaloneFamily(family, owner: owner) + let copied: CopiedRows + do { + copied = try await manager.withMetadataDriver( + scope: sourceScope, workload: .bulk + ) { sourceDriver in + guard let sourcePlugin = CompareMetadataService.pluginDriver(from: sourceDriver) else { + throw ObjectCopyError.refused(Self.noSourceDriver) + } + let readContext = sourceDriver.readOnlyRunContext(safeModeIsReadOnly: sourceIsReadOnly) + let outcome = try await BulkReadTimeout.lifted(on: [sourceDriver]) { + try await copier.copy(from: sourcePlugin, to: targetPlugin, readContext: readContext) { rows in + progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + } + } + return CopiedRows(outcome: outcome, joinedSessionTransaction: owner == .session) } - let outcome = try await copier.copy(from: sourcePlugin, to: targetPlugin) { rows in - progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + } catch { + // A failed table keeps the rows written before the failure, as it reports them. + do { + try await Self.commitRows(on: targetPlugin, committingEach: committingEach) + } catch let commitError { + Self.logger.error("Committing the rows before a copy failure failed: \(commitError.publicLogShape, privacy: .public)") } - return CopiedRows(outcome: outcome, joinedSessionTransaction: owner == .session) + throw error } + try await Self.commitRows(on: targetPlugin, committingEach: committingEach) + return copied + } + } + + // A commit that fails is rolled back: a pooled connection would otherwise carry the rows into its next lease. + nonisolated private static func commitRows( + on plugin: any PluginDatabaseDriver, + committingEach family: TransactionEngineFamily? + ) async throws { + guard let family else { return } + do { + try await AppDefinitionCommit.commitIfOpen(on: plugin, family: family) + } catch { + _ = try? await TaskCancellationShield.run { try await plugin.rollbackTransaction() } + throw error } } diff --git a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift index 549259fa11..ac55d9e63a 100644 --- a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift +++ b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift @@ -9,23 +9,16 @@ import SwiftUI import TableProAutocomplete import TableProPluginKit -/// Everything a `DatabaseType` knows by asking the plugin registry. -/// -/// It lives apart from the type itself because these are the members that make naming a database -/// type cost the whole plugin system at compile time. `DatabaseType` is a string a connection -/// carries and a value the display layer compares against a constant; keeping the lookups here -/// means a file that only does that does not compile against the registry, the snapshot table and -/// the theme engine behind them. +// Apart from `DatabaseType`, so a file that only names or compares a type does not compile against the registry, the +// snapshot table and the theme engine behind them. extension DatabaseType { - /// All registered database types, derived dynamically from the plugin metadata registry. static var allKnownTypes: [DatabaseType] { PluginMetadataRegistry.shared.allRegisteredTypeIds().map { DatabaseType(rawValue: $0) } } } extension DatabaseType { - /// Returns nil if rawValue doesn't match any registered type. init?(validating rawValue: String) { guard PluginMetadataRegistry.shared.hasType(rawValue) else { return nil } self.rawValue = rawValue @@ -33,14 +26,11 @@ extension DatabaseType { } extension DatabaseType { - /// Plugin type ID used for PluginManager lookup, resolved via the registry. var pluginTypeId: String { PluginMetadataRegistry.shared.pluginTypeId(for: rawValue) } - /// Genuinely a fact about the plugin binary rather than the database, so it asks by - /// `pluginTypeId`: Redshift is served by the bundled PostgreSQL plugin and there is nothing - /// of its own to download. + // Asks by `pluginTypeId`: Redshift is served by the bundled PostgreSQL plugin and has nothing of its own to download. var isDownloadablePlugin: Bool { PluginMetadataRegistry.shared.snapshot(forRegisteredTypeId: pluginTypeId)?.isDownloadable ?? false } @@ -49,8 +39,7 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.iconName ?? "cylinder" } - /// Returns the correct SwiftUI Image for this database type, handling both - /// SF Symbol names (e.g. "cylinder.fill") and asset catalog names (e.g. "mysql-icon"). + // `iconName` is an SF Symbol (`cylinder.fill`) or an asset catalog name (`mysql-icon`). var iconImage: Image { let name = iconName if NSImage(systemSymbolName: name, accessibilityDescription: nil) != nil { @@ -88,6 +77,10 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.verifiesServerWithSystemTrust ?? false } + var requestedTLSVerifiesIdentity: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.requestedTLSVerifiesIdentity ?? false + } + var supportsPerConnectionCertificatePaths: Bool { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.supportsPerConnectionCertificatePaths ?? true } @@ -98,7 +91,7 @@ extension DatabaseType { } var sslModeWhenTLSEnabled: SSLMode { - verifiesServerWithSystemTrust ? .verifyIdentity : .required + requestedTLSVerifiesIdentity ? .verifyIdentity : .required } func impliedSSLMode(forPort port: Int) -> SSLMode? { @@ -261,8 +254,8 @@ extension DatabaseType { .capabilities.supportsUserDefinedTypeBrowse ?? false } - /// The object kinds this engine fetches. The sidebar waits on these fetches before calling a - /// container empty, and keeps a section for one that failed; it lists a section only for rows. + // The object kinds this engine fetches. The sidebar waits on these fetches before calling a + // container empty, and keeps a section for one that failed; it lists a section only for rows. var declaredObjectKinds: Set { var kinds: Set = [] if supportsRoutines { diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index ee30d84cc6..bfb2c5a5a1 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -26,9 +26,12 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable private let pagination: PaginationCapability private let appliesRowScope: Bool private let commentableKinds: Set - private let cappedTables = OSAllocatedUnfairLock<[String]>(initialState: []) + private let cappedTables = OSAllocatedUnfairLock<[(table: String, limit: Int)]>(initialState: []) + // The caller reads the connection's level on the main actor, where the gate reads it. + private let safeModeIsReadOnly: Bool - init(driver: DatabaseDriver, databaseType: DatabaseType) { + init(driver: DatabaseDriver, databaseType: DatabaseType, safeModeIsReadOnly: Bool = false) { + self.safeModeIsReadOnly = safeModeIsReadOnly let snapshot = PluginMetadataRegistry.shared.snapshot(for: databaseType) self.supportsCascadeDrop = snapshot?.capabilities.supportsCascadeDrop ?? false self.lexicalFeatures = SQLLexicalResolver.executionFeatures(for: databaseType, session: driver.sessionLexicalState) @@ -59,17 +62,9 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable (driver as? PluginDriverAdapter)?.schemaPluginDriver } - /// One line per table that stopped at the engine's row ceiling, so a partial copy is never - /// reported as the whole table. + // One line per table whose rows stopped at a ceiling, so a partial copy is never reported as the whole table. var cappedTableWarnings: [String] { - guard let maximum = pagination.maximumRows else { return [] } - return cappedTables.withLock { $0 }.map { table in - String( - format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), - table, - maximum - ) - } + cappedTables.withLock { $0 }.map { StreamRowCutWarning.text(table: $0.table, limit: $0.limit) } } func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { @@ -88,7 +83,13 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable // A table export only reads, whatever the statement the driver chose classifies as. var context = PluginStatementContext() context.readOnly = true - return pluginDriver.streamExportRows(query: customQuery, context: context) + context.enforcesReadOnly = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) != nil + // The driver's own export query is not the app's capped SELECT, so only a limit it declares applies. + return reportingCut( + of: pluginDriver.streamExportRows(query: customQuery, context: context), + table: table, + engineCeiling: nil + ) } let query = leadingRowsQuery( columns: "*", @@ -132,26 +133,18 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable guard let adapter = driver as? PluginDriverAdapter else { return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } } - let stream = adapter.streamExportRows(query: query) - guard let maximum = pagination.maximumRows else { return stream } + let rows = adapter.streamExportRows(query: query, safeModeIsReadOnly: safeModeIsReadOnly) + return reportingCut(of: rows, table: table, engineCeiling: pagination.maximumRows) + } + + private func reportingCut( + of stream: AsyncThrowingStream, + table: String, + engineCeiling: Int? + ) -> AsyncThrowingStream { let cappedTables = cappedTables - return AsyncThrowingStream { continuation in - let task = Task { - var rowCount = 0 - do { - for try await element in stream { - if case .rows(let rows, _) = element { rowCount += rows.count } - continuation.yield(element) - } - if rowCount >= maximum { - cappedTables.withLock { $0.append(table) } - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - continuation.onTermination = { _ in task.cancel() } + return ExportRowCut.reporting(stream, engineCeiling: engineCeiling) { limit in + cappedTables.withLock { $0.append((table, limit)) } } } @@ -190,11 +183,21 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable return try await pluginDriver.fetchTableDDL(table: table, schema: exportSchema(for: databaseName)) } + // At Read-Only each read runs in a read-only transaction of its own, which an open one refuses, so a format's + // snapshot (`BEGIN`, `START TRANSACTION`, `COMMIT`) is not opened and the export reads table by table. func execute(query: String) async throws -> PluginQueryResult { - let result = try await driver.execute(query: query) - return mapToPluginResult(result) + guard let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) else { + return mapToPluginResult(try await driver.execute(query: query)) + } + let keyword = QueryClassifier.leadingKeyword(of: query, grammar: dbType.lexicalGrammar) + guard !Self.transactionKeywords.contains(keyword) else { + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + return mapToPluginResult(try await driver.execute(query: query, context: context)) } + private static let transactionKeywords: Set = ["BEGIN", "START", "COMMIT", "END", "ROLLBACK"] + func quoteIdentifier(_ identifier: String) -> String { driver.quoteIdentifier(identifier) } diff --git a/apple/TablePro/Core/Plugins/ExportRowCut.swift b/apple/TablePro/Core/Plugins/ExportRowCut.swift new file mode 100644 index 0000000000..a3be667953 --- /dev/null +++ b/apple/TablePro/Core/Plugins/ExportRowCut.swift @@ -0,0 +1,99 @@ +// +// ExportRowCut.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// A stream that ends at the engine's ceiling or the driver's declared `rowLimit` wrote a partial file. +// One consumer reads an export stream, so one call at a time reaches the iterator. +final class ExportRowCut: @unchecked Sendable { + private var iterator: AsyncThrowingStream.AsyncIterator + private let engineCeiling: Int? + private let reading: @Sendable (Element) -> (header: PluginStreamHeader?, rows: Int) + private var declaredLimit: Int? + private var rowCount = 0 + private var isSettled = false + + private init( + upstream: AsyncThrowingStream, + engineCeiling: Int?, + reading: @escaping @Sendable (Element) -> (header: PluginStreamHeader?, rows: Int) + ) { + self.iterator = upstream.makeAsyncIterator() + self.engineCeiling = engineCeiling + self.reading = reading + } + + private func next() async throws -> Element? { + guard let element = try await iterator.next() else { return nil } + let read = reading(element) + if let header = read.header { declaredLimit = header.rowLimit } + rowCount += read.rows + return element + } + + private func settle() -> Int? { + guard !isSettled else { return nil } + isSettled = true + return [engineCeiling, declaredLimit].compactMap { $0 }.filter { rowCount >= $0 }.min() + } + + // Pulled rather than pumped by a task of its own, which kept every row a slow writer had not taken yet. + static func reporting( + _ upstream: AsyncThrowingStream, + engineCeiling: Int?, + reading: @escaping @Sendable (Element) -> (header: PluginStreamHeader?, rows: Int), + onCut: @escaping @Sendable (Int) -> Void + ) -> AsyncThrowingStream { + let cut = ExportRowCut(upstream: upstream, engineCeiling: engineCeiling, reading: reading) + return AsyncThrowingStream(unfolding: { + guard let element = try await cut.next() else { + if let limit = cut.settle() { onCut(limit) } + return nil + } + return element + }) + } +} + +// One wording for every reader of a cut stream: export, Table Transfer, Object Copy and Data Compare. +enum StreamRowCutWarning { + static func text(table: String, limit: Int) -> String { + String( + format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), + table, + limit + ) + } +} + +extension ExportRowCut where Element == PluginStreamElement { + static func reporting( + _ upstream: AsyncThrowingStream, + engineCeiling: Int?, + onCut: @escaping @Sendable (Int) -> Void + ) -> AsyncThrowingStream { + reporting(upstream, engineCeiling: engineCeiling, reading: { element in + switch element { + case .header(let header): return (header, 0) + case .rows(let rows): return (nil, rows.count) + } + }, onCut: onCut) + } +} + +extension ExportRowCut where Element == PluginExportStreamElement { + static func reporting( + _ upstream: AsyncThrowingStream, + engineCeiling: Int?, + onCut: @escaping @Sendable (Int) -> Void + ) -> AsyncThrowingStream { + reporting(upstream, engineCeiling: engineCeiling, reading: { element in + if case .header(let header) = element { return (header, 0) } + if case .rows(let rows, _) = element { return (nil, rows.count) } + return (nil, 0) + }, onCut: onCut) + } +} diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index cb350fb44f..adb9b50c60 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -31,6 +31,8 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let sourceFieldsPerFoldedName: [String: Int] private let rowGenerator: SQLStatementGenerator? private let typedRowGenerator = OSAllocatedUnfairLock(uncheckedState: nil) + private let targetValues = OSAllocatedUnfairLock(initialState: nil) + private let binaryText: ImportBinaryText? private let rowWriting: ImportRowWriting private let tally: OSAllocatedUnfairLock private let immutableColumns = OSAllocatedUnfairLock?>(initialState: nil) @@ -40,7 +42,10 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let tableEditDialect: TableEditDialect? private let nameHazards = OSAllocatedUnfairLock<[String]>(initialState: []) private let ownsTransaction = OSAllocatedUnfairLock(initialState: false) + private let leavesTransactionOpen = OSAllocatedUnfairLock(initialState: false) + private let disabledForeignKeys = OSAllocatedUnfairLock(initialState: false) private let joinsTransaction: Bool + private let commitMode: ConnectionCommitMode private let transactionRecord = OSAllocatedUnfairLock(initialState: TransactionRecord()) private struct TransactionRecord { @@ -71,17 +76,22 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { columnMapping: [String: String] = [:], sourceFields: Set = [], rowWriting: ImportRowWriting = .insert, + binaryText: ImportBinaryText? = nil, joinsTransaction: Bool = false, + commitMode: ConnectionCommitMode = .automatic, importRun: ImportRun? = nil, fileIndex: Int? = 0, isCancelled: @escaping @Sendable () -> Bool = { false } ) { self.importControl = importRun.map { ImportSinkRun(run: $0, driver: driver, databaseType: databaseType, fileIndex: fileIndex, - targetTable: targetTable, targetSchema: targetSchema, joinsTransaction: joinsTransaction) + targetTable: targetTable, targetSchema: targetSchema, joinsTransaction: joinsTransaction, + commitMode: commitMode) } + self.commitMode = commitMode self.isCancelled = isCancelled self.rowWriting = rowWriting + self.binaryText = binaryText self.tally = OSAllocatedUnfairLock(initialState: ImportRowTally(mode: rowWriting.mode)) self.joinsTransaction = joinsTransaction self.driver = driver @@ -125,27 +135,50 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { ) } - // MySQL stores text bound to a BIT column as its bytes, so a 5 lands as 53 unless the INSERT knows the column is BIT. + // The driver gets each parameter's column type: Oracle stored `00123` bound to a VARCHAR2 as 123 without it, and + // MySQL stores text bound to a BIT column as its bytes, so a 5 lands as 53. private func writingGenerator() async throws -> SQLStatementGenerator? { - guard let rowGenerator, let targetTable, SQLTypeFamily.of(databaseType).readsBitsAsDecimal else { - return rowGenerator - } + guard let rowGenerator, let targetTable else { return rowGenerator } if let typed = typedRowGenerator.withLockUnchecked({ $0 }) { return typed } - let columns = try await importedData { - try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) - } - var typeNames: [String: String] = [:] - for column in columns where typeNames[column.name] == nil { - typeNames[column.name] = column.typeNameForClassification - } let typed = Self.makeRowGenerator( table: targetTable, schema: targetSchema, databaseType: databaseType, driver: driver, - columnTypeNames: typeNames + columnTypeNames: try await targetColumnValues().typeNames ) ?? rowGenerator typedRowGenerator.withLockUnchecked { $0 = typed } return typed } + // Read once per import, for the typed binding above and for what a file's text means in each column. + private func targetColumnValues() async throws -> ImportTargetValues { + if let known = targetValues.withLock({ $0 }) { return known } + guard let targetTable else { + throw PluginImportError.importFailed("No target table configured for row import") + } + // A BIT family and Oracle write wrong values without the types; elsewhere the import goes on as it did before + // the types were read, and the error stays out of the run's list. + let family = SQLTypeFamily.of(databaseType) + let needsTypes = family.readsBitsAsDecimal || family == .oracle + let read: [ColumnInfo]? = try await importedData { + do { + return try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) + } catch where !needsTypes && !(error is CancellationError) { + Self.logger.warning( + "Import writes untyped values: the target's columns could not be read: \(error.localizedDescription, privacy: .private)" + ) + return nil + } + } + let values = ImportTargetValues( + columns: (read ?? []).map { + (name: $0.name, typeName: $0.typeNameForClassification, isGenerated: $0.isGenerated) + }, + databaseType: databaseType, + binaryText: binaryText + ) + targetValues.withLock { $0 = values } + return values + } + func execute(statement: String) async throws { try await execute(statement: statement, line: 1) } @@ -201,6 +234,15 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } } + // The driver's Stop ends a copy at once (a CopyFail, and a cancel request when the server takes no data). + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + guard driver.supportsCopyFromStandardInput else { throw SQLFileParseError.copyFromStandardInput(line: line) } + let driver = self.driver + try await importedData(writes: true, statement: statement, interrupt: { try? driver.cancelQuery() }) { + _ = try await driver.copyFromStandardInput(statement: statement, rows: rows, context: nil) + } + } + // T-SQL scopes a variable and a TRY...CATCH to its batch, so a batch goes whole where the driver can send it so. private func send(_ statement: String, line: Int, grammar: SQLLexicalGrammar) async throws { guard grammar.contains(.batchSeparatorLines) else { @@ -228,17 +270,15 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed("Could not resolve SQL dialect for \(targetTable)") } - let row = mappedRow(values) - guard !row.columns.isEmpty else { - guard values.isEmpty else { - throw PluginImportError.importFailed(Self.unmappedRowMessage) - } - return - } + let row = try await writableRows([values])[0] guard !rowWriting.mode.matchesExistingRows else { try await writeMatching([row], generator: rowGenerator) return } + guard !row.columns.isEmpty else { + try await insertDefaultRows(1, generator: rowGenerator) + return + } guard let statement = rowGenerator.insertStatement(columns: row.columns, values: row.values) else { throw PluginImportError.importFailed( String(format: String(localized: "Could not build an INSERT for %@"), targetTable) @@ -246,7 +286,9 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } try await importedData(rows: 1, writes: true) { - _ = try await self.driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + _ = try await self.driver.executeParameterized( + query: statement.sql, parameters: statement.parameters, context: statement.context + ) self.tally.withLock { $0.inserted += 1 } } } @@ -259,11 +301,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed("Could not resolve SQL dialect for row import") } - let mapped = rows.map(mappedRow) - if zip(rows, mapped).contains(where: { !$0.0.isEmpty && $0.1.columns.isEmpty }) { - throw PluginImportError.importFailed(Self.unmappedRowMessage) - } - let writable = mapped.filter { !$0.columns.isEmpty } + let writable = try await writableRows(rows) if rowWriting.mode.matchesExistingRows { try await writeMatching(writable, generator: rowGenerator) @@ -277,9 +315,17 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { let values: [PluginCellValue] } - private func mappedRow(_ values: [String: PluginCellValue]) -> MappedRow { - let (columns, mappedValues) = mappedColumnsAndValues(values) - return MappedRow(columns: columns, values: mappedValues) + // A row with no field, such as JSON's `{}` for an all-NULL row, is a row of defaults; a row whose fields all + // missed the mapping is an error. + private func writableRows(_ rows: [[String: PluginCellValue]]) async throws -> [MappedRow] { + let target = try await targetColumnValues() + return try rows.map { values in + let (columns, mappedValues) = mappedColumnsAndValues(values) + if columns.isEmpty, !values.isEmpty { + throw PluginImportError.importFailed(Self.unmappedRowMessage) + } + return MappedRow(columns: columns, values: try target.values(mappedValues, columns: columns)) + } } private func insertGrouped(_ rows: [MappedRow], generator: SQLStatementGenerator) async throws { @@ -294,7 +340,30 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } index = next - try await insertGroup(groupValues, columns: columns, generator: generator) + if columns.isEmpty { + try await insertDefaultRows(groupValues.count, generator: generator) + } else { + try await insertGroup(groupValues, columns: columns, generator: generator) + } + } + } + + private func insertDefaultRows(_ count: Int, generator: SQLStatementGenerator) async throws { + let firstColumn = try await targetColumnValues().firstWritableColumn + guard let statement = generator.allDefaultsInsertStatement(firstColumn: firstColumn) else { + throw PluginImportError.importFailed(String( + format: String(localized: "This row has no values, and %@ has no way to insert a row of only defaults."), + databaseType.rawValue + )) + } + for _ in 0.. Void)? = nil, operation: () async throws -> T ) async throws -> T { if let importControl { return try await importControl.data( - rows: rows, writes: writes, deletes: deletes, statement: statement, operation: operation + rows: rows, writes: writes, deletes: deletes, statement: statement, interrupt: interrupt, + operation: operation ) } return try await operation() diff --git a/apple/TablePro/Core/Plugins/ImportSinkRun.swift b/apple/TablePro/Core/Plugins/ImportSinkRun.swift index bbad43e321..d0f50b8eaa 100644 --- a/apple/TablePro/Core/Plugins/ImportSinkRun.swift +++ b/apple/TablePro/Core/Plugins/ImportSinkRun.swift @@ -15,8 +15,12 @@ internal final class ImportSinkRun: @unchecked Sendable { private let targetTable: String? private let targetSchema: String? private let joinsTransaction: Bool + private let commitMode: ConnectionCommitMode private let lock = NSLock() private var ownsTransaction = false + private var leavesTransactionOpen = false + private var disabledForeignKeys = false + private var writesOutsideTransaction: Bool? private var finalized = false private var acknowledgedEnding: ImportTransactionOutcome = .untracked private var fileCommittedEarly = false @@ -38,7 +42,8 @@ internal final class ImportSinkRun: @unchecked Sendable { fileIndex: Int?, targetTable: String?, targetSchema: String?, - joinsTransaction: Bool + joinsTransaction: Bool, + commitMode: ConnectionCommitMode = .automatic ) { self.run = run self.driver = driver @@ -47,10 +52,15 @@ internal final class ImportSinkRun: @unchecked Sendable { self.targetTable = targetTable self.targetSchema = targetSchema self.joinsTransaction = joinsTransaction + self.commitMode = commitMode } internal func prepare() async throws { try await admitted(.begin) { + if self.commitMode == .manual { + let state = await self.driver.heldSessionTransactionState() + try ManualCommit.refuseUnheldWrite(supportsTransactions: self.driver.supportsTransactions, sessionState: state) + } if !self.run.snapshot.prepared { let state = await self.sessionState() self.run.record { @@ -90,9 +100,11 @@ internal final class ImportSinkRun: @unchecked Sendable { writes: Bool = false, deletes: Bool = false, statement: String? = nil, + interrupt: (@Sendable () -> Void)? = nil, operation: () async throws -> T ) async throws -> T { try await admitted(.data) { + if writes, self.commitMode == .manual { try self.refuseWriteOutsideTransaction() } if let statement { self.noteStatement(statement) } if writes, let index = self.fileIndex { self.run.record { $0.effects[index]?.admittedWrite = true } @@ -100,7 +112,12 @@ internal final class ImportSinkRun: @unchecked Sendable { let before = self.run.snapshot.currentState let hadAuthority = self.lock.withLock { self.ownsTransaction } do { - let value = try await operation() + let value: T + if let interrupt { + value = try await self.run.interruptible(by: interrupt, operation) + } else { + value = try await operation() + } if let index = self.fileIndex { self.run.record { $0.effects[index]?.submittedRows += rows @@ -122,6 +139,16 @@ internal final class ImportSinkRun: @unchecked Sendable { } } + // Decided at the first write, before a GO line resets the holder: outside a transaction each statement commits. + private func refuseWriteOutsideTransaction() throws { + let holder = run.snapshot.holder + let outside = lock.withLock { () -> Bool in + if writesOutsideTransaction == nil { writesOutsideTransaction = holder == .none } + return writesOutsideTransaction == true + } + if outside { throw ManualCommitUnwrappedImportRefusal() } + } + private func noteStatement(_ statement: String) { let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution if grammar.contains(.batchSeparatorLines) { @@ -171,17 +198,34 @@ internal final class ImportSinkRun: @unchecked Sendable { try await admitted(.data) {} } + // SQLite ignores the re-enable inside a held transaction, and a Roll Back undoes PostgreSQL's. + internal func noteForeignKeysDisabled() { + lock.withLock { disabledForeignKeys = true } + } + internal func begin() async throws { guard !joinsTransaction else { return } try await admitted(.begin) { let state = await self.driver.heldSessionTransactionState() - let owner = WriteTransactionOwner.resolve(supportsTransactions: self.driver.supportsTransactions, sessionState: state) - guard owner.opensTransaction else { + if self.commitMode == .manual { + guard !self.lock.withLock({ self.disabledForeignKeys }) else { throw ManualCommitForeignKeyRefusal() } + try ManualCommit.refuseUnheldWrite(supportsTransactions: self.driver.supportsTransactions, sessionState: state) + } + let owner = WriteTransactionOwner.resolve( + supportsTransactions: self.driver.supportsTransactions, + sessionState: state, + commitMode: self.commitMode + ) + guard owner.sendsBegin else { self.run.record { $0.currentState = ImportSessionState(state) } return } try await self.driver.beginTransaction(mode: .readWrite) - self.lock.withLock { self.ownsTransaction = true } + if owner == .held { ManualCommit.turnHold?.mark(true) } + self.lock.withLock { + self.ownsTransaction = true + self.leavesTransactionOpen = !owner.commitsAtEnd + } self.run.record { $0.holder = .owned; $0.ending = nil } await self.observe() } @@ -203,6 +247,13 @@ internal final class ImportSinkRun: @unchecked Sendable { markFinalized() return } + // Manual Commit: the rows stay pending in a transaction that is now the user's, as a joined one is. + if lock.withLock({ leavesTransactionOpen }) { + lock.withLock { ownsTransaction = false } + run.record { $0.holder = .joinedUser; $0.currentState = state } + markFinalized() + return + } do { try await driver.commitTransaction() await recordEnding(.committed) @@ -243,6 +294,7 @@ internal final class ImportSinkRun: @unchecked Sendable { } do { try await driver.rollbackTransaction() + ManualCommit.turnHold?.mark(false) await recordEnding(.rolledBack) lock.withLock { acknowledgedEnding = .rolledBack } markFinalized() diff --git a/apple/TablePro/Core/Plugins/ImportTargetValues.swift b/apple/TablePro/Core/Plugins/ImportTargetValues.swift new file mode 100644 index 0000000000..fdf052f14c --- /dev/null +++ b/apple/TablePro/Core/Plugins/ImportTargetValues.swift @@ -0,0 +1,100 @@ +// +// ImportTargetValues.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// How a format's export writes bytes as text, so its import can read them back. +enum ImportBinaryText: Sendable, Equatable { + case prefixedHex + case base64 + + init?(formatId: String) { + switch formatId { + case "csv", "xlsx": self = .prefixedHex + case "json": self = .base64 + default: return nil + } + } +} + +// A file holds text; the target column's type decides the value it stands for. +struct ImportTargetValues: Sendable { + let typeNames: [String: String] + // A row of defaults on Oracle names one column, and a virtual column refuses even DEFAULT. + let firstWritableColumn: String? + private let binaryColumns: Set + private let numericBooleanColumns: Set + private let binaryText: ImportBinaryText? + + init( + columns: [(name: String, typeName: String, isGenerated: Bool)], + databaseType: DatabaseType, + binaryText: ImportBinaryText? + ) { + var typeNames: [String: String] = [:] + for column in columns where typeNames[column.name] == nil { + typeNames[column.name] = column.typeName + } + let classifier = ColumnTypeClassifier(family: SQLTypeFamily.of(databaseType)) + let types = typeNames.mapValues { classifier.classify(rawTypeName: $0) } + // MySQL strict mode refuses `true` for an integer flag, SQL Server refuses `yes` for a bit, and SQLite keeps either as text. + let writesNumericBooleans = (try? resolveSQLDialect(for: databaseType))?.booleanLiteralStyle == .numeric + self.typeNames = typeNames + self.firstWritableColumn = columns.first { !$0.isGenerated }?.name + self.binaryColumns = Set(types.filter { $0.value.isBlobType }.keys) + self.numericBooleanColumns = writesNumericBooleans ? Set(types.filter { $0.value.isBooleanType }.keys) : [] + self.binaryText = binaryText + } + + func values(_ values: [PluginCellValue], columns: [String]) throws -> [PluginCellValue] { + try zip(columns, values).map { column, value in + guard case .text(let text) = value else { return value } + if binaryColumns.contains(column) { + return try binaryValue(text, column: column) + } + if numericBooleanColumns.contains(column), let digit = Self.booleanDigit(text) { + return .text(digit) + } + return value + } + } + + // Text not in the form the export writes is bound as text, as it always was: PostgreSQL's own JSON writes bytea as + // `\x...`, which the server reads from text. + private func binaryValue(_ text: String, column: String) throws -> PluginCellValue { + switch binaryText { + case .prefixedHex: + guard BlobFormattingService.hasHexPrefix(text) else { return .text(text) } + guard let bytes = BlobFormattingService.shared.bytes(fromPrefixedHex: text) else { + throw PluginImportError.importFailed(String( + format: String(localized: "%@ is a binary column, and this value is not 0x followed by hex digits."), + column + )) + } + return .bytes(bytes) + case .base64: + return Data(base64Encoded: text).map { .bytes($0) } ?? .text(text) + case nil: + return .text(text) + } + } + + // CSV reads t, f, y and n as booleans too. + static func booleanDigit(_ text: String) -> String? { + let word = text.trimmingCharacters(in: .whitespaces) + switch PluginSQLLiteral.booleanSynonym(for: word) { + case .isTrue: return "1" + case .isFalse: return "0" + case nil: break + @unknown default: break + } + switch word.lowercased() { + case "t", "y": return "1" + case "f", "n": return "0" + default: return nil + } + } +} diff --git a/apple/TablePro/Core/Plugins/ImportTypeMapper.swift b/apple/TablePro/Core/Plugins/ImportTypeMapper.swift index 9f88679c97..c9bc57bd03 100644 --- a/apple/TablePro/Core/Plugins/ImportTypeMapper.swift +++ b/apple/TablePro/Core/Plugins/ImportTypeMapper.swift @@ -55,11 +55,13 @@ enum ImportTypeMapper { } } + // SQLite stores a boolean as the integer 1 or 0. `BOOLEAN` has NUMERIC affinity and tells the import and the grid + // that the column is one. private static func sqliteType(_ type: PluginImportFieldType) -> String { switch type { case .integer: return "INTEGER" case .real: return "REAL" - case .boolean: return "INTEGER" + case .boolean: return "BOOLEAN" case .json, .text: return "TEXT" @unknown default: return "TEXT" } diff --git a/apple/TablePro/Core/Plugins/PluginCodeSignatureVerifier.swift b/apple/TablePro/Core/Plugins/PluginCodeSignatureVerifier.swift index a66fef198f..9aee96219d 100644 --- a/apple/TablePro/Core/Plugins/PluginCodeSignatureVerifier.swift +++ b/apple/TablePro/Core/Plugins/PluginCodeSignatureVerifier.swift @@ -1,8 +1,3 @@ -// -// PluginCodeSignatureVerifier.swift -// TablePro -// - import Foundation import os import Security @@ -19,8 +14,11 @@ enum PluginCodeSignatureVerifier { return teamId }() - /// The app keeps `disable-library-validation` for SQLite and DuckDB extensions, so dyld checks - /// nothing and this is the whole load decision for a plugin bundle. + static let validationFlags = kSecCSCheckAllArchitectures | kSecCSCheckNestedCode | kSecCSStrictValidate + | kSecCSRestrictSymlinks + + // The app keeps `disable-library-validation` for SQLite and DuckDB extensions, so dyld checks + // nothing and this is the whole load decision for a plugin bundle. static func verify(bundle: Bundle) throws { #if DEBUG if ProcessInfo.processInfo.environment["TABLEPRO_ALLOW_UNSIGNED_PLUGINS"] == "1" { @@ -30,37 +28,55 @@ enum PluginCodeSignatureVerifier { return } #endif + try verify(bundle: bundle, signingTeam: resolvedSigningTeamId) + } + + static func verify(bundle: Bundle, signingTeam: String) throws { + let requirement = try firstPartyRequirement(signingTeam: signingTeam) + var staticCode: SecStaticCode? let createStatus = SecStaticCodeCreateWithPath( bundle.bundleURL as CFURL, SecCSFlags(), &staticCode ) - guard createStatus == errSecSuccess, let code = staticCode else { throw PluginError.signatureInvalid(detail: describeOSStatus(createStatus)) } - let flags = SecCSFlags(rawValue: Self.validationFlags) - let status = SecStaticCodeCheckValidity(code, flags, requirement(Self.firstPartyRequirement)) + let flags = SecCSFlags(rawValue: validationFlags) + let status = SecStaticCodeCheckValidity(code, flags, requirement) guard status == errSecSuccess else { throw PluginError.signatureInvalid(detail: describeOSStatus(status)) } } - static let validationFlags = kSecCSCheckAllArchitectures | kSecCSCheckNestedCode | kSecCSStrictValidate - | kSecCSRestrictSymlinks - - private static var firstPartyRequirement: String { - "anchor apple generic and certificate leaf[subject.OU] = \"\(resolvedSigningTeamId)\"" + static func firstPartyRequirement(signingTeam team: String) throws -> SecRequirement { + // The team goes into the requirement text, where a quote could append an `or` clause. + guard isTeamIdentifier(team) else { + throw PluginError.signatureInvalid(detail: "TablePro's signing Team ID is not valid") + } + return try compiledRequirement("anchor apple generic and certificate leaf[subject.OU] = \"\(team)\"") } - private static func requirement(_ text: String) -> SecRequirement? { + static func compiledRequirement(_ text: String) throws -> SecRequirement { var requirement: SecRequirement? - SecRequirementCreateWithString(text as CFString, SecCSFlags(), &requirement) + let status = SecRequirementCreateWithString(text as CFString, SecCSFlags(), &requirement) + // A nil requirement makes SecStaticCodeCheckValidity accept any valid signature. + guard status == errSecSuccess, let requirement else { + throw PluginError.signatureInvalid( + detail: "TablePro's signing requirement could not be built (OSStatus \(status))" + ) + } return requirement } + private static func isTeamIdentifier(_ team: String) -> Bool { + team.unicodeScalars.count == 10 && team.unicodeScalars.allSatisfy { scalar in + ("A"..."Z").contains(scalar) || ("0"..."9").contains(scalar) + } + } + private static func teamIdFromBundleSignature() -> String? { var staticCode: SecStaticCode? let createStatus = SecStaticCodeCreateWithPath( diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+CellValue.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CellValue.swift new file mode 100644 index 0000000000..b8361d0ed5 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CellValue.swift @@ -0,0 +1,52 @@ +import Foundation +import os +import TableProNumberFormatting +import TableProPluginKit +import TableProUtilities + +extension PluginDriverAdapter { + private static let iso8601Formatter = OSAllocatedUnfairLock( + uncheckedState: { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter + }() + ) + + static func cellValue(for parameter: Any?) -> PluginCellValue { + guard let parameter else { return .null } + if let typed = parameter as? TypedParameterValue { return .text(typed.text) } + if let data = parameter as? Data { return .bytes(data) } + if let f = parameter as? Float { + guard f.isFinite else { return .null } + return .text(NumberText.text(for: f)) + } + if let f = parameter as? any BinaryFloatingPoint { + let d = Double(f) + guard d.isFinite else { return .null } + return .text(NumberText.text(for: d)) + } + return .text(stringValue(for: parameter)) + } + + private static func stringValue(for parameter: Any) -> String { + switch parameter { + case let s as String: + return s + case let b as Bool: + return b ? "1" : "0" + case let i as any BinaryInteger: + return String(i) + case let f as any BinaryFloatingPoint: + return NumberText.text(for: Double(f)) + case let d as Date: + return Self.iso8601Formatter.withLockUnchecked { $0.string(from: d) } + case let data as Data: + return data.hexEncoded + case let uuid as UUID: + return uuid.uuidString + default: + return String(describing: parameter) + } + } +} diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+CopyIn.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CopyIn.swift new file mode 100644 index 0000000000..df153629f4 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CopyIn.swift @@ -0,0 +1,27 @@ +// +// PluginDriverAdapter+CopyIn.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +extension PluginDriverAdapter { + var supportsCopyFromStandardInput: Bool { + schemaPluginDriver.capabilities.contains(.copyFromStandardInput) + } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult { + try StatementTextValidator.validate(statement) + // The rows come from the editor's text or an imported file, never from an external caller. + guard supportsCopyFromStandardInput, context?.origin != .external else { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + let sent = context.map { statementContext(statement, approvedAs: $0) } ?? statementContext(statement) + return mapQueryResult(try await schemaPluginDriver.copyFromStandardInput(statement: statement, rows: rows, context: sent)) + } +} diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+Export.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+Export.swift index 4e4313d3dd..15eb32cba2 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter+Export.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+Export.swift @@ -2,17 +2,27 @@ import Foundation import TableProPluginKit extension PluginDriverAdapter { - func streamRows(query: String) -> AsyncThrowingStream { + func streamRows(query: String, safeModeIsReadOnly: Bool = false) -> AsyncThrowingStream { if let error = StatementTextValidator.error(for: query) { return AsyncThrowingStream { $0.finish(throwing: error) } } - return schemaPluginDriver.streamRows(query: query, context: statementContext(query)) + return schemaPluginDriver.streamRows(query: query, context: streamContext(query, safeModeIsReadOnly)) } - func streamExportRows(query: String) -> AsyncThrowingStream { + func streamExportRows( + query: String, + safeModeIsReadOnly: Bool = false + ) -> AsyncThrowingStream { if let error = StatementTextValidator.error(for: query) { return AsyncThrowingStream { $0.finish(throwing: error) } } - return schemaPluginDriver.streamExportRows(query: query, context: statementContext(query)) + return schemaPluginDriver.streamExportRows(query: query, context: streamContext(query, safeModeIsReadOnly)) + } + + private func streamContext(_ query: String, _ safeModeIsReadOnly: Bool) -> PluginStatementContext { + var context = statementContext(query) + context.enforcesReadOnly = readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) != nil + context.readFunctionCalls(in: query, databaseType: connection.type) + return context } } diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+ReadOnlyMode.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+ReadOnlyMode.swift new file mode 100644 index 0000000000..dc8b03615b --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+ReadOnlyMode.swift @@ -0,0 +1,26 @@ +// +// PluginDriverAdapter+ReadOnlyMode.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +extension PluginDriverAdapter { + var enforcesReadOnlyMode: Bool { + schemaPluginDriver.capabilities.contains(.enforcesReadOnlyMode) + } + + func executeBoundedQuery(query: String, rowCap: Int, context: PluginStatementContext) async throws -> QueryResult? { + try StatementTextValidator.validate(query) + try refuseUnenforcedPolicy(context) + guard let pluginResult = try await schemaPluginDriver.executeBoundedQuery( + query: query, + rowCap: rowCap, + context: statementContext(query, approvedAs: context) + ) else { + return nil + } + return mapQueryResult(pluginResult) + } +} diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift index 8f28aa10e1..5dcb69f862 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift @@ -7,6 +7,10 @@ import Foundation import TableProPluginKit internal extension PluginDriverAdapter { + func schemaChangeToken() async throws -> String? { + try await schemaPluginDriver.schemaChangeToken() + } + func reviewSchemaChange( table: String, schema: String?, diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+ServerObjects.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+ServerObjects.swift new file mode 100644 index 0000000000..1994fe8817 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+ServerObjects.swift @@ -0,0 +1,36 @@ +// +// PluginDriverAdapter+ServerObjects.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +extension PluginDriverAdapter { + private static let serverObjectsLogger = Logger(subsystem: "com.TablePro", category: "PluginDriverAdapter") + + var serverObjectKinds: [PluginServerObjectKind] { schemaPluginDriver.serverObjectKinds } + + // A container kind is stamped with the schema it was read from, as routines are; a database kind + // names no schema and reads with none. + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [ServerObjectInfo] { + let isDatabaseScoped = schemaPluginDriver.serverObjectKinds.first { $0.id == kindId }?.scope == .database + let resolvedSchema = isDatabaseScoped ? nil : (schema ?? schemaPluginDriver.currentSchema) + do { + return try await schemaPluginDriver.fetchServerObjects(ofKind: kindId, schema: resolvedSchema) + .filter { $0.kindId == kindId } + .map { ServerObjectInfo($0).adoptingSchema(resolvedSchema) } + .sorted { ($0.schema ?? "", $0.name) < ($1.schema ?? "", $1.name) } + } catch { + Self.serverObjectsLogger.warning( + "fetchServerObjects failed kind=\(kindId, privacy: .public) error=\(error.publicLogShape, privacy: .public)" + ) + throw error + } + } + + func fetchServerObject(_ object: ServerObjectInfo) async throws -> ServerObjectInfo { + ServerObjectInfo(try await schemaPluginDriver.fetchServerObject(object.pluginObject)) + } +} diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 587b6fe793..a2355bc845 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -16,6 +16,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor private struct ConnectAttempt { let generation: UInt64 let gate: ConnectionSingleResumeGate + var deadline: ContinuousClock.Instant var task: Task? var isAbandoned = false } @@ -38,7 +39,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let effectiveQueryTimeoutSeconds: Int private let pluginDriver: any PluginDatabaseDriver // The Vault lease this driver signed in with. Its database user goes away when it ends. - private let credentialLease: VaultLease? + private let credentialLease: VaultLeaseHold? private let connectionDeadline: ConnectionDeadline private let timeoutEndpoint: ConnectionTimeoutEndpoint private let requiresHostDeadline: Bool @@ -96,50 +97,6 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor private static let logger = Logger(subsystem: "com.TablePro", category: "PluginDriverAdapter") - private static let iso8601Formatter = OSAllocatedUnfairLock( - uncheckedState: { - let formatter = ISO8601DateFormatter() - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter - }() - ) - - static func cellValue(for parameter: Any?) -> PluginCellValue { - guard let parameter else { return .null } - if let data = parameter as? Data { return .bytes(data) } - if let f = parameter as? Float { - guard f.isFinite else { return .null } - return .text(NumberText.text(for: f)) - } - if let f = parameter as? any BinaryFloatingPoint { - let d = Double(f) - guard d.isFinite else { return .null } - return .text(NumberText.text(for: d)) - } - return .text(stringValue(for: parameter)) - } - - private static func stringValue(for parameter: Any) -> String { - switch parameter { - case let s as String: - return s - case let b as Bool: - return b ? "1" : "0" - case let i as any BinaryInteger: - return String(i) - case let f as any BinaryFloatingPoint: - return NumberText.text(for: Double(f)) - case let d as Date: - return Self.iso8601Formatter.withLockUnchecked { $0.string(from: d) } - case let data as Data: - return data.hexEncoded - case let uuid as UUID: - return uuid.uuidString - default: - return String(describing: parameter) - } - } - init( connection: DatabaseConnection, pluginDriver: any PluginDatabaseDriver, @@ -147,7 +104,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor timeoutEndpoint: ConnectionTimeoutEndpoint? = nil, effectiveQueryTimeoutSeconds: Int? = nil, requiresHostDeadline: Bool? = nil, - credentialLease: VaultLease? = nil + credentialLease: VaultLeaseHold? = nil ) { self.connection = connection self.pluginDriver = pluginDriver @@ -201,7 +158,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let generation = state.nextConnectGeneration state.status = .connecting state.connectionWarning = nil - state.connectAttempt = ConnectAttempt(generation: generation, gate: gate) + state.connectAttempt = ConnectAttempt(generation: generation, gate: gate, deadline: connectionDeadline.instant) return generation } guard let generation else { @@ -215,12 +172,17 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let operation = Task { [self] in do { - try await pluginDriver.connect(reportingStage: guardedReport) + try await connectSigningIn(generation: generation, report: guardedReport) + credentialLease?.signInAccepted() let adopted = gate.resume(with: .success(())) finishConnect(generation: generation, succeeded: true, adopted: adopted, error: nil) } catch { // Every sign-in fails here, pooled and per-database ones included, so a rejected reused password goes. PasswordCommandCache.shared.forget(connectionId: connection.id) + // A user refused through the whole window is still on its way: the server may load it by the next try. + if !(error is VaultUserNotLoaded), ConnectionFailureClassifier.isAuthenticationFailure(error) { + credentialLease?.signInRefused(credentials: VaultLeaseHold.isRefusedSignIn(error)) + } let adopted = gate.resume(with: .failure(error)) finishConnect(generation: generation, succeeded: false, adopted: adopted, error: error) } @@ -230,10 +192,15 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let timeout: Task? if requiresHostDeadline { timeout = Task { [self] in - do { - try await ContinuousClock().sleep(until: connectionDeadline.instant) - } catch { - return + var instant = connectionDeadline.instant + while true { + do { + try await ContinuousClock().sleep(until: instant) + } catch { + return + } + guard let later = hostDeadline(of: generation), later > instant else { break } + instant = later } abandonConnect( generation: generation, @@ -300,6 +267,39 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor state.withLock { $0.disconnectInProgress = false } } + // A database can refuse a user Vault created moments ago until it loads it, so a young lease dials again. + private func connectSigningIn(generation: UInt64, report: @escaping ConnectionStageReporter) async throws { + guard let credentialLease else { + try await pluginDriver.connect(reportingStage: report) + return + } + try await credentialLease.dial( + { try await pluginDriver.connect(reportingStage: report) }, + waiting: { delay in + // The wait is the database's, not the connect's, so the host deadline moves out by it. + moveHostDeadline(of: generation, past: delay) + report(.custom(String(localized: "Waiting for the database to accept the new Vault user"))) + }, + isCurrent: { isCurrentConnect(generation) } + ) + } + + private func moveHostDeadline(of generation: UInt64, past wait: Duration) { + let instant = ContinuousClock.now + wait + .seconds(connectionDeadline.configuredSeconds) + state.withLock { state in + guard var attempt = state.connectAttempt, attempt.generation == generation else { return } + attempt.deadline = max(attempt.deadline, instant) + state.connectAttempt = attempt + } + } + + private func hostDeadline(of generation: UInt64) -> ContinuousClock.Instant? { + state.withLock { state in + guard let attempt = state.connectAttempt, attempt.generation == generation else { return nil } + return attempt.deadline + } + } + private func attach(_ task: Task, to generation: UInt64) { let shouldCancel = state.withLock { state in guard var attempt = state.connectAttempt, attempt.generation == generation else { @@ -928,6 +928,8 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor var providesBulkColumnFetch: Bool { pluginDriver.providesBulkColumnFetch } + var fetchesAllColumnsInOneRequest: Bool { pluginDriver.fetchesAllColumnsInOneRequest } + func fetchAllColumns() async throws -> [String: [ColumnInfo]] { let pluginResult = try await pluginDriver.fetchAllColumns(schema: pluginDriver.currentSchema) return pluginResult.mapValues { $0.map(ColumnInfo.init) } @@ -1317,7 +1319,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor // MARK: - Result Mapping - private func mapQueryResult(_ pluginResult: PluginQueryResult) -> QueryResult { + func mapQueryResult(_ pluginResult: PluginQueryResult) -> QueryResult { let columnTypes = mapColumnTypes( rawTypeNames: pluginResult.columnTypeNames, classificationHints: PluginResultColumnHints.hints( @@ -1439,7 +1441,7 @@ extension PluginDriverAdapter { let pluginResult = try await pluginDriver.executeParameterized( query: query, parameters: parameters.map(Self.cellValue(for:)), - context: statementContext(query, approvedAs: context) + context: statementContext(query, approvedAs: context, parameters: parameters) ) return mapQueryResult(pluginResult) } @@ -1456,19 +1458,20 @@ extension PluginDriverAdapter { query: query, rowCap: rowCap, parameters: parameters?.map(Self.cellValue(for:)), - context: statementContext(query, approvedAs: context) + context: statementContext(query, approvedAs: context, parameters: parameters) ) return mapQueryResult(pluginResult) } - fileprivate func statementContext(_ query: String, approvedAs caller: PluginStatementContext) -> PluginStatementContext { + func statementContext(_ query: String, approvedAs caller: PluginStatementContext, parameters: [Any?]? = nil) -> PluginStatementContext { var context = PluginStatementContext.statement(query, databaseType: connection.type, approvedAs: caller) context.lexicalFeatures = sessionLexicalFeatures + context.parameterKinds = TypedParameterValue.kinds(of: parameters) ?? context.parameterKinds return context } // A driver built before `.statementPolicy` would run an external write the approval did not cover. - fileprivate func refuseUnenforcedPolicy(_ context: PluginStatementContext) throws { + func refuseUnenforcedPolicy(_ context: PluginStatementContext) throws { guard context.origin == .external, !context.readOnly, !context.allowsDestructive || !context.allowsServerCode, PluginMetadataRegistry.shared.snapshot(for: connection.type)?.capabilities.statementPolicyInDriver == true, diff --git a/apple/TablePro/Core/Plugins/PluginManager+Lifecycle.swift b/apple/TablePro/Core/Plugins/PluginManager+Lifecycle.swift index 12bfefacba..b5067794fa 100644 --- a/apple/TablePro/Core/Plugins/PluginManager+Lifecycle.swift +++ b/apple/TablePro/Core/Plugins/PluginManager+Lifecycle.swift @@ -35,4 +35,29 @@ extension PluginManager { Self.logger.info("Uninstalled plugin '\(id)'") needsRestart = true } + + // Only a bundle in the user's plugins folder: one inside the app breaks its seal, and no registry install brings + // a bundled engine back. + func canRemoveRejectedPlugin(_ plugin: RejectedPlugin) -> Bool { + let bundle = plugin.url.standardizedFileURL + guard !bundle.lastPathComponent.isEmpty, bundle.lastPathComponent != "/" else { return false } + // The folder, not the bundle, is resolved: a bundle linked in from a build folder is still the user's to remove. + let folder = bundle.deletingLastPathComponent().resolvingSymlinksInPath().path + return folder == userPluginsDir.standardizedFileURL.resolvingSymlinksInPath().path + } + + // To the Trash rather than deleted, so a mistaken removal can be put back. + func removeRejectedPlugin( + _ plugin: RejectedPlugin, + moveToTrash: (URL) throws -> Void = { try FileManager.default.trashItem(at: $0, resultingItemURL: nil) } + ) throws { + guard canRemoveRejectedPlugin(plugin) else { throw PluginError.cannotUninstallBuiltIn } + if FileManager.default.fileExists(atPath: plugin.url.path) { + do { + try moveToTrash(plugin.url) + } catch CocoaError.fileNoSuchFile {} + } + removeFromRejected(url: plugin.url) + Self.logger.info("Moved rejected plugin '\(plugin.name, privacy: .public)' to the Trash") + } } diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift index 3e10fedf55..87a4fba493 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+BigtableDefaults.swift @@ -39,7 +39,8 @@ extension PluginMetadataRegistry { exactRowCountIsFullScan: true, supportsColumnSort: false, columnsAreSampled: true, - supportsPrincipalConnectionLimit: false + supportsPrincipalConnectionLimit: false, + dataCommands: bigtableDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -71,6 +72,9 @@ extension PluginMetadataRegistry { } } +// The driver's writes, each one MutateRow on the Bigtable Data API v2. +private let bigtableDataCommands: Set = ["INSERT", "PUT", "DELETE"] + private let bigtableCompletions: [CompletionEntry] = [ CompletionEntry(label: "GET row", insertText: "GET events/user#42"), CompletionEntry(label: "SCAN prefix", insertText: "SCAN events/user#?limit=20"), diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CommandEngineDataCommands.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CommandEngineDataCommands.swift new file mode 100644 index 0000000000..65c8788f4c --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CommandEngineDataCommands.swift @@ -0,0 +1,45 @@ +// +// PluginMetadataRegistry+CommandEngineDataCommands.swift +// TablePro +// + +import Foundation + +extension PluginMetadataRegistry { + // Redis 8.10.3 `COMMAND DOCS` with its modules (search, timeseries, bf, vectorset, ReJSON). A container command is + // named `CONTAINER|SUB`. Keys only: no index, alias, rule, user, function, session or server state. + static let redisDataCommands: Set = [ + "APPEND", "DECR", "DECRBY", "DELEX", "GETDEL", "GETEX", "GETSET", "INCR", "INCRBY", "INCRBYFLOAT", "INCREX", + "MSET", "MSETEX", "MSETNX", "PSETEX", "SET", "SETEX", "SETNX", "SETRANGE", + "COPY", "DEL", "EXPIRE", "EXPIREAT", "MOVE", "PERSIST", "PEXPIRE", "PEXPIREAT", "RENAME", "RENAMENX", + "RESTORE", "SORT", "UNLINK", + "HDEL", "HEXPIRE", "HEXPIREAT", "HGETDEL", "HGETEX", "HINCRBY", "HINCRBYFLOAT", "HMSET", "HPERSIST", + "HPEXPIRE", "HPEXPIREAT", "HSET", "HSETEX", "HSETNX", + "LINSERT", "LMOVE", "LMOVEM", "LMPOP", "LPOP", "LPUSH", "LPUSHX", "LREM", "LSET", "LTRIM", "RPOP", + "RPOPLPUSH", "RPUSH", "RPUSHX", "BLMOVE", "BLMOVEM", "BLMPOP", "BLPOP", "BRPOP", "BRPOPLPUSH", + "SADD", "SDIFFSTORE", "SINTERSTORE", "SMOVE", "SPOP", "SREM", "SUNIONSTORE", + "ZADD", "ZDIFFSTORE", "ZINCRBY", "ZINTERSTORE", "ZMPOP", "ZPOPMAX", "ZPOPMIN", "ZRANGESTORE", "ZREM", + "ZREMRANGEBYLEX", "ZREMRANGEBYRANK", "ZREMRANGEBYSCORE", "ZUNIONSTORE", "BZMPOP", "BZPOPMAX", "BZPOPMIN", + "XACK", "XACKDEL", "XADD", "XAUTOCLAIM", "XCFGSET", "XCLAIM", "XDEL", "XDELEX", "XGROUP|CREATE", + "XGROUP|CREATECONSUMER", "XGROUP|DELCONSUMER", "XGROUP|DESTROY", "XGROUP|SETID", "XNACK", "XREADGROUP", + "XSETID", "XTRIM", + "BITFIELD", "BITOP", "SETBIT", "GEOADD", "GEORADIUS", "GEORADIUSBYMEMBER", "GEOSEARCHSTORE", "PFADD", + "PFMERGE", + "ARDEL", "ARDELRANGE", "ARINSERT", "ARMSET", "ARRING", "ARSEEK", "ARSET", + "JSON.ARRAPPEND", "JSON.ARRINSERT", "JSON.ARRPOP", "JSON.ARRTRIM", "JSON.CLEAR", "JSON.DEL", "JSON.FORGET", + "JSON.MERGE", "JSON.MSET", "JSON.NUMINCRBY", "JSON.NUMMULTBY", "JSON.NUMPOWBY", "JSON.SET", "JSON.STRAPPEND", + "JSON.TOGGLE", + "TS.ADD", "TS.ALTER", "TS.CREATE", "TS.DECRBY", "TS.DEL", "TS.INCRBY", "TS.MADD", + "FT.ADD", "FT.SAFEADD", "FT.DEL", "FT.SUGADD", "FT.SUGDEL", + "BF.ADD", "BF.INSERT", "BF.LOADCHUNK", "BF.MADD", "BF.RESERVE", "CF.ADD", "CF.ADDNX", "CF.COMPACT", "CF.DEL", + "CF.INSERT", "CF.INSERTNX", "CF.LOADCHUNK", "CF.RESERVE", "CMS.INCRBY", "CMS.INITBYDIM", "CMS.INITBYPROB", + "CMS.MERGE", "TDIGEST.ADD", "TDIGEST.CREATE", "TDIGEST.MERGE", "TDIGEST.RESET", "TOPK.ADD", "TOPK.INCRBY", + "TOPK.RESERVE", + "VADD", "VREM", "VSETATTR", + ] + + // etcd v3.6 API reference, in the driver's etcdctl notation. `create` and `mv` are the driver's own KV verbs. + static let etcdDataCommands: Set = [ + "put", "create", "mv", "del", "lease grant", "lease keep-alive", + ] +} diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift index 298a4c1a62..8f2bbbe41e 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift @@ -5,22 +5,29 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar import TableProSQLVocabulary import TableProUtilities -/// What the app knows about a database type before its plugin loads. -/// -/// The primary type ids here are overwritten by `buildMetadataSnapshot` the moment the plugin -/// registers, so these are the pre-load answer for those, apart from an explain list the plugin -/// leaves empty, which keeps the one here. For a variant id they are the whole -/// answer: `registerVariant` keeps the curated entry and ignores the plugin's own statics, which -/// is the only reason MariaDB, TiDB, Databend, OceanBase, Redshift, CockroachDB and PGlite can differ from -/// the plugin that drives them. +// What the app knows about a database type before its plugin loads. +// The primary type ids here are overwritten by `buildMetadataSnapshot` the moment the plugin +// registers, so these are the pre-load answer for those, apart from an explain list the plugin +// leaves empty, which keeps the one here. For a variant id they are the whole +// answer: `registerVariant` keeps the curated entry and ignores the plugin's own statics, which +// is the only reason MariaDB, TiDB, Databend, OceanBase, Redshift, CockroachDB and PGlite can differ from +// the plugin that drives them. extension PluginMetadataRegistry { - /// SQLite's own extensions. libSQL and Turso name a new local file with them too, since the file - /// their driver creates is a SQLite database. + // SQLite's own extensions. libSQL and Turso name a new local file with them too, since the file + // their driver creates is a SQLite database. static let sqliteFileExtensions = ["db", "db3", "s3db", "sl3", "sqlite", "sqlite3", "sqlitedb"] + // Engines where a function a SELECT calls can change data. Read-Only refuses a call off the built-in list only when + // the live driver cannot run the statement in the engine's read-only mode. MySQL and MariaDB are listed by type + // because an OceanBase server answers as MySQL, so only the driver's own bit tells them apart. + static let queryFunctionsCanWrite = Set( + SQLReadOnlyFunctionCalls.queryFunctionsCanWrite.map(DatabaseType.init(rawValue:)) + ) + // swiftlint:disable:next function_body_length static func curatedDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { let mysqlDialect = SQLDialectDescriptor( @@ -192,8 +199,8 @@ extension PluginMetadataRegistry { ) ] - /// Cloud SQL signs IAM principals in to MySQL and PostgreSQL only, so SQL Server and the - /// MySQL and PostgreSQL forks keep the AWS-only picker. + // Cloud SQL signs IAM principals in to MySQL and PostgreSQL only, so SQL Server and the + // MySQL and PostgreSQL forks keep the AWS-only picker. let cloudIAMFields = GoogleCloudSQLAuthFields.standardWithAWS() + [AWSAuthFields.rdsEndpointField()] let postgresqlAuthFields = KerberosAuthFields.standardWithAWSAndGoogleCloud() + [ @@ -201,8 +208,8 @@ extension PluginMetadataRegistry { AWSAuthFields.rdsEndpointField() ] - /// The MySQL plugin's types only. PostgreSQL shares the IAM fields and must not pick this up: - /// its driver holds no releasable resource, so the setting would do nothing. + // The MySQL plugin's types only. PostgreSQL shares the IAM fields and must not pick this up: + // its driver holds no releasable resource, so the setting would do nothing. let mysqlIdleReleaseField = ConnectionField( id: "mysqlIdleReleaseMinutes", label: String(localized: "Release the Server Connection After (minutes, 0 to keep it)"), @@ -409,6 +416,7 @@ extension PluginMetadataRegistry { supportsDatabaseTriggerBrowse: true, supportsUserDefinedTypeBrowse: true, defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true, readOnlyEnforcement: .sessionDefault ), schema: PluginMetadataSnapshot.SchemaInfo( @@ -470,7 +478,8 @@ extension PluginMetadataRegistry { supportsDropSchema: true, supportsCreateSchema: true, supportsSchemaOwner: true, - defaultSSLMode: .preferred + defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", @@ -548,6 +557,7 @@ extension PluginMetadataRegistry { supportsCheckConstraintEditing: false, supportsGeneratedColumns: true, defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true, readOnlyEnforcement: .sessionDefault ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift index d8106a1092..6e29479517 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift @@ -8,7 +8,6 @@ import TableProPluginKit import TableProUtilities extension PluginMetadataRegistry { - /// The curated snapshot for DuckDB, alongside its connection fields in the sibling file. func duckdbPluginDefaults( dialect: SQLDialectDescriptor, columnTypes: [String: [String]] @@ -47,6 +46,7 @@ extension PluginMetadataRegistry { // httpfs checks a server against the system CA bundle when no CA is given, so a URL asking // for TLS with no mode means a verified connection, never Required's unchecked one. verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, supportsConnectionPooling: false, localFilePathField: .additionalField("duckdbFilePath"), serverEndpointFields: ServerEndpointFields(hostFieldId: "duckdbHost", portFieldId: "duckdbPort"), @@ -64,16 +64,15 @@ extension PluginMetadataRegistry { systemDatabaseNames: ["system", "temp"], systemSchemaNames: [], fileExtensions: ["duckdb", "ddb", "parquet", "csv", "tsv", "json", "ndjson"], - /// Only DuckDB's own storage. The data formats above are recognised by name - /// alone, because `duckdb_open` picks their reader from the extension and - /// refuses a Parquet file called anything else. - /// - /// `DUCK` sits behind the header's eight-byte checksum, followed by the storage - /// version as a little-endian `uint64`. Four bytes on their own are not enough - /// to name a format, and `SELECT 'DUCK';` spells them at exactly that offset, - /// so the version's high six bytes have to be zero as well. Storage versions - /// are still in the sixties, and a version past 65535 would cost recognition by - /// content rather than break it. + // Only DuckDB's own storage. The data formats above are recognised by name + // alone, because `duckdb_open` picks their reader from the extension and + // refuses a Parquet file called anything else. + // `DUCK` sits behind the header's eight-byte checksum, followed by the storage + // version as a little-endian `uint64`. Four bytes on their own are not enough + // to name a format, and `SELECT 'DUCK';` spells them at exactly that offset, + // so the version's high six bytes have to be zero as well. Storage versions + // are still in the sixties, and a version past 65535 would cost recognition by + // content rather than break it. fileSignatures: [.magic("DUCK", at: 8).andZeroes(at: 14, count: 6)], databaseGroupingStrategy: .bySchema, structureColumnFields: [.name, .type, .nullable, .defaultValue, .autoIncrement, .comment] diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+ElasticsearchDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+ElasticsearchDefaults.swift index ff8979dbe2..8335a590d7 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+ElasticsearchDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+ElasticsearchDefaults.swift @@ -30,7 +30,8 @@ extension PluginMetadataRegistry { supportsQueryProgress: false, requiresReconnectForDatabaseSwitch: false, supportsDropDatabase: false, - supportsOpportunisticTLS: false + supportsOpportunisticTLS: false, + dataCommands: elasticsearchDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -76,6 +77,18 @@ private let elasticsearchCompletions: [CompletionEntry] = [ CompletionEntry(label: "aggs", insertText: "\"aggs\": {\n \"name\": {\n \"terms\": { \"field\": \"field\" }\n }\n}"), ] +// The Document APIs of the REST API spec, 8.19: index, create, update, delete, bulk, update and delete by query. +// `_reindex` is left out: it can create its destination index and make the cluster fetch from another host. +private let elasticsearchDataCommands: Set = [ + "PUT /{target}/_doc/{id}", "POST /{target}/_doc/{id}", "POST /{target}/_doc", + "PUT /{target}/_create/{id}", "POST /{target}/_create/{id}", + "POST /{target}/_update/{id}", + "DELETE /{target}/_doc/{id}", + "POST /_bulk", "PUT /_bulk", "POST /{target}/_bulk", "PUT /{target}/_bulk", + "POST /{target}/_update_by_query", + "POST /{target}/_delete_by_query", +] + private let elasticsearchColumnTypes: [String: [String]] = [ "Text": ["text", "keyword", "match_only_text", "search_as_you_type"], "Numeric": ["long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long"], diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+FirebaseDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+FirebaseDefaults.swift index 7b1ef747a5..aac3f6a0e0 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+FirebaseDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+FirebaseDefaults.swift @@ -75,7 +75,8 @@ extension PluginMetadataRegistry { exactRowCountIsFullScan: realtimeDatabase, supportsMatchAnyFilters: !realtimeDatabase, columnsAreSampled: true, - supportsPrincipalConnectionLimit: false + supportsPrincipalConnectionLimit: false, + dataCommands: realtimeDatabase ? realtimeDatabaseDataCommands : firestoreDataCommands ) } @@ -104,6 +105,12 @@ extension PluginMetadataRegistry { } } +// Firestore REST v1 as the driver sends it: createDocument on a collection, patch and delete on a document. +private let firestoreDataCommands: Set = ["POST /{collection}", "PATCH /{document}", "DELETE /{document}"] + +// Realtime Database REST reference: PUT, PATCH, POST and DELETE on a node. +private let realtimeDatabaseDataCommands: Set = ["PUT /{path}", "PATCH /{path}", "POST /{path}", "DELETE /{path}"] + private let firebaseCompletions: [CompletionEntry] = [ CompletionEntry(label: "GET collection", insertText: "GET users?pageSize=20"), CompletionEntry(label: "GET document", insertText: "GET users/alice"), diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift index 16e6804ba0..4bbd5fd4e0 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift @@ -41,7 +41,8 @@ extension PluginMetadataRegistry { defaultSSLMode: .verifyIdentity, supportsOpportunisticTLS: false, tlsImpliedPorts: [443], - verifiesServerWithSystemTrust: true + verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift index 2905785999..57d569afc9 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift @@ -7,13 +7,11 @@ import Foundation import TableProPluginKit extension PluginMetadataRegistry { - /// The curated snapshot for Kafka. - /// - /// A registry-only driver is not installed until someone connects with it, so without an - /// entry here the type is absent from the New Connection picker and there is no way to - /// create the connection that would trigger the install. The connection fields are - /// duplicated from `KafkaConnectionField` for the same reason: the plugin that owns them - /// is not loaded yet. + // A registry-only driver is not installed until someone connects with it, so without an + // entry here the type is absent from the New Connection picker and there is no way to + // create the connection that would trigger the install. The connection fields are + // duplicated from `KafkaConnectionField` for the same reason: the plugin that owns them + // is not loaded yet. func kafkaPluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { [ ("Kafka", PluginMetadataSnapshot( @@ -43,9 +41,11 @@ extension PluginMetadataRegistry { defaultSSLMode: .disabled, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, supportsColumnSort: false, supportsRawFilter: false, - statementPolicyInDriver: true + statementPolicyInDriver: true, + dataCommands: kafkaDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -87,13 +87,16 @@ private let kafkaCompletions: [CompletionEntry] = [ CompletionEntry(label: "DESCRIBE TOPIC", insertText: "DESCRIBE TOPIC \"topic\"") ] +// KafkaQL (`TableProKafkaCore/KafkaQL.swift`): CONSUME, SHOW and DESCRIBE read, DROP TOPIC drops the topic. +private let kafkaDataCommands: Set = ["PRODUCE"] + private let kafkaColumnTypes: [String: [String]] = [ "Message": ["TEXT", "BLOB", "JSON"], "Coordinates": ["INTEGER", "BIGINT", "TIMESTAMP"] ] -/// Kept identical to `KafkaConnectionField.fields()` in the plugin. The two cannot share code: -/// this list has to exist before the bundle that owns it is downloaded. +// Kept identical to `KafkaConnectionField.fields()` in the plugin. The two cannot share code: +// this list has to exist before the bundle that owns it is downloaded. private func kafkaConnectionFields() -> [ConnectionField] { [ ConnectionField( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift index c6543201af..1d58c1df50 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MemcachedDefaults.swift @@ -28,7 +28,8 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, supportsColumnSort: false, supportsRawFilter: false, - readOnlyEnforcement: .clientGuard + readOnlyEnforcement: .clientGuard, + dataCommands: memcachedDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -59,6 +60,11 @@ extension PluginMetadataRegistry { } } +// memcached `doc/protocol.txt`, the text protocol commands `MemcachedCommandParser` reads. +private let memcachedDataCommands: Set = [ + "set", "add", "replace", "append", "prepend", "cas", "touch", "incr", "decr", "gat", "gats", "delete", +] + private let memcachedCompletions: [CompletionEntry] = [ "get", "gets", "gat", "set", "add", "replace", "append", "prepend", "cas", "touch", "delete", "incr", "decr", ].map { CompletionEntry(label: $0, insertText: "\($0) ") } + [ diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MongoDBDataCommands.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MongoDBDataCommands.swift new file mode 100644 index 0000000000..044700515c --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+MongoDBDataCommands.swift @@ -0,0 +1,21 @@ +// +// PluginMetadataRegistry+MongoDBDataCommands.swift +// TablePro +// + +import Foundation + +extension PluginMetadataRegistry { + // mongosh CRUD methods the plugin's prelude defines, and the write commands of MongoDB 4.0 to 8.x + // (https://www.mongodb.com/docs/manual/reference/command/, Query and Write Operation Commands), named as + // `QueryClassifier.mongoDataCommands` names them. + static let mongoDBDataCommands: Set = [ + "db.collection.insertOne", "db.collection.insertMany", "db.collection.insert", + "db.collection.updateOne", "db.collection.updateMany", "db.collection.update", + "db.collection.replaceOne", "db.collection.save", + "db.collection.deleteOne", "db.collection.deleteMany", "db.collection.remove", + "db.collection.findOneAndUpdate", "db.collection.findOneAndReplace", "db.collection.findOneAndDelete", + "db.collection.bulkWrite", + "insert", "update", "delete", "findAndModify", "findandmodify" + ] +} diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+Neo4jDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+Neo4jDefaults.swift index 29e51549c7..b28024673e 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+Neo4jDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+Neo4jDefaults.swift @@ -39,7 +39,8 @@ extension PluginMetadataRegistry { supportsDropColumn: false, supportsAddIndex: false, supportsDropIndex: false, - supportsModifyPrimaryKey: false + supportsModifyPrimaryKey: false, + dataCommands: neo4jDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "Labels", @@ -70,6 +71,13 @@ extension PluginMetadataRegistry { } } +// Cypher Manual clauses index (Cypher 25) and the Operations Manual's WRITE procedures, Neo4j 4.2 to 2026.x, named as +// `QueryClassifier.cypherDataCommands` names them. +private let neo4jDataCommands: Set = [ + "CREATE", "MERGE", "SET", "REMOVE", "DELETE", "DETACH DELETE", "FOREACH", "INSERT", + "db.create.setnodevectorproperty", "db.create.setrelationshipvectorproperty", "db.create.setvectorproperty" +] + private let neo4jColumnTypes: [String: [String]] = [ "Scalar": ["STRING", "INTEGER", "FLOAT", "BOOLEAN", "BYTES"], "Temporal": ["DATE", "LOCAL TIME", "ZONED TIME", "LOCAL DATETIME", "ZONED DATETIME", "DURATION"], diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index 95c510298b..162405890a 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -33,8 +33,10 @@ extension PluginMetadataRegistry { supportsModifyPrimaryKey: false, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, columnsAreSampled: true, - supportsPrincipalConnectionLimit: false + supportsPrincipalConnectionLimit: false, + dataCommands: qdrantDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -66,6 +68,19 @@ extension PluginMetadataRegistry { } } +// The point operations of the Qdrant OpenAPI, v1.9 to v1.20. +private let qdrantDataCommands: Set = [ + "PUT /collections/{collection_name}/points", + "POST /collections/{collection_name}/points/delete", + "POST /collections/{collection_name}/points/payload", + "PUT /collections/{collection_name}/points/payload", + "POST /collections/{collection_name}/points/payload/delete", + "POST /collections/{collection_name}/points/payload/clear", + "PUT /collections/{collection_name}/points/vectors", + "POST /collections/{collection_name}/points/vectors/delete", + "POST /collections/{collection_name}/points/batch" +] + private let qdrantCompletions: [CompletionEntry] = [ CompletionEntry(label: "GET /collections", insertText: "GET /collections"), CompletionEntry(label: "GET /collections/{name}", insertText: "GET /collections/docs"), diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 6adb6fe2d9..8936d1b620 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -7,9 +7,9 @@ import Foundation import TableProPluginKit extension PluginMetadataRegistry { - /// SQL Server refuses `=` on these (Msg 402 for `ntext`, `text`, `image` and `xml`, 403 for the spatial types), so - /// a keyless match compares them through a cast, which is exact for each. A `hierarchyid` reads as its bytes, which - /// the cast to `varbinary(max)` compares without an implicit conversion back to the type. + // SQL Server refuses `=` on these (Msg 402 for `ntext`, `text`, `image` and `xml`, 403 for the spatial types), so + // a keyless match compares them through a cast, which is exact for each. A `hierarchyid` reads as its bytes, which + // the cast to `varbinary(max)` compares without an implicit conversion back to the type. static let mssqlRowMatchTextTypePrefixes = ["NTEXT", "TEXT", "XML", "IMAGE", "GEOGRAPHY", "GEOMETRY", "HIERARCHYID"] // Measured on ClickHouse 25.8: a JSON value or a NaN never equals the text it was read as, and concat() of it does. @@ -17,13 +17,13 @@ extension PluginMetadataRegistry { "JSON", "OBJECT(", "DYNAMIC", "FLOAT", "BFLOAT16", "NULLABLE(FLOAT", "NULLABLE(BFLOAT16", ] - /// `sql_variant` refuses `=` too (Msg 206), and no cast keeps its base type: the integer `1` and the string `N'1'` - /// cast to the same text, so a keyless match on it could pick either row. + // `sql_variant` refuses `=` too (Msg 206), and no cast keeps its base type: the integer `1` and the string `N'1'` + // cast to the same text, so a keyless match on it could pick either row. static let mssqlRowMatchExcludedTypePrefixes = ["SQL_VARIANT"] - /// Oracle cannot compare a LOB, `LONG`, `LONG RAW`, `BFILE`, `XMLTYPE`, `JSON`, `VECTOR` or spatial value with `=` - /// (ORA-22848 on a CLOB), so a keyless match leaves them out. Matched against the bare type name, which an - /// owner-qualified column such as `"MDSYS"."SDO_GEOMETRY"` classifies by. + // Oracle cannot compare a LOB, `LONG`, `LONG RAW`, `BFILE`, `XMLTYPE`, `JSON`, `VECTOR` or spatial value with `=` + // (ORA-22848 on a CLOB), so a keyless match leaves them out. Matched against the bare type name, which an + // owner-qualified column such as `"MDSYS"."SDO_GEOMETRY"` classifies by. static let oracleRowMatchExcludedTypePrefixes = [ "CLOB", "NCLOB", "BLOB", "LONG", "BFILE", "XMLTYPE", "JSON", "VECTOR", "SDO_" ] @@ -79,7 +79,8 @@ extension PluginMetadataRegistry { patternFiltersReadAnyColumn: true, filtersTypedByReadRows: true, readOnlyEnforcement: .clientGuard, - statementPolicyInDriver: true + statementPolicyInDriver: true, + dataCommands: Self.mongoDBDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", @@ -181,7 +182,8 @@ extension PluginMetadataRegistry { supportsDropDatabase: false, supportsOpportunisticTLS: false, supportsColumnSort: false, - readOnlyEnforcement: .clientGuard + readOnlyEnforcement: .clientGuard, + dataCommands: Self.redisDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", @@ -602,6 +604,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, tlsImpliedPorts: [8_443, 443], verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, readOnlyEnforcement: .engineEnforced ), schema: PluginMetadataSnapshot.SchemaInfo( @@ -737,8 +740,8 @@ extension PluginMetadataRegistry { defaultPrimaryKeyColumn: nil, immutableColumns: [], systemDatabaseNames: [ - "system", "system_schema", "system_auth", - "system_distributed", "system_traces", "system_virtual_schema" + "system", "system_schema", "system_auth", "system_distributed", "system_distributed_everywhere", + "system_traces", "system_views", "system_virtual_schema", "system_replicated_keys" ], systemSchemaNames: [], fileExtensions: [], @@ -783,8 +786,8 @@ extension PluginMetadataRegistry { defaultPrimaryKeyColumn: nil, immutableColumns: [], systemDatabaseNames: [ - "system", "system_schema", "system_auth", - "system_distributed", "system_traces", "system_virtual_schema" + "system", "system_schema", "system_auth", "system_distributed", "system_distributed_everywhere", + "system_traces", "system_views", "system_virtual_schema", "system_replicated_keys" ], systemSchemaNames: [], fileExtensions: [], @@ -832,7 +835,8 @@ extension PluginMetadataRegistry { supportsDropDatabase: false, supportsOpportunisticTLS: false, supportsColumnSort: false, - supportsRawFilter: false + supportsRawFilter: false, + dataCommands: Self.etcdDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+SurrealDBDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+SurrealDBDefaults.swift index c0907de2e7..550e30e225 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+SurrealDBDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+SurrealDBDefaults.swift @@ -33,7 +33,10 @@ extension PluginMetadataRegistry { requiresReconnectForDatabaseSwitch: false, supportsDropDatabase: true, supportsDropSchema: true, - supportsOpportunisticTLS: false + supportsOpportunisticTLS: false, + // The statements that make, change and remove records (SurrealQL statements reference, 2.x and 3.x). + // CREATE makes records only: `CREATE TABLE` does not parse. + dataCommands: ["CREATE", "INSERT", "UPDATE", "UPSERT", "DELETE", "RELATE"] ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift index 974471c1f2..d147b1d00d 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift @@ -37,7 +37,8 @@ extension PluginMetadataRegistry { defaultSSLMode: .disabled, supportsOpportunisticTLS: false, tlsImpliedPorts: [443], - verifiesServerWithSystemTrust: true + verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true ), schema: Self.trinoSchemaInfo, editor: PluginMetadataSnapshot.EditorConfig( @@ -88,6 +89,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, tlsImpliedPorts: [443], verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, // OFFSET is off by default (offset_clause_enabled), and the grammar takes neither // FETCH NEXT nor LIMIT before OFFSET. pagination: .leadingRowsOnly(maximumRows: nil) diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TypesenseDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TypesenseDefaults.swift index 0fbb78d139..362ef7bfcb 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TypesenseDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TypesenseDefaults.swift @@ -30,7 +30,8 @@ extension PluginMetadataRegistry { supportsQueryProgress: false, requiresReconnectForDatabaseSwitch: false, supportsDropDatabase: false, - supportsOpportunisticTLS: false + supportsOpportunisticTLS: false, + dataCommands: typesenseDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -61,6 +62,16 @@ extension PluginMetadataRegistry { } } +// The document routes of the Typesense server, v26.0 to v30.2. +private let typesenseDataCommands: Set = [ + "POST /collections/{collection}/documents", + "POST /collections/{collection}/documents/import", + "PATCH /collections/{collection}/documents/{id}", + "PATCH /collections/{collection}/documents", + "DELETE /collections/{collection}/documents/{id}", + "DELETE /collections/{collection}/documents", +] + private let typesenseCompletions: [CompletionEntry] = [ CompletionEntry(label: "GET /collections", insertText: "GET /collections"), CompletionEntry(label: "GET /collections/:name", insertText: "GET /collections/my-collection"), diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift index c6ed93f3b2..219b17658b 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+WeaviateDefaults.swift @@ -33,7 +33,8 @@ extension PluginMetadataRegistry { supportsModifyPrimaryKey: false, supportsOpportunisticTLS: false, supportsRawFilter: false, - supportsPrincipalConnectionLimit: false + supportsPrincipalConnectionLimit: false, + dataCommands: weaviateDataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", @@ -65,6 +66,21 @@ extension PluginMetadataRegistry { } } +// The object and batch operations of the Weaviate OpenAPI, v1.25.0 to v1.40.5. +private let weaviateDataCommands: Set = [ + "POST /v1/objects", + "PUT /v1/objects/{className}/{id}", "PATCH /v1/objects/{className}/{id}", "DELETE /v1/objects/{className}/{id}", + "PUT /v1/objects/{id}", "PATCH /v1/objects/{id}", "DELETE /v1/objects/{id}", + "POST /v1/objects/{className}/{id}/references/{propertyName}", + "PUT /v1/objects/{className}/{id}/references/{propertyName}", + "DELETE /v1/objects/{className}/{id}/references/{propertyName}", + "POST /v1/objects/{id}/references/{propertyName}", + "PUT /v1/objects/{id}/references/{propertyName}", + "DELETE /v1/objects/{id}/references/{propertyName}", + "POST /v1/batch/objects", "DELETE /v1/batch/objects", + "POST /v1/batch/references" +] + private let weaviateCompletions: [CompletionEntry] = [ CompletionEntry( label: "Get", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index c50ece4e4d..77b7e765db 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -73,93 +73,96 @@ struct PluginMetadataSnapshot: Sendable { var defaultSSLMode: SSLMode = .disabled var supportsOpportunisticTLS: Bool = true var tlsImpliedPorts: [Int] = [] + // Verify Identity with no CA file checks the server against the system trust store. var verifiesServerWithSystemTrust: Bool = false + // A URL or port that asks for TLS without naming a mode gets Verify Identity rather than Required. Kept apart from + // `verifiesServerWithSystemTrust`: a libpq server on a private CA that connects with `ssl=true` today would stop. + var requestedTLSVerifiesIdentity: Bool = false var supportsPerConnectionCertificatePaths: Bool = true var supportsClientKeyPassphrase: Bool = false var supportsConnectionPooling: Bool = true - /// Whether two pooled drivers for the same connection sit on one physical session. Snowflake - /// keys its session on the account and role and deliberately not on the database, so every - /// pooled scope shares one mutable `currentDatabase`: a statement that selects a database on - /// behalf of one entry selects it for all of them. + // Whether two pooled drivers for the same connection sit on one physical session. Snowflake + // keys its session on the account and role and deliberately not on the database, so every + // pooled scope shares one mutable `currentDatabase`: a statement that selects a database on + // behalf of one entry selects it for all of them. var pooledDriversShareOneSession: Bool = false var authenticationIsDatabaseScoped: Bool = false - /// Whether a connection that names no database has nothing to browse until one is chosen. A - /// MySQL session opened without one has no current database: `SHOW TABLES` there is - /// `ERROR 1046 No database selected`. An engine whose session falls back to a default - /// database, such as ClickHouse's `default`, leaves this false. + // Whether a connection that names no database has nothing to browse until one is chosen. A + // MySQL session opened without one has no current database: `SHOW TABLES` there is + // `ERROR 1046 No database selected`. An engine whose session falls back to a default + // database, such as ClickHouse's `default`, leaves this false. var browsingRequiresSelectedDatabase: Bool = false var pagination: PaginationCapability = .offset - /// Whether an exact count reads the whole table with no estimate to stand in for it, and only the driver - /// knows how to count it. DynamoDB bills a `Scan` of every item and has no `COUNT(*)`; Cassandra reads - /// every partition, because `LIMIT` does not bound `COUNT(*)`, and a filtered count needs CQL the host - /// cannot write. Such an engine is counted only when the user asks, and only by its driver. + // Whether an exact count reads the whole table with no estimate to stand in for it, and only the driver + // knows how to count it. DynamoDB bills a `Scan` of every item and has no `COUNT(*)`; Cassandra reads + // every partition, because `LIMIT` does not bound `COUNT(*)`, and a filtered count needs CQL the host + // cannot write. Such an engine is counted only when the user asks, and only by its driver. var exactRowCountIsFullScan: Bool = false - /// Whether a table can be ordered by any of its columns. CQL orders rows only by clustering columns inside - /// one partition, so a Cassandra table offers no column sort and a query result sorts the rows it holds. + // Whether a table can be ordered by any of its columns. CQL orders rows only by clustering columns inside + // one partition, so a Cassandra table offers no column sort and a query result sorts the rows it holds. var supportsColumnSort: Bool = true - /// Whether filters can ask for rows matching any one of several conditions. CQL has no `OR`. + // Whether filters can ask for rows matching any one of several conditions. CQL has no `OR`. var supportsMatchAnyFilters: Bool = true - /// Whether a filter row can hold a condition written in the engine's own query language. A - /// driver that answers one with an error, such as etcd or DynamoDB, or reads it as a column - /// name, is not offered the row at all. + // Whether a filter row can hold a condition written in the engine's own query language. A + // driver that answers one with an error, such as etcd or DynamoDB, or reads it as a column + // name, is not offered the row at all. var supportsRawFilter: Bool = true - /// Whether a table's columns are a sample of its rows rather than a declared schema. A - /// MongoDB collection lists the fields found in its first documents, so a field missing from - /// one side's list says nothing about whether that side holds it. + // Whether a table's columns are a sample of its rows rather than a declared schema. A + // MongoDB collection lists the fields found in its first documents, so a field missing from + // one side's list says nothing about whether that side holds it. var columnsAreSampled: Bool = false - /// Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, - /// so a search can send number and date columns too. MongoDB's builder converts the value before it matches. + // Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, + // so a search can send number and date columns too. MongoDB's builder converts the value before it matches. var patternFiltersReadAnyColumn: Bool = false - /// Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a - /// filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same - /// filter another way: a date compared as a string counts no rows. + // Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a + // filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same + // filter another way: a date compared as a string counts no rows. var filtersTypedByReadRows: Bool = false var isEngineReadOnly: Bool = false - /// Which connection field carries the path of the local database file this driver opens, - /// for the types that open one. Nil for every driver that reaches its database over the - /// network, which is what makes it the test for "can this connection name a remote file". + // Which connection field carries the path of the local database file this driver opens, + // for the types that open one. Nil for every driver that reaches its database over the + // network, which is what makes it the test for "can this connection name a remote file". var localFilePathField: LocalFilePathField? - /// The plugin fields that name the server, for a type the built-in Host and Port do not - /// reach. Nil for every type whose driver dials `config.host`, which is what keeps a tunnel - /// forwarding and rewriting the built-in endpoint there. + // The plugin fields that name the server, for a type the built-in Host and Port do not + // reach. Nil for every type whose driver dials `config.host`, which is what keeps a tunnel + // forwarding and rewriting the built-in endpoint there. var serverEndpointFields: ServerEndpointFields? - /// Fields a driver older than this app ignores to the user's cost. A connection that sets one - /// away from its default, while it is in use, is refused when the loaded driver does not - /// declare it: DuckDB's SSL Mode read by a driver from before it existed would connect as - /// Automatic, which is plain HTTP on a loopback host, however the user set it. + // Fields a driver older than this app ignores to the user's cost. A connection that sets one + // away from its default, while it is in use, is refused when the loaded driver does not + // declare it: DuckDB's SSL Mode read by a driver from before it existed would connect as + // Automatic, which is plain HTTP on a loopback host, however the user set it. var fieldsRequiringDriverSupport: [String] = [] - /// The extensions a new database file can be named with, the first one preferred. Empty - /// when the driver refuses a path with nothing at it, as Beancount does, so the form offers - /// no way to name one. DuckDB lists only its own format: a missing Parquet or CSV path is - /// refused rather than created. + // The extensions a new database file can be named with, the first one preferred. Empty + // when the driver refuses a path with nothing at it, as Beancount does, so the form offers + // no way to name one. DuckDB lists only its own format: a missing Parquet or CSV path is + // refused rather than created. var newDatabaseFileExtensions: [String] = [] var supportsSOCKSProxy: Bool { supportsSSH } - /// `cloudflared access tcp` forwards a loopback port just as an SSH tunnel does. A stored flag - /// drifted: it defaulted to true in the built-in snapshots, so a registry type the plugin - /// declares without SSH offered Cloudflare Tunnel until that plugin was installed. + // `cloudflared access tcp` forwards a loopback port just as an SSH tunnel does. A stored flag + // drifted: it defaulted to true in the built-in snapshots, so a registry type the plugin + // declares without SSH offered Cloudflare Tunnel until that plugin was installed. var supportsCloudflareTunnel: Bool { supportsSSH } - /// A tunnel command forwards a loopback port to the server the connection names, so it - /// applies wherever an SSH tunnel would. Computed for the same reason `supportsSOCKSProxy` - /// is: a stored flag would need an opt-out line in every hand-written snapshot. + // A tunnel command forwards a loopback port to the server the connection names, so it + // applies wherever an SSH tunnel would. Computed for the same reason `supportsSOCKSProxy` + // is: a stored flag would need an opt-out line in every hand-written snapshot. var supportsTunnelCommand: Bool { supportsSSH } - /// Whether this type may point at a file on an SSH server instead of a local one. - /// - /// Deliberately not derived from `localFilePathField`. Beancount opens a local file and must - /// still be excluded: a ledger is a graph of files reached through `include`, so one file - /// out of it either fails to load or presents incomplete accounts, which is worse. + // Whether this type may point at a file on an SSH server instead of a local one. + // Deliberately not derived from `localFilePathField`. Beancount opens a local file and must + // still be excluded: a ledger is a graph of files reached through `include`, so one file + // out of it either fails to load or presents incomplete accounts, which is worse. var supportsRemoteDatabaseFile: Bool = false - /// Whether this type can run its statements on an SSH server against a live database file, - /// rather than only fetching a read-only copy. True for SQLite alone today, because the - /// remote agent opens the file with the server's `libsqlite3`. + // Whether this type can run its statements on an SSH server against a live database file, + // rather than only fetching a read-only copy. True for SQLite alone today, because the + // remote agent opens the file with the server's `libsqlite3`. var supportsRemoteDatabaseSession: Bool = false var supportsPrincipalConnectionLimit: Bool = true @@ -176,6 +179,12 @@ struct PluginMetadataSnapshot: Sendable { // cannot read its language in full. A loaded driver without `.statementPolicy` runs no external write. var statementPolicyInDriver: Bool = false + // What the assistant's Edit mode runs besides reads on an engine with a query language of its own: the commands + // that change data, from the engine's command reference, named as `QueryClassifier.dataCommands(of:)` names + // them. Edit refuses every other command. Nil on an engine whose language is SQL, which the SQL reading decides, + // and on an engine nobody curated, where Edit runs reads only. + var dataCommands: Set? + static let defaults = CapabilityFlags( supportsSchemaSwitching: false, supportsImport: true, @@ -211,24 +220,24 @@ struct PluginMetadataSnapshot: Sendable { let systemDatabaseNames: [String] let systemSchemaNames: [String] let fileExtensions: [String] - /// Curated in the app rather than declared by the plugin: claiming a format from the - /// system also needs a `CFBundleDocumentTypes` entry only the app bundle can make. + // Curated in the app rather than declared by the plugin: claiming a format from the + // system also needs a `CFBundleDocumentTypes` entry only the app bundle can make. let fileSignatures: [DatabaseFileSignature] let databaseGroupingStrategy: GroupingStrategy let structureColumnFields: [StructureColumnField] let implicitSchemaName: String? let rowMatchExcludedTypePrefixes: [String] - /// Column types whose keyless row match has to compare the server's own text rendering - /// rather than the value itself. A grid reads every cell back as text, and on these types - /// that text does not compare equal to what it was read from: measured on MySQL 8.4.11 and - /// OceanBase 4.4.2.1, a `FLOAT` holding 1.1 reads back `1.1` and `WHERE f = '1.1'` matches - /// no row, because the server widens the stored single to a different double than it parses - /// the literal into. `JSON` never compares equal to a string at all. Listing a type here - /// wraps only the keyless comparison, and only for the types measured to need it: `CONCAT` - /// over a `BIT` column returns its raw bytes and breaks a match that works today. + // Column types whose keyless row match has to compare the server's own text rendering + // rather than the value itself. A grid reads every cell back as text, and on these types + // that text does not compare equal to what it was read from: measured on MySQL 8.4.11 and + // OceanBase 4.4.2.1, a `FLOAT` holding 1.1 reads back `1.1` and `WHERE f = '1.1'` matches + // no row, because the server widens the stored single to a different double than it parses + // the literal into. `JSON` never compares equal to a string at all. Listing a type here + // wraps only the keyless comparison, and only for the types measured to need it: `CONCAT` + // over a `BIT` column returns its raw bytes and breaks a match that works today. let rowMatchTextTypePrefixes: [String] - /// Teradata and Snowflake compare database names without regard to case, so `dbc` is the - /// system database `DBC`. Every other engine keeps case: MySQL can hold a user database `MySQL`. + // Teradata and Snowflake compare database names without regard to case, so `dbc` is the + // system database `DBC`. Every other engine keeps case: MySQL can hold a user database `MySQL`. let systemDatabaseNamesIgnoreCase: Bool // Nil for an engine that keeps the case of an unquoted name, or whose rule is not curated yet. let identifierRules: SQLIdentifierRules? @@ -313,10 +322,10 @@ struct PluginMetadataSnapshot: Sendable { let category: DatabaseCategory let tagline: String let hidesBuiltInPassword: Bool - /// The driver takes no container name on the connection, so the built-in field would - /// be a second, meaningless place to type one: an embedded engine reads it from the - /// file it opens, Redis numbers its databases through its own field, and a key-value - /// store may have no container at all. + // The driver takes no container name on the connection, so the built-in field would + // be a second, meaningless place to type one: an embedded engine reads it from the + // file it opens, Redis numbers its databases through its own field, and a key-value + // store may have no container at all. let hidesBuiltInDatabase: Bool let defaultUnixSocketPath: String? // Non-nil means the form offers Connect Using > Socket. @@ -505,11 +514,10 @@ final class PluginMetadataRegistry: @unchecked Sendable { registerBuiltInDefaults() } - /// The curated table the app ships, before any plugin or test registers over it. - /// - /// The live snapshots are not that table: a loaded plugin replaces its entry, and a test can - /// register a synthetic engine into the same shared registry. Anything that means to check the - /// table itself reads this instead of iterating `allRegisteredTypeIds()`. + // The curated table the app ships, before any plugin or test registers over it. + // The live snapshots are not that table: a loaded plugin replaces its entry, and a test can + // register a synthetic engine into the same shared registry. Anything that means to check the + // table itself reads this instead of iterating `allRegisteredTypeIds()`. func builtInDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { Self.curatedDefaults() + registryPluginDefaults() } @@ -542,12 +550,12 @@ final class PluginMetadataRegistry: @unchecked Sendable { registerLocked(snapshot: snapshot, forTypeId: typeId, preserveIcon: preserveIcon) } - /// Registers an additional database type served by a multi-type plugin (Redshift, - /// CockroachDB, PGlite on the PostgreSQL plugin). A plugin's statics are per-class, so - /// they cannot express per-type facts like PGlite's disabled SSL or single-connection limit. - /// The curated built-in entry is therefore authoritative for a variant; the plugin fills the - /// EXPLAIN variants the curated entry leaves open, and supplies the editor config. A variant - /// with no curated entry falls back to deriving its snapshot from the plugin. + // Registers an additional database type served by a multi-type plugin (Redshift, + // CockroachDB, PGlite on the PostgreSQL plugin). A plugin's statics are per-class, so + // they cannot express per-type facts like PGlite's disabled SSL or single-connection limit. + // The curated built-in entry is therefore authoritative for a variant; the plugin fills the + // EXPLAIN variants the curated entry leaves open, and supplies the editor config. A variant + // with no curated entry falls back to deriving its snapshot from the plugin. func registerVariant( pluginSnapshot: PluginMetadataSnapshot, forTypeId typeId: String, @@ -611,33 +619,30 @@ final class PluginMetadataRegistry: @unchecked Sendable { } } - /// A raw lookup by the exact id a snapshot was registered under, for the callers that hold an - /// id rather than a type: registration itself, and iteration over `allRegisteredTypeIds()`. - /// - /// It is deliberately not named `snapshot(forTypeId:)` any more. That spelling read as the - /// way to ask about a `DatabaseType`, so 62 call sites passed it `databaseType.pluginTypeId` - /// and every variant was answered with its primary's facts. Asking about a type is - /// `snapshot(for:)`; this overload cannot be reached from a `DatabaseType` without first - /// choosing which id you mean, which is the point. + // A raw lookup by the exact id a snapshot was registered under, for the callers that hold an + // id rather than a type: registration itself, and iteration over `allRegisteredTypeIds()`. + // It is deliberately not named `snapshot(forTypeId:)` any more. That spelling read as the + // way to ask about a `DatabaseType`, so 62 call sites passed it `databaseType.pluginTypeId` + // and every variant was answered with its primary's facts. Asking about a type is + // `snapshot(for:)`; this overload cannot be reached from a `DatabaseType` without first + // choosing which id you mean, which is the point. func snapshot(forRegisteredTypeId typeId: String) -> PluginMetadataSnapshot? { lock.lock() defer { lock.unlock() } return snapshots[typeId] } - /// The snapshot describing a database type, which for a variant is its own curated entry - /// rather than the entry of the plugin that serves it. Reaching a snapshot through - /// `pluginTypeId` asks the primary instead, which is how a Redshift tab came to be told - /// PostgreSQL's `ILIKE` folds non-ASCII, and how PGlite came to be offered the SSH, SSL, - /// Cloudflare Tunnel and SOCKS panes its own entry declares it does not support. - /// - /// The fallback covers a type registered by a plugin with no curated entry of its own, where - /// the primary's snapshot is the only one there is. - /// - /// This is the only way to read a snapshot for a `DatabaseType`. `pluginTypeId` answers a - /// different question, "which plugin serves this type", and belongs to driver lookup alone. - /// The entry the app ships for a type, kept after a plugin registers over it: what a driver - /// built with this app declares, against which an older loaded driver can be told apart. + // The snapshot describing a database type, which for a variant is its own curated entry + // rather than the entry of the plugin that serves it. Reaching a snapshot through + // `pluginTypeId` asks the primary instead, which is how a Redshift tab came to be told + // PostgreSQL's `ILIKE` folds non-ASCII, and how PGlite came to be offered the SSH, SSL, + // Cloudflare Tunnel and SOCKS panes its own entry declares it does not support. + // The fallback covers a type registered by a plugin with no curated entry of its own, where + // the primary's snapshot is the only one there is. + // This is the only way to read a snapshot for a `DatabaseType`. `pluginTypeId` answers a + // different question, "which plugin serves this type", and belongs to driver lookup alone. + // The entry the app ships for a type, kept after a plugin registers over it: what a driver + // built with this app declares, against which an older loaded driver can be told apart. func curatedSnapshot(for databaseType: DatabaseType) -> PluginMetadataSnapshot? { lock.lock() defer { lock.unlock() } @@ -665,42 +670,37 @@ final class PluginMetadataRegistry: @unchecked Sendable { // MARK: - Dynamic Type Registration - /// Registers an alias type ID that maps to a primary type ID. - /// Used for multi-type plugins (e.g., MariaDB → MySQL, Redshift → PostgreSQL). + // For multi-type plugins: MariaDB to MySQL, Redshift to PostgreSQL. func registerTypeAlias(_ aliasTypeId: String, primaryTypeId: String) { lock.lock() defer { lock.unlock() } reverseTypeIndex[aliasTypeId] = primaryTypeId } - /// The inverse of `registerTypeAlias`. `unregister(typeId:)` drops a snapshot and leaves the - /// alias behind, so a test that registers one has nothing to undo it with. + // The inverse of `registerTypeAlias`. `unregister(typeId:)` drops a snapshot and leaves the + // alias behind, so a test that registers one has nothing to undo it with. func removeTypeAlias(_ aliasTypeId: String) { lock.lock() defer { lock.unlock() } reverseTypeIndex.removeValue(forKey: aliasTypeId) } - /// Returns all registered type IDs (sorted for deterministic UI ordering). func allRegisteredTypeIds() -> [String] { lock.lock() defer { lock.unlock() } return Array(snapshots.keys).sorted() } - /// Resolves a database type raw value to its plugin type ID for driver lookup. - /// For multi-type plugins (MySQL serves MariaDB), maps the alias to the primary. - /// - /// This is the answer to "which plugin serves this type", never "what is this type like". - /// A snapshot read for a `DatabaseType` belongs in `snapshot(for:)`, which asks the variant - /// first; `snapshot(forTypeId:)` is for a bare id that is already the one you want. + // For multi-type plugins (MySQL serves MariaDB), maps the alias to the primary. + // This is the answer to "which plugin serves this type", never "what is this type like". + // A snapshot read for a `DatabaseType` belongs in `snapshot(for:)`, which asks the variant + // first; `snapshot(forTypeId:)` is for a bare id that is already the one you want. func pluginTypeId(for rawValue: String) -> String { lock.lock() defer { lock.unlock() } return reverseTypeIndex[rawValue] ?? rawValue } - /// Checks if a type ID is registered (has a snapshot). func hasType(_ typeId: String) -> Bool { lock.lock() defer { lock.unlock() } @@ -709,8 +709,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { // MARK: - Snapshot Builder - /// Builds a PluginMetadataSnapshot from a DriverPlugin's protocol properties. - /// Used by PluginManager to self-register plugins at load time. + // PluginManager registers each plugin with this at load time. func buildMetadataSnapshot( from driverType: any DriverPlugin.Type, isDownloadable: Bool = false @@ -802,6 +801,8 @@ final class PluginMetadataRegistry: @unchecked Sendable { tlsImpliedPorts: existingSnapshot?.capabilities.tlsImpliedPorts ?? [], verifiesServerWithSystemTrust: existingSnapshot?.capabilities .verifiesServerWithSystemTrust ?? false, + requestedTLSVerifiesIdentity: existingSnapshot?.capabilities + .requestedTLSVerifiesIdentity ?? false, supportsPerConnectionCertificatePaths: existingSnapshot?.capabilities .supportsPerConnectionCertificatePaths ?? true, supportsClientKeyPassphrase: existingSnapshot?.capabilities.supportsClientKeyPassphrase ?? false, @@ -834,7 +835,8 @@ final class PluginMetadataRegistry: @unchecked Sendable { supportsRoleLogin: existingSnapshot?.capabilities.supportsRoleLogin ?? true, binaryLengthFunction: existingSnapshot?.capabilities.binaryLengthFunction, readOnlyEnforcement: existingSnapshot?.capabilities.readOnlyEnforcement ?? .loginPrivileges, - statementPolicyInDriver: existingSnapshot?.capabilities.statementPolicyInDriver ?? false + statementPolicyInDriver: existingSnapshot?.capabilities.statementPolicyInDriver ?? false, + dataCommands: existingSnapshot?.capabilities.dataCommands ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: driverType.defaultSchemaName, diff --git a/apple/TablePro/Core/Plugins/PluginStatementContext+Statement.swift b/apple/TablePro/Core/Plugins/PluginStatementContext+Statement.swift index d900e3ab28..0ae23fab0f 100644 --- a/apple/TablePro/Core/Plugins/PluginStatementContext+Statement.swift +++ b/apple/TablePro/Core/Plugins/PluginStatementContext+Statement.swift @@ -25,6 +25,14 @@ extension PluginStatementContext { ) -> PluginStatementContext { var context = caller context.readOnly = caller.readOnly || statement(query, databaseType: databaseType).readOnly + context.readFunctionCalls(in: query, databaseType: databaseType) return context } + + // At Read-Only a session holding a transaction runs a read in it, outside the read-only mode, when the read calls + // only built-in functions. + mutating func readFunctionCalls(in query: String, databaseType: DatabaseType) { + guard enforcesReadOnly else { return } + callsUnlistedFunction = ReadOnlyFunctionCalls.callsUnlistedFunction(query, databaseType: databaseType) + } } diff --git a/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift b/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift index d3a8ecfef7..8247482ecf 100644 --- a/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift +++ b/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift @@ -172,32 +172,10 @@ final class QueryResultExportDataSource: PluginExportDataSource, @unchecked Send streamExportRows(table: object.name, databaseName: object.databaseName) } + // Pulled rather than pumped by a task of its own, which kept every filled chunk the writer had not taken yet. func streamExportRows(table: String, databaseName: String) -> AsyncThrowingStream { - let stream = streamRows(table: table, databaseName: databaseName) - let documents = exportDocuments - return AsyncThrowingStream { continuation in - let task = Task { - var offset = 0 - do { - for try await element in stream { - switch element { - case .header(let header): - continuation.yield(.header(header)) - case .rows(let rows): - let end = offset + rows.count - let aligned = end <= documents.count ? Array(documents[offset...AsyncIterator + private let documents: [PluginExportDocument?] + private var offset = 0 + + init(_ stream: AsyncThrowingStream, documents: [PluginExportDocument?]) { + iterator = stream.makeAsyncIterator() + self.documents = documents + } + + func next() async throws -> PluginExportStreamElement? { + guard let element = try await iterator.next() else { return nil } + switch element { + case .header(let header): + return .header(header) + case .rows(let rows): + let end = offset + rows.count + let aligned = end <= documents.count ? Array(documents[offset.. SQLLexicalGrammar> + private let readReporter = OSAllocatedUnfairLock<(@Sendable (Int64, Int64) -> Void)?>(initialState: nil) private let family: TransactionEngineFamily private let parser = SQLFileParser() @@ -58,28 +59,41 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { grammar.withLock { $0 = provider } } + // Bytes read from the file the parser opens, which for a `.gz` is the decompressed copy. + func reportsReading(to handler: @escaping @Sendable (_ bytesRead: Int64, _ fileSize: Int64) -> Void) { + readReporter.withLock { $0 = handler } + } + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { let fileURL = try await resolveURL() let grammar = self.grammar.withLock { $0 } - let parsed = parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar) + let onRead = readReporter.withLock { $0 } + let parsed = parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar, onRead: onRead) + guard !sendsFileBytesUnchanged else { return parsed } + let rewriting = TranscodedDumpStatements(parsed, rewriter: rewriter(grammar: grammar)) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) + } + + func elements() async throws -> AsyncThrowingStream { + let fileURL = try await resolveURL() + let grammar = self.grammar.withLock { $0 } + let onRead = readReporter.withLock { $0 } + let parsed = parser.parseElements(url: fileURL, encoding: encoding, grammar: grammar, onRead: onRead) guard !sendsFileBytesUnchanged else { return parsed } - return Self.sentAsUTF8(parsed, decodedFrom: encoding, family: family, grammar: grammar) + let rewriting = TranscodedDumpElements(parsed, rewriter: rewriter(grammar: grammar)) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) } private var sendsFileBytesUnchanged: Bool { encoding == .utf8 || encoding == .ascii } - private static func sentAsUTF8( - _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, - decodedFrom encoding: String.Encoding, - family: TransactionEngineFamily, - grammar: @escaping @Sendable () -> SQLLexicalGrammar - ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - let rewriting = TranscodedDumpStatements(statements, decodedFrom: encoding, family: family, grammar: grammar) - return AsyncThrowingStream(unfolding: { - try await rewriting.next() - }) + private func rewriter(grammar: @escaping @Sendable () -> SQLLexicalGrammar) -> TranscodedStatementRewriter { + TranscodedStatementRewriter(encoding: encoding, family: family, grammar: grammar) } /// `ownsDecompressedFile` answers only whether the caller handed over a file to delete. A file @@ -158,34 +172,52 @@ internal enum SqlFileImportError: LocalizedError, Equatable { } private final class TranscodedDumpStatements: @unchecked Sendable { - private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") - private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator - private let encoding: String.Encoding - private let family: TransactionEngineFamily - private let grammar: @Sendable () -> SQLLexicalGrammar + private let rewriter: TranscodedStatementRewriter - init( - _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, - decodedFrom encoding: String.Encoding, - family: TransactionEngineFamily, - grammar: @escaping @Sendable () -> SQLLexicalGrammar - ) { + init(_ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, rewriter: TranscodedStatementRewriter) { self.statements = statements.makeAsyncIterator() - self.encoding = encoding - self.family = family - self.grammar = grammar + self.rewriter = rewriter } func next() async throws -> (statement: String, lineNumber: Int)? { guard let (statement, lineNumber) = try await statements.next() else { return nil } + return (try rewriter.rewritten(statement, at: lineNumber), lineNumber) + } +} + +// COPY rows pass unchanged: the parser hands them out as UTF-8 already. +private final class TranscodedDumpElements: @unchecked Sendable { + private var elements: AsyncThrowingStream.AsyncIterator + private let rewriter: TranscodedStatementRewriter + + init(_ elements: AsyncThrowingStream, rewriter: TranscodedStatementRewriter) { + self.elements = elements.makeAsyncIterator() + self.rewriter = rewriter + } + + func next() async throws -> PluginImportElement? { + guard let element = try await elements.next() else { return nil } + guard case let .statement(statement, lineNumber) = element else { return element } + return .statement(try rewriter.rewritten(statement, at: lineNumber), lineNumber: lineNumber) + } +} + +private struct TranscodedStatementRewriter: Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") + + let encoding: String.Encoding + let family: TransactionEngineFamily + let grammar: @Sendable () -> SQLLexicalGrammar + + func rewritten(_ statement: String, at lineNumber: Int) throws -> String { let grammar = self.grammar() let literals = try rewrittenLiterals(of: statement, at: lineNumber, grammar: grammar) ?? statement guard let rewritten = SQLClientEncodingRewriter.rewritten(literals, family: family, grammar: grammar) else { - return (literals, lineNumber) + return literals } Self.logger.info("Kept the session's client encoding at line \(lineNumber, privacy: .public), since the import sends UTF-8") - return (rewritten, lineNumber) + return rewritten } private func rewrittenLiterals(of statement: String, at lineNumber: Int, grammar: SQLLexicalGrammar) throws -> String? { diff --git a/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift b/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift index 35ec58638b..fa822d1bba 100644 --- a/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift +++ b/apple/TablePro/Core/Plugins/StreamingQueryExportDataSource.swift @@ -18,11 +18,19 @@ final class StreamingQueryExportDataSource: PluginExportDataSource, @unchecked S private let query: String private let driver: DatabaseDriver private let dbType: DatabaseType + private let safeModeIsReadOnly: Bool + private let rowCut = OSAllocatedUnfairLock(initialState: nil) private static let logger = Logger(subsystem: "com.TablePro", category: "StreamingQueryExport") - init(query: String, driver: DatabaseDriver, databaseType: DatabaseType) { + // Where the driver stopped the stream at a limit it declared, so the file is partial. + var driverRowCut: Int? { + rowCut.withLock { $0 } + } + + init(query: String, driver: DatabaseDriver, databaseType: DatabaseType, safeModeIsReadOnly: Bool = false) { self.query = query + self.safeModeIsReadOnly = safeModeIsReadOnly self.driver = driver self.dbType = databaseType self.databaseTypeId = databaseType.rawValue @@ -33,7 +41,11 @@ final class StreamingQueryExportDataSource: PluginExportDataSource, @unchecked S guard let adapter = driver as? PluginDriverAdapter else { return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } } - return adapter.streamRows(query: query) + let rowCut = rowCut + let rows = adapter.streamRows(query: query, safeModeIsReadOnly: safeModeIsReadOnly) + return ExportRowCut.reporting(rows, engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + } } func streamExportRows(for object: PluginExportTable) -> AsyncThrowingStream { @@ -44,7 +56,11 @@ final class StreamingQueryExportDataSource: PluginExportDataSource, @unchecked S guard let adapter = driver as? PluginDriverAdapter else { return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed("No plugin driver available")) } } - return adapter.streamExportRows(query: query) + let rowCut = rowCut + let rows = adapter.streamExportRows(query: query, safeModeIsReadOnly: safeModeIsReadOnly) + return ExportRowCut.reporting(rows, engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + } } func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { diff --git a/apple/TablePro/Core/Process/PipeLineSplitter.swift b/apple/TablePro/Core/Process/PipeLineSplitter.swift new file mode 100644 index 0000000000..67c66175a0 --- /dev/null +++ b/apple/TablePro/Core/Process/PipeLineSplitter.swift @@ -0,0 +1,34 @@ +// +// PipeLineSplitter.swift +// TablePro +// + +import Foundation + +// Splits on LF only and decodes whole lines, so a chunk that ends inside a UTF-8 sequence loses nothing +// and a U+2028 inside a JSON string does not cut the line. +struct PipeLineSplitter: Sendable { + private var pending = Data() + + mutating func lines(appending chunk: Data) -> [String] { + pending.append(chunk) + var lines: [String] = [] + var lineStart = pending.startIndex + while let newline = pending[lineStart...].firstIndex(of: 0x0A) { + lines.append(Self.decode(pending[lineStart ..< newline])) + lineStart = pending.index(after: newline) + } + pending.removeSubrange(pending.startIndex ..< lineStart) + return lines + } + + mutating func finish() -> String? { + defer { pending = Data() } + return pending.isEmpty ? nil : Self.decode(pending) + } + + private static func decode(_ bytes: Data) -> String { + let line = bytes.last == 0x0D ? bytes.dropLast() : bytes + return String(decoding: line, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + } +} diff --git a/apple/TablePro/Core/Process/PipeReader.swift b/apple/TablePro/Core/Process/PipeReader.swift index 916db0a0e0..7ef5581ec0 100644 --- a/apple/TablePro/Core/Process/PipeReader.swift +++ b/apple/TablePro/Core/Process/PipeReader.swift @@ -5,23 +5,32 @@ import Foundation import os -import TableProUtilities +import TableProProcess internal final class PipeReader: @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "PipeReader") private let handle: FileHandle private let descriptor: Int32 - private let lock = NSLock() + // Recursive: an owner may stop the reader from inside its own consumer or end callback. + private let lock = NSRecursiveLock() private var consumer: (@Sendable (Data) -> Void)? + private var atEnd: (@Sendable ((any Error)?) -> Void)? init(_ handle: FileHandle) { self.handle = handle descriptor = handle.fileDescriptor } - func start(delivering consumer: @escaping @Sendable (Data) -> Void) { - lock.withLock { self.consumer = consumer } + // `atEnd` gets nil at end of file or the read error, after the last chunk, and never once the owner stopped the reader. + func start( + delivering consumer: @escaping @Sendable (Data) -> Void, + atEnd: (@Sendable ((any Error)?) -> Void)? = nil + ) { + lock.withLock { + self.consumer = consumer + self.atEnd = atEnd + } handle.readabilityHandler = { _ in self.readArrivedBytes() } @@ -47,6 +56,7 @@ internal final class PipeReader: @unchecked Sendable { handle.readabilityHandler = nil lock.lock() defer { lock.unlock() } + atEnd = nil guard let consumer else { return } self.consumer = nil drain(consumer) @@ -59,7 +69,7 @@ internal final class PipeReader: @unchecked Sendable { do { let chunk = try DescriptorRead.availableBytes(from: descriptor) guard !chunk.isEmpty else { - endReading() + endReading(failure: nil) return } consumer(chunk) @@ -67,12 +77,15 @@ internal final class PipeReader: @unchecked Sendable { return } catch { Self.logger.error("Reading a pipe failed: \(error.publicLogShape, privacy: .public)") - endReading() + endReading(failure: error) } } - private func endReading() { + private func endReading(failure: (any Error)?) { consumer = nil handle.readabilityHandler = nil + let finished = atEnd + atEnd = nil + finished?(failure) } } diff --git a/apple/TablePro/Core/Process/SupervisedProcessRunner.swift b/apple/TablePro/Core/Process/SupervisedProcessRunner.swift index c22c0d2d41..8858b20775 100644 --- a/apple/TablePro/Core/Process/SupervisedProcessRunner.swift +++ b/apple/TablePro/Core/Process/SupervisedProcessRunner.swift @@ -30,7 +30,7 @@ final class ProcessSupervisedRunner: SupervisedProcessRunner, @unchecked Sendabl private static let forcedTerminationGrace = Duration.seconds(2) - private var partialLine = "" + private var stderrSplitter = PipeLineSplitter() private var wasRequested = false private var terminationResult: SubprocessTermination? private var terminationContinuations: [CheckedContinuation] = [] @@ -121,14 +121,8 @@ final class ProcessSupervisedRunner: SupervisedProcessRunner, @unchecked Sendabl } private func ingestStderr(_ chunk: Data) { - guard let text = String(data: chunk, encoding: .utf8) else { return } stateLock.lock() - partialLine += text - var lines: [String] = [] - while let newlineIndex = partialLine.firstIndex(of: "\n") { - lines.append(String(partialLine[.. Int in + source.withUnsafeBytes { input -> Int in + guard let destinationBase = destination.bindMemory(to: UInt8.self).baseAddress, + let inputBase = input.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return compression_encode_buffer(destinationBase, capacity, inputBase, source.count, nil, COMPRESSION_ZLIB) + } + } + // 0 means the output did not fit `capacity`; raw DEFLATE of this text never grows past it. + precondition(written > 0, "RemoteSQLiteAgentSource could not compress the agent program") + deflated.removeSubrange(written.. String { - let encoded = Data(python.utf8).base64EncodedString() - let loop = interpreters.map { interpreter -> String in + let branches = interpreters.map { interpreter -> String in let present = #"command -v "\#(interpreter)" >/dev/null 2>&1"# - let run = #"exec "\#(interpreter)" -I -S -c "import base64,sys;exec(base64.b64decode(sys.argv[1]))" \#(encoded)"# + let run = #"exec "\#(interpreter)" -I -S -c "\#(interpreterBootstrap)" "$b""# return "if \(present); then \(run); fi" } .joined(separator: "; ") - return "sh -c '" + loop + "; echo " + RemoteSQLiteWire.noPythonNotice + "; exit 0'" + return "sh -c 'b=" + encodedProgram + "; " + branches + "; echo " + RemoteSQLiteWire.noPythonNotice + "; exit 0'" } static let python = #""" diff --git a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift index 13651ee288..74a1690600 100644 --- a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift +++ b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteExecChannelOpener.swift @@ -8,16 +8,16 @@ import Foundation import TableProSSHTransport -/// Opens one session channel and starts a command on it, one non-blocking attempt at a time, so -/// `SSHForwardChannelOpenPump` can drive it to a decision within an app-owned deadline exactly as it -/// drives a forwarding open. A forwarding channel opens in a single call; an exec channel is two -/// steps, the channel open and the `exec` request, and either can report `LIBSSH2_ERROR_EAGAIN`, so -/// the phase is remembered between attempts. -/// -/// The opener owns the channel until it hands it back as `.opened`. If the pump times out or is -/// cancelled after the channel was created but before it was handed over, the caller must call -/// `abort()` to free it, because dropping the Swift reference does not free the libssh2 channel and -/// the session stays live. +// Opens one session channel and starts a command on it, one non-blocking attempt at a time, so +// `SSHForwardChannelOpenPump` can drive it to a decision within an app-owned deadline exactly as it +// drives a forwarding open. A forwarding channel opens in a single call; an exec channel is two +// steps, the channel open and the `exec` request, and either can report `LIBSSH2_ERROR_EAGAIN`, so +// the phase is remembered between attempts. +// +// The opener owns the channel until it hands it back as `.opened`. If the pump times out or is +// cancelled after the channel was created but before it was handed over, the caller takes it back +// with `abandon()` and frees it, because dropping the Swift reference does not free the libssh2 +// channel and the session stays live. final class RemoteSQLiteExecChannelOpener: SSHForwardChannelOpening { private enum Phase { case opening @@ -28,18 +28,32 @@ final class RemoteSQLiteExecChannelOpener: SSHForwardChannelOpening { private let session: OpaquePointer private let command: String - private let sessionQueue: DispatchQueue + private let pump: SSHSessionPump + private let turn: SSHSendTurn private var phase: Phase = .opening private var channel: OpaquePointer? - init(session: OpaquePointer, command: String, sessionQueue: DispatchQueue) { + private static let abandonSeconds: TimeInterval = 5 + + init(session: OpaquePointer, command: String, pump: SSHSessionPump) { self.session = session self.command = command - self.sessionQueue = sessionQueue + self.pump = pump + self.turn = SSHSendTurn(pump: pump) } func attemptOpen() -> SSHForwardChannelAttempt { - sessionQueue.sync { + let attempt = attemptOnce() + if case .wouldBlock(let directions) = attempt { + turn.waitsToSend(directions.contains(.outbound)) + } else { + turn.end() + } + return attempt + } + + private func attemptOnce() -> SSHForwardChannelAttempt { + pump.perform { switch phase { case .opening: return openChannel() @@ -53,10 +67,24 @@ final class RemoteSQLiteExecChannelOpener: SSHForwardChannelOpening { } } - /// Frees a channel that was opened but never handed to the relay, for an open the pump timed out - /// or cancelled. A no-op once the channel has been handed over or already freed. - func abort() { - sessionQueue.sync { freeChannel() } + // Hands back a channel that was opened but never handed to the relay, for an open that failed, timed out or + // was cancelled; nil once handed over. A request still half sent sits in the channel's own buffer, and only + // this opener's next attempt finishes it, so the attempts go on first while they wait to send. + func abandon(transportFD: Int32) -> OpaquePointer? { + let deadline = Date().addingTimeInterval(Self.abandonSeconds) + while Date() < deadline { + let attempt = attemptOpen() + if case .opened(let opened) = attempt { return opened } + guard case .wouldBlock(let directions) = attempt, directions.contains(.outbound) else { break } + _ = pollReady(fd: transportFD, directions: .outbound, timeoutMs: LibSSH2ForwardChannel.openPollTimeoutMs) + } + turn.end() + return pump.perform { + let abandoned = channel + channel = nil + phase = .finished + return abandoned + } } private func openChannel() -> SSHForwardChannelAttempt { @@ -78,14 +106,13 @@ final class RemoteSQLiteExecChannelOpener: SSHForwardChannelOpening { if rc == LIBSSH2_ERROR_EAGAIN { return .wouldBlock(RelayDirections(libssh2BlockDirections: libssh2_session_block_directions(session))) } - let message = LibSSH2ForwardChannel.lastErrorMessage(session: session) - freeChannel() - return .failed(code: rc, message: message) + phase = .finished + return .failed(code: rc, message: LibSSH2ForwardChannel.lastErrorMessage(session: session)) } - /// Discards the agent's standard error rather than merging it into the protocol stream. A merge - /// would corrupt the framed replies; leaving it unread would let a large Python traceback fill - /// the channel window and stall the relay, which reads standard output alone. + // Discards the agent's standard error rather than merging it into the protocol stream. A merge + // would corrupt the framed replies; leaving it unread would let a large Python traceback fill + // the channel window and stall the relay, which reads standard output alone. private func configure() -> SSHForwardChannelAttempt { guard let channel else { return .failed(code: 0, message: "no channel to configure") } let rc = libssh2_channel_handle_extended_data2(channel, LIBSSH2_CHANNEL_EXTENDED_DATA_IGNORE) @@ -104,12 +131,4 @@ final class RemoteSQLiteExecChannelOpener: SSHForwardChannelOpening { } return .wouldBlock(RelayDirections(libssh2BlockDirections: libssh2_session_block_directions(session))) } - - private func freeChannel() { - if let channel { - libssh2_channel_free(channel) - self.channel = nil - } - phase = .finished - } } diff --git a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift index d44e922406..0d65c420ac 100644 --- a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift +++ b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTransportManager.swift @@ -26,7 +26,6 @@ actor RemoteSQLiteTransportManager: TunnelManaging { private var tunnels: [UUID: RemoteSQLiteTunnel] = [:] private var creations = TunnelCreationGenerations() private static let tunnelRegistry = OSAllocatedUnfairLock(initialState: [UUID: RemoteSQLiteTunnel]()) - private var appNapActivity: NSObjectProtocol? private init() {} @@ -118,7 +117,6 @@ actor RemoteSQLiteTransportManager: TunnelManaging { Self.tunnelRegistry.withLock { $0[connectionId] = tunnel } tunnel.startForwarding() tunnel.startKeepAlive() - updateAppNapState() Self.logger.info("Remote SQLite session ready for \(connectionId) on 127.0.0.1:\(tunnel.localPort)") return RemoteSQLiteEndpoint(port: tunnel.localPort, token: token) @@ -133,7 +131,6 @@ actor RemoteSQLiteTransportManager: TunnelManaging { private func discardTunnel(connectionId: UUID) async throws { guard let tunnel = tunnels.removeValue(forKey: connectionId) else { return } Self.tunnelRegistry.withLock { $0[connectionId] = nil } - updateAppNapState() tunnel.close() } @@ -165,7 +162,6 @@ actor RemoteSQLiteTransportManager: TunnelManaging { private func handleTunnelDeath(connectionId: UUID) async { guard tunnels.removeValue(forKey: connectionId) != nil else { return } Self.tunnelRegistry.withLock { $0[connectionId] = nil } - updateAppNapState() Self.logger.warning("Remote SQLite session died for \(connectionId)") await DatabaseManager.shared.handleRemoteSQLiteTunnelDied(connectionId: connectionId) } @@ -197,16 +193,4 @@ actor RemoteSQLiteTransportManager: TunnelManaging { } return listenFD } - - private func updateAppNapState() { - if !tunnels.isEmpty && appNapActivity == nil { - appNapActivity = ProcessInfo.processInfo.beginActivity( - options: .userInitiatedAllowingIdleSystemSleep, - reason: "Remote SQLite session keepalive requires timely execution" - ) - } else if tunnels.isEmpty, let activity = appNapActivity { - ProcessInfo.processInfo.endActivity(activity) - appNapActivity = nil - } - } } diff --git a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift index 5d7e3fb833..a38d51ac14 100644 --- a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift +++ b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift @@ -9,15 +9,13 @@ import os import TableProSSHTransport -/// A loopback listener whose accepted clients each get a fresh exec channel running the SQLite -/// agent on the SSH server, so the SQLite plugin can reach a live database on that server as if it -/// were a local port. -/// -/// It mirrors `LibSSH2Tunnel`, which forwards a TCP port over one authenticated session: the same -/// accept loop, keepalive, teardown latch and byte counter. What differs is the cargo. A forwarding -/// tunnel opens a direct-tcpip channel per client; this opens a session channel and runs a command -/// on it, and it admits a client only after the client presents the per-connection token, because -/// nothing behind this listener authenticates the way a database port does. +// A loopback listener whose accepted clients each get a fresh exec channel running the SQLite agent +// on the SSH server, so the SQLite plugin can reach a live database there as if it were a local port. +// +// It mirrors `LibSSH2Tunnel`: the same acceptor, keepalive, teardown latch and byte counter. What +// differs is the cargo. A forwarding tunnel opens a direct-tcpip channel per client; this opens a +// session channel and runs a command on it, and it admits a client only after the client presents the +// per-connection token, because nothing behind this listener authenticates the way a database port does. final class RemoteSQLiteTunnel: @unchecked Sendable { let connectionId: UUID let localPort: Int @@ -35,19 +33,15 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { private var session: OpaquePointer { chain.session } private var socketFD: Int32 { chain.socketFD } - private var forwardingTask: Task? + private var acceptor: SSHListenerAcceptor? private var keepAliveTask: Task? private let aliveLatch = TeardownLatch() - /// The relays still running, so teardown frees the session only once none of them can touch - /// it. A group rather than a collection of tasks: a relay leaves it by finishing, which is the - /// one thing a `[Task]` pruned on `isCancelled` never noticed, so a tunnel that had served - /// clients carried every one of them until it closed. It is also all a task was ever worth - /// here, since the relay runs on `relayQueue` outside the task's cancellation scope and stops - /// on `aliveLatch` rather than on `Task.isCancelled`. + // Each accepted client from accept until its channel is freed or given up, so teardown frees the + // session only once none of them can touch it. private let clientRelays = DispatchGroup() - private let sessionQueue: DispatchQueue + private let pump: SSHSessionPump private let relayQueue: DispatchQueue private let acceptQueue: DispatchQueue @@ -56,7 +50,10 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { private let byteCounter = TransportByteCounter() private static let relayBufferSize = 32_768 - private static let acceptPollTimeoutMs: Int32 = 200 + + private struct ChannelHandle: @unchecked Sendable { + let pointer: OpaquePointer + } private static let admissionTimeoutSeconds = 5 init( @@ -76,7 +73,10 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { self.token = token self.relayDeadlines = ConnectionRelayDeadlineProvider(initialDeadline: connectionDeadline) self.createdAt = Date() - self.sessionQueue = DispatchQueue(label: "com.TablePro.rsqlite.session.\(connectionId.uuidString)", qos: .utility) + self.pump = SSHSessionPump( + transportFD: chain.socketFD, + queue: DispatchQueue(label: "com.TablePro.rsqlite.session.\(connectionId.uuidString)", qos: .utility) + ) self.relayQueue = DispatchQueue( label: "com.TablePro.rsqlite.relay.\(connectionId.uuidString)", qos: .utility, attributes: .concurrent ) @@ -96,48 +96,36 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { // MARK: - Forwarding func startForwarding() { - sessionQueue.sync { libssh2_session_set_blocking(session, 0) } + pump.perform { libssh2_session_set_blocking(session, 0) } - forwardingTask = Task.detached { [weak self] in - guard let self else { return } - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.acceptQueue.async { [weak self] in - defer { continuation.resume() } - guard let self else { return } - Self.logger.info("Remote SQLite listener started on 127.0.0.1:\(self.localPort)") - while self.isRunning { - guard let clientFD = self.acceptClient() else { - if self.isRunning { continue } - break - } - self.spawnClient( - clientFD: clientFD, - budget: self.relayDeadlines.next() - ) - } - } + // A handler already running when the latch is claimed may still spawn one relay; teardown + // waits for the cancel handler before the relay group, so it waits for that relay too. + let acceptor = SSHListenerAcceptor(listenFD: listenFD, queue: acceptQueue) { [weak self] client in + guard let self, self.isRunning else { + Darwin.close(client.descriptor) + return } + self.spawnClient(clientFD: client.descriptor, budget: self.relayDeadlines.next()) } + self.acceptor = acceptor + acceptor.start() + Self.logger.info("Remote SQLite listener started on 127.0.0.1:\(self.localPort)") } func startKeepAlive() { - sessionQueue.sync { libssh2_keepalive_config(session, 1, 30) } + pump.perform { LibSSH2KeepAlive.configure(session: session) } keepAliveTask = Task.detached { [weak self] in guard let self else { return } while !Task.isCancelled && self.isRunning { - let failed = await withCheckedContinuation { (continuation: CheckedContinuation) in - self.sessionQueue.async { - var secondsToNext: Int32 = 0 - let rc = libssh2_keepalive_send(self.session, &secondsToNext) - continuation.resume(returning: sshKeepAliveDidFail(rc)) - } + let outcome = await self.pump.performAsync { + LibSSH2KeepAlive.send(session: self.session, transportFD: self.socketFD) } - if failed { + if outcome.failed { Self.logger.warning("Remote SQLite keepalive failed, marking dead") self.markDead() break } - try? await Task.sleep(for: .seconds(10)) + try? await Task.sleep(for: .seconds(outcome.secondsToNext)) } } } @@ -149,13 +137,13 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { performTeardown() } - /// Synchronous cleanup for app termination, which waits for nothing because the process is - /// exiting. No descriptor is closed and the session is not freed: the relays and the accept - /// loop may still be polling them, and the OS reclaims every one of them anyway. + // App termination: waits for nothing, closes no descriptor and frees no session, since a relay or + // the accept handler may still be using them and the OS reclaims them all. func closeSync() { guard aliveLatch.claim() else { return } - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() + pump.stop() shutdown(socketFD, SHUT_RDWR) } @@ -165,34 +153,33 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { onDeath?(connectionId) } - /// Breaks every blocking wait first, then frees the session and the descriptors only once - /// every task that could still be polling them has exited. `shutdown` unblocks a poll at - /// once without releasing the descriptor number, which another thread would otherwise be - /// free to receive from the kernel and poll by mistake. `shutdown` does not wake a poll on a - /// listening socket on Darwin, so the accept loop ends on the latch instead, inside one - /// `acceptPollTimeoutMs`, and its descriptor closes once it has. - /// - /// The relays are waited on after the accept loop, not alongside it, because the accept loop - /// is the only thing that starts one: once it has ended, the group can only empty. + // `shutdown` wakes every open or free waiting on the transport but keeps the descriptor number, + // so no other socket can take it mid-poll. The listening socket closes only after the acceptor's + // cancel handler ran, which GCD runs once no accept handler is running. The clients are awaited + // after that, because the accept handler is the only thing that starts one: the group can only + // empty. The pump is awaited last, so no source of it can fire on the descriptors closed below. private func performTeardown() { - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() + pump.stop() shutdown(socketFD, SHUT_RDWR) + let pump = self.pump let chain = self.chain let relayQueue = self.relayQueue let clientRelays = self.clientRelays let listenFD = self.listenFD - let forwardingTask = self.forwardingTask + let acceptor = self.acceptor let keepAliveTask = self.keepAliveTask let connectionId = self.connectionId Task.detached { - await forwardingTask?.value + await acceptor?.waitForCancellation() await keepAliveTask?.value await withCheckedContinuation { (continuation: CheckedContinuation) in clientRelays.notify(queue: relayQueue) { continuation.resume() } } + await pump.waitUntilStopped() Darwin.close(listenFD) LibSSH2TunnelFactory.cleanupChain(chain, reason: "Closing remote SQLite session") @@ -202,19 +189,6 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { // MARK: - Accept - private func acceptClient() -> Int32? { - var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) - guard poll(&pollFD, 1, Self.acceptPollTimeoutMs) > 0, pollFD.revents & Int16(POLLIN) != 0 else { - return nil - } - var clientAddr = sockaddr_in() - var addrLen = socklen_t(MemoryLayout.size) - let clientFD = withUnsafeMutablePointer(to: &clientAddr) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { accept(listenFD, $0, &addrLen) } - } - return clientFD >= 0 ? clientFD : nil - } - private func spawnClient( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget @@ -222,19 +196,22 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { let clientRelays = self.clientRelays clientRelays.enter() relayQueue.async { [weak self] in - defer { clientRelays.leave() } guard let self else { Darwin.close(clientFD) + clientRelays.leave() return } - self.admitAndRelay(clientFD: clientFD, budget: budget) + if !self.admitAndRelay(clientFD: clientFD, budget: budget) { + clientRelays.leave() + } } } + // True once the relay owns the client, which then leaves `clientRelays` when its channel is freed. private func admitAndRelay( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget - ) { + ) -> Bool { let connectionDeadline = budget.deadline guard let line = Self.readAdmissionLine(clientFD, deadline: connectionDeadline), RemoteSQLiteAdmission.isAuthorized(line: line, token: token) else { @@ -243,11 +220,11 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { } Self.logger.warning("Remote SQLite client rejected: bad or missing token") Darwin.close(clientFD) - return + return false } - let opener = RemoteSQLiteExecChannelOpener(session: session, command: command, sessionQueue: sessionQueue) - let pump = SSHForwardChannelOpenPump( + let opener = RemoteSQLiteExecChannelOpener(session: session, command: command, pump: pump) + let openPump = SSHForwardChannelOpenPump( opener: opener, isActive: { [weak self] in guard self?.isRunning == true else { return false } @@ -268,7 +245,7 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { } ) - let outcome = pump.run() + let outcome = openPump.run() let initialDeadlineTimedOut: Bool switch outcome { case .timedOut: @@ -283,22 +260,26 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { if case .opened(let channel) = outcome { Darwin.close(clientFD) freeChannel(channel) - return + return false } } switch outcome { case .opened(let channel): - runRelay(clientFD: clientFD, channel: channel) + relayChannel(clientFD: clientFD, channel: channel) + return true case .failed(let code, let message): Self.logger.error("Remote SQLite exec channel failed to open, libssh2 \(code): \(message)") Darwin.close(clientFD) case .timedOut, .cancelled: - /// The channel may have opened before the deadline and be waiting on the exec request. - /// The pump never took ownership, so free it here rather than leak it on the live session. - opener.abort() Darwin.close(clientFD) } + // The channel may have opened and then failed or run out of time on the exec request. The open + // pump never took ownership, so free it here rather than leak it on the live session. + if let channel = opener.abandon(transportFD: socketFD) { + freeAbandonedChannel(channel) + } + return false } private func recordConnectionTimeout(_ deadline: ConnectionDeadline) { @@ -347,26 +328,24 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { return nil } - private func runRelay(clientFD: Int32, channel: OpaquePointer) { - let relay = SSHChannelRelay( + private func relayChannel(clientFD: Int32, channel: OpaquePointer) { + let clientRelays = self.clientRelays + let handle = ChannelHandle(pointer: channel) + pump.relay( localFD: clientFD, - transportFD: socketFD, - channelIO: LibSSH2ChannelIO(channel: channel, session: session, sessionQueue: sessionQueue), + channel: LibSSH2ChannelIO(channel: channel, session: session), bufferSize: Self.relayBufferSize, - isActive: { [weak self] in self?.isRunning ?? false }, byteCounter: byteCounter - ) - - let termination = relay.run() - Darwin.close(clientFD) - - guard isRunning else { return } + ) { [weak self] termination in + defer { clientRelays.leave() } + guard let self, self.isRunning else { return } - freeChannel(channel) + self.freeChannel(handle.pointer) - if termination == .transportHangup { - Self.logger.info("Remote SQLite transport hung up for \(self.connectionId)") - markDead() + if termination == .transportHangup { + Self.logger.info("Remote SQLite transport hung up for \(self.connectionId)") + self.markDead() + } } } @@ -376,11 +355,25 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { _ = LibSSH2ChannelFree.run( channel: channel, session: session, - sessionQueue: sessionQueue, + pump: pump, transportFD: socketFD, - sendEOF: true, deadline: .distantFuture, isActive: { [weak self] in self?.isRunning ?? false } ) } + + // A channel given up on mid-open may hold its exec request half sent, which no call finishes once the + // open stopped, so its free has the keep-alive's limit and a stuck one ends the session. + private func freeAbandonedChannel(_ channel: OpaquePointer) { + let outcome = LibSSH2ChannelFree.run( + channel: channel, + session: session, + pump: pump, + transportFD: socketFD, + isActive: { [weak self] in self?.isRunning ?? false } + ) + guard outcome == .timedOut else { return } + Self.logger.warning("Remote SQLite channel did not close within \(Int(LibSSH2ChannelFree.deadlineSeconds))s, marking dead") + markDead() + } } diff --git a/apple/TablePro/Core/RoutineDebugging/RoutineDebugController.swift b/apple/TablePro/Core/RoutineDebugging/RoutineDebugController.swift index 41f98644ee..ad0a0eeccb 100644 --- a/apple/TablePro/Core/RoutineDebugging/RoutineDebugController.swift +++ b/apple/TablePro/Core/RoutineDebugging/RoutineDebugController.swift @@ -770,6 +770,7 @@ internal final class RoutineDebugController: ObservableObject { configuration.traits = DataGridEngineTraits(databaseType: databaseType) configuration.showRowNumbers = true configuration.supportsValueFilter = false + configuration.colorsValuesByType = true configuration.commands = [.freezeColumns, .displayAs] let grid = HeldRowsGridAdapter(configuration: configuration, sortsHeldRows: true) grid.setRows(rows) diff --git a/apple/TablePro/Core/RowGeneration/RowGenerationColumnReader.swift b/apple/TablePro/Core/RowGeneration/RowGenerationColumnReader.swift index beb1cf185f..3c8a962a06 100644 --- a/apple/TablePro/Core/RowGeneration/RowGenerationColumnReader.swift +++ b/apple/TablePro/Core/RowGeneration/RowGenerationColumnReader.swift @@ -111,9 +111,9 @@ internal enum RowGenerationColumnReader { CrossEngineDefaultValue.strippingCast(value).uppercased().hasPrefix("NEXTVAL(") { return .assigned } - // SQLite's INTEGER PRIMARY KEY is the rowid: left out, it takes the next one. A WITHOUT ROWID table has none. - if family == .sqlite, hasRowID, primaryKey == [info.name], - info.dataType.trimmingCharacters(in: .whitespaces).uppercased() == "INTEGER" { + if SQLiteRowidAlias.isAlias( + column: info.name, declaredType: info.dataType, primaryKey: primaryKey, family: family, hasRowID: hasRowID + ) { return .assigned } return .none diff --git a/apple/TablePro/Core/SSH/LibSSH2ChannelFree.swift b/apple/TablePro/Core/SSH/LibSSH2ChannelFree.swift index 22bec799c0..bdcaa5d937 100644 --- a/apple/TablePro/Core/SSH/LibSSH2ChannelFree.swift +++ b/apple/TablePro/Core/SSH/LibSSH2ChannelFree.swift @@ -16,29 +16,43 @@ nonisolated internal enum LibSSH2ChannelFree { // The keep-alive's own limit: a packet still unsent by then holds every other send on the session. static let deadlineSeconds: TimeInterval = 30 - // Another relay can read the CLOSE reply off the shared socket and leave it quiet, so free runs - // again after every wait. An EOF that would block is sent again first: libssh2 keeps it as the - // session's next packet and turns every other send away until the same call sends it. + // Another relay can read the CLOSE reply off the shared socket and leave it quiet, so free runs again after + // every wait. The EOF goes first, on its own: channel_close would send it anyway, from a packet on its stack + // that a retry from another frame cannot finish (stackPacketReadiness), so it waits for room to go whole. A + // wait on another packet is bounded by the poll timeout, as only that packet's own call can finish it. static func run( channel: OpaquePointer, session: OpaquePointer, - sessionQueue: DispatchQueue, + pump: SSHSessionPump, transportFD: Int32, - sendEOF: Bool = false, deadline: Date = Date().addingTimeInterval(deadlineSeconds), isActive: @escaping () -> Bool ) -> SSHChannelFreeOutcome { - var eofPending = sendEOF + var eofSent = false + let turn = SSHSendTurn(pump: pump) + defer { turn.end() } return SSHChannelFreePump( attemptFree: { - sessionQueue.sync { - if eofPending { + let attempt: SSHChannelFreeAttempt = pump.perform { + if !eofSent { + switch stackPacketReadiness(session: session, transportFD: transportFD) { + case .socketFull: return .wouldBlock(.outbound) + case .sendPending: return .wouldBlock(.inbound) + case .ready, .gone: break + } guard libssh2_channel_send_eof(channel) != LIBSSH2_ERROR_EAGAIN else { return blocked(session) } - eofPending = false + eofSent = true } guard libssh2_channel_free(channel) == LIBSSH2_ERROR_EAGAIN else { return .finished } return blocked(session) } + // Past the EOF and the CLOSE the free only waits for the server's reply. + if case .wouldBlock(let directions) = attempt { + turn.waitsToSend(!eofSent || directions.contains(.outbound)) + } else { + turn.end() + } + return attempt }, isActive: isActive, deadline: deadline, diff --git a/apple/TablePro/Core/SSH/LibSSH2ChannelIO.swift b/apple/TablePro/Core/SSH/LibSSH2ChannelIO.swift index 5d6111c364..8d76a048c5 100644 --- a/apple/TablePro/Core/SSH/LibSSH2ChannelIO.swift +++ b/apple/TablePro/Core/SSH/LibSSH2ChannelIO.swift @@ -12,11 +12,10 @@ import Foundation import CLibSSH2 import TableProSSHTransport -// Every call goes through the serial sessionQueue: libssh2 is not thread-safe per session. -nonisolated internal struct LibSSH2ChannelIO: SSHChannelIO { +// libssh2 is not thread-safe per session: the session's SSHSessionPump makes every call on its queue. +nonisolated internal struct LibSSH2ChannelIO: SSHChannelIO, @unchecked Sendable { let channel: OpaquePointer let session: OpaquePointer - let sessionQueue: DispatchQueue // libssh2 hands a transport error (-43 on EOF, -13 on DISCONNECT, -12, a failed re-key) to every channel call; only these end just one channel. private static let channelScopedErrors: Set = [ @@ -45,20 +44,34 @@ nonisolated internal struct LibSSH2ChannelIO: SSHChannelIO { return channelScopedErrors.contains(returnCode) ? .closed : .transportFailed } + // A read after the server's EOF answers EAGAIN only while its window adjust waits to send (channel.c, + // ssh2_channel_read), and freeing the channel with that packet half sent leaves every later send refused. func read(into buffer: UnsafeMutablePointer, count: Int) -> ChannelReadResult { - let result = sessionQueue.sync { Int(tablepro_libssh2_channel_read(channel, buffer, count)) } - if result == Int(LIBSSH2_ERROR_EAGAIN), sessionQueue.sync(execute: { libssh2_channel_eof(channel) }) != 0 { + let result = Int(tablepro_libssh2_channel_read(channel, buffer, count)) + if result == Int(LIBSSH2_ERROR_EAGAIN), libssh2_channel_eof(channel) != 0, !blockDirections().contains(.outbound) { return .closed } return Self.readResult(result) } func write(_ buffer: UnsafePointer, count: Int) -> ChannelWriteResult { - Self.writeResult(sessionQueue.sync { Int(tablepro_libssh2_channel_write(channel, buffer, count)) }) + Self.writeResult(Int(tablepro_libssh2_channel_write(channel, buffer, count))) } func blockDirections() -> RelayDirections { - RelayDirections(libssh2BlockDirections: sessionQueue.sync { libssh2_session_block_directions(session) }) + RelayDirections(libssh2BlockDirections: libssh2_session_block_directions(session)) + } + + // read_avail counts the bytes another call already queued for this channel (channel.c, libssh2_channel_window_read_ex). + func hasQueuedInput() -> Bool { + var queued: UInt = 0 + _ = libssh2_channel_window_read_ex(channel, &queued, nil) + return queued > 0 || libssh2_channel_eof(channel) != 0 + } + + // ssh2_channel_write answers EAGAIN with no window before it builds a packet, and with window once the packet is built. + func hasSendWindow() -> Bool { + libssh2_channel_window_write_ex(channel, nil) > 0 } } diff --git a/apple/TablePro/Core/SSH/LibSSH2ForwardChannel.swift b/apple/TablePro/Core/SSH/LibSSH2ForwardChannel.swift index bc5ca4063c..19bb161e59 100644 --- a/apple/TablePro/Core/SSH/LibSSH2ForwardChannel.swift +++ b/apple/TablePro/Core/SSH/LibSSH2ForwardChannel.swift @@ -12,17 +12,28 @@ import Foundation import CLibSSH2 import TableProSSHTransport -/// Bridges `SSHForwardChannelOpening` to libssh2. Each attempt takes the session queue -/// only long enough to try the open and read back the error and block directions, so a -/// slow open never holds the queue against relays or keep-alive. +// Bridges `SSHForwardChannelOpening` to libssh2. Each attempt takes the session queue +// only long enough to try the open and read back the error and block directions, so a +// slow open never holds the queue against relays or keep-alive. nonisolated internal struct LibSSH2ForwardChannelOpener: SSHForwardChannelOpening { let session: OpaquePointer let destination: SSHForwardDestination let originPort: Int - let sessionQueue: DispatchQueue + let pump: SSHSessionPump + let turn: SSHSendTurn func attemptOpen() -> SSHForwardChannelAttempt { - sessionQueue.sync { + let attempt = openOnce() + if case .wouldBlock(let directions) = attempt { + turn.waitsToSend(directions.contains(.outbound)) + } else { + turn.end() + } + return attempt + } + + private func openOnce() -> SSHForwardChannelAttempt { + pump.perform { if let channel = LibSSH2ForwardChannel.open( session: session, destination: destination, @@ -41,9 +52,9 @@ nonisolated internal struct LibSSH2ForwardChannelOpener: SSHForwardChannelOpenin } } -/// Opens the channel that carries forwarded traffic to the destination. The two libssh2 -/// entry points behave identically to the caller: both return a channel that reads and -/// writes the same way, and both signal "not ready yet" with `LIBSSH2_ERROR_EAGAIN`. +// Opens the channel that carries forwarded traffic to the destination. The two libssh2 +// entry points behave identically to the caller: both return a channel that reads and +// writes the same way, and both signal "not ready yet" with `LIBSSH2_ERROR_EAGAIN`. nonisolated internal enum LibSSH2ForwardChannel { // Another open or relay can read this channel's reply into libssh2's queue and leave the socket // quiet, so a quiet wait is a retry and only the deadline ends an open. diff --git a/apple/TablePro/Core/SSH/LibSSH2JumpRelay.swift b/apple/TablePro/Core/SSH/LibSSH2JumpRelay.swift index cf66fe5429..83602c02d7 100644 --- a/apple/TablePro/Core/SSH/LibSSH2JumpRelay.swift +++ b/apple/TablePro/Core/SSH/LibSSH2JumpRelay.swift @@ -12,13 +12,15 @@ import Foundation import TableProSSHTransport nonisolated internal enum LibSSH2JumpRelay { - // The relay is the only thing that touches these once it starts, which the compiler cannot see + // The hop's pump is the only thing that touches these once it starts, which the compiler cannot see // through an OpaquePointer. nonisolated private struct Handles: @unchecked Sendable { let channel: OpaquePointer let session: OpaquePointer } + // The hop session carries one channel, relayed to `localFD`, which the pump closes. The fence finishes once + // the pump has stopped, so teardown frees the channel and session only after the pump's last libssh2 call. static func start( channel: OpaquePointer, localFD: Int32, @@ -26,36 +28,20 @@ nonisolated internal enum LibSSH2JumpRelay { session: OpaquePointer, sessionQueue: DispatchQueue ) -> SSHJumpRelayFence { - let relayQueue = DispatchQueue( - label: "com.TablePro.ssh.hop-relay", - qos: .utility - ) let handles = Handles(channel: channel, session: session) + let pump = SSHSessionPump(transportFD: transportFD, queue: sessionQueue) let fence = SSHJumpRelayFence { - shutdown(localFD, SHUT_RDWR) - shutdown(transportFD, SHUT_RDWR) + pump.stop() } - relayQueue.async { - defer { - fence.finish { - shutdown(localFD, SHUT_RDWR) - Darwin.close(localFD) - } - } - - let relay = SSHChannelRelay( - localFD: localFD, - transportFD: transportFD, - channelIO: LibSSH2ChannelIO( - channel: handles.channel, - session: handles.session, - sessionQueue: sessionQueue - ), - bufferSize: 32_768, - isActive: { fence.isActive } - ) - - _ = relay.run() + pump.whenStopped { + fence.finish() + } + pump.relay( + localFD: localFD, + channel: LibSSH2ChannelIO(channel: handles.channel, session: handles.session), + bufferSize: 32_768 + ) { _ in + pump.stop() } return fence } diff --git a/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift b/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift index e62fb7a0dc..5dfb190016 100644 --- a/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift +++ b/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift @@ -143,21 +143,28 @@ internal final class SFTPSerialSessionGate: @unchecked Sendable { } } -/// An authenticated SSH session carrying the SFTP subsystem and nothing else. -/// -/// A sibling of `LibSSH2Tunnel` rather than a mode of it: a tunnel owns a listening socket and an -/// accept loop, both meaningless here. Both types build their session through -/// `LibSSH2TunnelFactory.buildAuthenticatedChain`, so SFTP inherits password, public key, agent, -/// keyboard-interactive, jump hosts, TOTP and `~/.ssh/config` resolution without restating any of -/// it, and tears down through the same `cleanupChain`. -/// -/// libssh2 is not thread-safe per session, so every call runs on one serial queue. +// An authenticated SSH session carrying the SFTP subsystem and nothing else. +// +// A sibling of `LibSSH2Tunnel` rather than a mode of it: a tunnel owns a listening socket and its +// acceptor, both meaningless here. Both types build their session through +// `LibSSH2TunnelFactory.buildAuthenticatedChain`, so SFTP inherits password, public key, agent, +// keyboard-interactive, jump hosts, TOTP and `~/.ssh/config` resolution without restating any of +// it, and tears down through the same `cleanupChain`. +// +// libssh2 is not thread-safe per session, so every call runs on one serial queue. final class LibSSH2SFTPSession: @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SFTP") - /// Measured against OpenSSH 10.3: 32 KiB requests give ~94 MiB/s up and ~320 MiB/s down on - /// loopback while keeping each `libssh2_sftp_read` inside one SFTP packet. - private static let chunkSize = 32 * 1_024 + // libssh2_sftp_read keeps up to four times its buffer in flight (`max_read_ahead` in sftp.c, + // itself capped at 8 MiB) and returns once some bytes are in, so the buffer sets how much one + // round trip carries, not how long a read waits. 2 MiB reaches libssh2's cap; 32 KiB is a floor + // so a tiny or empty file still reads in a normal buffer. + private static let minimumChunkSize = 32 * 1_024 + private static let maximumChunkSize = 2 * 1_024 * 1_024 + + static func chunkSize(forFileSize fileSize: UInt64) -> Int { + Int(min(max(fileSize, UInt64(minimumChunkSize)), UInt64(maximumChunkSize))) + } private let chain: LibSSH2TunnelFactory.AuthenticatedChain private let sftp: OpaquePointer @@ -479,7 +486,7 @@ final class LibSSH2SFTPSession: @unchecked Sendable { } var hasher = SHA256() - var buffer = [UInt8](repeating: 0, count: Self.chunkSize) + var buffer = [UInt8](repeating: 0, count: Self.chunkSize(forFileSize: expected.size)) var received: UInt64 = 0 while true { diff --git a/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift b/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift index 5b8ca05346..43cf592a3d 100644 --- a/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift +++ b/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift @@ -10,9 +10,8 @@ import CLibSSH2 import TableProSSHTransport -/// Represents an active SSH tunnel backed by libssh2. -/// Each instance owns a TCP socket, libssh2 session, a local listening socket, -/// and the forwarding/keep-alive tasks. +// An SSH tunnel backed by libssh2. It owns the TCP socket, the session, the local listening socket +// with its acceptor, and the keep-alive task. internal final class LibSSH2Tunnel: @unchecked Sendable { let connectionId: UUID let localPort: Int @@ -28,27 +27,21 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { // Jump host chain (in connection order) private let jumpChain: [JumpHop] - private var forwardingTask: Task? + private var acceptor: SSHListenerAcceptor? private var keepAliveTask: Task? private let aliveLatch = TeardownLatch() - /// The relays still running, so teardown frees the session only once none of them can touch - /// it. A group rather than a collection of tasks: a relay leaves it by finishing, which is the - /// one thing a `[Task]` pruned on `isCancelled` never noticed, so a tunnel that had served - /// clients carried every one of them until it closed. It is also all a task was ever worth - /// here, since the relay runs on `relayQueue` outside the task's cancellation scope and stops - /// on `aliveLatch` rather than on `Task.isCancelled`. + // Each accepted client from accept until its channel is freed or given up, so teardown frees the + // session only once none of them can touch it. private let clientRelays = DispatchGroup() - /// Serial queue for all libssh2 calls on this tunnel's session. - /// libssh2 is not thread-safe per session, so every call must be serialized. - private let sessionQueue: DispatchQueue + // Every libssh2 call on the session, and the relays of its channels. + private let pump: SSHSessionPump - /// Concurrent queue for relay I/O (poll, send, recv — no libssh2 calls). - /// Individual libssh2 calls within each relay are dispatched to `sessionQueue`. + // Concurrent: channel opens, which wait on the transport between attempts. private let relayQueue: DispatchQueue - /// Dedicated queue for the accept loop (poll + accept only, no libssh2 calls). + // Serial: the acceptor's handler, which accepts and makes no libssh2 call. private let acceptQueue: DispatchQueue /// Callback invoked when the tunnel dies (keep-alive failure, etc.) @@ -73,9 +66,9 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { private static let relayBufferSize = 32_768 // 32KB - /// How long the accept loop waits per poll before rechecking `isRunning`. Small enough that - /// noticing a client the kernel already accepted costs a slim part of its connect budget. - internal static let acceptPollTimeoutMs: Int32 = 200 + private struct ChannelHandle: @unchecked Sendable { + let pointer: OpaquePointer + } init( connectionId: UUID, @@ -98,9 +91,9 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { self.timeoutEndpoint = timeoutEndpoint self.jumpChain = jumpChain self.createdAt = Date() - self.sessionQueue = DispatchQueue( - label: "com.TablePro.ssh.session.\(connectionId.uuidString)", - qos: .utility + self.pump = SSHSessionPump( + transportFD: socketFD, + queue: DispatchQueue(label: "com.TablePro.ssh.session.\(connectionId.uuidString)", qos: .utility) ) self.relayQueue = DispatchQueue( label: "com.TablePro.ssh.relay.\(connectionId.uuidString)", @@ -121,67 +114,48 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { // MARK: - Forwarding func startForwarding(destination: SSHForwardDestination) { - sessionQueue.sync { libssh2_session_set_blocking(session, 0) } + pump.perform { libssh2_session_set_blocking(session, 0) } - forwardingTask = Task.detached { [weak self] in - guard let self else { return } - - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.acceptQueue.async { [weak self] in - defer { continuation.resume() } - guard let self else { return } - - let target = destination.logDescription - - Self.logger.info( - "Forwarding started on port \(self.localPort) -> \(target)" - ) - - while self.isRunning { - guard let client = self.acceptClient() else { - if self.isRunning { - continue - } - break - } - - self.spawnClient( - clientFD: client, - budget: self.relayDeadlines.next(), - destination: destination - ) - } - - Self.logger.info("Forwarding loop ended for port \(self.localPort)") - } + // A handler already running when the latch is claimed may still spawn one relay; teardown + // waits for the cancel handler before the relay group, so it waits for that relay too. + let acceptor = SSHListenerAcceptor(listenFD: listenFD, queue: acceptQueue) { [weak self] client in + guard let self, self.isRunning else { + Darwin.close(client.descriptor) + return } + self.spawnClient( + clientFD: client.descriptor, + budget: self.relayDeadlines.next(), + destination: destination + ) } + self.acceptor = acceptor + acceptor.start() + + let target = destination.logDescription + Self.logger.info("Forwarding started on port \(self.localPort) -> \(target)") } // MARK: - Keep-Alive func startKeepAlive() { - sessionQueue.sync { libssh2_keepalive_config(session, 1, 30) } + pump.perform { LibSSH2KeepAlive.configure(session: session) } keepAliveTask = Task.detached { [weak self] in guard let self else { return } while !Task.isCancelled && self.isRunning { - let failed = await withCheckedContinuation { (continuation: CheckedContinuation) in - self.sessionQueue.async { - var secondsToNext: Int32 = 0 - let rc = libssh2_keepalive_send(self.session, &secondsToNext) - continuation.resume(returning: sshKeepAliveDidFail(rc)) - } + let outcome = await self.pump.performAsync { + LibSSH2KeepAlive.send(session: self.session, transportFD: self.socketFD) } - if failed { + if outcome.failed { Self.logger.warning("Keep-alive failed, marking tunnel dead") self.markDead() break } - try? await Task.sleep(for: .seconds(10)) + try? await Task.sleep(for: .seconds(outcome.secondsToNext)) } } } @@ -202,22 +176,19 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { aliveLatch.claim() } - /// Breaks every blocking wait first, then frees the session and the descriptors only once - /// every task that could still be polling them has exited. `shutdown` unblocks a poll at - /// once without releasing the descriptor number, which another thread would otherwise be - /// free to receive from the kernel and poll by mistake. `shutdown` does not wake a poll on a - /// listening socket on Darwin, so the accept loop ends on the latch instead, inside one - /// `acceptPollTimeoutMs`, and its descriptor closes once it has. - /// - /// The relays are waited on after the accept loop, not alongside it, because the accept loop - /// is the only thing that starts one: once it has ended, the group can only empty. + // `shutdown` wakes every open or free waiting on the transport but keeps the descriptor number, + // so no other socket can take it mid-poll. The listening socket closes only after the acceptor's + // cancel handler ran, which GCD runs once no accept handler is running. The clients are awaited + // after that, because the accept handler is the only thing that starts one: the group can only + // empty. The pump is awaited last, so no source of it can fire on the descriptors closed below. private func performTeardown() { - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() + pump.stop() shutdown(socketFD, SHUT_RDWR) - let sessionQueue = self.sessionQueue + let pump = self.pump let relayQueue = self.relayQueue let clientRelays = self.clientRelays let session = self.session @@ -225,18 +196,19 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { let listenFD = self.listenFD let jumpChain = self.jumpChain let connectionId = self.connectionId - let forwardingTask = self.forwardingTask + let acceptor = self.acceptor let keepAliveTask = self.keepAliveTask Task.detached { - await forwardingTask?.value + await acceptor?.waitForCancellation() await keepAliveTask?.value await withCheckedContinuation { (continuation: CheckedContinuation) in clientRelays.notify(queue: relayQueue) { continuation.resume() } } + await pump.waitUntilStopped() // Free before close: a closed number can go to another connection's socket, and this // session's DISCONNECT would land in it. - sessionQueue.sync { + pump.perform { libssh2_session_set_blocking(session, 1) tablepro_libssh2_session_disconnect(session, "Closing tunnel") libssh2_session_free(session) @@ -262,14 +234,14 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { } } - /// Synchronous cleanup for app termination, which waits for nothing because the process is - /// exiting. No descriptor is closed and the session is not freed: the relays and the accept - /// loop may still be polling them, and the OS reclaims every one of them anyway. + // App termination: waits for nothing, closes no descriptor and frees no session, since a relay or + // the accept handler may still be using them and the OS reclaims them all. func closeSync() { guard consumeAliveLatch() else { return } - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() + pump.stop() shutdown(socketFD, SHUT_RDWR) @@ -287,42 +259,23 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { onDeath?(connectionId) } - private func acceptClient() -> Int32? { - var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) - let pollResult = poll(&pollFD, 1, Self.acceptPollTimeoutMs) - - guard pollResult > 0, pollFD.revents & Int16(POLLIN) != 0 else { - return nil - } - - var clientAddr = sockaddr_in() - var addrLen = socklen_t(MemoryLayout.size) - - let clientFD = withUnsafeMutablePointer(to: &clientAddr) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - accept(listenFD, $0, &addrLen) - } - } - - guard clientFD >= 0 else { return nil } - return clientFD - } - - /// Open the channel and relay the client, off the accept loop so a slow open cannot - /// stall the next accept, and off `sessionQueue` between attempts so it cannot stall - /// the relays and keep-alive that share the session. + // Off the session queue between open attempts, so a slow open cannot stall the relays and the + // keep-alive that share the session. private func openAndRelay( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget, destination: SSHForwardDestination ) { let connectionDeadline = budget.deadline - let pump = SSHForwardChannelOpenPump( + let turn = SSHSendTurn(pump: pump) + defer { turn.end() } + let openPump = SSHForwardChannelOpenPump( opener: LibSSH2ForwardChannelOpener( session: session, destination: destination, originPort: localPort, - sessionQueue: sessionQueue + pump: pump, + turn: turn ), isActive: { [weak self] in guard self?.isRunning == true else { return false } @@ -343,7 +296,7 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { } ) - let outcome = pump.run() + let outcome = openPump.run() let relayTimedOut: Bool switch outcome { case .timedOut: @@ -360,6 +313,7 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { if case .opened(let channel) = outcome { Darwin.close(clientFD) freeChannel(channel) + clientRelays.leave() return } } @@ -375,8 +329,10 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { deadlineSeconds: deadlineSeconds ) handleChannelOpenOutcome(outcome, clientFD: clientFD) { channel in - runRelay(clientFD: clientFD, channel: channel, destination: destination) + relayChannel(clientFD: clientFD, channel: channel, destination: destination) } + if case .opened = outcome { return } + clientRelays.leave() } func consumeLastForwardFailure() -> (any Error)? { @@ -411,9 +367,8 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { } } - /// Opens the channel and relays one accepted client, off the accept loop so a slow - /// open cannot delay the next accept. The loop runs on `relayQueue` (concurrent); - /// individual libssh2 calls are dispatched to `sessionQueue` (serial) for thread safety. + // On `relayQueue`, off the accept handler, so a slow open cannot delay the next accept. The client + // stays in `clientRelays` until its relay ended and its channel was freed. private func spawnClient( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget, @@ -422,46 +377,42 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { let clientRelays = self.clientRelays clientRelays.enter() relayQueue.async { [weak self] in - defer { clientRelays.leave() } guard let self else { Darwin.close(clientFD) + clientRelays.leave() return } self.openAndRelay(clientFD: clientFD, budget: budget, destination: destination) } } - /// Blocking relay loop. Runs on `relayQueue`; libssh2 calls go through `sessionQueue`. - /// Every termination is logged, not just a hangup: a channel that opens only after the - /// database client already gave up ends as `.localClosed`, and staying silent about that - /// leaves no trace of a tunnel that worked but arrived too late. - private func runRelay(clientFD: Int32, channel: OpaquePointer, destination: SSHForwardDestination) { - let relay = SSHChannelRelay( + // Every termination is logged, not just a hangup: a channel that opens only after the database + // client already gave up ends as `.localClosed`, and staying silent about that leaves no trace + // of a tunnel that worked but arrived too late. + private func relayChannel(clientFD: Int32, channel: OpaquePointer, destination: SSHForwardDestination) { + let startedAt = Date() + let target = destination.logDescription + let clientRelays = self.clientRelays + let handle = ChannelHandle(pointer: channel) + pump.relay( localFD: clientFD, - transportFD: socketFD, - channelIO: LibSSH2ChannelIO(channel: channel, session: session, sessionQueue: sessionQueue), + channel: LibSSH2ChannelIO(channel: channel, session: session), bufferSize: Self.relayBufferSize, - isActive: { [weak self] in self?.isRunning ?? false }, byteCounter: byteCounter - ) - - let startedAt = Date() - let termination = relay.run() - let elapsed = Date().timeIntervalSince(startedAt) - - let target = destination.logDescription - Self.logger.debug( - "Relay to \(target) ended as \(String(describing: termination)) after \(elapsed, format: .fixed(precision: 1))s" - ) - - Darwin.close(clientFD) - guard self.isRunning else { return } + ) { [weak self] termination in + defer { clientRelays.leave() } + let elapsed = Date().timeIntervalSince(startedAt) + Self.logger.debug( + "Relay to \(target) ended as \(String(describing: termination)) after \(elapsed, format: .fixed(precision: 1))s" + ) + guard let self, self.isRunning else { return } - freeChannel(channel) + self.freeChannel(handle.pointer) - if termination == .transportHangup { - Self.logger.info("SSH transport hung up, marking tunnel dead for \(self.connectionId)") - markDead() + if termination == .transportHangup { + Self.logger.info("SSH transport hung up, marking tunnel dead for \(self.connectionId)") + self.markDead() + } } } @@ -469,7 +420,7 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { let outcome = LibSSH2ChannelFree.run( channel: channel, session: session, - sessionQueue: sessionQueue, + pump: pump, transportFD: socketFD, isActive: { [weak self] in self?.isRunning ?? false } ) diff --git a/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift b/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift index 048ccbac8d..d1f5c51706 100644 --- a/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift +++ b/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift @@ -20,13 +20,17 @@ internal struct SSHTunnelCredentials: Sendable { let keyboardInteractivePromptProvider: (any KeyboardInteractivePromptProvider)? } +internal struct LocalTunnelListener: Sendable { + let port: Int + let descriptor: Int32 +} + /// Creates fully-connected and authenticated SSH tunnels using libssh2. internal enum LibSSH2TunnelFactory { internal static let logger = Logger(subsystem: "com.TablePro", category: "LibSSH2TunnelFactory") - /// A single session opens more than one connection through the tunnel: the query - /// connection plus the metadata pool. A backlog that cannot hold them resets the - /// overflow before the accept loop reaches it. + // One session opens more than one connection through the tunnel: the query connection plus the + // metadata pool. A backlog that cannot hold them resets the overflow before the acceptor reaches it. private static let listenBacklogSize: Int32 = 16 // MARK: - Global Init @@ -40,12 +44,36 @@ internal enum LibSSH2TunnelFactory { // MARK: - Public + // Takes ownership of the listener: a failure closes it, success hands it to the tunnel. static func createTunnel( connectionId: UUID, config: SSHConfiguration, credentials: SSHTunnelCredentials, destination: SSHForwardDestination, - localPort: Int, + listener: LocalTunnelListener, + deadline: ConnectionDeadline + ) async throws -> LibSSH2Tunnel { + do { + return try await makeTunnel( + connectionId: connectionId, + config: config, + credentials: credentials, + destination: destination, + listener: listener, + deadline: deadline + ) + } catch { + Darwin.close(listener.descriptor) + throw error + } + } + + private static func makeTunnel( + connectionId: UUID, + config: SSHConfiguration, + credentials: SSHTunnelCredentials, + destination: SSHForwardDestination, + listener: LocalTunnelListener, deadline: ConnectionDeadline ) async throws -> LibSSH2Tunnel { _ = initialized @@ -74,21 +102,14 @@ internal enum LibSSH2TunnelFactory { deadline: deadline, attempt: attempt ) - - let listenFD = try bindListenSocket(port: localPort) - do { - try attempt.check(for: endpoint) - } catch { - Darwin.close(listenFD) - throw error - } + try attempt.check(for: endpoint) let tunnel = LibSSH2Tunnel( connectionId: connectionId, - localPort: localPort, + localPort: listener.port, session: chain.session, socketFD: chain.socketFD, - listenFD: listenFD, + listenFD: listener.descriptor, connectionDeadline: deadline, timeoutEndpoint: chain.timeoutEndpoint, jumpChain: chain.jumpHops.map { hop in @@ -103,7 +124,7 @@ internal enum LibSSH2TunnelFactory { ) logger.info( - "Tunnel created: \(config.host) -> 127.0.0.1:\(localPort) -> \(destination.logDescription)" + "Tunnel created: \(config.host) -> 127.0.0.1:\(listener.port) -> \(destination.logDescription)" ) attempt.finish() @@ -326,7 +347,7 @@ internal enum LibSSH2TunnelFactory { ) } } catch { - // Clean up nextSession and fds[1]; relay task owns fds[0] + // Clean up nextSession and fds[1]; the hop's pump owns fds[0] tablepro_libssh2_session_disconnect(nextSession, "Error") libssh2_session_free(nextSession) Darwin.close(fds[1]) @@ -589,8 +610,8 @@ internal enum LibSSH2TunnelFactory { libssh2_session_free(chain.session) Darwin.close(chain.socketFD) - // Wake every relay before waiting. A relay may be blocked in `poll` or a serialized - // libssh2 call, and neither a Swift Task cancellation nor freeing the handle is safe. + // Stop every hop before waiting. A hop's pump may be inside a libssh2 call, and freeing a + // handle under it is not safe. for hop in chain.jumpHops.reversed() { hop.relay.stop() shutdown(hop.socket, SHUT_RDWR) @@ -598,8 +619,8 @@ internal enum LibSSH2TunnelFactory { for hop in chain.jumpHops.reversed() { hop.relay.wait() } - // Only after the fence has drained can cleanup free the handles. The relay owns and closes - // its socketpair endpoint; this side owns the hop transport socket. + // Only after the fence has drained can cleanup free the handles. The hop's pump closes its + // socketpair endpoint; this side owns the hop transport socket. for hop in chain.jumpHops.reversed() { libssh2_channel_free(hop.channel) tablepro_libssh2_session_disconnect(hop.session, reason) @@ -1069,12 +1090,15 @@ internal enum LibSSH2TunnelFactory { probeQueue.sync { libssh2_session_set_blocking(session, 0) } defer { probeQueue.sync { libssh2_session_set_blocking(session, 1) } } + let probePump = SSHSessionPump(transportFD: socketFD, queue: probeQueue) + defer { probePump.stop() } let pump = SSHForwardChannelOpenPump( opener: LibSSH2ForwardChannelOpener( session: session, destination: destination, originPort: 0, - sessionQueue: probeQueue + pump: probePump, + turn: SSHSendTurn(pump: probePump) ), isActive: { !deadline.isExpired && !Task.isCancelled }, deadline: .distantFuture, @@ -1159,7 +1183,18 @@ internal enum LibSSH2TunnelFactory { // MARK: - Local Socket - private static func bindListenSocket(port: Int) throws -> Int32 { + // Bound before the SSH login, so a port collision never repeats the host key, passphrase or TOTP step. + static func bindLocalListener(candidates: [Int]) throws -> LocalTunnelListener { + for port in candidates { + if let descriptor = try bindListenSocket(port: port) { + return LocalTunnelListener(port: port, descriptor: descriptor) + } + } + throw SSHTunnelError.noAvailablePort + } + + // nil when the port is taken; any other failure throws, since the next port would fail the same way. + private static func bindListenSocket(port: Int) throws -> Int32? { let listenFD = socket(AF_INET, SOCK_STREAM, 0) guard listenFD >= 0 else { throw SSHTunnelError.tunnelCreationFailed("Failed to create listening socket") @@ -1180,12 +1215,18 @@ internal enum LibSSH2TunnelFactory { } guard bindResult == 0 else { + let bindError = errno Darwin.close(listenFD) - throw SSHTunnelError.tunnelCreationFailed("Port \(port) already in use") + if bindError == EADDRINUSE { return nil } + throw SSHTunnelError.tunnelCreationFailed( + "Could not bind local port \(port): \(String(cString: strerror(bindError)))" + ) } guard listen(listenFD, Self.listenBacklogSize) == 0 else { + let listenError = errno Darwin.close(listenFD) + if listenError == EADDRINUSE { return nil } throw SSHTunnelError.tunnelCreationFailed("Failed to listen on port \(port)") } return listenFD diff --git a/apple/TablePro/Core/SSH/SSHKeepAliveResult.swift b/apple/TablePro/Core/SSH/SSHKeepAliveResult.swift index d22ea11684..d272658354 100644 --- a/apple/TablePro/Core/SSH/SSHKeepAliveResult.swift +++ b/apple/TablePro/Core/SSH/SSHKeepAliveResult.swift @@ -7,10 +7,13 @@ // the iOS build is the check. // +import Foundation + import CLibSSH2 +import TableProSSHTransport -/// The libssh2 return codes the keep-alive has to tell apart, named here so callers and tests -/// can work with them without pulling the C module in. +// The libssh2 return codes the keep-alive has to tell apart, named here so callers and tests +// can work with them without pulling the C module in. nonisolated internal enum SSHKeepAliveCode { static let sent: Int32 = 0 static let wouldBlock: Int32 = LIBSSH2_ERROR_EAGAIN @@ -18,10 +21,66 @@ nonisolated internal enum SSHKeepAliveCode { static let socketDisconnect: Int32 = LIBSSH2_ERROR_SOCKET_DISCONNECT } -/// Whether a keep-alive return code means the tunnel is dead. The session is non-blocking for -/// the whole time it forwards, so libssh2 answers `LIBSSH2_ERROR_EAGAIN` when the send would -/// block on a transport that is busy carrying query results. That is a healthy tunnel under -/// load, not a dead one, and tearing it down drops every connection running through it. +// Whether a keep-alive return code means the tunnel is dead. The session is non-blocking for +// the whole time it forwards, so libssh2 answers `LIBSSH2_ERROR_EAGAIN` when the send would +// block on a transport that is busy carrying query results. That is a healthy tunnel under +// load, not a dead one, and tearing it down drops every connection running through it. nonisolated internal func sshKeepAliveDidFail(_ resultCode: Int32) -> Bool { resultCode != SSHKeepAliveCode.sent && resultCode != SSHKeepAliveCode.wouldBlock } + +nonisolated internal struct SSHKeepAliveOutcome: Sendable { + let failed: Bool + let secondsToNext: Int +} + +nonisolated internal enum StackPacketReadiness { + case ready + case socketFull + case sendPending + case gone +} + +// libssh2 keeps a packet it could not send whole until a call with the same buffer address sends the rest, and +// turns every other send away until then (transport_send_existing). The keep-alive and the channel EOF build +// their packets on their own stack, and a retry from another frame cannot finish them, so one that blocks stops +// every channel on the session (apple/scripts/probes/check-libssh2-keepalive-pending.sh). Such a packet goes out +// only when nothing else is waiting to send and the socket has room for it whole. +nonisolated internal func stackPacketReadiness(session: OpaquePointer, transportFD: Int32) -> StackPacketReadiness { + switch transportRoom(transportFD) { + case .gone: + return .gone + case .full: + return .socketFull + case .room: + let pending = libssh2_session_block_directions(session) & LIBSSH2_SESSION_BLOCK_OUTBOUND != 0 + return pending ? .sendPending : .ready + } +} + +nonisolated internal enum LibSSH2KeepAlive { + // Without a reply asked for: libssh2 never reads one, and each REQUEST_FAILURE sshd sends back stays at the + // head of the session's packet list for good (channel.c, ssh2_channel_read skips it without unlinking it). + static let wantsReply: Int32 = 0 + static let intervalSeconds: UInt32 = 30 + + private static let retrySeconds = 1 + + static func configure(session: OpaquePointer) { + libssh2_keepalive_config(session, wantsReply, intervalSeconds) + } + + // libssh2 sends at most once per interval and answers how long until the next one is due. + static func send(session: OpaquePointer, transportFD: Int32) -> SSHKeepAliveOutcome { + switch stackPacketReadiness(session: session, transportFD: transportFD) { + case .gone: + return SSHKeepAliveOutcome(failed: true, secondsToNext: retrySeconds) + case .socketFull, .sendPending: + return SSHKeepAliveOutcome(failed: false, secondsToNext: retrySeconds) + case .ready: + var secondsToNext: Int32 = 0 + let resultCode = libssh2_keepalive_send(session, &secondsToNext) + return SSHKeepAliveOutcome(failed: sshKeepAliveDidFail(resultCode), secondsToNext: max(1, Int(secondsToNext))) + } + } +} diff --git a/apple/TablePro/Core/SSH/SSHTunnelManager.swift b/apple/TablePro/Core/SSH/SSHTunnelManager.swift index 553ea82277..d0d48c91f3 100644 --- a/apple/TablePro/Core/SSH/SSHTunnelManager.swift +++ b/apple/TablePro/Core/SSH/SSHTunnelManager.swift @@ -18,17 +18,14 @@ actor SSHTunnelManager: TunnelManaging { private var tunnels: [UUID: LibSSH2Tunnel] = [:] private var creations = TunnelCreationGenerations() - private let portRangeStart = 60_000 - private let portRangeEnd = 65_000 + private let localPortCandidates: @Sendable () -> [Int] /// Static registry for synchronous termination during app shutdown private static let tunnelRegistry = OSAllocatedUnfairLock(initialState: [UUID: LibSSH2Tunnel]()) - /// Prevents App Nap from throttling SSH keepalive timers while tunnels are active. - /// Held as long as at least one tunnel exists; released when the last tunnel closes. - private var appNapActivity: NSObjectProtocol? - - private init() {} + init(localPortCandidates: @escaping @Sendable () -> [Int] = { Array(60_000...65_000).shuffled() }) { + self.localPortCandidates = localPortCandidates + } func createTunnel( connectionId: UUID, @@ -79,64 +76,50 @@ actor SSHTunnelManager: TunnelManaging { keyboardInteractivePromptProvider: nil ) - // Try ports until one works - for localPort in localPortCandidates() { - try deadline.check(endpoint: timeoutEndpoint) - do { - let creationTask = Task.detached { - try await LibSSH2TunnelFactory.createTunnel( - connectionId: connectionId, - config: config, - credentials: credentials, - destination: destination, - localPort: localPort, - deadline: deadline - ) - } - let tunnel = try await withTaskCancellationHandler { - try await creationTask.value - } onCancel: { - creationTask.cancel() - } - - do { - try deadline.check(endpoint: timeoutEndpoint) - } catch { - tunnel.close() - throw error - } - // A newer createTunnel or a close for this connection came while this one dialed, so it must not adopt. - guard creations.isNewest(generation, for: connectionId) else { - tunnel.close() - throw CancellationError() - } - - tunnel.onDeath = { [weak self] id in - Task { [weak self] in - await self?.handleTunnelDeath(connectionId: id) - } - } - - tunnels[connectionId] = tunnel - Self.tunnelRegistry.withLock { $0[connectionId] = tunnel } + try deadline.check(endpoint: timeoutEndpoint) + let listener = try LibSSH2TunnelFactory.bindLocalListener(candidates: localPortCandidates()) + let creationTask = Task.detached { + try await LibSSH2TunnelFactory.createTunnel( + connectionId: connectionId, + config: config, + credentials: credentials, + destination: destination, + listener: listener, + deadline: deadline + ) + } + let tunnel = try await withTaskCancellationHandler { + try await creationTask.value + } onCancel: { + creationTask.cancel() + } - tunnel.startForwarding(destination: destination) - tunnel.startKeepAlive() + do { + try deadline.check(endpoint: timeoutEndpoint) + } catch { + tunnel.close() + throw error + } + // A newer createTunnel or a close for this connection came while this one dialed, so it must not adopt. + guard creations.isNewest(generation, for: connectionId) else { + tunnel.close() + throw CancellationError() + } - updateAppNapState() - Self.logger.info("Tunnel created for \(connectionId) on local port \(localPort)") - return localPort - } catch let error as SSHTunnelError { - if case .tunnelCreationFailed(let msg) = error, - msg.contains("already in use") { - Self.logger.notice("Port \(localPort) in use, trying another") - continue - } - throw error + tunnel.onDeath = { [weak self] id in + Task { [weak self] in + await self?.handleTunnelDeath(connectionId: id) } } - throw SSHTunnelError.noAvailablePort + tunnels[connectionId] = tunnel + Self.tunnelRegistry.withLock { $0[connectionId] = tunnel } + + tunnel.startForwarding(destination: destination) + tunnel.startKeepAlive() + + Self.logger.info("Tunnel created for \(connectionId) on local port \(listener.port)") + return listener.port } // Also ends a creation still dialing, so nothing it builds is adopted after the close returned. @@ -148,7 +131,6 @@ actor SSHTunnelManager: TunnelManaging { private func discardTunnel(connectionId: UUID) async throws { guard let tunnel = tunnels.removeValue(forKey: connectionId) else { return } Self.tunnelRegistry.withLock { $0[connectionId] = nil } - updateAppNapState() tunnel.close() } @@ -157,7 +139,6 @@ actor SSHTunnelManager: TunnelManaging { let currentTunnels = tunnels tunnels.removeAll() Self.tunnelRegistry.withLock { $0.removeAll(); return } - updateAppNapState() for (_, tunnel) in currentTunnels { tunnel.close() @@ -222,39 +203,12 @@ actor SSHTunnelManager: TunnelManaging { tunnels[connectionId]?.consumeLastForwardFailure() } - /// Check if an error message indicates a local port bind failure - static func isLocalPortBindFailure(_ errorMessage: String) -> Bool { - errorMessage.lowercased().contains("already in use") - } - // MARK: - Private - private func localPortCandidates() -> [Int] { - Array(portRangeStart...portRangeEnd).shuffled() - } - private func handleTunnelDeath(connectionId: UUID) async { guard tunnels.removeValue(forKey: connectionId) != nil else { return } Self.tunnelRegistry.withLock { $0[connectionId] = nil } - updateAppNapState() Self.logger.warning("Tunnel died for connection \(connectionId)") await DatabaseManager.shared.handleSSHTunnelDied(connectionId: connectionId) } - - // MARK: - App Nap Prevention - - /// Acquires or releases an App Nap activity token based on whether tunnels exist. - private func updateAppNapState() { - if !tunnels.isEmpty && appNapActivity == nil { - appNapActivity = ProcessInfo.processInfo.beginActivity( - options: .userInitiatedAllowingIdleSystemSleep, - reason: "SSH tunnel keepalive requires timely execution" - ) - Self.logger.debug("App Nap prevention acquired") - } else if tunnels.isEmpty, let activity = appNapActivity { - ProcessInfo.processInfo.endActivity(activity) - appNapActivity = nil - Self.logger.debug("App Nap prevention released") - } - } } diff --git a/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift b/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift index 87b4c54dc8..90985b1650 100644 --- a/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift +++ b/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift @@ -589,6 +589,13 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { } for fk in workingForeignKeys where isStaged(.foreignKey(fk.id)) && fk.isValid { + // Create Table refuses this too; otherwise the server does, after a MySQL save's earlier DDL already applied. + guard fk.columns.count == fk.referencedColumns.count else { + validationErrors[.foreignKey(fk.id)] = String( + localized: "A foreign key needs as many referenced columns as it has columns." + ) + continue + } for columnName in fk.columns where !namesAColumn(columnName, in: columnNames) { validationErrors[.foreignKey(fk.id)] = String( format: String(localized: "Foreign key references a column that does not exist: %@"), columnName diff --git a/apple/TablePro/Core/SchemaTracking/StructureRebase.swift b/apple/TablePro/Core/SchemaTracking/StructureRebase.swift index 9134449a8f..c28c9b9da7 100644 --- a/apple/TablePro/Core/SchemaTracking/StructureRebase.swift +++ b/apple/TablePro/Core/SchemaTracking/StructureRebase.swift @@ -105,18 +105,35 @@ enum StructureRebase { let entityName = entity[keyPath: name] if !entityName.isEmpty, idsBeforeSave[entityName] == nil { idsBeforeSave[entityName] = entity.id } } + let earlierByID = Dictionary(earlier.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) + let loadedNames = Set(loaded.map { $0[keyPath: name] }) + func reads(_ entity: Entity, as held: Entity?) -> Bool { + guard let held else { return false } + var probe = entity + probe[keyPath: id] = held.id + return probe == held + } + // A deleted definition and another one that takes its name: the drop runs first, so the table still holds the + // deleted one while a rename into the name has not run, or while it reads as it did before the save. A rename + // whose old name another definition takes leaves no such evidence, and the renamed one is matched as before. + func stillHoldsDeleted(_ entity: Entity, deleted before: UUID, taker after: UUID) -> Bool { + if let renamedFrom = earlierByID[after]?[keyPath: name], renamedFrom != entity[keyPath: name] { + return idsAfterSave[renamedFrom] == nil && loadedNames.contains(renamedFrom) + } + return reads(entity, as: earlierByID[before]) + } var used = Set() let current: [Entity] = loaded.map { entity in let entityName = entity[keyPath: name] var candidates = [idsAfterSave[entityName], idsBeforeSave[entityName]].compactMap { $0 } + if let before = idsBeforeSave[entityName], deleted.contains(before), + let after = idsAfterSave[entityName], after != before, + stillHoldsDeleted(entity, deleted: before, taker: after) { + candidates = [before, after] + } if entityName.isEmpty { // A definition with no name is the one it was when nothing else about it changed. - candidates = earlier.filter { held in - var probe = entity - probe[keyPath: id] = held.id - return probe == held - } - .map(\.id) + candidates = earlier.filter { reads(entity, as: $0) }.map(\.id) } guard let kept = candidates.first(where: { !used.contains($0) }) else { return entity } used.insert(kept) @@ -126,7 +143,6 @@ enum StructureRebase { } let currentByID = Dictionary(current.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) - let earlierByID = Dictionary(earlier.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) var rebasedWorking: [Entity] = [] var changes: [(key: SchemaChangeIdentifier, change: SchemaChange)] = [] for entity in working { diff --git a/apple/TablePro/Core/Scripting/ScriptingError.swift b/apple/TablePro/Core/Scripting/ScriptingError.swift index 724b255795..a6901cd737 100644 --- a/apple/TablePro/Core/Scripting/ScriptingError.swift +++ b/apple/TablePro/Core/Scripting/ScriptingError.swift @@ -86,7 +86,7 @@ internal enum ScriptingError: LocalizedError { return .refused(String(localized: "The operation was cancelled.")) case .invalidArgument(let detail): return .badRequest(detail) - case .timeout(let detail, _), .expired(let detail), .dataSourceError(let detail): + case .timeout(let detail, _), .expired(let detail), .dataSourceError(let detail), .stoppedWhileRunning(let detail): return .failed(detail) } } diff --git a/apple/TablePro/Core/Services/ColumnType.swift b/apple/TablePro/Core/Services/ColumnType.swift index 63145b21f9..38c5696c28 100644 --- a/apple/TablePro/Core/Services/ColumnType.swift +++ b/apple/TablePro/Core/Services/ColumnType.swift @@ -8,17 +8,12 @@ import Foundation -/// Which editor an array's elements get. -/// -/// A Bool could only say that an array has a per-element editor, never which one, and the two are -/// not the same control: a scalar element fits a one-line field, a JSON element is a document and -/// needs the JSON viewer. +// Not a Bool: a scalar element fits a one-line field, a JSON element is a document and needs the JSON viewer. enum ArrayElementEditor: Equatable, Sendable { case scalar case json } -/// Represents the semantic type of a database column enum ColumnType: Equatable, Sendable { case text(rawType: String?) case integer(rawType: String?) @@ -35,7 +30,6 @@ enum ColumnType: Equatable, Sendable { case vector(rawType: String?) indirect case array(rawType: String?, element: ColumnType) - /// Raw database type name (e.g., "LONGTEXT", "VARCHAR(255)", "CLOB") var rawType: String? { switch self { case .text(let raw), .integer(let raw), .decimal(let raw), @@ -50,7 +44,6 @@ enum ColumnType: Equatable, Sendable { // MARK: - Display Properties - /// Human-readable name for this column type var displayName: String { switch self { case .text: return "Text" @@ -70,7 +63,6 @@ enum ColumnType: Equatable, Sendable { } } - /// Whether this type represents a JSON value that should use JSON editor var isJsonType: Bool { switch self { case .json: @@ -80,7 +72,6 @@ enum ColumnType: Equatable, Sendable { } } - /// Whether this type represents a date/time value that should be formatted var isDateType: Bool { switch self { case .date, .timestamp, .datetime: @@ -99,8 +90,6 @@ enum ColumnType: Equatable, Sendable { || base == "TIME WITH TIME ZONE" } - /// Whether this type represents long text that should use multi-line editor - /// Checks for TEXT, LONGTEXT, MEDIUMTEXT, TINYTEXT, CLOB types var isLongText: Bool { guard let raw = rawType?.uppercased() else { return false @@ -119,14 +108,12 @@ enum ColumnType: Equatable, Sendable { return false } - /// Whether this type is a very large text type that should be excluded from browse queries. - /// Only MEDIUMTEXT (16MB), LONGTEXT (4GB), and CLOB — not plain TEXT (65KB) or TINYTEXT (255B). + // Kept out of browse queries: MEDIUMTEXT runs to 16MB and LONGTEXT to 4GB, where TEXT stops at 65KB. var isVeryLongText: Bool { guard let raw = rawType?.uppercased() else { return false } return raw == "MEDIUMTEXT" || raw == "LONGTEXT" || raw == "CLOB" } - /// Whether this type is an enum column var isEnumType: Bool { switch self { case .enumType: @@ -136,7 +123,6 @@ enum ColumnType: Equatable, Sendable { } } - /// Whether this type is a SET column var isSetType: Bool { switch self { case .set: @@ -146,7 +132,7 @@ enum ColumnType: Equatable, Sendable { } } - /// True from the result set alone, before the allowed values arrive on the metadata round trip. + // True from the result set alone, before the allowed values arrive on the metadata round trip. var isEnumOrSetType: Bool { isEnumType || isSetType } @@ -165,13 +151,7 @@ enum ColumnType: Equatable, Sendable { } } - /// Whether `''` is a value of this type, so an emptied cell can be written as one. - /// - /// A date, number, boolean, JSON document, geometry or array has no empty value. Sent `''`, - /// MySQL without strict mode stores its zero value (`0000-00-00 00:00:00`, `0`), SQL Server - /// stores `1900-01-01` or `0`, SQLite keeps the text, and a strict MySQL or PostgreSQL refuses - /// the save. An empty SET is the empty set, and an enum has an empty value only when one of - /// its labels is the empty string. + // Sent '', non-strict MySQL stores a zero value, SQL Server 1900-01-01 or 0, SQLite keeps the text, and strict engines refuse it. var hasEmptyValue: Bool { switch self { case .text(let rawType): @@ -185,14 +165,13 @@ enum ColumnType: Equatable, Sendable { } } - /// Types the classifier files under `.text`, either by name or because nothing else matched, - /// that hold no empty string either. + // Filed under .text by the classifier, by name or as the fallback, yet none of them holds an empty string. private static let nonCharacterTextTypes: Set = [ "YEAR", "UUID", "TIMEUUID", "UNIQUEIDENTIFIER", "INET", "CIDR", "MACADDR", "MACADDR8", "IPV4", "IPV6", "INTERVAL" ] - /// A column the engine types as a UUID. Cassandra's `timeuuid` is one, though it takes version 1 only. + // Cassandra's `timeuuid` counts, though it takes version 1 only. var isUUIDType: Bool { guard case .text(let rawType) = self else { return false } return Self.uuidTypeNames.contains(Self.baseTypeName(rawType)) @@ -205,16 +184,26 @@ enum ColumnType: Equatable, Sendable { private static let uuidTypeNames: Set = ["UUID", "UNIQUEIDENTIFIER", "TIMEUUID"] - /// `Nullable(UUID)`, `year(4)` and `INTERVAL DAY TO SECOND` all name their type in the first word. + // `Nullable(UUID)`, `year(4)` and `INTERVAL DAY TO SECOND` all name their type in the first word. private static func baseTypeName(_ rawType: String?) -> String { guard var name = rawType?.uppercased() else { return "" } - while let wrapper = ["NULLABLE(", "LOWCARDINALITY("].first(where: { name.hasPrefix($0) }) { + while let wrapper = typeWrappers.first(where: { name.hasPrefix($0) }) { name.removeFirst(wrapper.count) } return String(name.prefix { $0.isLetter || $0.isNumber || $0 == "_" }) } - /// The element type of an array column, if this is one + // ClickHouse's `Nullable(String)` and `LowCardinality(Nullable(String))` are the type they wrap. + static func unwrappedTypeName(_ rawType: String?) -> String? { + guard var name = rawType?.trimmingCharacters(in: .whitespaces) else { return nil } + while name.hasSuffix(")"), let wrapper = typeWrappers.first(where: { name.uppercased().hasPrefix($0) }) { + name = String(name.dropFirst(wrapper.count).dropLast()).trimmingCharacters(in: .whitespaces) + } + return name + } + + private static let typeWrappers = ["NULLABLE(", "LOWCARDINALITY("] + var arrayElement: ColumnType? { switch self { case .array(_, let element): return element @@ -222,11 +211,7 @@ enum ColumnType: Equatable, Sendable { } } - /// The editor this array's elements get, or nil where they cannot be edited one at a time. - /// - /// A JSON element is included because PostgreSQL's array quoting round-trips it exactly: the - /// literal a `jsonb[]` cell carries parses to its elements and re-serializes byte for byte, - /// SQL NULL and JSON null included. Binary, spatial and nested arrays stay out. + // JSON elements qualify because PostgreSQL's array quoting round-trips a `jsonb[]` literal byte for byte, NULLs included. var arrayElementEditor: ArrayElementEditor? { guard let element = arrayElement else { return nil } switch element { @@ -239,7 +224,6 @@ enum ColumnType: Equatable, Sendable { } } - /// Whether this array's elements can be edited one at a time var supportsElementEditing: Bool { arrayElementEditor != nil } var isVectorType: Bool { @@ -257,7 +241,6 @@ enum ColumnType: Equatable, Sendable { return dimensions } - /// Compact lowercase badge label for sidebar var badgeLabel: String { switch self { case .boolean: return "bool" @@ -278,12 +261,7 @@ enum ColumnType: Equatable, Sendable { } } - /// The same type with the labels the catalog declared, reaching inside an array to its element. - /// - /// The classifier cannot know them: a column's labels arrive separately in - /// `TableRows.columnEnumValues`. Injecting them on the scalar cases alone left an `ENUM[]` - /// column's element editor with no vocabulary, so it offered free-form text where the grid - /// offers the declared labels. + // Reaches into arrays: setting the labels on the scalar cases alone left an `ENUM[]` element editor offering free text. func withAllowedValues(_ values: [String]) -> ColumnType { switch self { case .enumType(let rawType, _): @@ -297,7 +275,6 @@ enum ColumnType: Equatable, Sendable { } } - /// The allowed enum/set values, if known var enumValues: [String]? { switch self { case .enumType(_, let values), .set(_, let values): diff --git a/apple/TablePro/Core/Services/ColumnTypeClassifier.swift b/apple/TablePro/Core/Services/ColumnTypeClassifier.swift index cb2fb3502d..becc872c02 100644 --- a/apple/TablePro/Core/Services/ColumnTypeClassifier.swift +++ b/apple/TablePro/Core/Services/ColumnTypeClassifier.swift @@ -10,7 +10,7 @@ import Foundation struct ColumnTypeClassifier { - /// The engine's type family, when the caller knows it. Without one a `BIT` is a boolean. + // Without a family a `BIT` of any width is a boolean. let family: SQLTypeFamily? init(family: SQLTypeFamily? = nil) { @@ -22,10 +22,7 @@ struct ColumnTypeClassifier { if Self.isAngleBracketCompositeType(stripped) { return .json(rawType: rawTypeName) } - /// `extractBaseAndParams` splits at the first `(` and keeps nothing after the last `)`, so - /// a parameterized array's `[]` never reached the test below: `numeric(10,2)[]` classified - /// as `.decimal` and `bit(8)[]` as `.boolean`, which put a boolean dropdown on an array - /// column. PostgreSQL spells a domain's array this way through `format_type`. + // `extractBaseAndParams` drops what follows the last `)`, so `bit(8)[]` would read as its element. if stripped.hasSuffix("[]") { let elementRaw = String(stripped.dropLast(2)) guard !elementRaw.isEmpty else { return .text(rawType: rawTypeName) } @@ -34,8 +31,7 @@ struct ColumnTypeClassifier { let (base, params) = extractBaseAndParams(stripped) - /// The app's own spelling for an enum array, `ENUM[](mood)`, puts the `[]` on the base and - /// the labels in the parentheses, so it is the one array form the suffix test above misses. + // The app spells an enum array `ENUM[](mood)`, with the `[]` on the base, which the suffix test misses. if base.hasSuffix("[]") { let elementBase = String(base.dropLast(2)) guard !elementBase.isEmpty else { return .text(rawType: rawTypeName) } @@ -59,6 +55,11 @@ struct ColumnTypeClassifier { return .text(rawType: rawTypeName) } + // A MySQL `BIT(8)` reads back as the number it holds, such as 170; only one bit is a flag. + if upper == "BIT", family?.readsBitsAsDecimal == true, let width = params.flatMap({ Int($0) }), width > 1 { + return .integer(rawType: rawTypeName) + } + if let factory = Self.typeLookup[upper] { return factory(rawTypeName) } @@ -72,8 +73,7 @@ struct ColumnTypeClassifier { private static let caseSensitiveBytesTypeName = "BYTES" - /// Elasticsearch names an object array `nested` and a flat key-value object `flattened`, and - /// both hold JSON the grid should offer its viewer for. + // Elasticsearch's `nested` is an object array and `flattened` a key-value object, both JSON the grid offers its viewer for. private static let structuredTypeNames: Set = ["VARIANT", "OBJECT", "ARRAY", "NESTED", "FLATTENED"] private static func isAngleBracketCompositeType(_ value: String) -> Bool { @@ -95,8 +95,7 @@ struct ColumnTypeClassifier { return value } - /// MySQL's catalog spells a column `INT UNSIGNED` or `INT(10) UNSIGNED ZEROFILL`, while its - /// result metadata says `INT`. Without this the catalog spelling fell through to text. + // MySQL's catalog says `INT(10) UNSIGNED ZEROFILL` where its result metadata says `INT`; unstripped, it fell through to text. private func stripTrailingAttributes(_ value: String) -> String { var stripped = value while let attribute = Self.trailingAttributes.first(where: { stripped.uppercased().hasSuffix(" \($0)") }) { @@ -212,7 +211,7 @@ struct ColumnTypeClassifier { map[key] = { .json(rawType: $0) } } - /// Oracle's `LONG RAW` holds bytes, and no pattern arm below reads the name as binary. + // Oracle's `LONG RAW` holds bytes, and no pattern arm below reads the name as binary. for key in [ "BLOB", "BYTEA", "BINARY", "VARBINARY", "RAW", "IMAGE", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "LONG RAW" @@ -226,11 +225,7 @@ struct ColumnTypeClassifier { map["SET"] = { .set(rawType: $0, values: nil) } - /// GEOMCOLLECTION is MySQL 8.0.11's own spelling, which its catalog reports for a column - /// declared either way. GEO_POINT and GEO_SHAPE are Elasticsearch's, SDO_GEOMETRY Oracle's, - /// ST_GEOMETRY Teradata's, and RING ClickHouse's. Every one of them fell through to a - /// pattern arm before this: GEO_POINT reached `hasSuffix("INT")` and classified as an - /// integer, which drove its alignment, its sort comparator and its filter operators. + // Engine spellings that fell to a pattern arm before, where GEO_POINT matched `hasSuffix("INT")` and classified as an integer. for key in [ "GEOMETRY", "POINT", "LINESTRING", "POLYGON", "MULTIPOINT", "MULTILINESTRING", "MULTIPOLYGON", @@ -244,8 +239,7 @@ struct ColumnTypeClassifier { map[key] = { .vector(rawType: $0) } } - /// An Oracle `BFILE` cell is the `BFILENAME('DIR', 'file')` call that points at the file, never its bytes, - /// so it edits as text; the hex editor would write bytes the driver refuses for a BFILE. + // An Oracle `BFILE` cell is the `BFILENAME('DIR', 'file')` call, never its bytes, so it edits as text. for key in [ "TEXT", "VARCHAR", "CHAR", "NVARCHAR", "NCHAR", "NTEXT", "VARCHAR2", "NVARCHAR2", "CLOB", "NCLOB", "ROWID", "UROWID", "REF", "BFILE", diff --git a/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement+Teradata.swift b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement+Teradata.swift new file mode 100644 index 0000000000..f9f31f294c --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement+Teradata.swift @@ -0,0 +1,78 @@ +// +// AutocommitOnlyStatement+Teradata.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar +import TableProUtilities + +internal extension AutocommitOnlyStatement { + // Inside a transaction Teradata takes nothing after DDL but its end: failure 3932 in Teradata mode, error 3722 in + // ANSI. DDL is what the SQL Data Definition Language manual lists, less HELP and SHOW; the ANSI transaction rules + // name DATABASE and SET SESSION, and the Data Control Language manual puts GRANT, REVOKE and GIVE under the same + // rule. Unmeasured: there is no Teradata server here. + static func matchesTeradata(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) + guard let keyword = cursor.next()?.word else { return false } + if teradataDefinitionKeywords.contains(keyword) { return true } + let following = cursor.next()?.word ?? "" + switch keyword { + case "BEGIN", "END": + return teradataLoggingTargets.contains(following) + case "CHECKPOINT": + return following == "ISOLATED" + case "COLLECT": + return teradataStatisticsWords.contains(following) + case "DELETE": + return following == "DATABASE" || following == "USER" + case "SET": + guard following == "QUERY_BAND" else { return teradataSessionSettings.contains(following) } + return queryBandScope(&cursor) != "TRANSACTION" + default: + return false + } + } + + // DATABASE, SET SESSION DATABASE and its SS abbreviation move the session's default database. + static func movesDatabase(_ statement: String, family: TransactionEngineFamily, grammar: SQLLexicalGrammar) -> Bool { + guard family == .teradata else { return false } + var cursor = SQLTokenCursor(statement as NSString, grammar: grammar) + guard let keyword = cursor.next()?.word else { return false } + switch keyword { + case "DATABASE": + return true + case "SET": + guard cursor.next()?.word == "SESSION" else { return false } + return cursor.next()?.word == "DATABASE" + case "SS": + return cursor.next()?.word == "DATABASE" + default: + return false + } + } +} + +private extension AutocommitOnlyStatement { + // CT is the CREATE TABLE abbreviation the CREATE TABLE syntax diagram gives. + static let teradataDefinitionKeywords: Set = [ + "CREATE", "CT", "ALTER", "DROP", "RENAME", "REPLACE", "COMMENT", "MODIFY", "DATABASE", "FLUSH", "LOGGING", + "INCREMENTAL", "GRANT", "REVOKE", "GIVE" + ] + + static let teradataLoggingTargets: Set = ["LOGGING", "QUERY", "ISOLATED"] + + static let teradataStatisticsWords: Set = ["STATISTICS", "STATS", "STAT", "SUMMARY"] + + static let teradataSessionSettings: Set = ["SESSION", "ROLE", "TIME", "TRANSFORM"] + + // SET QUERY_BAND ... FOR TRANSACTION is DML and FOR SESSION is DDL, and the FOR clause follows the band. + static func queryBandScope(_ cursor: inout SQLTokenCursor) -> String? { + var scope: String? + while let token = cursor.next() { + guard token.word == "FOR", cursor.parenDepth == 0 else { continue } + scope = cursor.next()?.word + } + return scope + } +} diff --git a/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift index 5266aed1b5..dad6484b4e 100644 --- a/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift +++ b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift @@ -45,6 +45,8 @@ internal enum AutocommitOnlyStatement { return matchesDuckDB(statement, grammar: grammar) case .sqlServer: return matchesSQLServer(statement, grammar: grammar) + case .teradata: + return matchesTeradata(statement, grammar: grammar) case .oracle, .redis, .other: return false } diff --git a/apple/TablePro/Core/Services/Execution/BatchCommitStatement.swift b/apple/TablePro/Core/Services/Execution/BatchCommitStatement.swift index 5eec82effb..3fad4836c7 100644 --- a/apple/TablePro/Core/Services/Execution/BatchCommitStatement.swift +++ b/apple/TablePro/Core/Services/Execution/BatchCommitStatement.swift @@ -35,6 +35,8 @@ internal enum BatchCommitStatement { case "END": guard let follower = cursor.next()?.word else { return true } return transactionNouns.contains(follower) + case "ET" where grammar.contains(.transactionShortForms): + return cursor.next() == nil default: return false } diff --git a/apple/TablePro/Core/Services/Execution/BatchStatementRun.swift b/apple/TablePro/Core/Services/Execution/BatchStatementRun.swift index 3f3c01bb6d..186f91a400 100644 --- a/apple/TablePro/Core/Services/Execution/BatchStatementRun.swift +++ b/apple/TablePro/Core/Services/Execution/BatchStatementRun.swift @@ -75,15 +75,19 @@ internal enum BatchStatementRun { gate: BatchClaimGate, failureSQL: (Statement) -> String, isCommitPoint: (Statement) -> Bool, + endsItsTransaction: (Statement) -> Bool = { _ in false }, + closesTransaction: (Statement) -> Bool = { _ in false }, + movesDatabase: (Statement) -> Bool = { _ in false }, serverError: (Output) -> String? = { _ in nil }, replanned: @escaping @MainActor @Sendable (_ ranCount: Int) async -> [Statement]? = { _ in nil }, commitOutput: ServerOutputBox = ServerOutputBox(), execute: @escaping @MainActor @Sendable (Statement) async throws -> Output ) async -> BatchStatementOutcome { - let opensTransaction = plan.opensTransaction && driver.supportsTransactions - if opensTransaction { + let sentBegin = plan.sendsBegin && driver.supportsTransactions + if sentBegin { do { try await driver.beginTransaction(mode: mode) + if plan == .heldTransaction { ManualCommit.turnHold?.mark(true) } } catch { return .failed(results: [], failure: .transactionStart, errorDescription: error.localizedDescription) } @@ -92,13 +96,23 @@ internal enum BatchStatementRun { var results: [Output] = [] var queue = statements var position = 0 + var databaseMove: String? while position < queue.count { let statement = queue[position] guard !Task.isCancelled, gate.isCurrent() else { - await rollback(driver: driver, connectionId: connectionId, plan: plan, opensTransaction: opensTransaction) + await rollback(driver: driver, connectionId: connectionId, plan: plan, sentBegin: sentBegin) return .cancelled(results: plan.keepsExecutedStatements ? results : []) } - let ran = await runStatement( + let moves = movesDatabase(statement) + // `.unknown` also stands for a statement nobody asked about. + let heldBefore: PluginSessionTransactionState = moves ? await driver.heldSessionTransactionState() : .unknown + // A new run is pinned back to the tab's database, so the rest of this one would land in the wrong one. + if heldBefore.holdsUserTransaction { + return .failed( + results: results, failure: .databaseMoveAwaitsCommit(sql: failureSQL(statement)), errorDescription: "" + ) + } + var ran = await runStatement( statement, isCommitPoint: isCommitPoint(statement), driver: driver, @@ -106,14 +120,20 @@ internal enum BatchStatementRun { gate: gate, execute: execute ) + // An ANSI session holds the move in a transaction of its own, which the app's own switch commits the same way. + if heldBefore == .idle, case .committed(let result) = ran, + await driver.heldSessionTransactionState() == .inTransaction { + ran = await committingMove(result, driver: driver, connectionId: connectionId, gate: gate) + } + if moves, case .committed = ran { databaseMove = failureSQL(statement) } switch ran { case .stopped: - await rollback(driver: driver, connectionId: connectionId, plan: plan, opensTransaction: opensTransaction) + await rollback(driver: driver, connectionId: connectionId, plan: plan, sentBegin: sentBegin) return .cancelled(results: plan.keepsExecutedStatements ? results : []) case .committed(let result): results.append(result) if let errorDescription = serverError(result) { - await rollback(driver: driver, connectionId: connectionId, plan: plan, opensTransaction: opensTransaction) + await rollback(driver: driver, connectionId: connectionId, plan: plan, sentBegin: sentBegin) return .failed( results: results, failure: .batch(sql: failureSQL(statement)), @@ -127,7 +147,7 @@ internal enum BatchStatementRun { errorDescription: failure.errorDescription ) case .failed(let failure): - await rollback(driver: driver, connectionId: connectionId, plan: plan, opensTransaction: opensTransaction) + await rollback(driver: driver, connectionId: connectionId, plan: plan, sentBegin: sentBegin) return .failed( results: results, failure: .statement(sql: failureSQL(statement)), @@ -138,9 +158,18 @@ internal enum BatchStatementRun { if position < queue.count, let rest = await replanned(position) { queue = Array(queue.prefix(position)) + rest } + // The next statement would be refused, inside BT with the whole block rolled back, so the user commits first. + if position < queue.count, !Task.isCancelled, gate.isCurrent(), endsItsTransaction(statement), + !closesTransaction(queue[position]), await driver.heldSessionTransactionState() == .inTransaction { + return .failed( + results: results, + failure: .awaitsCommit(sql: failureSQL(statement), databaseMove: databaseMove), + errorDescription: "" + ) + } } - guard opensTransaction else { return .completed(results: results) } + guard plan.commitsAtEnd, sentBegin else { return .completed(results: results) } return await commit( results: results, driver: driver, connectionId: connectionId, gate: gate, plan: plan, output: commitOutput ) @@ -177,6 +206,29 @@ internal enum BatchStatementRun { ) { @Sendable in try await execute(statement) } } + // Protected like a script's own COMMIT, and the run carries on after it. + private static func committingMove( + _ result: Output, + driver: DatabaseDriver, + connectionId: UUID, + gate: BatchClaimGate + ) async -> BatchCommitOutcome { + let committed = await BatchCommitPoint.run( + driver: driver, + connectionId: connectionId, + gate: gate, + exit: .resumes + ) { @Sendable in try await driver.commitTransaction() } + switch committed { + case .stopped: + return .stopped + case .committed: + return .committed(result) + case .failed(let failure): + return .failed(failure) + } + } + /// The app's own commit, and the last thing the run does. The mark it takes is held until the /// claim settles rather than released here: between the server's answer and the settle there is /// no statement left to stop, and a Stop landing in that gap would drop the results of work the @@ -198,7 +250,7 @@ internal enum BatchStatementRun { switch committed { case .stopped: - await rollback(driver: driver, connectionId: connectionId, plan: plan, opensTransaction: true) + await rollback(driver: driver, connectionId: connectionId, plan: plan, sentBegin: true) return .cancelled(results: plan.keepsExecutedStatements ? results : []) case .committed: output.store(await ServerOutputCapture.drain(driver)) @@ -211,7 +263,7 @@ internal enum BatchStatementRun { results: results, failure: .commitOutcomeUnknown, errorDescription: failure.errorDescription ) } - await rollback(driver: driver, connectionId: connectionId, plan: plan, opensTransaction: true) + await rollback(driver: driver, connectionId: connectionId, plan: plan, sentBegin: true) return .failed(results: results, failure: .commit, errorDescription: failure.errorDescription) } } @@ -229,15 +281,16 @@ internal enum BatchStatementRun { driver: DatabaseDriver, connectionId: UUID, plan: BatchTransactionPlan, - opensTransaction: Bool + sentBegin: Bool ) async { guard plan.rollsBackAfterStop, driver.supportsTransactions else { return } let token = DatabaseManager.shared.beginProtectedWrite(on: driver, for: connectionId) defer { DatabaseManager.shared.endProtectedWrite(token, for: connectionId) } do { try await TaskCancellationShield.run { @Sendable in try await driver.rollbackTransaction() } + if plan == .heldTransaction { ManualCommit.turnHold?.mark(false) } } catch { - guard opensTransaction else { + guard sentBegin else { batchLog.debug("No open script transaction to roll back: \(error.publicLogShape, privacy: .public)") return } diff --git a/apple/TablePro/Core/Services/Execution/BatchTransactionPlan.swift b/apple/TablePro/Core/Services/Execution/BatchTransactionPlan.swift index cb9d4df71f..13f736b9f3 100644 --- a/apple/TablePro/Core/Services/Execution/BatchTransactionPlan.swift +++ b/apple/TablePro/Core/Services/Execution/BatchTransactionPlan.swift @@ -13,14 +13,17 @@ import TableProPluginKit /// `PRAGMA foreign_keys`. Such a batch opens nothing, commits nothing, and rolls nothing back, /// because there is no transaction of its own to roll back to. /// -/// `.sessionTransaction` is the answer the text cannot reach: it comes from the driver, once the -/// lease is held, through ``joining(_:)``. +/// `.sessionTransaction` and `.heldTransaction` are the answers the text cannot reach: they come +/// from the driver and the connection's commit mode, once the lease is held, through +/// ``joining(_:commitMode:)``. /// /// The switches over this are exhaustive on purpose. No case may inherit `.appTransaction`'s begin /// or `.scriptTransaction`'s rollback by falling into a `default:` arm. internal enum BatchTransactionPlan: Equatable, Sendable { /// The app opens the transaction, commits it, and rolls it back on a failure or a Stop. case appTransaction + // Manual Commit: the app opens the transaction and leaves it for the user to commit; a failure or a Stop rolls it back. + case heldTransaction /// The script opens its own transaction. The app opens none and still rolls back what a failed /// script left open. case scriptTransaction @@ -31,24 +34,43 @@ internal enum BatchTransactionPlan: Equatable, Sendable { /// ending in `COMMIT` commits. case sessionTransaction - internal var opensTransaction: Bool { + internal var sendsBegin: Bool { switch self { - case .appTransaction: + case .appTransaction, .heldTransaction: return true case .scriptTransaction, .autocommit, .sessionTransaction: return false } } + internal var commitsAtEnd: Bool { + switch self { + case .appTransaction: + return true + case .heldTransaction, .scriptTransaction, .autocommit, .sessionTransaction: + return false + } + } + internal var rollsBackAfterStop: Bool { switch self { - case .appTransaction, .scriptTransaction: + case .appTransaction, .heldTransaction, .scriptTransaction: return true case .autocommit, .sessionTransaction: return false } } + // What the session holds once the run ends is the user's to see: the transaction is still open. + internal var readsSessionEndState: Bool { + switch self { + case .heldTransaction, .sessionTransaction: + return true + case .appTransaction, .scriptTransaction, .autocommit: + return false + } + } + /// Whether the statements that ran before a failure or a Stop stay in place. The failure banner /// says so, and a stopped run keeps their results and their history rather than dropping them. internal var keepsExecutedStatements: Bool { @@ -65,8 +87,9 @@ internal enum BatchTransactionPlan: Equatable, Sendable { /// A session that answers `.unknown` keeps the wrap for a plain batch, but a self-managed /// script stops being rolled back: the transaction its text left open may predate the run, and /// rolling that back discards work the run never did. - internal func joining(_ session: PluginSessionTransactionState) -> BatchTransactionPlan { + internal func joining(_ session: PluginSessionTransactionState, commitMode: ConnectionCommitMode) -> BatchTransactionPlan { guard session.permitsAppTransaction else { return .sessionTransaction } + if self == .appTransaction, commitMode == .manual, session == .idle { return .heldTransaction } guard session == .unknown, self == .scriptTransaction else { return self } return .sessionTransaction } diff --git a/apple/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift b/apple/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift index 6ee9a857c5..b53e8cfb5f 100644 --- a/apple/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift +++ b/apple/TablePro/Core/Services/Execution/BatchTransactionPolicy.swift @@ -14,7 +14,8 @@ import TableProUtilities /// Three answers, in precedence order. A script that opens a transaction of its own or turns the /// commit mode off manages its own, so the app opens none. A script holding a statement the engine /// refuses inside a transaction block runs in autocommit. Everything else is wrapped, which is what -/// gives a failed batch its rollback. +/// gives a failed batch its rollback, except at Read-Only, where there is no write to roll back and +/// each statement runs in a read-only transaction of its own, which a wrap would refuse. /// /// An engine the app cannot open a transaction on at all /// (``TransactionEngineFamily/wrapsBatchInTransaction``) never reaches the SQL rules, because they @@ -30,11 +31,12 @@ internal enum BatchTransactionPolicy { internal static func plan( for statements: [String], databaseType: DatabaseType, - grammar: SQLLexicalGrammar + grammar: SQLLexicalGrammar, + safeModeIsReadOnly: Bool = false ) -> BatchTransactionPlan { let family = TransactionEngineFamily.of(databaseType) guard family.wrapsBatchInTransaction else { return queuedBlockPlan(for: statements, grammar: grammar) } - var runsInAutocommit = false + var runsInAutocommit = safeModeIsReadOnly var holdsASavepoint = false for statement in statements { let text = statement as NSString @@ -79,6 +81,8 @@ internal enum BatchTransactionPolicy { return SqlBlockStructure.beginStartsTransaction(followedBy: cursor.next()?.word) case "START": return cursor.next()?.word == "TRANSACTION" + case "BT": + return grammar.contains(.transactionShortForms) case "XA": let following = cursor.next()?.word return following == "START" || following == "BEGIN" diff --git a/apple/TablePro/Core/Services/Execution/CopyDataPieces.swift b/apple/TablePro/Core/Services/Execution/CopyDataPieces.swift new file mode 100644 index 0000000000..86300eea35 --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/CopyDataPieces.swift @@ -0,0 +1,26 @@ +// +// CopyDataPieces.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +enum CopyDataPieces { + static let pieceSize = 64 * 1_024 + + // A piece may end inside a row, even inside a character: the server reads a copy's data as one byte stream. + static func reading(_ text: String) -> PluginCopyData { + let data = Data(text.utf8) + let offset = OSAllocatedUnfairLock(initialState: data.startIndex) + return PluginCopyData { + offset.withLock { start in + guard start < data.endIndex else { return nil } + let end = min(start + Self.pieceSize, data.endIndex) + defer { start = end } + return data.subdata(in: start.. SafeModeLevel private let forcesWriteResolver: @Sendable (DatabaseType) async -> Bool private let connectionNameResolver: @Sendable (UUID) async -> String? + private let readOnlyFunctionPolicyResolver: @Sendable (UUID, DatabaseType) async -> ReadOnlyFunctionPolicy + private let bareNameCallResolver: @Sendable (OperationRequest, SQLBareNameReading) async -> Bool private let auditLog: any ExecutionAuditLogging /// The end of a sentence macOS writes, not a sentence of its own. `LAContext` shows the reason @@ -29,6 +31,12 @@ internal actor DefaultExecutionGate: ExecutionGate { safeModeLevelResolver: @escaping @Sendable (UUID) async -> SafeModeLevel, forcesWriteResolver: @escaping @Sendable (DatabaseType) async -> Bool, connectionNameResolver: @escaping @Sendable (UUID) async -> String? = { _ in nil }, + readOnlyFunctionPolicyResolver: @escaping @Sendable (UUID, DatabaseType) async -> ReadOnlyFunctionPolicy = { _, _ in + ReadOnlyFunctionPolicy() + }, + bareNameCallResolver: @escaping @Sendable (OperationRequest, SQLBareNameReading) async -> Bool = { _, reading in + reading.callsOutright + }, auditLog: any ExecutionAuditLogging = ExecutionAuditLog.shared ) { self.confirming = confirming @@ -36,6 +44,8 @@ internal actor DefaultExecutionGate: ExecutionGate { self.safeModeLevelResolver = safeModeLevelResolver self.forcesWriteResolver = forcesWriteResolver self.connectionNameResolver = connectionNameResolver + self.readOnlyFunctionPolicyResolver = readOnlyFunctionPolicyResolver + self.bareNameCallResolver = bareNameCallResolver self.auditLog = auditLog } @@ -74,6 +84,11 @@ internal actor DefaultExecutionGate: ExecutionGate { )) } + if level.blocksAllWrites, request.kind != .metadataRead, let sql = request.sql, + let refusal = await readOnlyFunctionRefusal(request, sql: sql) { + return refusal + } + /// Narrower than `effectiveWrite`, which is true for every statement on a driver that /// cannot be opened read-only. A caller asking to confirm its writes means the ones that /// actually write, not every `GET` sent to Redis. @@ -129,6 +144,33 @@ internal actor DefaultExecutionGate: ExecutionGate { ) } + // The classifier tiers `SELECT purge_orders()` as a read. Where the driver cannot run it in the engine's read-only + // mode and a function on the engine can write, a call off the built-in list is one the gate cannot tell from a + // write, so Read-Only refuses it like one, whatever the caller was cleared for. Oracle and Dameng also call a + // function named bare, as in `SELECT purge_orders FROM dual`, which the session's catalog tells from a column. + // Nil runs it. + private func readOnlyFunctionRefusal(_ request: OperationRequest, sql: String) async -> OperationDecision? { + let policy = await readOnlyFunctionPolicyResolver(request.connectionId, request.databaseType) + guard policy.functionsCanWrite else { return nil } + var callsFunction = ReadOnlyFunctionCalls.callsUnlistedFunction(sql, databaseType: request.databaseType) + if !callsFunction, let reading = ReadOnlyFunctionCalls.bareNameReading(sql, databaseType: request.databaseType), + !reading.namesNothing { + callsFunction = await bareNameCallResolver(request, reading) + } + guard callsFunction else { return nil } + guard let level = policy.levelThatRunsIt else { + return .denied(reason: String( + localized: "Cannot run this query: it calls a function that can change data, and Safe Mode is held at Read-Only for this connection." + )) + } + return .denied(reason: String( + format: String( + localized: "Cannot run this query at Read-Only: it calls a function that can change data. Switch Safe Mode to %@ to run it." + ), + level.displayName + )) + } + private func resolveEffectiveWrite(_ request: OperationRequest, tier: QueryTier?) async -> Bool { if request.kind == .metadataRead { return false diff --git a/apple/TablePro/Core/Services/Execution/ExecutableBatch.swift b/apple/TablePro/Core/Services/Execution/ExecutableBatch.swift index eacca11002..e9e675964a 100644 --- a/apple/TablePro/Core/Services/Execution/ExecutableBatch.swift +++ b/apple/TablePro/Core/Services/Execution/ExecutableBatch.swift @@ -72,6 +72,9 @@ enum QueryExecutionRoute { /// /// A driver that cannot send batches runs statement by statement, and a `GO n` it cannot honour is refused rather /// than run once: expanding a count up to `Int32.max` into statements would stall the app before any of them ran. + /// + /// A run holding a `COPY ... FROM STDIN`'s rows goes statement by statement, the one path that sends them: the + /// single path streams rows into the grid, and a batch is sent as text. static func resolve( _ batches: [ExecutableBatch], sendsBatchesWhole: Bool, @@ -79,9 +82,10 @@ enum QueryExecutionRoute { ) -> QueryExecutionRoute? { let statements = batches.flatMap(\.statements) guard let first = statements.first else { return nil } - guard sendsBatchesWhole else { + let carriesCopyData = statements.contains { $0.copyData != nil } + guard sendsBatchesWhole, !carriesCopyData else { guard batches.allSatisfy({ $0.repeatCount == 1 }) else { return .needsBatchDriver } - return statements.count == 1 ? .single(first) : .statements(statements) + return statements.count == 1 && !carriesCopyData ? .single(first) : .statements(statements) } if statements.count == 1, batches.first?.repeatCount == 1, isPlainQuery(first.sql) { return .single(first) diff --git a/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift b/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift index 15a906ccee..d0fd50ad1b 100644 --- a/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift +++ b/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift @@ -17,27 +17,93 @@ internal enum ExecutionAuditVerification: Equatable, Sendable { case broken(atSequence: Int) } -/// Append-only log of every authorization decision, written from inside `ExecutionGate`. -/// -/// An actor rather than a lock: the gate is called from many tasks, and the sequence number and the -/// previous hash have to be read and advanced together or two concurrent decisions produce two -/// records claiming the same position. +// The file stays the array older builds read and rewrite, so records go in over its closing bracket. internal actor ExecutionAuditLog: ExecutionAuditLogging { internal static let shared = ExecutionAuditLog() private static let logger = Logger(subsystem: "com.TablePro", category: "ExecutionAudit") + private static let tailWindow: UInt64 = 65_536 + private static let openBracket = UInt8(ascii: "[") + private static let closeBracket = UInt8(ascii: "]") + private static let openBrace = UInt8(ascii: "{") + private static let closeBrace = UInt8(ascii: "}") + private static let comma = UInt8(ascii: ",") + private static let maxRecordLength = 4_096 + + // What the file holds: its last record's link, and where its closing bracket sits. + internal struct ChainHead: Equatable, Sendable { + let nextSequence: Int + let lastHash: String + let closingBracketOffset: UInt64 + let fileSize: UInt64 + } + + private enum Disk { + case noRecords + case chain(ChainHead) + case unavailable + } + + // A decision before it has a place in the chain; it is linked to the head read when it is written. + private struct Draft: Sendable { + let recordedAt: Date + let connectionId: UUID + let kind: String + let caller: String + let outcome: ExecutionAuditRecord.Outcome + let effectiveWrite: Bool + let statementDigest: String? + + func linked(sequence: Int, previousHash: String) -> ExecutionAuditRecord { + ExecutionAuditRecord( + sequence: sequence, + recordedAt: recordedAt, + connectionId: connectionId, + kind: kind, + caller: caller, + outcome: outcome, + effectiveWrite: effectiveWrite, + statementDigest: statementDigest, + previousHash: previousHash + ) + } + + func isWritten(as record: ExecutionAuditRecord) -> Bool { + record.recordedAt == recordedAt && record.connectionId == connectionId && record.kind == kind + && record.caller == caller && record.outcome == outcome && record.effectiveWrite == effectiveWrite + && record.statementDigest == statementDigest + } + } + + // Identifies the file this actor last wrote, so a file replaced or edited since is read again. + private struct FileStamp: Equatable { + let inode: UInt64 + let size: UInt64 + let modifiedSeconds: Int + let modifiedNanoseconds: Int + + init(_ info: stat) { + inode = UInt64(info.st_ino) + size = UInt64(info.st_size) + modifiedSeconds = info.st_mtimespec.tv_sec + modifiedNanoseconds = info.st_mtimespec.tv_nsec + } + } private let fileURL: URL - private var records: [ExecutionAuditRecord] = [] - private var loaded = false + private let retryInterval: Duration + private var head: ChainHead? + private var stamp: FileStamp? + // Decided but not on disk yet, oldest first. + private var pending: [Draft] = [] + private var retryAfter: ContinuousClock.Instant? - internal init(fileURL: URL = ExecutionAuditLog.defaultFileURL()) { + internal init(fileURL: URL = ExecutionAuditLog.defaultFileURL(), retryInterval: Duration = .seconds(5)) { self.fileURL = fileURL + self.retryInterval = retryInterval } - /// Resolved through `AppStorageEnvironment` like every other store, so a UI test appends to its - /// own sandbox. The chain is hash linked, which makes a foreign entry indistinguishable from a - /// real one rather than merely noise. + // Inside the storage environment, so a UI test appends to its own sandbox and never to the real chain. internal static func defaultFileURL() -> URL { let directory = AppStorageEnvironment.shared.supportDirectory try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -45,8 +111,6 @@ internal actor ExecutionAuditLog: ExecutionAuditLogging { } internal func record(request: OperationRequest, decision: OperationDecision) async { - load() - let outcome: ExecutionAuditRecord.Outcome let effectiveWrite: Bool switch decision { @@ -58,23 +122,21 @@ internal actor ExecutionAuditLog: ExecutionAuditLogging { effectiveWrite = false } - let record = ExecutionAuditRecord( - sequence: records.count, + pending.append(Draft( recordedAt: Date(), connectionId: request.connectionId, kind: String(describing: request.kind), caller: Self.describe(request.caller), outcome: outcome, effectiveWrite: effectiveWrite, - statementDigest: request.sql.map(ExecutionAuditRecord.sha256), - previousHash: records.last?.hash ?? ExecutionAuditRecord.genesisHash - ) - records.append(record) - persist() + statementDigest: request.sql.map(ExecutionAuditRecord.sha256) + )) + // A failure keeps the decision in `pending`; a disk that stays full is not retried on every authorization. + if let retryAfter, ContinuousClock.now < retryAfter { return } + retryAfter = writePending() ? nil : ContinuousClock.now + retryInterval } - /// A label, not the payload. An MCP client's label and an AI session id are caller-supplied and - /// can carry anything, so only the channel is recorded. + // A label, not the payload: an MCP client's label and an AI session id are caller-supplied and can carry anything. private static func describe(_ caller: OperationCaller) -> String { switch caller { case .userInterface: "userInterface" @@ -88,37 +150,317 @@ internal actor ExecutionAuditLog: ExecutionAuditLogging { } internal func entries() -> [ExecutionAuditRecord] { - load() - return records + (try? readAllRecords()) ?? [] } internal func verify() -> ExecutionAuditVerification { - load() + guard FileManager.default.fileExists(atPath: fileURL.path(percentEncoded: false)) else { + return .intact(count: 0) + } + guard let records = try? readAllRecords() else { return .broken(atSequence: 0) } if let broken = ExecutionAuditRecord.firstBrokenSequence(in: records) { return .broken(atSequence: broken) } return .intact(count: records.count) } - private func load() { - guard !loaded else { return } - loaded = true - guard let data = try? Data(contentsOf: fileURL) else { return } + private func readAllRecords() throws -> [ExecutionAuditRecord] { + guard let data = try? Data(contentsOf: fileURL) else { return [] } + return try JSONDecoder().decode([ExecutionAuditRecord].self, from: data) + } + + // MARK: - Appending + + private func writePending() -> Bool { + guard FileManager.default.fileExists(atPath: fileURL.path(percentEncoded: false)) else { + return startChain() + } + let handle: FileHandle + do { + handle = try FileHandle(forUpdating: fileURL) + } catch { + Self.logger.error("Execution audit log could not be opened: \(error.publicLogShape, privacy: .public)") + return false + } + defer { try? handle.close() } + + switch disk(through: handle) { + case .unavailable: + return false + case .noRecords: + return startChain() + case .chain(let current): + return append(after: current, through: handle) + } + } + + private func linkPending(after current: ChainHead?) -> [ExecutionAuditRecord] { + var sequence = current?.nextSequence ?? 0 + var previousHash = current?.lastHash ?? ExecutionAuditRecord.genesisHash + return pending.map { draft in + let record = draft.linked(sequence: sequence, previousHash: previousHash) + sequence += 1 + previousHash = record.hash + return record + } + } + + private func disk(through handle: FileHandle) -> Disk { do { - records = try JSONDecoder().decode([ExecutionAuditRecord].self, from: data) + let current = try Self.stamp(of: handle) + guard current.size > 0 else { return .noRecords } + if let head, current == stamp { + return .chain(head) + } + stamp = nil + head = nil + let read: (head: ChainHead, last: ExecutionAuditRecord?) + if let found = try Self.readHead(from: handle, fileSize: current.size) { + read = found + } else if let repaired = try repairTail(of: handle, fileSize: current.size) { + read = repaired + } else { + return setAsideUnreadableLog() ? .noRecords : .unavailable + } + dropWritten(upTo: read.last) + return .chain(read.head) } catch { - // A log that cannot be read is itself a finding, so it is reported rather than - // silently replaced. Starting a fresh chain over it would destroy the evidence. - Self.logger.error("Execution audit log could not be decoded: \(error.localizedDescription)") + Self.logger.error("Execution audit log could not be read: \(error.publicLogShape, privacy: .public)") + return .unavailable } } - private func persist() { + // An append that failed after its bytes landed must not write its records a second time. + private func dropWritten(upTo last: ExecutionAuditRecord?) { + guard let last, let index = pending.lastIndex(where: { $0.isWritten(as: last) }) else { return } + pending.removeFirst(index + 1) + } + + private func append(after current: ChainHead, through handle: FileHandle) -> Bool { + guard !pending.isEmpty else { return true } + let records = linkPending(after: current) + guard let encoded = Self.encode(records), let last = records.last else { return false } + var tail = Data() + if current.nextSequence > 0 { + tail.append(Self.comma) + } + tail.append(encoded) + tail.append(Self.closeBracket) + + do { + try handle.seek(toOffset: current.closingBracketOffset) + try handle.write(contentsOf: tail) + let newSize = current.closingBracketOffset + UInt64(tail.count) + if newSize < current.fileSize { + try handle.truncate(atOffset: newSize) + } + pending = [] + head = ChainHead( + nextSequence: last.sequence + 1, + lastHash: last.hash, + closingBracketOffset: newSize - 1, + fileSize: newSize + ) + stamp = try? Self.stamp(of: handle) + return true + } catch { + head = nil + stamp = nil + Self.restoreClosingBracket(of: handle, at: current.closingBracketOffset) + Self.logger.error("Execution audit log could not be appended to: \(error.publicLogShape, privacy: .public)") + return false + } + } + + // Puts the array back as it was, so neither this build nor an older one finds an unterminated file. + private static func restoreClosingBracket(of handle: FileHandle, at offset: UInt64) { + do { + try handle.seek(toOffset: offset) + try handle.write(contentsOf: Data([closeBracket])) + try handle.truncate(atOffset: offset + 1) + } catch { + logger.error("Execution audit log could not be restored after a failed append") + } + } + + private func startChain() -> Bool { + head = nil + stamp = nil + guard !pending.isEmpty else { return true } + let records = linkPending(after: nil) + guard let encoded = Self.encode(records), let last = records.last else { return false } + var data = Data([Self.openBracket]) + data.append(encoded) + data.append(Self.closeBracket) do { - let data = try JSONEncoder().encode(records) try data.write(to: fileURL, options: .atomic) + pending = [] + head = ChainHead( + nextSequence: last.sequence + 1, + lastHash: last.hash, + closingBracketOffset: UInt64(data.count - 1), + fileSize: UInt64(data.count) + ) + stamp = try? Self.stamp(atPath: fileURL.path(percentEncoded: false)) + return true + } catch { + Self.logger.error("Execution audit log could not be written: \(error.publicLogShape, privacy: .public)") + return false + } + } + + // A file cut short by a crash keeps the records before the cut; a copy of it as found goes aside first. + private func repairTail(of handle: FileHandle, fileSize: UInt64) throws -> (head: ChainHead, last: ExecutionAuditRecord?)? { + let start = fileSize > Self.tailWindow ? fileSize - Self.tailWindow : 0 + try handle.seek(toOffset: start) + let bytes = [UInt8](try handle.read(upToCount: Int(fileSize - start)) ?? Data()) + guard let last = Self.lastRecord(in: bytes, endingAtOrBefore: bytes.count - 1) else { return nil } + + let destination = LocalSQLiteStoreOpener.setAsideURL(for: fileURL, at: Date()) + do { + try FileManager.default.copyItem(at: fileURL, to: destination) + } catch { + Self.logger.error("Execution audit log could not be copied before its repair: \(error.publicLogShape, privacy: .public)") + return nil + } + + let closingOffset = start + UInt64(last.end) + 1 + try handle.seek(toOffset: closingOffset) + try handle.write(contentsOf: Data([Self.closeBracket])) + try handle.truncate(atOffset: closingOffset + 1) + Self.logger.fault("Execution audit log ended mid-record; repaired after its last whole record, original kept aside") + let head = ChainHead( + nextSequence: last.record.sequence + 1, + lastHash: last.record.hash, + closingBracketOffset: closingOffset, + fileSize: closingOffset + 1 + ) + return (head, last.record) + } + + // The unreadable file is evidence: it is kept under a free name, and nothing is written while it cannot be moved. + private func setAsideUnreadableLog() -> Bool { + let destination = LocalSQLiteStoreOpener.setAsideURL(for: fileURL, at: Date()) + do { + try FileManager.default.moveItem(at: fileURL, to: destination) + Self.logger.fault("Execution audit log was unreadable and was set aside; a new chain starts") + return true } catch { - Self.logger.error("Execution audit log could not be written: \(error.localizedDescription)") + Self.logger.fault("Unreadable execution audit log could not be set aside: \(error.publicLogShape, privacy: .public)") + return false + } + } + + private static func encode(_ records: [ExecutionAuditRecord]) -> Data? { + let encoder = JSONEncoder() + var joined = Data() + do { + for (index, record) in records.enumerated() { + if index > 0 { + joined.append(comma) + } + joined.append(try encoder.encode(record)) + } + return joined + } catch { + logger.error("Execution audit record could not be encoded: \(error.publicLogShape, privacy: .public)") + return nil + } + } + + // MARK: - Reading the tail + + private static func stamp(of handle: FileHandle) throws -> FileStamp { + var info = stat() + guard fstat(handle.fileDescriptor, &info) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + return FileStamp(info) + } + + private static func stamp(atPath path: String) throws -> FileStamp { + var info = stat() + guard stat(path, &info) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + return FileStamp(info) + } + + private static func readHead( + from handle: FileHandle, + fileSize: UInt64 + ) throws -> (head: ChainHead, last: ExecutionAuditRecord?)? { + let start = fileSize > tailWindow ? fileSize - tailWindow : 0 + try handle.seek(toOffset: start) + let bytes = [UInt8](try handle.read(upToCount: Int(fileSize - start)) ?? Data()) + guard let head = chainHead(inTail: Data(bytes), startingAt: start, fileSize: fileSize) else { return nil } + guard head.nextSequence > 0, + let closing = lastNonWhitespaceIndex(in: bytes, before: bytes.count), + let beforeClosing = lastNonWhitespaceIndex(in: bytes, before: closing) + else { return (head, nil) } + return (head, decodedRecord(in: bytes, endingAt: beforeClosing)) + } + + internal static func chainHead(inTail tail: Data, startingAt start: UInt64, fileSize: UInt64) -> ChainHead? { + let bytes = [UInt8](tail) + guard let closing = lastNonWhitespaceIndex(in: bytes, before: bytes.count), + bytes[closing] == closeBracket, + let beforeClosing = lastNonWhitespaceIndex(in: bytes, before: closing) + else { return nil } + + let closingOffset = start + UInt64(closing) + if bytes[beforeClosing] == openBracket { + guard start == 0, lastNonWhitespaceIndex(in: bytes, before: beforeClosing) == nil else { return nil } + return ChainHead( + nextSequence: 0, + lastHash: ExecutionAuditRecord.genesisHash, + closingBracketOffset: closingOffset, + fileSize: fileSize + ) + } + guard let record = decodedRecord(in: bytes, endingAt: beforeClosing) else { return nil } + return ChainHead( + nextSequence: record.sequence + 1, + lastHash: record.hash, + closingBracketOffset: closingOffset, + fileSize: fileSize + ) + } + + internal static func lastRecord( + in bytes: [UInt8], + endingAtOrBefore limit: Int + ) -> (record: ExecutionAuditRecord, end: Int)? { + var end = limit + while end > 0 { + if let record = decodedRecord(in: bytes, endingAt: end) { + return (record, end) + } + end -= 1 + } + return nil + } + + // Records are flat and short, so one starts at the nearest `{` before its `}`, right after a `,` or the `[`. + private static func decodedRecord(in bytes: [UInt8], endingAt end: Int) -> ExecutionAuditRecord? { + guard end > 0, end < bytes.count, bytes[end] == closeBrace, + let start = bytes[max(0, end - maxRecordLength) ..< end].lastIndex(of: openBrace), + let separator = lastNonWhitespaceIndex(in: bytes, before: start), + bytes[separator] == comma || bytes[separator] == openBracket + else { return nil } + return try? JSONDecoder().decode(ExecutionAuditRecord.self, from: Data(bytes[start ... end])) + } + + private static func lastNonWhitespaceIndex(in bytes: [UInt8], before end: Int) -> Int? { + var index = end - 1 + while index >= 0 { + switch bytes[index] { + case 0x20, 0x09, 0x0A, 0x0D: + index -= 1 + default: + return index + } } + return nil } } diff --git a/apple/TablePro/Core/Services/Execution/ExecutionGateProvider.swift b/apple/TablePro/Core/Services/Execution/ExecutionGateProvider.swift index 733f1eb502..9353562da0 100644 --- a/apple/TablePro/Core/Services/Execution/ExecutionGateProvider.swift +++ b/apple/TablePro/Core/Services/Execution/ExecutionGateProvider.swift @@ -4,6 +4,8 @@ // import Foundation +import TableProPluginKit +import TableProSQLGrammar internal enum ExecutionGateProvider { /// A driver that declares no read-only mode gets every statement treated as a write, unless the @@ -13,24 +15,80 @@ internal enum ExecutionGateProvider { !supportsReadOnlyMode && !QueryClassifier.enginesWithProvenReads.contains(databaseType) } + // The level the gate decides with. A run reads it here too, so the statement carries the level it was approved under. + @MainActor + static func effectiveSafeModeLevel(for connectionId: UUID, in manager: DatabaseManager? = nil) -> SafeModeLevel { + let connectionLevel: SafeModeLevel + switch (manager ?? .shared).connectionState(connectionId) { + case .live(_, let session): + connectionLevel = session.safeModeLevel + case .stored(let connection): + connectionLevel = connection.safeModeLevel + case .unknown: + connectionLevel = .silent + } + return ManagedPolicyResolver.effectiveSafeModeLevel( + connectionLevel: connectionLevel, + policy: ManagedPolicyReader.shared + ) + } + + @MainActor + static func runsAtReadOnly(_ connectionId: UUID, in manager: DatabaseManager? = nil) -> Bool { + effectiveSafeModeLevel(for: connectionId, in: manager).blocksAllWrites + } + + @MainActor + static func readOnlyFunctionPolicy( + connectionId: UUID, + databaseType: DatabaseType, + in manager: DatabaseManager? = nil + ) -> ReadOnlyFunctionPolicy { + let state = (manager ?? .shared).connectionState(connectionId) + let (liveDriver, connection): (DatabaseDriver?, DatabaseConnection?) = switch state { + case .live(let driver, let session): (driver, session.connection) + case .stored(let stored): (nil, stored) + case .unknown: (nil, nil) + } + let floor: SafeModeLevel? = if let connection { + AgentModeSafeModeFloor.effectiveFloor(for: connection)?.level + } else { + ManagedPolicyResolver.minimumSafeModeLevel(policy: ManagedPolicyReader.shared) + } + return ReadOnlyFunctionPolicy( + functionsCanWrite: liveDriver?.enforcesReadOnlyMode != true + && PluginMetadataRegistry.queryFunctionsCanWrite.contains(databaseType), + floor: floor + ) + } + + // A session holding a transaction or a table lock runs a read as it is, since the read-only mode would end what it + // holds, so a function the read calls is checked as on a driver without the mode. + static func liveReadOnlyFunctionPolicy( + connectionId: UUID, + databaseType: DatabaseType, + in manager: DatabaseManager? = nil + ) async -> ReadOnlyFunctionPolicy { + let (policy, driver): (ReadOnlyFunctionPolicy, DatabaseDriver?) = await MainActor.run { + let policy = readOnlyFunctionPolicy(connectionId: connectionId, databaseType: databaseType, in: manager) + guard case .live(let driver, _) = (manager ?? .shared).connectionState(connectionId) else { + return (policy, nil) + } + return (policy, driver) + } + guard !policy.functionsCanWrite, PluginMetadataRegistry.queryFunctionsCanWrite.contains(databaseType), + let driver + else { return policy } + var live = policy + live.functionsCanWrite = await driver.heldSessionTransactionState() != .idle + return live + } + static let shared: ExecutionGate = DefaultExecutionGate( confirming: AlertOperationConfirming(), authenticating: BiometricOperationAuthenticating(), safeModeLevelResolver: { connectionId in - let connectionLevel: SafeModeLevel = await MainActor.run { - switch DatabaseManager.shared.connectionState(connectionId) { - case .live(_, let session): - return session.safeModeLevel - case .stored(let connection): - return connection.safeModeLevel - case .unknown: - return .silent - } - } - return ManagedPolicyResolver.effectiveSafeModeLevel( - connectionLevel: connectionLevel, - policy: ManagedPolicyReader.shared - ) + await effectiveSafeModeLevel(for: connectionId) }, forcesWriteResolver: { databaseType in let supportsReadOnlyMode = await MainActor.run { @@ -49,6 +107,45 @@ internal enum ExecutionGateProvider { return nil } } + }, + readOnlyFunctionPolicyResolver: { connectionId, databaseType in + await liveReadOnlyFunctionPolicy(connectionId: connectionId, databaseType: databaseType) + }, + bareNameCallResolver: { request, reading in + await callsBareFunction(request, reading: reading) } ) + + static let callableCatalog = SQLCallableCatalogCache() + + // Asks the session's catalog. A catalog that cannot be read leaves the name unexplained, so it counts as a call. + static func callsBareFunction( + _ request: OperationRequest, + reading: SQLBareNameReading, + catalog: SQLCallableCatalogCache = callableCatalog, + metadata: (any ScopedMetadataProviding)? = nil + ) async -> Bool { + let (provider, resolved) = await MainActor.run { + let provider: any ScopedMetadataProviding = metadata ?? DatabaseManager.shared + return (provider, request.scope ?? request.runsIn ?? provider.browseScope(for: request.connectionId)) + } + guard let scope = resolved else { return true } + do { + return try await catalog.callsFunction(reading, connectionId: scope.connectionId, schema: scope.schema) { query in + try await provider.withMetadataRead(scope: scope, workload: .interactive) { driver in + try await driver.executeCatalogRead(query: query).rows.map { row in + row.map { cell -> String? in + switch cell { + case .null: return nil + case .text(let text): return text + case .bytes(let data): return String(bytes: data, encoding: .utf8) + } + } + } + } + } + } catch { + return true + } + } } diff --git a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift index 493ef67dde..13c37d6591 100644 --- a/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift +++ b/apple/TablePro/Core/Services/Execution/ExternalStatementGate.swift @@ -32,9 +32,10 @@ internal enum ExternalStatementGate { internal let loadsExtensions: Bool internal let allowsDestructive: Bool internal let allowsMultiStatement: Bool - /// What this transport offers instead, appended to the destructive refusal. MCP has a tool - /// for it; AppleScript confirms interactively and never reaches the refusal. + // Appended to the destructive refusal: MCP has a tool for it, AppleScript confirms and never reaches it. internal let destructiveAlternative: String? + // The assistant's Edit mode: queries and row changes, no schema, privilege, session or transaction statement. + internal let allowsOnlyDataStatements: Bool internal init( sql: String, @@ -44,7 +45,8 @@ internal enum ExternalStatementGate { loadsExtensions: Bool, allowsDestructive: Bool, allowsMultiStatement: Bool = false, - destructiveAlternative: String? = nil + destructiveAlternative: String? = nil, + allowsOnlyDataStatements: Bool = false ) { self.sql = sql self.connectionId = connectionId @@ -54,6 +56,7 @@ internal enum ExternalStatementGate { self.allowsDestructive = allowsDestructive self.allowsMultiStatement = allowsMultiStatement self.destructiveAlternative = destructiveAlternative + self.allowsOnlyDataStatements = allowsOnlyDataStatements } } @@ -102,6 +105,11 @@ internal enum ExternalStatementGate { ) } + if statement.allowsOnlyDataStatements, + !QueryClassifier.isDataStatement(statement.sql, databaseType: statement.databaseType) { + throw ExternalStatementGateError.denied(dataStatementRefusal) + } + return classification } @@ -120,6 +128,14 @@ internal enum ExternalStatementGate { localized: "This statement is destructive, like DROP, TRUNCATE or a DELETE with no WHERE." ) + internal static let dataStatementRefusal = String( + localized: """ + Edit mode runs only queries and data changes, such as SELECT, INSERT, UPDATE, DELETE and MERGE. \ + A schema, privilege, session or transaction statement, or a change to the database's own tables, \ + needs Agent mode, or run it yourself in a query tab. + """ + ) + internal static let notDestructiveRefusal = String( localized: """ This tool runs only destructive statements, like DROP, TRUNCATE or a DELETE with no WHERE. \ @@ -350,6 +366,15 @@ internal struct ApprovedOperation: Sendable { context = Self.context(for: pending) } + // A read stops once its rows pass the caller's answer budget. A write runs to its end whatever it returns. + internal func context(readingAtMost byteLimit: Int?) -> PluginStatementContext { + var bounded = context + if !kind.declaresWrite { + bounded.resultByteLimit = byteLimit + } + return bounded + } + private static func context(for pending: PendingOperation) -> PluginStatementContext { let graded = pending.gradedSQL.map { [$0] } ?? pending.statements.map(\.sql) let approvedAsDestructive = pending.kind.declaresDestructive diff --git a/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift b/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift new file mode 100644 index 0000000000..f0e6bdaeae --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift @@ -0,0 +1,39 @@ +// +// GatedOperationFailure.swift +// TablePro +// + +import Foundation + +// Cancel at the gate's confirmation is the user's own choice: it shows nothing and the sheet keeps its edits. +internal enum GatedOperationFailure: Equatable { + case cancelledByUser + case refused(String) + case failed(String) + + internal init(_ error: any Error) { + guard let gateError = error as? ExecutionGateError else { + self = .failed(error.localizedDescription) + return + } + switch gateError { + case .cancelledByUser: + self = .cancelledByUser + case .denied(let reason): + self = .refused(reason) + } + } + + internal var message: String? { + switch self { + case .cancelledByUser: nil + case .refused(let reason), .failed(let reason): reason + } + } + + // A refusal and a Cancel both stop before the first statement: nothing to report or reload. + internal var reportsFailure: Bool { + if case .failed = self { return true } + return false + } +} diff --git a/apple/TablePro/Core/Services/Execution/ManualCommit.swift b/apple/TablePro/Core/Services/Execution/ManualCommit.swift new file mode 100644 index 0000000000..fc3cf36993 --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/ManualCommit.swift @@ -0,0 +1,348 @@ +// +// ManualCommit.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit +import TableProSQLGrammar +import TableProUtilities + +// A refusal an external caller reads as it is: it quotes nothing the server said. +internal protocol ManualCommitRefusing: LocalizedError {} + +internal struct ManualCommitRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "This connection can't hold this change until you commit, so it did not run. Turn off Manual Commit in the Database menu to run it." + ) + } +} + +internal struct ManualCommitMixedRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "This run mixes changes with a statement that can't run in a transaction. Run that statement on its own, or turn off Manual Commit." + ) + } +} + +internal struct ManualCommitSessionOptionsRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "This change sets session options around its statements, so it can't wait for Commit. Turn off Manual Commit in the Database menu to run it." + ) + } +} + +internal struct ManualCommitForeignKeyRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "This import turns off foreign key checks, which can't wait for Commit. Turn off Disable foreign key checks or Manual Commit to run it." + ) + } +} + +internal struct ManualCommitUnwrappedImportRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "This import runs each statement on its own, so nothing can wait for Commit. Turn on Wrap in transaction or turn off Manual Commit." + ) + } +} + +internal struct ManualCommitTransactionControlRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { ManualCommit.transactionControlRefusalMessage } +} + +internal struct OpenTransactionRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { ManualCommit.openTransactionRefusalMessage } +} + +internal struct AbortingTransactionRefusal: ManualCommitRefusing, Equatable { + var errorDescription: String? { + String( + localized: "A transaction is open on this connection, and a failed statement would abort it. Commit or roll back in TablePro first." + ) + } +} + +// Set by `openIfNeeded` for the session turn it runs in, so the turn records the hold its BEGIN opened. +internal final class ManualCommitTurnHold: @unchecked Sendable { + private let lock = NSLock() + private var held = false + + var isHeld: Bool { lock.withLock { held } } + + func mark(_ value: Bool) { + lock.withLock { held = value } + } +} + +// One statement's run under its connection's commit mode, read on the main actor before the lease. +internal struct ManualCommitRun: Sendable { + let commitMode: ConnectionCommitMode + let databaseType: DatabaseType + let grammar: SQLLexicalGrammar + let connectionId: UUID + + nonisolated func running( + _ statements: [String], + on driver: DatabaseDriver, + _ body: () async throws -> T + ) async throws -> T { + let opened = try await ManualCommit.openIfNeeded( + on: driver, statements: statements, databaseType: databaseType, grammar: grammar, commitMode: commitMode + ) + do { + return try await body() + } catch { + if opened { await ManualCommit.rollBackOpened(on: driver, connectionId: connectionId) } + throw error + } + } +} + +// The hold a Manual Commit connection opens before the first write the user or an agent sends: BEGIN, never COMMIT. +internal enum ManualCommit { + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "ManualCommit") + + @TaskLocal internal static var turnHold: ManualCommitTurnHold? + + internal enum Decision: Equatable { + case runAsIs + case hold + case refuse + case refuseMixed + + var refusal: (any Error)? { + switch self { + case .runAsIs, .hold: return nil + case .refuse: return ManualCommitRefusal() + case .refuseMixed: return ManualCommitMixedRefusal() + } + } + } + + // These commit schema and maintenance statements on their own, so nothing can wait there; Redshift's TRUNCATE + // and CockroachDB's DDL commit the open transaction too. + nonisolated internal static func isAvailable(for databaseType: DatabaseType) -> Bool { + guard TableEditDialect.of(databaseType)?.commitsDDLImplicitly != true else { return false } + let family = TransactionEngineFamily.of(databaseType) + return family != .redshift && family != .cockroach + } + + nonisolated internal static func effectiveMode( + _ stored: ConnectionCommitMode, + databaseType: DatabaseType + ) -> ConnectionCommitMode { + isAvailable(for: databaseType) ? stored : .automatic + } + + internal static var unavailableExplanation: String { + String(localized: "This database commits schema and maintenance statements on its own, so changes can't wait for Commit.") + } + + // Exactly `.idle` holds: `.unknown` and a table lock would let the write commit on its own. + nonisolated internal static func decision( + commitMode: ConnectionCommitMode, + statements: [String], + databaseType: DatabaseType, + grammar: SQLLexicalGrammar, + supportsTransactions: Bool, + sessionState: PluginSessionTransactionState + ) -> Decision { + guard commitMode == .manual, writes(statements, databaseType: databaseType) else { return .runAsIs } + let family = TransactionEngineFamily.of(databaseType) + guard supportsTransactions, family.wrapsBatchInTransaction else { return .refuse } + switch BatchTransactionPolicy.plan(for: statements, databaseType: databaseType, grammar: grammar) { + case .appTransaction, .heldTransaction: + break + case .autocommit: + let mixed = statements.contains { statement in + !AutocommitOnlyStatement.matches(statement, family: family, grammar: grammar) + && writes([statement], databaseType: databaseType) + } + return mixed ? .refuseMixed : .runAsIs + case .scriptTransaction, .sessionTransaction: + return .runAsIs + } + switch sessionState { + case .idle: + return .hold + case .inTransaction, .abortedTransaction: + return .runAsIs + case .holdsSessionLocks, .unknown: + return .refuse + @unknown default: + return .refuse + } + } + + // A held decision runs held; anything else keeps the plan an automatic connection would use. + nonisolated internal static func runPlan( + text plan: BatchTransactionPlan, + decision: Decision, + sessionState: PluginSessionTransactionState + ) -> BatchTransactionPlan { + decision == .hold ? .heldTransaction : plan.joining(sessionState, commitMode: .automatic) + } + + // Inside the caller's lease, so nothing runs between the state read and the BEGIN. True when it sent the BEGIN. + nonisolated internal static func openIfNeeded( + on driver: DatabaseDriver, + statements: [String], + databaseType: DatabaseType, + grammar: SQLLexicalGrammar, + commitMode: ConnectionCommitMode + ) async throws -> Bool { + guard commitMode == .manual else { return false } + let state = await driver.heldSessionTransactionState() + let decided = decision( + commitMode: commitMode, + statements: statements, + databaseType: databaseType, + grammar: grammar, + supportsTransactions: driver.supportsTransactions, + sessionState: state + ) + if let refusal = decided.refusal { throw refusal } + guard decided == .hold else { return false } + let mode = OperationKind.worst(of: statements, databaseType: databaseType).transactionAccessMode + try await driver.beginTransaction(mode: mode) + turnHold?.mark(true) + return true + } + + // The app's own writes on a Manual Commit session: one the session can't hold is refused before it runs. + nonisolated internal static func refuseUnheldWrite( + supportsTransactions: Bool, + sessionState: PluginSessionTransactionState + ) throws { + guard supportsTransactions else { throw ManualCommitRefusal() } + switch sessionState { + case .idle, .inTransaction, .abortedTransaction: + return + case .holdsSessionLocks, .unknown: + throw ManualCommitRefusal() + @unknown default: + throw ManualCommitRefusal() + } + } + + // A timed-out read after the turn's own BEGIN still means the transaction it opened. + nonisolated internal static func recordedState( + read: PluginSessionTransactionState, + turnOpenedHold: Bool + ) -> PluginSessionTransactionState { + read == .unknown && turnOpenedHold ? .inTransaction : read + } + + // A failed first write leaves nothing worth keeping, and on PostgreSQL an aborted block that needs a Roll Back. + @MainActor + internal static func rollBackOpened(on driver: DatabaseDriver, connectionId: UUID) async { + let token = DatabaseManager.shared.beginProtectedWrite(on: driver, for: connectionId) + defer { DatabaseManager.shared.endProtectedWrite(token, for: connectionId) } + do { + try await TaskCancellationShield.run { @Sendable in try await driver.rollbackTransaction() } + turnHold?.mark(false) + } catch { + logger.error("Rolling back the held transaction failed: \(error.publicLogShape, privacy: .public)") + } + } + + // A procedure or dynamic batch can end the transaction unseen, and so can a T-SQL batch, which needs no `;` before COMMIT. + nonisolated internal static func mayEndTransactionUnseen( + _ statements: [String], + databaseType: DatabaseType, + grammar: SQLLexicalGrammar + ) -> Bool { + let scansTokens = TransactionEngineFamily.of(databaseType) == .sqlServer + return statements.contains { statement in + CatalogChangeClassifier.effect(of: statement, databaseType: databaseType, grammar: grammar).runsUnseenStatements + || (scansTokens && mentionsTransactionControl(statement, grammar: grammar)) + } + } + + // END is left out: it closes every T-SQL block and every CASE. + nonisolated private static func mentionsTransactionControl(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + let text = statement as NSString + var start = 0 + while start < text.length { + var cursor = SQLTokenCursor(text.substring(from: start) as NSString, grammar: grammar) + var previous: String? + while let token = cursor.next() { + guard let word = token.word else { + previous = nil + continue + } + if controlWords.contains(word) { return true } + if word == "TRAN" || word == "TRANSACTION", let previous, transactionOpeners.contains(previous) { + return true + } + previous = word + } + start += cursor.location + 1 + } + return false + } + + private static let controlWords: Set = ["COMMIT", "ROLLBACK", "SAVEPOINT", "RELEASE", "ABORT", "XA"] + private static let transactionOpeners: Set = ["BEGIN", "SAVE", "START", "DISTRIBUTED"] + + // BEGIN and START are read the way a batch is planned, so a PL/SQL or T-SQL block is not taken for one. + nonisolated internal static func controlsTransaction( + _ statements: [String], + databaseType: DatabaseType, + grammar: SQLLexicalGrammar + ) -> Bool { + if startsTransaction(statements, databaseType: databaseType, grammar: grammar) { return true } + return statements.contains { statement in + if CatalogChangeClassifier.effect(of: statement, databaseType: databaseType, grammar: grammar).endsTransaction { + return true + } + return transactionControlKeywords.contains(QueryClassifier.leadingKeyword(of: statement, grammar: grammar)) + } + } + + nonisolated internal static func startsTransaction( + _ statements: [String], + databaseType: DatabaseType, + grammar: SQLLexicalGrammar + ) -> Bool { + BatchTransactionPolicy.plan(for: statements, databaseType: databaseType, grammar: grammar) == .scriptTransaction + } + + nonisolated private static func writes(_ statements: [String], databaseType: DatabaseType) -> Bool { + OperationKind.worst(of: statements, databaseType: databaseType).declaresWrite + } + + private static let transactionControlKeywords: Set = [ + "COMMIT", "ROLLBACK", "END", "ABORT", "SAVEPOINT", "RELEASE", "XA", + ] + + internal static var transactionEndedNotice: String { + String(localized: "The open transaction on this connection ended while this statement ran.") + } + + internal static var statementOpenedNotice: String { + String( + localized: "This change is pending in a transaction the statement opened. Other statements are refused until it is committed or rolled back." + ) + } + + internal static var pendingNotice: String { + String(localized: "Manual Commit is on for this connection, so this change is pending until the user commits it in TablePro.") + } + + internal static var transactionControlRefusalMessage: String { + String(localized: "Manual Commit is on for this connection. Commit or roll back in TablePro.") + } + + internal static var openTransactionRefusalMessage: String { + String( + localized: "A transaction is already open on this connection, or it cannot say whether one is. Commit or roll it back first." + ) + } +} diff --git a/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift b/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift index d4b7700a1e..9aa97a005d 100644 --- a/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift +++ b/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift @@ -19,6 +19,10 @@ internal enum MultiStatementFailure: Equatable, Sendable { /// and committed its row once the lock was released. Nothing here can say whether the /// transaction took, so nothing here may report a rollback. case commitOutcomeUnknown + // Stopped after `sql`, which takes nothing after it but the transaction's end; `databaseMove` is an earlier move that ran. + case awaitsCommit(sql: String, databaseMove: String? = nil) + // Stopped before `sql`, a database move, because the session holds a transaction the move cannot join. + case databaseMoveAwaitsCommit(sql: String) func ranStatementCount(executedCount: Int, totalCount: Int) -> Int { switch self { @@ -26,7 +30,7 @@ internal enum MultiStatementFailure: Equatable, Sendable { return 0 case .statement: return min(executedCount + 1, totalCount) - case .batch, .commit, .commitOutcomeUnknown: + case .batch, .commit, .commitOutcomeUnknown, .awaitsCommit, .databaseMoveAwaitsCommit: return executedCount } } @@ -80,9 +84,102 @@ internal struct MultiStatementFailureContext: Equatable, Sendable { ) case .commitOutcomeUnknown: return unknownCommitReport() + case .awaitsCommit(_, let databaseMove): + return awaitsCommitReport(databaseMove: databaseMove) + case .databaseMoveAwaitsCommit: + return databaseMoveReport() } } + // The bar's Commit, as a typed COMMIT is refused after a DATABASE; a move is redone, as a new run is pinned to the tab's database. + private func awaitsCommitReport(databaseMove: String?) -> MultiStatementFailureReport { + let next = executedCount + 1 + let message: String + if plan == .scriptTransaction { + // The rest has to open its own BT again, or the script's ET ends a block the app opened for it. + let opening = databaseMove.map { "\(Self.statementText($0)); BT" } ?? "BT" + message = String( + format: String( + localized: """ + Statement %1$d/%2$d is a DDL statement, and the session takes nothing after it but COMMIT \ + or ROLLBACK. The run stopped before statement %3$d. Click Commit on the Transaction Open \ + bar, then begin the rest with “%4$@”. + """ + ), + executedCount, totalCount, next, opening + ) + } else if let databaseMove { + message = String( + format: String( + localized: """ + Statement %1$d/%2$d is a DDL statement, and the session takes nothing after it but COMMIT \ + or ROLLBACK. The run stopped before statement %3$d. Click Commit on the Transaction Open \ + bar. A new run starts in the tab's database, so begin the rest with “%4$@”. + """ + ), + executedCount, totalCount, next, Self.statementText(databaseMove) + ) + } else { + message = String( + format: String( + localized: """ + Statement %1$d/%2$d is a DDL statement, and the session takes nothing after it but COMMIT \ + or ROLLBACK. The run stopped before statement %3$d. Click Commit on the Transaction Open \ + bar, then run the rest. + """ + ), + executedCount, totalCount, next + ) + } + return MultiStatementFailureReport( + message: message, + resultLabel: String(localized: "Stopped"), + failedStatementIndex: next <= totalCount ? executedCount : nil, + failedSQL: nil + ) + } + + // Run again from the move itself: a new run is pinned to the tab's database until the move runs. + private func databaseMoveReport() -> MultiStatementFailureReport { + let position = executedCount + 1 + let message: String + if plan == .scriptTransaction { + // A rerun from the move would leave the script's BT behind, and what ran before the BT is already committed. + message = String( + format: String( + localized: """ + Statement %1$d/%2$d moves the session to another database, which this session takes only \ + outside a transaction. The run stopped before it. Click Roll Back on the Transaction Open \ + bar, move the statement above the script's BT, then run the script again from there. + """ + ), + position, totalCount + ) + } else { + message = String( + format: String( + localized: """ + Statement %1$d/%2$d moves the session to another database, which this session takes only \ + outside a transaction. The run stopped before it. Click Commit or Roll Back on the \ + Transaction Open bar, then run the script again from statement %1$d. + """ + ), + position, totalCount + ) + } + return MultiStatementFailureReport( + message: message, + resultLabel: String(localized: "Stopped"), + failedStatementIndex: position <= totalCount ? executedCount : nil, + failedSQL: nil + ) + } + + private static func statementText(_ sql: String) -> String { + let trimmed = sql.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.hasSuffix(";") ? String(trimmed.dropLast()) : trimmed + } + /// Worded without a rollback in it. The app sent the commit, the connection went before the /// answer, and the server decides on its own: the same kill that rolled a commit back under a /// read lock was ignored by a commit waiting on `binlog_group_commit_sync_delay`, which then @@ -133,7 +230,7 @@ internal struct MultiStatementFailureContext: Equatable, Sendable { /// What became of the statements that ran before the failure. private func standingWorkNote(position: Int) -> String? { switch plan { - case .appTransaction, .scriptTransaction: + case .appTransaction, .heldTransaction, .scriptTransaction: return nil case .autocommit: return appliedWorkNote(position: position) @@ -157,7 +254,7 @@ internal struct MultiStatementFailureContext: Equatable, Sendable { private func appliedWorkNote(position: Int) -> String? { switch unit { case .statement: - return appliedStatementsNote() + return sessionState.openTransactionNotice ?? appliedStatementsNote() case .batch: return sessionState.openTransactionNotice ?? appliedBatchesNote(position: position) } diff --git a/apple/TablePro/Core/Services/Execution/OperationRequest.swift b/apple/TablePro/Core/Services/Execution/OperationRequest.swift index 12b3010de3..26925996de 100644 --- a/apple/TablePro/Core/Services/Execution/OperationRequest.swift +++ b/apple/TablePro/Core/Services/Execution/OperationRequest.swift @@ -16,6 +16,8 @@ internal struct OperationRequest: Sendable { let scope: DatabaseScope? // What the confirmation shows, values written in. Empty means `sql` alone. let statements: [String] + // Where a tab's statement runs, which its confirmation does not name: Oracle and Dameng resolve a bare name there. + let runsIn: DatabaseScope? init( connectionId: UUID, @@ -26,7 +28,8 @@ internal struct OperationRequest: Sendable { capabilities: CallerCapabilities, operationDescription: String, scope: DatabaseScope? = nil, - statements: [String] = [] + statements: [String] = [], + runsIn: DatabaseScope? = nil ) { self.connectionId = connectionId self.databaseType = databaseType @@ -37,5 +40,6 @@ internal struct OperationRequest: Sendable { self.operationDescription = operationDescription self.scope = scope self.statements = statements + self.runsIn = runsIn } } diff --git a/apple/TablePro/Core/Services/Execution/ReadOnlyFunctionCalls.swift b/apple/TablePro/Core/Services/Execution/ReadOnlyFunctionCalls.swift new file mode 100644 index 0000000000..37ae31f7ec --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/ReadOnlyFunctionCalls.swift @@ -0,0 +1,103 @@ +// +// ReadOnlyFunctionCalls.swift +// TablePro +// + +import Foundation +import TableProPluginKit +import TableProSQLGrammar + +// What Read-Only needs to know about a connection before it lets a read call a function. +internal struct ReadOnlyFunctionPolicy: Sendable, Equatable { + // The live driver cannot run a statement in the engine's read-only mode, and a function on this engine can write. + var functionsCanWrite = false + // What holds the level up: a read-only engine or file, a configuration profile, Agent mode. + var floor: SafeModeLevel? + + var levelThatRunsIt: SafeModeLevel? { + SafeModeFloor.levelThatWrites(above: floor) + } +} + +// The Mac's reading of `SQLReadOnlyFunctionCalls`, with the engine's types from its plugin's dialect. +internal enum ReadOnlyFunctionCalls { + static func builtins(for databaseType: DatabaseType) -> Set { + SQLReadOnlyFunctionCalls.builtins(forDatabaseTypeId: databaseType.rawValue) + } + + static func callsUnlistedFunction(_ sql: String, databaseType: DatabaseType) -> Bool { + SQLReadOnlyFunctionCalls.callsUnlistedFunction( + sql, + databaseTypeId: databaseType.rawValue, + readings: databaseType.lexicalReadings, + typeNames: Set(PluginMetadataRegistry.shared.snapshot(for: databaseType)?.editor.sqlDialect?.dataTypes ?? []) + ) + } + + // Nil on an engine that calls a function only with parentheses. + static func bareNameReading(_ sql: String, databaseType: DatabaseType) -> SQLBareNameReading? { + guard SQLBareNameCalls.engines.contains(databaseType.rawValue) else { return nil } + return SQLBareNameCalls.read(sql, databaseTypeId: databaseType.rawValue, readings: databaseType.lexicalReadings) + } +} + +// Export, Transfer To, Data Compare and Copy Objects read rows outside the editor, so text the user wrote into them, a +// row filter or the query an export runs again, meets the gate's Read-Only decision before any row is read. +internal enum ReadOnlyReadGate { + @MainActor + static func authorize( + _ statements: [String], + connectionId: UUID, + databaseType: DatabaseType, + gate: any ExecutionGate, + operationDescription: String = String(localized: "Export"), + canPrompt: Bool = true, + manager: DatabaseManager? = nil, + runsIn: DatabaseScope? = nil + ) async throws { + guard !statements.isEmpty, ExecutionGateProvider.runsAtReadOnly(connectionId, in: manager) else { return } + let decision = await gate.authorize( + OperationRequest( + connectionId: connectionId, + databaseType: databaseType, + sql: statements.joined(separator: ";\n"), + kind: .readQuery, + caller: .userInterface, + capabilities: canPrompt ? .interactiveUser : CallerCapabilities.interactiveUser.union(.cannotPrompt), + operationDescription: operationDescription, + statements: statements, + runsIn: runsIn + ) + ) + if let denial = decision.denialError { throw denial } + } + + // The read each filtered object runs, in the form the gate reads it. + static func rowScopeStatements( + _ objects: [ExportObjectItem], + databaseType: DatabaseType, + quote: (String) -> String + ) -> [String] { + objects.compactMap { object in + filterStatement( + table: object.name, + schema: object.databaseName, + filter: object.rowScope.sanitizedFilter, + databaseType: databaseType, + quote: quote + ) + } + } + + static func filterStatement( + table: String, + schema: String?, + filter: String?, + databaseType: DatabaseType, + quote: (String) -> String + ) -> String? { + guard let filter, !filter.isEmpty else { return nil } + let name = SchemaQualifiedName.render(name: table, schema: schema, databaseType: databaseType, quote: quote) + return "SELECT * FROM \(name) WHERE \(filter)" + } +} diff --git a/apple/TablePro/Core/Services/Execution/TransactionEndStatement.swift b/apple/TablePro/Core/Services/Execution/TransactionEndStatement.swift new file mode 100644 index 0000000000..d7797aca30 --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/TransactionEndStatement.swift @@ -0,0 +1,25 @@ +// +// TransactionEndStatement.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar +import TableProUtilities + +// A statement that ends the session's transaction, by commit or by rollback: all Teradata takes after DDL. +internal enum TransactionEndStatement { + private static let rollbackWords: Set = ["ROLLBACK", "ABORT"] + + internal static func matches(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + if BatchCommitStatement.matches(statement, grammar: grammar) { return true } + var cursor = SQLTokenCursor(statement, grammar: grammar) + guard let keyword = cursor.next()?.word, rollbackWords.contains(keyword) else { return false } + // ROLLBACK TO SAVEPOINT keeps the transaction open. + return cursor.next()?.word != "TO" + } + + internal static func matches(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + matches(statement as NSString, grammar: grammar) + } +} diff --git a/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift b/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift index bc761bd61d..b228340039 100644 --- a/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift +++ b/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift @@ -27,6 +27,7 @@ internal enum TransactionEngineFamily: String, Hashable, Sendable, CaseIterable case sqlServer case oracle case redis + case teradata case other internal static func of(_ type: DatabaseType) -> TransactionEngineFamily { @@ -64,6 +65,35 @@ internal enum TransactionEngineFamily: String, Hashable, Sendable, CaseIterable self == .oracle } + // After a failed statement the block takes nothing but a rollback, so one error costs every pending change. + internal var failedStatementAbortsTransaction: Bool { + switch self { + // Teradata mode rolls back the whole transaction on any failed request, ANSI mode on a failure such as a + // deadlock; the family cannot see the session's mode, so it takes the stricter one. + case .postgres, .redshift, .cockroach, .duckdb, .teradata: + return true + case .mysql, .sqlite, .sqlServer, .oracle, .redis, .other: + return false + } + } + + // Teradata holds a definition statement in a transaction in an ANSI session, and inside one takes nothing after + // it but COMMIT or ROLLBACK: Error 3722 in ANSI mode, Failure 3932 inside BT, which rolls the whole block back. + internal var definitionEndsTransaction: Bool { + self == .teradata + } + + // A Teradata ANSI session opens a transaction with any request and ends it only with COMMIT or ROLLBACK, so work + // the app runs without a BEGIN of its own is still pending, and a pooled connection's logoff rolls it back. + internal var unwrappedWorkStaysOpen: Bool { + self == .teradata + } + + // Redshift and DuckDB have no savepoint to take a failed statement back to. + internal var takesBackAFailedStatement: Bool { + self == .postgres || self == .cockroach + } + private static let familiesByTypeId: [String: TransactionEngineFamily] = [ "PostgreSQL": .postgres, "PGlite": .postgres, @@ -83,6 +113,7 @@ internal enum TransactionEngineFamily: String, Hashable, Sendable, CaseIterable "DuckDB": .duckdb, "SQL Server": .sqlServer, "Oracle": .oracle, - "Redis": .redis + "Redis": .redis, + "Teradata": .teradata ] } diff --git a/apple/TablePro/Core/Services/Execution/WriteTransactionOwner.swift b/apple/TablePro/Core/Services/Execution/WriteTransactionOwner.swift index 74cd43e9b4..e0f2587eb6 100644 --- a/apple/TablePro/Core/Services/Execution/WriteTransactionOwner.swift +++ b/apple/TablePro/Core/Services/Execution/WriteTransactionOwner.swift @@ -14,31 +14,55 @@ import TableProPluginKit /// transaction of its own then commits their pending work on PostgreSQL, implicitly commits it on /// MySQL and aborts it on DuckDB, with nothing raised and the write reporting success. /// -/// This is the one place that decision is made. ``BatchTransactionPlan/joining(_:)`` is the same -/// decision for a multi-statement run, where the statement text has a say as well. +/// This is the one place that decision is made. ``BatchTransactionPlan/joining(_:commitMode:)`` is +/// the same decision for a multi-statement run, where the statement text has a say as well. internal enum WriteTransactionOwner: Equatable { /// The app opens the transaction, verifies inside it, and commits or rolls it back. case app + // Manual Commit: the app opens the transaction and leaves it open for the user to commit, rolling back only a failure. + case held /// The session already holds one. The write joins it and sends no `BEGIN`, `COMMIT` or /// `ROLLBACK`: ending it belongs to whoever opened it. case session /// The engine has no transactions, so every statement is on the server as it runs. case none + // No default for the mode: a writer off the session's turn must say `.automatic`, or its hold is never seen. internal static func resolve( supportsTransactions: Bool, - sessionState: PluginSessionTransactionState + sessionState: PluginSessionTransactionState, + commitMode: ConnectionCommitMode ) -> WriteTransactionOwner { guard supportsTransactions else { return .none } + if commitMode == .manual, sessionState == .idle { return .held } return sessionState.permitsAppTransaction ? .app : .session } - internal var opensTransaction: Bool { - self == .app + internal var sendsBegin: Bool { + switch self { + case .app, .held: + return true + case .session, .none: + return false + } + } + + internal var commitsAtEnd: Bool { + switch self { + case .app: + return true + case .held, .session, .none: + return false + } } /// Whether the statements that already ran can still be taken back by the app. internal var canRollBack: Bool { - self == .app + switch self { + case .app, .held: + return true + case .session, .none: + return false + } } } diff --git a/apple/TablePro/Core/Services/Export/BulkReadTimeout.swift b/apple/TablePro/Core/Services/Export/BulkReadTimeout.swift new file mode 100644 index 0000000000..64245263cb --- /dev/null +++ b/apple/TablePro/Core/Services/Export/BulkReadTimeout.swift @@ -0,0 +1,58 @@ +// +// BulkReadTimeout.swift +// TablePro +// + +import Foundation +import os + +// A driver's row stream waits for its reader, so a bulk read lasts as long as the copy, transfer or compare consuming +// it, and the session's interactive query timeout would end it part way. The export lifts the timeout the same way. +enum BulkReadTimeout { + private static let logger = Logger(subsystem: "com.TablePro", category: "BulkReadTimeout") + + static func lifted( + on drivers: [DatabaseDriver], + _ operation: () async throws -> T + ) async throws -> T { + var restored: [(DatabaseDriver, Int)] = [] + for driver in drivers { + restored.append((driver, await restoredSeconds(for: driver))) + await apply(0, on: driver) + } + do { + let value = try await operation() + await restore(restored) + return value + } catch { + await restore(restored) + throw error + } + } + + // In a task of its own, so a Stop that cancelled the read still puts the timeout back on the pooled connection. + private static func restore(_ timeouts: [(DatabaseDriver, Int)]) async { + await Task { + for (driver, seconds) in timeouts { await apply(seconds, on: driver) } + }.value + } + + static func restoredSeconds(for driver: DatabaseDriver) async -> Int { + if let adapter = driver as? PluginDriverAdapter { return adapter.effectiveQueryTimeoutSeconds } + let configured = driver.connection.queryTimeoutSeconds + return await MainActor.run { + ConnectionTimeoutPolicy.effectiveQueryTimeoutSeconds( + configuredSeconds: configured, + globalSeconds: AppSettingsManager.shared.general.queryTimeoutSeconds + ) + } + } + + private static func apply(_ seconds: Int, on driver: DatabaseDriver) async { + do { + try await driver.applyQueryTimeout(seconds) + } catch { + logger.warning("Query timeout \(seconds) not applied for a bulk read: \(error.localizedDescription, privacy: .private)") + } + } +} diff --git a/apple/TablePro/Core/Services/Export/ExportService.swift b/apple/TablePro/Core/Services/Export/ExportService.swift index b63587ffd7..3fdee14f15 100644 --- a/apple/TablePro/Core/Services/Export/ExportService.swift +++ b/apple/TablePro/Core/Services/Export/ExportService.swift @@ -77,6 +77,7 @@ final class ExportService: ObservableObject { private let databaseType: DatabaseType? private let effectiveQueryTimeoutSeconds: Int private let exportPlugin: @MainActor (String) -> (any ExportFormatPlugin)? + var executionGate: any ExecutionGate = ExecutionGateProvider.shared init( driver: DatabaseDriver, @@ -207,15 +208,29 @@ final class ExportService: ObservableObject { } try throwIfStopped() - let dataSource = ExportDataSourceAdapter(driver: driver, databaseType: databaseType) - let rowObjects = objects.filter { plugin.readsRows(optionValues: $0.optionValues, kind: $0.kind) } + try await ReadOnlyReadGate.authorize( + ReadOnlyReadGate.rowScopeStatements(rowObjects, databaseType: databaseType) { driver.quoteIdentifier($0) }, + connectionId: driver.connection.id, + databaseType: databaseType, + gate: executionGate + ) + try throwIfStopped() + + let safeModeIsReadOnly = ExecutionGateProvider.runsAtReadOnly(driver.connection.id) + let dataSource = ExportDataSourceAdapter( + driver: driver, + databaseType: databaseType, + safeModeIsReadOnly: safeModeIsReadOnly + ) + state.readsRows = !rowObjects.isEmpty state.totalRows = await fetchTotalRowCount( for: rowObjects.filter { $0.kind.carriesRows }, driver: driver, databaseType: databaseType, - dataSource: dataSource + dataSource: dataSource, + context: driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) ) try throwIfStopped() @@ -429,11 +444,16 @@ final class ExportService: ObservableObject { currentProgress = nil } try throwIfStopped() + try await ReadOnlyReadGate.authorize( + [query], connectionId: driver.connection.id, databaseType: databaseType, gate: executionGate + ) + try throwIfStopped() let dataSource = StreamingQueryExportDataSource( query: LeadingRowsStatement.resolve(query, rowCap: nil, databaseType: databaseType).sql, driver: driver, - databaseType: databaseType + databaseType: databaseType, + safeModeIsReadOnly: ExecutionGateProvider.runsAtReadOnly(driver.connection.id) ) let nsProgress = Progress(totalUnitCount: Int64(max(estimatedRows, 1))) @@ -475,16 +495,21 @@ final class ExportService: ObservableObject { let capWarning = Self.leadingRowsCapWarning( exportedRows: progress.processedRows, - pagination: PaginationCapability.of(databaseType) + pagination: PaginationCapability.of(databaseType), + driverCut: dataSource.driverRowCut ) state.warnings = result.warnings + [capWarning].compactMap { $0 } state.notes = result.notes } - /// A query result exported from an engine that returns only its leading rows stops at the - /// engine's ceiling, so a file that reached it is named as partial rather than passing as whole. - static func leadingRowsCapWarning(exportedRows: Int, pagination: PaginationCapability) -> String? { - guard let maximum = pagination.maximumRows, exportedRows >= maximum else { return nil } + // A file that reached the engine's ceiling, or the limit the driver cut its stream at, is named as partial. + static func leadingRowsCapWarning( + exportedRows: Int, + pagination: PaginationCapability, + driverCut: Int? = nil + ) -> String? { + let engineCut = pagination.maximumRows.flatMap { exportedRows >= $0 ? $0 : nil } + guard let maximum = engineCut ?? driverCut else { return nil } return String( format: String(localized: "Only the first %lld rows were exported, the most this database returns from one query."), maximum @@ -511,9 +536,15 @@ final class ExportService: ObservableObject { for tables: [ExportObjectItem], driver: DatabaseDriver, databaseType: DatabaseType, - dataSource: ExportDataSourceAdapter + dataSource: ExportDataSourceAdapter, + context: PluginStatementContext? ) async -> Int { guard !tables.isEmpty else { return 0 } + // At Read-Only a count runs read-only too: a view's COUNT(*) runs the view. + let runCount: (String) async throws -> QueryResult = { query in + guard let context else { return try await driver.execute(query: query) } + return try await driver.execute(query: query, context: context) + } var total = 0 var failedCount = 0 @@ -555,7 +586,7 @@ final class ExportService: ObservableObject { let batchQuery = unionParts.joined(separator: " UNION ALL ") do { - let result = try await driver.execute(query: batchQuery) + let result = try await runCount(batchQuery) for row in result.rows { if let cell = row.first, let count = Int(cell.asText ?? "0") { total = Self.saturatingSum(total, count) @@ -566,7 +597,7 @@ final class ExportService: ObservableObject { guard !isStopRequested else { return total } do { let tableRef = qualifiedTableRef(for: table, driver: driver, databaseType: databaseType) - let result = try await driver.execute(query: "SELECT COUNT(*) FROM \(tableRef)") + let result = try await runCount("SELECT COUNT(*) FROM \(tableRef)") if let cell = result.rows.first?.first, let count = Int(cell.asText ?? "0") { total = Self.saturatingSum(total, count) } diff --git a/apple/TablePro/Core/Services/Export/ImportRun.swift b/apple/TablePro/Core/Services/Export/ImportRun.swift index 70fa682d39..37f0faa411 100644 --- a/apple/TablePro/Core/Services/Export/ImportRun.swift +++ b/apple/TablePro/Core/Services/Export/ImportRun.swift @@ -43,6 +43,7 @@ internal final class ImportRun: @unchecked Sendable { private var waiters: [UUID: CheckedContinuation] = [:] private var ledger = ImportLedger() private var onChange: (@Sendable () -> Void)? + private var interrupt: (@Sendable () -> Void)? internal var currentPhase: ImportRunPhase { lock.withLock { phase } @@ -107,11 +108,15 @@ internal final class ImportRun: @unchecked Sendable { if changed { notify() } } + // A COPY streams its whole block as one call, so a pause would wait for all of it. While one runs, Stop ends it + // and the run, as any failure would: its rows cannot be resumed, and an owned transaction can only roll back. @discardableResult internal func requestPause() -> Bool { - let changed = lock.withLock { () -> Bool in + transition { phase in guard phase == .running else { return false } - if inFlight == 0 { + if interrupt != nil { + phase = .stopping(.rollBack) + } else if inFlight == 0 { generation += 1 phase = .paused(generation: generation) } else { @@ -119,8 +124,25 @@ internal final class ImportRun: @unchecked Sendable { } return true } - if changed { notify() } - return changed + } + + // `interrupt` ends the call early; it runs when the run stops while the call is in flight. A Stop that came after + // the call was admitted stops it here, before it starts. + internal func interruptible( + by interrupt: @escaping @Sendable () -> Void, + _ body: () async throws -> T + ) async throws -> T { + let starts = lock.withLock { () -> Bool in + guard phase == .running else { return false } + self.interrupt = interrupt + return true + } + guard starts else { + stop(.rollBack) + throw PluginImportCancellationError() + } + defer { lock.withLock { self.interrupt = nil } } + return try await body() } // An answer to a question asked at `generation`: ignored once that pause has ended. @@ -278,7 +300,11 @@ internal final class ImportRun: @unchecked Sendable { private func transition(_ change: (inout ImportRunPhase) -> Bool) -> Bool { let wakes = lock.withLock { () -> [CheckedContinuation]? in guard change(&phase) else { return nil } - if case .stopping(let choice) = phase { chosenStop = choice } + if case .stopping(let choice) = phase { + chosenStop = choice + // Under the lock: the interrupted call cannot return and let the next statement start meanwhile. + interrupt?() + } switch phase { case .pausing, .paused: return [] diff --git a/apple/TablePro/Core/Services/Export/ImportService.swift b/apple/TablePro/Core/Services/Export/ImportService.swift index 3f0e86c491..6959fc99ef 100644 --- a/apple/TablePro/Core/Services/Export/ImportService.swift +++ b/apple/TablePro/Core/Services/Export/ImportService.swift @@ -27,6 +27,8 @@ struct ImportState { // A row import counts rows through the statement counter. var countsRows = false var totalIsEstimate = true + var fileBytesRead: Int64 = 0 + var fileBytes: Int64 = 0 } // MARK: - Import Service @@ -85,6 +87,8 @@ final class ImportService: ObservableObject { func requestStop() { run.requestPause() + // A Stop during a COPY stops the run at once, and the file's reader has to see it too. + if run.stopChoice != nil { currentProgress?.cancel() } syncStopState() } @@ -183,6 +187,7 @@ final class ImportService: ObservableObject { decompressedURL: decompressedURL, ownsDecompressedFile: ownsDecompressedFile ) + sqlSource?.reportsReading(to: fileReadingReporter()) let source: any PluginImportSource if let sqlSource { source = sqlSource @@ -304,6 +309,26 @@ final class ImportService: ObservableObject { return [counts, status] } + // The parser reads a chunk every 64 KiB, so the main actor hears only of a move of a whole permille. + private func fileReadingReporter() -> @Sendable (Int64, Int64) -> Void { + let lastPermille = OSAllocatedUnfairLock(initialState: -1) + return { [weak self] read, size in + let permille = size > 0 ? min(1_000, read * 1_000 / size) : 0 + let moved = lastPermille.withLock { last in + guard last != permille else { return false } + last = permille + return true + } + guard moved else { return } + Task { @MainActor [weak self] in + guard let self else { return } + self.state.fileBytes = size + // Hops can land out of order, and the bar never moves back. + self.state.fileBytesRead = max(self.state.fileBytesRead, read) + } + } + } + // MARK: - Several files func importFiles( @@ -413,7 +438,13 @@ final class ImportService: ObservableObject { policy: RowImportBatchPolicy ) async throws -> RowImportBatchReport { let started = Date() - let owner = ImportDataSinkAdapter(driver: driver, databaseType: connection.type, importRun: run, fileIndex: nil) + let owner = ImportDataSinkAdapter( + driver: driver, + databaseType: connection.type, + commitMode: DatabaseManager.shared.commitMode(for: connection.id), + importRun: run, + fileIndex: nil + ) var sessionScope: PluginSessionScope? var outcomes = [RowImportBatchReport.FileOutcome](repeating: .notRun, count: files.count) var ending = RowImportBatchReport.Ending.completed @@ -501,6 +532,7 @@ final class ImportService: ObservableObject { targetTable: file.targetTable, columnMapping: file.columnMapping, sourceFields: file.sourceFields, + binaryText: ImportBinaryText(formatId: file.formatId), joinsTransaction: true, importRun: run, fileIndex: index, @@ -644,6 +676,8 @@ final class ImportService: ObservableObject { let sink = ImportDataSinkAdapter( driver: driver, databaseType: connection.type, targetTable: targetTable, columnMapping: columnMapping, sourceFields: sourceFields, rowWriting: rowWriting, + binaryText: ImportBinaryText(formatId: type(of: plugin).formatId), + commitMode: DatabaseManager.shared.commitMode(for: connection.id), importRun: run, isCancelled: { progress.isCancelled } ) defer { diff --git a/apple/TablePro/Core/Services/Export/RowImportTableMaker.swift b/apple/TablePro/Core/Services/Export/RowImportTableMaker.swift index 3e8c0b0efb..795793dde5 100644 --- a/apple/TablePro/Core/Services/Export/RowImportTableMaker.swift +++ b/apple/TablePro/Core/Services/Export/RowImportTableMaker.swift @@ -81,17 +81,16 @@ internal struct RowImportTableMaker { ) } } - for step in steps { - switch step { - case .create(let table, let statements, let sql): - for statement in statements { - try await runOnLeasedDriver(statement, scope: scope, table: table, clearing: false, run: importRun) - createdAny = true - } - onCreated(table, sql) - case .clear(let table, let sql): - try await runOnLeasedDriver(sql, scope: scope, table: table, clearing: true, run: importRun) + // Every create before any clear: on Teradata ANSI a clear leaves its DELETE pending, which refuses a later create. + for case .create(let table, let statements, let sql) in steps { + for statement in statements { + try await runOnLeasedDriver(statement, scope: scope, table: table, clearing: false, run: importRun) + createdAny = true } + onCreated(table, sql) + } + for case .clear(let table, let sql) in steps { + try await runOnLeasedDriver(sql, scope: scope, table: table, clearing: true, run: importRun) } return receipt } @@ -173,19 +172,37 @@ internal struct RowImportTableMaker { if let run { _ = try await run.admit(.data) } defer { run?.release() } let route = DatabaseManager.shared.executionRoute(for: scope) + let refusesOpenTransaction = !route.isPooled try await DatabaseManager.shared.withScopedDriver( scope: scope, route: route, cancellation: .protectedWrite ) { driver in if let run { - try await Self.executePreparation(sql, table: table, clearing: clearing, driver: driver, run: run) + try await Self.executePreparation( + sql, table: table, clearing: clearing, refusesOpenTransaction: refusesOpenTransaction, + driver: driver, run: run + ) } else { - _ = try await driver.execute(query: sql) + try await Self.executeStep(sql, clearing: clearing, refusesOpenTransaction: refusesOpenTransaction, on: driver) } } } + // A clear belongs to the import's own transaction; only a table this makes is committed on its own. + nonisolated private static func executeStep( + _ sql: String, clearing: Bool, refusesOpenTransaction: Bool, on driver: DatabaseDriver + ) async throws { + guard !clearing else { + _ = try await driver.execute(query: sql) + return + } + _ = try await AppDefinitionCommit.run(on: driver, refusesOpenTransaction: refusesOpenTransaction) { + try await driver.execute(query: sql) + } + } + nonisolated private static func executePreparation( - _ sql: String, table: TableScope, clearing: Bool, driver: DatabaseDriver, run: ImportRun + _ sql: String, table: TableScope, clearing: Bool, refusesOpenTransaction: Bool, driver: DatabaseDriver, + run: ImportRun ) async throws { let id = UUID() let rollback: ImportTableRollback @@ -208,7 +225,7 @@ internal struct RowImportTableMaker { $0.preparations[id] = ImportPreparationEffect(table: table, clearing: clearing) } do { - _ = try await driver.execute(query: sql) + try await executeStep(sql, clearing: clearing, refusesOpenTransaction: refusesOpenTransaction, on: driver) let state = driver.supportsTransactions ? ImportSessionState(await driver.heldSessionTransactionState()) : .idle run.record { diff --git a/apple/TablePro/Core/Services/Export/TableTransferService.swift b/apple/TablePro/Core/Services/Export/TableTransferService.swift index 98d3a27644..2e4806418c 100644 --- a/apple/TablePro/Core/Services/Export/TableTransferService.swift +++ b/apple/TablePro/Core/Services/Export/TableTransferService.swift @@ -138,7 +138,8 @@ final class TableTransferService: ObservableObject { /// drivers, so every statement lands on the database the sheet named rather than wherever /// either shared driver was last parked. The gate asks before either lease is taken, so a /// prompt nobody has answered yet holds no connection. `confirmedByCaller` is a delete of the - /// destination's rows the sheet already confirmed in its own words; Touch ID still applies. + /// destination's rows the sheet already confirmed in its own words; Touch ID still applies. At + /// Read-Only on the source, a row filter meets the gate first, as a Run of it would. func run( request: Request, source: DatabaseScope, @@ -148,6 +149,18 @@ final class TableTransferService: ObservableObject { manager: DatabaseManager = .shared, gate: ExecutionGate = ExecutionGateProvider.shared ) async throws { + let sourceSession = manager.driver(for: source.connectionId) + try await ReadOnlyReadGate.authorize( + ReadOnlyReadGate.rowScopeStatements(request.objects, databaseType: request.sourceType) { name in + sourceSession?.quoteIdentifier(name) ?? name + }, + connectionId: source.connectionId, + databaseType: request.sourceType, + gate: gate, + manager: manager, + runsIn: source + ) + var capabilities = CallerCapabilities.interactiveUser if confirmedByCaller { capabilities.insert(.confirmationPreCleared) @@ -177,11 +190,13 @@ final class TableTransferService: ObservableObject { workload: .bulk, cancellation: .protectedWrite ) { destinationDriver in - try await self.transfer( - request: request, - sourceDriver: sourceDriver, - destinationDriver: destinationDriver - ) + try await BulkReadTimeout.lifted(on: [sourceDriver]) { + try await self.transfer( + request: request, + sourceDriver: sourceDriver, + destinationDriver: destinationDriver + ) + } } } } @@ -204,7 +219,11 @@ final class TableTransferService: ObservableObject { isCancelled = false } - let source = ExportDataSourceAdapter(driver: sourceDriver, databaseType: request.sourceType) + let source = ExportDataSourceAdapter( + driver: sourceDriver, + databaseType: request.sourceType, + safeModeIsReadOnly: ExecutionGateProvider.runsAtReadOnly(sourceDriver.connection.id) + ) /// The destination can be the user's own editor session, so the scopes close on every exit. let scopes = await openSessionScopes( for: request, objects: rowObjects, sourceDriver: sourceDriver, destinationDriver: destinationDriver) @@ -273,7 +292,7 @@ final class TableTransferService: ObservableObject { // PostgreSQL and DuckDB read a BIT as its digits and MySQL as its number, so 00000101 bound to a MySQL BIT stores 101. // Read before the row stream opens: a catalog read queued behind the source's own open stream waits for the whole // table, and behind DuckDB's bounded one never ends. The catalog, because a fresh PostgreSQL session's header names - // a bit column `unknown` and MySQL's drops the width. + // a bit column `unknown` and MySQL's drops the width of a BIT(1). static func bitColumnKinds( for object: ExportObjectItem, source: ExportDataSourceAdapter, diff --git a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift index 41f93a634b..056d832464 100644 --- a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift @@ -9,26 +9,23 @@ import Foundation import TableProPluginKit enum BlobDisplayContext { - /// Data grid cell: compact single-line "0x48656C6C6F..." case grid - /// Sidebar detail view: full multi-line hex dump case detail - /// Copy to clipboard: compact hex case copy - /// Editable hex in sidebar: space-separated hex bytes "48 65 6C 6C 6F" case edit } -@MainActor -final class BlobFormattingService { +final class BlobFormattingService: Sendable { static let shared = BlobFormattingService() private init() {} func format(_ value: String, for context: BlobDisplayContext) -> String? { switch context { - case .grid, .copy: + case .grid: return value.formattedAsCompactHex() + case .copy: + return value.formattedAsCompactHex(maxBytes: .max) case .detail: return value.formattedAsHexDump() case .edit: @@ -37,8 +34,16 @@ final class BlobFormattingService { } func format(_ data: Data, for context: BlobDisplayContext) -> String? { - let value = String(data: data, encoding: .isoLatin1) ?? "" - return format(value, for: context) + switch context { + case .grid: + return data.formattedAsCompactHex() + case .copy: + return data.formattedAsCompactHex(maxBytes: .max) + case .detail: + return data.formattedAsHexDump() + case .edit: + return data.formattedAsEditableHex() + } } func parseHex(_ hexString: String) -> String? { @@ -68,12 +73,44 @@ final class BlobFormattingService { return String(data: data, encoding: .isoLatin1) } - /// Whether the given column type requires BLOB formatting. + static func hasHexPrefix(_ text: String) -> Bool { + text.hasPrefix("0x") || text.hasPrefix("0X") + } + + // The form a copy, a CSV export and an XLSX export write bytes in, read strictly: a copy cut short with `…` + // or text that only starts like hex is not bytes. + func bytes(fromPrefixedHex text: String) -> Data? { + guard Self.hasHexPrefix(text) else { return nil } + let digits = text.utf8.dropFirst(2) + guard digits.count.isMultiple(of: 2) else { return nil } + var bytes = [UInt8]() + bytes.reserveCapacity(digits.count / 2) + var high: UInt8? + for character in digits { + guard let nibble = Self.nibble(character) else { return nil } + if let pending = high { + bytes.append(pending << 4 | nibble) + high = nil + } else { + high = nibble + } + } + return Data(bytes) + } + + private static func nibble(_ character: UInt8) -> UInt8? { + switch character { + case UInt8(ascii: "0")...UInt8(ascii: "9"): return character - UInt8(ascii: "0") + case UInt8(ascii: "a")...UInt8(ascii: "f"): return character - UInt8(ascii: "a") + 10 + case UInt8(ascii: "A")...UInt8(ascii: "F"): return character - UInt8(ascii: "A") + 10 + default: return nil + } + } + func requiresFormatting(columnType: ColumnType) -> Bool { columnType.isBlobType } - /// Format a value if the column type is a BLOB type; otherwise return the original value. func formatIfNeeded(_ value: String, columnType: ColumnType?, for context: BlobDisplayContext) -> String { guard let columnType, requiresFormatting(columnType: columnType) else { return value diff --git a/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift index b1794b9b94..2fd4e12284 100644 --- a/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift @@ -2,46 +2,112 @@ // CellDisplayFormatter.swift // TablePro // -// Pure formatter that transforms raw cell values into display-ready strings. -// Used by the data grid coordinator's display cache to compute values once per cell. -// import Foundation import TableProPluginKit -@MainActor enum CellDisplayFormatter { static let maxDisplayLength = 10_000 + @MainActor static func format( _ rawValue: PluginCellValue, columnType: ColumnType?, displayFormat: ValueDisplayFormat? = nil, databaseType: DatabaseType? = nil ) -> String? { + resolve( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer, + checksRendering: false + ).text + } + + static func format( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer + ) -> String? { + resolve( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: dates, + checksRendering: false + ).text + } + + @MainActor + static func display( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType? + ) -> (text: String?, isFormatted: Bool) { + resolve( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer, + checksRendering: true + ) + } + + // `isFormatted` is false wherever the cell falls back to how its column reads without the display format. + static func display( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer + ) -> (text: String?, isFormatted: Bool) { + resolve( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: dates, + checksRendering: true + ) + } + + // Find, the value filter and the display order discard the flag, and the JSON and PHP checks read the whole value. + private static func resolve( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer, + checksRendering: Bool + ) -> (text: String?, isFormatted: Bool) { switch rawValue { case .null: - return nil + return (nil, false) case .bytes(let data): if let displayFormat, displayFormat.isApplicable(to: columnType, databaseType: databaseType), let formatted = ValueDisplayFormatter.apply(data, format: displayFormat, columnType: columnType) { - return fitToCell(formatted) + return (fitToCell(formatted), true) } - return BlobFormattingService.shared.format(data, for: .grid) + return (BlobFormattingService.shared.format(data, for: .grid), false) case .text(let value): - guard !value.isEmpty else { return value } - var displayValue = value + guard !value.isEmpty else { return (value, false) } if let displayFormat, displayFormat != .raw, displayFormat.isApplicable(to: columnType, databaseType: databaseType), - let formatted = ValueDisplayFormatter.apply(value, format: displayFormat, columnType: columnType) { - displayValue = formatted - } else if let columnType { + let formatted = ValueDisplayFormatter.apply(value, format: displayFormat, columnType: columnType, dates: dates) { + return (fitToCell(formatted), checksRendering && rendered(formatted, from: value, by: displayFormat)) + } + var displayValue = value + if let columnType { if columnType.isDateType { - if let formatted = DateFormattingService.shared.format( - dateString: displayValue, - columnType: columnType - ) { + if let formatted = dates.format(dateString: displayValue, columnType: columnType) { displayValue = formatted } } else if columnType.isVectorType { @@ -52,13 +118,27 @@ enum CellDisplayFormatter { ) } } - return fitToCell(displayValue) + return (fitToCell(displayValue), false) + } + } + + // The UUID, timestamp, JSON and PHP formats hand back a text value they cannot read unchanged rather than nil. + private static func rendered(_ formatted: String, from value: String, by format: ValueDisplayFormat) -> Bool { + switch format { + case .uuid, .uuidSwappedTime: + return UUID(uuidString: formatted) != nil + case .unixTimestamp, .unixTimestampMillis: + return formatted != value + case .json: + return value.isValidJson + case .phpSerialized: + return CellValueContentDetector.detect(value) == .phpSerialized + case .raw, .text, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: + return true } } - /// A row is one line, so anything a format produces is capped and stripped of line breaks - /// before it reaches a cell. Binary read as text is the case that needs it: a blob carries - /// newlines and runs to megabytes where hex never did. A lone CR stays for the cell to mark. + // Binary read as text carries newlines and runs to megabytes, so all output is capped and flattened. A lone CR stays for the cell to mark. private static func fitToCell(_ value: String) -> String { var displayValue = value let nsDisplay = displayValue as NSString diff --git a/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift b/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift new file mode 100644 index 0000000000..6b37d06584 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift @@ -0,0 +1,118 @@ +import Foundation + +// One column shape's formatters: the pattern as the user chose it, and the same pattern with a slot +// for the sub-second digits a value may carry, used only for a value that arrived with a fraction. +private struct TemporalFormatters { + let plain: DateFormatter + let withFractionalSeconds: DateFormatter? +} + +// Not Sendable: each value sets its formatter's zone, so one renderer serves one thread. Work off the +// main actor builds its own from the format option. +final class DateDisplayRenderer { + let format: DateFormatOption + + private let dateTimeFormatters: TemporalFormatters + private let dateOnlyFormatters: TemporalFormatters + private let timeOnlyFormatters: TemporalFormatters + + // The only formatter that renders an instant rather than a wall clock the value carries, so its zone + // is the reader's. `.autoupdatingCurrent` follows a system zone change; a captured `.current` is + // frozen for the life of the process, measured. + private let instantFormatter: DateFormatter + + private let formatCache = NSCache() + + init(format: DateFormatOption) { + self.format = format + dateTimeFormatters = Self.createFormatters(for: format, components: .dateAndTime) + dateOnlyFormatters = Self.createFormatters(for: format, components: .dateOnly) + timeOnlyFormatters = Self.createFormatters(for: format, components: .timeOnly) + instantFormatter = Self.createFormatter(pattern: format.formatString(for: .dateAndTime)) + instantFormatter.timeZone = .autoupdatingCurrent + formatCache.countLimit = 100_000 + } + + func format(_ date: Date) -> String { + instantFormatter.string(from: date) + } + + // Nil when the text does not parse. The value renders in the zone it was written in, not the + // reader's, so the grid and the cell editor agree on the day. A value with an offset prints that + // offset as the literal text it arrived with: a pattern token prints the formatter's zone, which + // would give a naive `DATETIME` an offset the database never held. The cache key needs no zone, + // since the zone is read off the value. + func format(dateString: String, columnType: ColumnType?) -> String? { + let components = temporalComponents(for: columnType) + let cacheKey = "\(components.rawValue)|\(dateString)" as NSString + if let cached = formatCache.object(forKey: cacheKey) { + return cached.length == 0 ? nil : cached as String + } + + guard let parsed = DatabaseDateParser.parse(dateString) else { + formatCache.setObject("" as NSString, forKey: cacheKey) + return nil + } + var result = render(parsed, with: formatters(for: components)) + if self.format.rendersTime(for: components), let suffix = parsed.layout.timeZoneSuffix { + result += suffix + } + formatCache.setObject(result as NSString, forKey: cacheKey) + return result + } + + // A column with no type reads as a full timestamp, the grid's fallback before the editor and the + // formatter shared this vocabulary. + private func temporalComponents(for columnType: ColumnType?) -> TemporalComponents { + guard let columnType else { return .dateAndTime } + return DateEditingService.components(for: columnType) + } + + private func formatters(for components: TemporalComponents) -> TemporalFormatters { + switch components { + case .dateOnly: return dateOnlyFormatters + case .timeOnly: return timeOnlyFormatters + case .dateAndTime: return dateTimeFormatters + } + } + + // The value's own sub-second digits are spliced in, not rendered, so `.789012` stays `.789012` and two + // rows a microsecond apart never share a value filter entry. A fraction of only zeros is padding + // (MySQL writes one on every `DATETIME(6)` row) and is dropped. + private func render(_ parsed: ParsedTemporalValue, with formatters: TemporalFormatters) -> String { + guard let fractionalSeconds = parsed.layout.fractionalSeconds, + fractionalSeconds.contains(where: { $0 != "0" && $0 != "." }), + let marked = formatters.withFractionalSeconds + else { + formatters.plain.timeZone = parsed.timeZone + return formatters.plain.string(from: parsed.wholeSecond) + } + marked.timeZone = parsed.timeZone + return marked.string(from: parsed.wholeSecond) + .replacingOccurrences(of: String(DateFormatOption.fractionalSecondsMarker), with: fractionalSeconds) + } + + // A fixed pattern needs a fixed locale and calendar: the user's region would render timestamps in + // the Buddhist or Japanese calendar and change the value the grid writes back to SQL. + private static let fixedPatternLocale = Locale(identifier: "en_US_POSIX") + + private static func createFormatters( + for format: DateFormatOption, + components: TemporalComponents + ) -> TemporalFormatters { + TemporalFormatters( + plain: createFormatter(pattern: format.formatString(for: components)), + withFractionalSeconds: format.fractionalSecondsFormatString(for: components) + .map(createFormatter(pattern:)) + ) + } + + private static func createFormatter(pattern: String) -> DateFormatter { + let formatter = DateFormatter() + formatter.dateFormat = pattern + formatter.locale = fixedPatternLocale + formatter.calendar = Calendar(identifier: .gregorian) + formatter.timeZone = TimeZone.current + return formatter + } +} diff --git a/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift b/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift index 25e5751407..67449a233a 100644 --- a/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift @@ -2,73 +2,33 @@ // DateFormattingService.swift // TablePro // -// Centralized date formatting service that respects user settings. -// Thread-safe singleton that formats dates according to DataGridSettings.dateFormat. -// import Combine import Foundation -/// The pair of formatters one column shape needs: the pattern as the user chose it, and the same -/// pattern with a slot for the sub-second digits a value may carry. -private struct TemporalFormatters { - let plain: DateFormatter - let withFractionalSeconds: DateFormatter? -} - -/// Centralized date formatting service that respects user settings @MainActor final class DateFormattingService { static let shared = DateFormattingService() - // MARK: - Properties - - /// Cached formatters for the current user-selected format. - /// - /// Two per column shape, not one: the marked variant carries a quoted slot after the seconds - /// and is used only for a value that arrived with a fraction, so a value without one pays - /// neither the second format nor the splice. - private var dateTimeFormatters: TemporalFormatters - private var dateOnlyFormatters: TemporalFormatters - private var timeOnlyFormatters: TemporalFormatters - - /// The one formatter that renders an instant rather than a wall clock the value already - /// carries, so it is the only one whose zone is the reader's. - /// - /// It holds `.autoupdatingCurrent`, which follows a system time zone change with no - /// reassignment; a captured `.current` is frozen for the life of the process, measured. Its - /// pattern still tracks the user's chosen format, so `updateFormat` rewrites that and never - /// the zone. - private let instantFormatter: DateFormatter + // Replaced on a format change rather than reconfigured, so no text cached under the old format survives. + private(set) var renderer: DateDisplayRenderer private var systemTimeZoneObserver: NSObjectProtocol? - /// Rises each time the Mac's zone moves, so a display cache built in the old one compares - /// unequal and is replaced rather than repaired. + // Rises each time the Mac's zone moves, so a display cache built in the old one compares unequal and + // is replaced rather than repaired. private(set) var systemTimeZoneGeneration = 0 - /// Current date format option - private(set) var currentFormat: DateFormatOption - - /// Cache for formatted date strings to avoid repeated parsing - private let formatCache = NSCache() - - // MARK: - Initialization + var currentFormat: DateFormatOption { renderer.format } + // AppSettingsManager sets the user's format once it has loaded. private init() { - // Will be updated by AppSettingsManager after it completes initialization - self.currentFormat = .iso8601 - self.dateTimeFormatters = Self.createFormatters(for: .iso8601, components: .dateAndTime) - self.dateOnlyFormatters = Self.createFormatters(for: .iso8601, components: .dateOnly) - self.timeOnlyFormatters = Self.createFormatters(for: .iso8601, components: .timeOnly) - self.instantFormatter = Self.createFormatter(pattern: DateFormatOption.iso8601.formatString(for: .dateAndTime)) - self.instantFormatter.timeZone = .autoupdatingCurrent - formatCache.countLimit = 100_000 + renderer = DateDisplayRenderer(format: .iso8601) observeSystemTimeZone() } - /// `NSSystemTimeZoneDidChange` documents no posting thread, and this app has already shipped a - /// crash from assuming one, so the observer names the main queue and hops explicitly. + // `NSSystemTimeZoneDidChange` documents no posting thread, and this app has already shipped a crash + // from assuming one, so the observer names the main queue and hops explicitly. private func observeSystemTimeZone() { systemTimeZoneObserver = NotificationCenter.default.addObserver( forName: .NSSystemTimeZoneDidChange, @@ -82,122 +42,16 @@ final class DateFormattingService { } } - // MARK: - Public Methods - - /// Update the date format (called by AppSettingsManager when settings change) func updateFormat(_ format: DateFormatOption) { guard format != currentFormat else { return } - currentFormat = format - dateTimeFormatters = Self.createFormatters(for: format, components: .dateAndTime) - dateOnlyFormatters = Self.createFormatters(for: format, components: .dateOnly) - timeOnlyFormatters = Self.createFormatters(for: format, components: .timeOnly) - instantFormatter.dateFormat = format.formatString(for: .dateAndTime) - // Clear cache when format changes since all cached values are now stale - formatCache.removeAllObjects() + renderer = DateDisplayRenderer(format: format) } - /// Format a date using current user settings - /// - Parameter date: The date to format - /// - Returns: Formatted date string func format(_ date: Date) -> String { - instantFormatter.string(from: date) + renderer.format(date) } - /// Format a string date value (parse then format). - /// - /// The value is rendered in the zone it was written in, not the reader's. A value carrying an - /// offset names an instant in that offset, and the cell editor opens it there, so formatting it - /// somewhere else made the grid and the picker disagree about which day the row held. A value - /// with no offset is naive and parses in GMT, so it prints the wall clock it was written with - /// whatever zone the reader is in. - /// - /// The offset itself is printed with it, because a wall clock alone does not name an instant. - /// It is the literal text the value arrived with, off the same layout the write-back reads. A - /// pattern token cannot serve: it prints the formatter's own zone, which for a value carrying - /// no offset is the reader's, so a `DATETIME` would gain an offset the database never held. - /// - /// The cache key needs no zone: the zone is read off the value, so the string determines it. - /// - Parameter dateString: Date string from database (ISO 8601, MySQL timestamp, etc.) - /// - Parameter columnType: Column type, used to pick date-only / time-only / datetime variant - /// - Returns: Formatted date string, or nil if unparseable func format(dateString: String, columnType: ColumnType? = nil) -> String? { - let components = temporalComponents(for: columnType) - let cacheKey = "\(components.rawValue)|\(dateString)" as NSString - if let cached = formatCache.object(forKey: cacheKey) { - return cached.length == 0 ? nil : cached as String - } - - guard let parsed = DatabaseDateParser.parse(dateString) else { - formatCache.setObject("" as NSString, forKey: cacheKey) - return nil - } - var result = render(parsed, with: formatters(for: components)) - if currentFormat.rendersTime(for: components), let suffix = parsed.layout.timeZoneSuffix { - result += suffix - } - formatCache.setObject(result as NSString, forKey: cacheKey) - return result - } - - /// A column with no type at all is read as a full timestamp, which is what the grid's own - /// fallback did before the editor and the formatter shared this vocabulary. - private func temporalComponents(for columnType: ColumnType?) -> TemporalComponents { - guard let columnType else { return .dateAndTime } - return DateEditingService.components(for: columnType) - } - - private func formatters(for components: TemporalComponents) -> TemporalFormatters { - switch components { - case .dateOnly: return dateOnlyFormatters - case .timeOnly: return timeOnlyFormatters - case .dateAndTime: return dateTimeFormatters - } - } - - /// The value's own sub-second digits are spliced in, not rendered, so `.5` stays `.5` and - /// `.789012` stays `.789012`. Two rows one microsecond apart used to draw the same text, which - /// also collapsed them into one entry in the column's value filter. - /// - /// A fraction of nothing but zeros is padding rather than precision, and MySQL writes one on - /// every row of a `DATETIME(6)` where PostgreSQL trims it at the server. It is dropped, so such - /// a column reads exactly as it did before and only a value that carries real digits grows. - private func render(_ parsed: ParsedTemporalValue, with formatters: TemporalFormatters) -> String { - guard let fractionalSeconds = parsed.layout.fractionalSeconds, - fractionalSeconds.contains(where: { $0 != "0" && $0 != "." }), - let marked = formatters.withFractionalSeconds - else { - formatters.plain.timeZone = parsed.timeZone - return formatters.plain.string(from: parsed.wholeSecond) - } - marked.timeZone = parsed.timeZone - return marked.string(from: parsed.wholeSecond) - .replacingOccurrences(of: String(DateFormatOption.fractionalSecondsMarker), with: fractionalSeconds) - } - - // MARK: - Private Helper Methods - - /// `dateFormat` is a fixed pattern, so it must be evaluated against a fixed locale and - /// calendar. Following the user's region renders database timestamps in the Buddhist or - /// Japanese calendar and reformats the very value the grid round-trips back to SQL. - private static let fixedPatternLocale = Locale(identifier: "en_US_POSIX") - - private static func createFormatters( - for format: DateFormatOption, - components: TemporalComponents - ) -> TemporalFormatters { - TemporalFormatters( - plain: createFormatter(pattern: format.formatString(for: components)), - withFractionalSeconds: format.fractionalSecondsFormatString(for: components) - .map(createFormatter(pattern:)) - ) - } - - private static func createFormatter(pattern: String) -> DateFormatter { - let formatter = DateFormatter() - formatter.dateFormat = pattern - formatter.locale = fixedPatternLocale - formatter.calendar = Calendar(identifier: .gregorian) - formatter.timeZone = TimeZone.current - return formatter + renderer.format(dateString: dateString, columnType: columnType) } } diff --git a/apple/TablePro/Core/Services/Formatting/JsonReindenter.swift b/apple/TablePro/Core/Services/Formatting/JsonReindenter.swift index a34098ad0f..c6e40a8bb1 100644 --- a/apple/TablePro/Core/Services/Formatting/JsonReindenter.swift +++ b/apple/TablePro/Core/Services/Formatting/JsonReindenter.swift @@ -21,6 +21,15 @@ internal enum JsonReindenter { reindentIfValid(source, indent: indent) ?? source } + // An edit keeps the stored value's layout: a value stored on one line stays compact, a formatted one takes the + // editor's lines rather than losing every line break. + static func staged(_ edited: String, keepingLayoutOf stored: String?) -> String { + guard let stored, stored.trimmingCharacters(in: .whitespacesAndNewlines).contains(where: \.isNewline) else { + return normalize(edited) + } + return edited + } + static func normalize(_ source: String) -> String { guard (source as NSString).length <= maxLength else { return source } guard let node = JsonSyntaxParser.parse(source) else { return source } diff --git a/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift b/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift index deddba5282..ebf788e835 100644 --- a/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift +++ b/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift @@ -261,7 +261,14 @@ internal struct SQLTokenFormatter { directiveLines: directiveLines ) } - return text.trimmingCharacters(in: .whitespacesAndNewlines) + guard stream.last?.type == .directive else { return text.trimmingCharacters(in: .whitespacesAndNewlines) } + // A directive's text stays as written to its end: a COPY's last row can end in a tab that is a column. + let start = text.firstIndex { !$0.isWhitespace } ?? text.endIndex + var end = text.endIndex + while end > start, text[text.index(before: end)] == "\n" { + end = text.index(before: end) + } + return String(text[start.. = Set("(),;.".utf16) private static let operatorUnits: Set = Set("=<>+-*/%&|^~!@#?:$".utf16) - private static let delimiterWord = "DELIMITER" static func isOperatorUnit(_ unit: UInt16) -> Bool { operatorUnits.contains(unit) @@ -329,6 +328,7 @@ private struct Lexer { let grammar: SQLLexicalGrammar let tokenizer: SQLTokenizer private let batchSeparatorEnds: [Int: Int] + private let psqlTextEnds: [Int: Int] private var delimiter = ";" private var atStatementStart = true private var tokens: [SQLToken] = [] @@ -344,6 +344,11 @@ private struct Lexer { ends[separator.range.location] = NSMaxRange(separator.range) } self.batchSeparatorEnds = ends + var psqlEnds: [Int: Int] = [:] + for range in SQLStatementScanner.psqlScriptText(in: text as String, grammar: grammar) { + psqlEnds[range.location] = NSMaxRange(range) + } + self.psqlTextEnds = psqlEnds } mutating func run() -> [SQLToken] { @@ -356,20 +361,21 @@ private struct Lexer { private mutating func step() { let unit = text.character(at: index) + // A `GO` line, a `COPY ... FROM STDIN` with its rows and a psql `\restrict` line keep their text. + if let end = batchSeparatorEnds[index] ?? psqlTextEnds[index] { + let start = index + index = end + emit(.directive, from: start) + return + } if isBlank(unit) { let start = index - while index < length, isBlank(text.character(at: index)) { + while index < length, isBlank(text.character(at: index)), psqlTextEnds[index] == nil { index += 1 } emit(.whitespace, from: start) return } - if let end = batchSeparatorEnds[index] { - let start = index - index = end - emit(.directive, from: start) - return - } if unit == Self.slash, grammar.contains(.slashLineTerminators), SQLStatementScanner.isSlashLine(text, at: index, length: length) { let start = index @@ -440,23 +446,16 @@ private struct Lexer { // `DELIMITER //` at the start of a statement, on a line of its own, changes what ends a statement, as the mysql // client and the import read a script. private mutating func readDelimiterDirective() -> Bool { - guard grammar.contains(.delimiterDirective), atStatementStart, isAtLineStart(index) else { return false } - let wordLength = (Self.delimiterWord as NSString).length - guard index + wordLength < length, - substring(from: index, to: index + wordLength).uppercased() == Self.delimiterWord, - isLineBlank(text.character(at: index + wordLength)) + guard grammar.contains(.delimiterDirective), atStatementStart, isAtLineStart(index), + let directive = SQLDelimiterDirective.line(startingAt: index, in: text, length: length) else { return false } - var end = index + wordLength - while end < length, text.character(at: end) != Self.newline { - end += 1 + let end = NSMaxRange(directive.range) + if let newDelimiter = directive.delimiter { + delimiter = newDelimiter } - let line = substring(from: index, to: end).trimmingCharacters(in: .whitespaces) - let newDelimiter = String(line.dropFirst(Self.delimiterWord.count)).trimmingCharacters(in: .whitespaces) - guard !newDelimiter.isEmpty else { return false } - delimiter = newDelimiter - append(SQLToken(type: .directive, value: line)) + append(SQLToken(type: .directive, value: substring(from: index, to: end).trimmingCharacters(in: .whitespaces))) index = end return true } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 3732e7d5b0..96a4ee5e54 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -12,14 +12,23 @@ import TableProUtilities // per cell, and it has to stay reachable without dragging the preference layer behind it. // Resolving *which* format a column carries is a separate job with separate dependencies, and it // lives in `ValueDisplayFormatService`. -@MainActor enum ValueDisplayFormatter { + @MainActor + static func apply( + _ rawValue: String, + format: ValueDisplayFormat, + columnType: ColumnType? = nil + ) -> String? { + apply(rawValue, format: format, columnType: columnType, dates: DateFormattingService.shared.renderer) + } + // nil where the format cannot render this value, so the caller falls back to how the column // would have read without it rather than showing a value the format did not actually produce. static func apply( _ rawValue: String, format: ValueDisplayFormat, - columnType: ColumnType? = nil + columnType: ColumnType?, + dates: DateDisplayRenderer ) -> String? { switch format { case .raw: @@ -31,9 +40,9 @@ enum ValueDisplayFormatter { case .uuidSwappedTime: return formatAsUuid(rawValue, order: .swappedTime) case .unixTimestamp: - return formatAsTimestamp(rawValue, divideBy: 1) + return formatAsTimestamp(rawValue, divideBy: 1, dates: dates) case .unixTimestampMillis: - return formatAsTimestamp(rawValue, divideBy: 1_000) + return formatAsTimestamp(rawValue, divideBy: 1_000, dates: dates) case .json, .phpSerialized: return rawValue case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: @@ -105,11 +114,11 @@ enum ValueDisplayFormatter { return "\(p1)-\(p2)-\(p3)-\(p4)-\(p5)" } - private static func formatAsTimestamp(_ rawValue: String, divideBy divisor: Double) -> String { + private static func formatAsTimestamp(_ rawValue: String, divideBy divisor: Double, dates: DateDisplayRenderer) -> String { guard let numericValue = Double(rawValue) else { return rawValue } let seconds = numericValue / divisor let date = Date(timeIntervalSince1970: seconds) - return DateFormattingService.shared.format(date) + return dates.format(date) } } diff --git a/apple/TablePro/Core/Services/Highlight/HighlightCondition.swift b/apple/TablePro/Core/Services/Highlight/HighlightCondition.swift index 094ad197df..82181fb47c 100644 --- a/apple/TablePro/Core/Services/Highlight/HighlightCondition.swift +++ b/apple/TablePro/Core/Services/Highlight/HighlightCondition.swift @@ -10,8 +10,8 @@ struct HighlightCondition { static let searchLimit = 10_000 private enum ValueKind { - case numeric - case boolean + // A boolean column can hold numbers: MySQL's TINYINT(1) stores any integer from -128 to 127. + case numberOrBoolean case text } @@ -130,10 +130,8 @@ struct HighlightCondition { private var prefersNumbers: Bool { switch valueKind { - case .numeric: + case .numberOrBoolean: return true - case .boolean: - return false case .text: return isOrderingOperator } @@ -141,7 +139,7 @@ struct HighlightCondition { private var prefersBooleans: Bool { switch valueKind { - case .numeric, .boolean: + case .numberOrBoolean: return true case .text: return false @@ -194,12 +192,7 @@ struct HighlightCondition { regularExpression(pattern, ignoresCase: false) != nil } - /// `searchLimit` caps the haystack in `searchable(_:)`, not the pattern. Capping the pattern - /// looked like a guard against a runaway expression and is not one: cost comes from a pattern's - /// shape rather than its length, and `(a+)+$` compiles in 0.01ms. Measured against this cap, - /// a valid 160,000-unit alternation compiles in 8.85ms, once per rule set rather than per row, - /// so the only thing the cap achieved was to refuse a pattern the user wrote and then report it - /// as invalid. Nil now means the expression will not compile, and nothing else. + // No length cap: cost follows a pattern's shape, and a valid 160,000-unit alternation compiled in 8.85ms. private static func regularExpression(_ pattern: String, ignoresCase: Bool) -> NSRegularExpression? { guard !pattern.isEmpty else { return nil } return try? NSRegularExpression(pattern: pattern, options: ignoresCase ? [.caseInsensitive] : []) @@ -207,10 +200,8 @@ struct HighlightCondition { private static func valueKind(for columnType: ColumnType?) -> ValueKind { switch columnType { - case .integer, .decimal: - return .numeric - case .boolean: - return .boolean + case .integer, .decimal, .boolean: + return .numberOrBoolean case .text, .date, .timestamp, .datetime, .blob, .json, .enumType, .set, .spatial, .vector, .array, .none: return .text } diff --git a/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift b/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift index 42e8bc3191..ba42c1e068 100644 --- a/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift +++ b/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift @@ -7,12 +7,7 @@ import Combine import Foundation import os -/// Every agent session the app is holding, for as long as the app is running. -/// -/// Sessions used to be reached through a field on the window's trailing pane, which made a -/// session's lifetime the window's: closing a window, disconnecting, or losing a session each -/// destroyed a transcript the user never asked to lose. They live here instead, so a window is one -/// of the places a session can be looked at rather than the thing that owns it. +// Owns every agent session, so closing a window or disconnecting never destroys a transcript. @MainActor internal final class AgentSessionRegistry: ObservableObject { internal static let shared = AgentSessionRegistry() @@ -24,20 +19,10 @@ internal final class AgentSessionRegistry: ObservableObject { private let store: AgentSessionStore private let services: AppServices - /// Serialises writes. `persist()` used to fire an unordered task per mutation, and actor - /// reentrancy let a stale snapshot land last. + // One writer: unordered tasks let a stale snapshot land last. private var writeTask: Task? - /// Which session each connection is showing. - /// - /// Held explicitly rather than derived. Deriving it from `sessions(for:)` answered the oldest - /// one every time, so New Session appended a row nothing switched to and Open Session moved a - /// timestamp nothing read: both panes stayed bound to the first session for ever. - /// - /// Showing nothing is a state of its own. Closing or deleting the session on screen with no other - /// live one to hand to used to fall back to the last session in the list, which was the one just - /// closed, so the column went on drawing a stopped session with a composer that still took - /// messages. + // `.nothing` is a state of its own, or a closed last session would stay drawn with a live composer. private var displayed: [UUID: DisplayedSession] = [:] private enum DisplayedSession: Equatable { @@ -45,14 +30,10 @@ internal final class AgentSessionRegistry: ObservableObject { case nothing } - /// Going to work moves a session up the rail, and the rail observes this registry rather than - /// each session, so the registry has to say so, and write the new stamp so the order survives a - /// relaunch. + // The rail observes the registry, not each session, so a session going to work is announced here. private var activityCancellables: [UUID: AnyCancellable] = [:] - /// Restore runs here, synchronously, which is the whole point of the store being a plain struct. - /// A window that opens while a load is suspended finds nothing, mints a session, and is then - /// joined by the stored one; doing it in the initialiser means no window can exist yet. + // Restored synchronously, before any window exists to mint a second session beside a stored one. internal init(store: AgentSessionStore = AgentSessionStore(), services: AppServices = .live) { self.store = store self.services = services @@ -78,8 +59,7 @@ internal final class AgentSessionRegistry: ObservableObject { } } - /// `@Published` announces a change before it is stored, so the write waits for the next turn to - /// read the new stamp rather than the one it replaces. + // `@Published` announces before it stores, so the write reads the stamp on the next turn. private func observeActivity(of session: AgentSession) { activityCancellables[session.id] = session.$lastActiveAt .dropFirst() @@ -96,8 +76,7 @@ internal final class AgentSessionRegistry: ObservableObject { // MARK: - Reading - /// A connection's sessions, the one that last went to work first, which is the order the rail - /// lists them in. Two stamped in the same instant keep the order they were added in, newest first. + // The rail's order. Two stamped in the same instant list the one added last first. internal func sessions(for connectionId: UUID) -> [AgentSession] { sessions.enumerated() .filter { $0.element.connectionId == connectionId } @@ -114,15 +93,7 @@ internal final class AgentSessionRegistry: ObservableObject { sessions.first { $0.id == id } } - /// The session a connection's two presentations share. - /// - /// Both the trailing-pane chat and the agent-mode conversation column resolve through here, so - /// they cannot end up rendering two different sessions of the same connection. Reading never - /// creates: opening a connection window starts no session and loads no transcript. - /// - /// With nothing named it is the latest live session, then the latest of any, which is how a - /// window opened after a relaunch, where every session comes back stopped, carries on with the - /// last one rather than starting another. + // Falls back to the latest of any, so a window after a relaunch, with every session stopped, carries on. internal func currentSession(for connectionId: UUID) -> AgentSession? { let owned = sessions(for: connectionId) switch displayed[connectionId] { @@ -138,8 +109,6 @@ internal final class AgentSessionRegistry: ObservableObject { return owned.first { !$0.status.isEnded } ?? owned.first } - /// Names the session a connection's two panes render. The pane render key carries it, so a - /// switch repaints rather than comparing equal. internal func setDisplayedSession(_ sessionId: UUID, for connectionId: UUID) { guard sessions.contains(where: { $0.id == sessionId && $0.connectionId == connectionId }) else { return @@ -147,9 +116,7 @@ internal final class AgentSessionRegistry: ObservableObject { displayed[connectionId] = .session(sessionId) } - /// Shows the latest live session in place of one that is ending or going away, and nothing when - /// no other live one is left. A stopped session is shown only once someone opens it, which - /// resumes it. + // A stopped session is shown only once someone opens it, which resumes it. private func handOverDisplay(from session: AgentSession) { let next = sessions(for: session.connectionId).first { $0.id != session.id && !$0.status.isEnded } displayed[session.connectionId] = next.map { .session($0.id) } ?? .nothing @@ -173,7 +140,6 @@ internal final class AgentSessionRegistry: ObservableObject { return session } - /// The session to show for a connection, starting one only when the caller means to. @discardableResult internal func resolveSession(for connectionId: UUID, startingIfNeeded: Bool) -> AgentSession? { if let existing = currentSession(for: connectionId) { @@ -193,11 +159,7 @@ internal final class AgentSessionRegistry: ObservableObject { return startSession(for: connectionId) } - /// Ends one session and keeps its transcript. The conversation stays in the history and the - /// session stays in the rail, to be opened again. - /// - /// The session on screen is handed over as it stops, which is what Close Session means. Nothing - /// used to tell the panes, so they went on drawing a stopped session with a live composer. + // The panes are handed over first, so no window shows the session when its memory is released. internal func stopSession(id: UUID) { guard let session = session(id: id) else { return } let isShown = currentSession(for: session.connectionId) === session @@ -206,32 +168,33 @@ internal final class AgentSessionRegistry: ObservableObject { if isShown { handOverDisplay(from: session) } + session.release() persist() } - /// Ends every session on a connection once nothing is showing it any more. - /// - /// Called from the connection's own teardown rather than from a window's, because a connection - /// can be hosted by two windows and one of them closing is not the connection going away. + // A connection can be hosted by two windows, and one of them closing is not the connection going away. internal func stopSessionsIfUnhosted(for connectionId: UUID) { guard WindowManager.shared.workspaces(for: connectionId).isEmpty else { return } stopSessions(for: connectionId) } - /// Ends every session on a connection. Disconnect and window close reach this; neither is the - /// user discarding a conversation, so nothing is deleted. + // Ended sessions are released too: a failed one, or one stopped while on screen, still holds its transcript. internal func stopSessions(for connectionId: UUID) { - let owned = sessions(for: connectionId).filter { !$0.status.isEnded } - guard !owned.isEmpty else { return } - for session in owned { + let owned = sessions(for: connectionId) + let live = owned.filter { !$0.status.isEnded } + for session in live { session.stop() detachRemoteTools(from: session.id) } - persist() + for session in owned { + session.release() + } + if !live.isEmpty { + persist() + } } - /// Discards a session and the conversation behind it. Only an explicit user action reaches here, - /// and it has been asked to confirm by then. + // Only an explicit, confirmed user action reaches here. internal func removeSession(id: UUID) { guard let index = sessions.firstIndex(where: { $0.id == id }) else { return } let session = sessions[index] @@ -246,35 +209,25 @@ internal final class AgentSessionRegistry: ObservableObject { handOverDisplay(from: session) } if let conversationId { - let storage = services.aiChatStorage - Task { await storage.delete(conversationId) } + session.viewModel.deleteStoredConversations([conversationId]) } persist() } // MARK: - Outside MCP servers - /// Connects the outside MCP servers this session's connection allows, and registers their tools. - /// - /// Started rather than awaited. A server on the other side of a network is not something a - /// session's first paint may wait on, and a session with no remote tools yet is a session the - /// model simply has not been offered them in; they appear on the turn after they land. - /// - /// Attaching an already-attached session is how a resumed one keeps its tools, so this is safe - /// to call on every resolve. + // Not awaited: a first paint never waits on a remote server. Attaching again keeps a resumed session's tools. private func attachRemoteTools(to session: AgentSession) { Task { await MCPRemoteToolCoordinator.shared.attach(session: session) } } - /// Drops this session's claim on every server. The tools stay registered while another session - /// still holds one, so ending one conversation cannot take them from another mid-turn. + // The tools stay registered while another session still holds a server. private func detachRemoteTools(from sessionId: UUID) { Task { await MCPRemoteToolCoordinator.shared.detach(sessionId: sessionId) } } // MARK: - Persistence - /// One writer, so the last snapshot taken is the last one written. internal func persist() { let records = sessions.map(\.record) let store = store @@ -286,10 +239,7 @@ internal final class AgentSessionRegistry: ObservableObject { } } - /// Writes now, on this thread, and returns once it is on disk. - /// - /// `persist()` books the write for the next turn, which is right for an ordinary mutation and - /// wrong for anything that has to know the write landed. + // For a caller that has to know the write landed; `persist()` books it for the next turn. internal func persistNow() { writeTask?.cancel() writeTask = nil @@ -299,11 +249,7 @@ internal final class AgentSessionRegistry: ObservableObject { store.save(sessions.map(\.record)) } - /// Writes on the way out. - /// - /// An actor hop at terminate may never be scheduled, so a session killed mid-reply used to come - /// back with its last turn missing. A session still working when the app went away is recorded - /// as failed, because it was: its reply never finished. + // A hop at terminate may never run. A session still working is recorded as failed: its reply never finished. internal func persistSynchronouslyForTermination() { for session in sessions where session.status == .working { session.mark(.failed) diff --git a/apple/TablePro/Core/Services/Infrastructure/ClipboardService.swift b/apple/TablePro/Core/Services/Infrastructure/ClipboardService.swift index 111178a6bf..a4840ee40f 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ClipboardService.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ClipboardService.swift @@ -64,6 +64,8 @@ protocol ClipboardProvider { func readGridRows() -> GridRowsClipboardPayload? func readGridCells() -> GridCellsClipboardPayload? func writeText(_ text: String) + // A requirement, so a call on `ClipboardService.shared` reaches a provider's own concealed write. + func writeSecretText(_ text: String) func writeCsv(_ csv: String) func writeImage(_ image: NSImage) func writeRows(tsv: String, html: String?, gridRows: GridRowsClipboardPayload) @@ -75,8 +77,7 @@ protocol ClipboardProvider { } extension ClipboardProvider { - /// Text a clipboard-history app should not retain. Providers that cannot express that - /// fall back to a plain write rather than refusing to copy. + // A provider that cannot mark text concealed still copies it rather than refusing. func writeSecretText(_ text: String) { writeText(text) } diff --git a/apple/TablePro/Core/Services/Infrastructure/ConnectionCloseAction.swift b/apple/TablePro/Core/Services/Infrastructure/ConnectionCloseAction.swift index f14c1cfce1..5cc66ab9fc 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ConnectionCloseAction.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ConnectionCloseAction.swift @@ -9,10 +9,12 @@ import Foundation // Disconnect keeps the connection to reconnect, an entry close takes one container, this ends the connection and every entry. @MainActor internal enum ConnectionCloseAction { + // Asked in this order. The transaction comes last and is read then: a Save under Manual Commit lands in a held one. internal enum Decision: Equatable { case closeImmediately - case confirmEndingTransaction + case confirmRunningWork case confirmUnsavedWork + case confirmEndingTransaction } // Every window hosting the connection, every tab in each: saving only the selected tab closed over the rest. @@ -34,112 +36,84 @@ internal enum ConnectionCloseAction { return MainContentCommandActions.unsavedWorkMessage(keepsTabsOpen: keepsTabsOpen) } - // The transaction is asked before unsaved work, whose Save writes at once. - internal static func decision(hasSession: Bool, holdsTransaction: Bool, hasUnsavedWork: Bool) -> Decision { - guard hasSession else { return .closeImmediately } - if holdsTransaction { return .confirmEndingTransaction } - return hasUnsavedWork ? .confirmUnsavedWork : .closeImmediately - } - - internal static func transactionMessage(for databases: [String]) -> String? { - guard let first = databases.first else { return nil } - guard databases.count > 1 else { - return String( - format: String(localized: "The database “%@” has an open transaction. Closing rolls it back and discards its uncommitted changes."), - first - ) - } - return String( - format: String(localized: "These databases have open transactions: %@. Closing rolls them back and discards their uncommitted changes."), - ListFormatter.localizedString(byJoining: databases.map { "“\($0)”" }) - ) + // The Save alert names the running work, so the two never ask separately. + internal static func decision(hasSession: Bool, hasRunningWork: Bool, hasUnsavedWork: Bool) -> Decision { + if hasSession, hasUnsavedWork { return .confirmUnsavedWork } + if hasRunningWork { return .confirmRunningWork } + return hasSession ? .confirmEndingTransaction : .closeImmediately } internal static func close(connectionId: UUID, registry: RunningWorkRegistry = .shared) async { // Every window hosting the connection: a torn-off tab keeps its edits in its own coordinator. let coordinators = WindowManager.shared.coordinators(for: connectionId) let coordinator = coordinators.first ?? WindowManager.shared.coordinator(for: connectionId) + let hasSession = coordinator != nil // Read at each decision: the window stays editable while the alerts are awaited. let hasUnsavedWork = { coordinators.contains { $0.hasAnyUnsavedWork() } || (coordinators.isEmpty && coordinator?.hasAnyUnsavedWork() == true) } - let holding = await DatabaseManager.shared.databasesHoldingTransaction(for: connectionId) - var decision = decision( - hasSession: coordinator != nil, - holdsTransaction: !holding.isEmpty, - hasUnsavedWork: hasUnsavedWork() - ) // Shown, then asked; a close that closes nothing puts the user back where they were. let wasShowing = WindowManager.shared.shownConnection(besides: connectionId) + func restore() { + WindowManager.shared.show(wasShowing, inWindowHosting: connectionId) + } let work = registry.items(in: .connection(connectionId)) - var hasAnsweredForWork = false - // With nothing unsaved, one alert names the work and any open transaction. - if !work.isEmpty, !hasUnsavedWork(), let prompt = RunningWorkPrompt.make( - action: .close, - work: work, - otherLosses: RunningWorkPrompt.transactionLines(for: holding) - ) { - guard await RunningWorkApproval.confirm(prompt, window: reveal(connectionId: connectionId)) else { - WindowManager.shared.show(wasShowing, inWindowHosting: connectionId) - return + var namedWork = false + var decision = decision(hasSession: hasSession, hasRunningWork: !work.isEmpty, hasUnsavedWork: hasUnsavedWork()) + + if decision == .confirmRunningWork { + if let prompt = RunningWorkPrompt.make(action: .close, work: work) { + guard await RunningWorkApproval.confirm(prompt, window: reveal(connectionId: connectionId)) else { + restore() + return + } } - hasAnsweredForWork = true - // Another window on the connection stayed editable while the alert was up. - decision = Self.decision(hasSession: coordinator != nil, holdsTransaction: false, hasUnsavedWork: hasUnsavedWork()) + namedWork = true + decision = Self.decision(hasSession: hasSession, hasRunningWork: false, hasUnsavedWork: hasUnsavedWork()) } - if decision == .confirmEndingTransaction, let coordinator, let message = transactionMessage(for: holding) { - let confirmed = await AlertHelper.confirmDestructive( - title: String(format: String(localized: "Close the connection “%@”?"), coordinator.connection.name), - message: message, - confirmButton: String(localized: "Close"), + + if decision == .confirmUnsavedWork { + let targets = coordinators.isEmpty ? [coordinator].compactMap { $0 } : coordinators + switch await AlertHelper.confirmSaveChanges( + message: RunningWorkPrompt.saveMessage( + unsavedWorkMessage(across: targets), + action: .close, + alsoStopping: namedWork ? [] : work + ), window: reveal(connectionId: connectionId) - ) - guard confirmed else { - WindowManager.shared.show(wasShowing, inWindowHosting: connectionId) - return - } - decision = Self.decision(hasSession: true, holdsTransaction: false, hasUnsavedWork: hasUnsavedWork()) - } - guard decision == .confirmUnsavedWork else { - // The edits were saved meanwhile, so the work still needs its own answer. - if !hasAnsweredForWork, let prompt = RunningWorkPrompt.make(action: .close, work: work) { - guard await RunningWorkApproval.confirm(prompt, window: reveal(connectionId: connectionId)) else { - WindowManager.shared.show(wasShowing, inWindowHosting: connectionId) + ) { + case .save: + guard await saveEveryWindowsWork(across: coordinators, fallback: coordinator) else { + restore() return } + case .dontSave: + break + case .cancel: + restore() + return } - registry.cancel(work) - WindowManager.shared.closeWindow(for: connectionId) - return + decision = .confirmEndingTransaction } - // Work is stopped only once the close is certain: a cancelled or failed save keeps it running. - let presentingWindow = reveal(connectionId: connectionId) - let targets = coordinators.isEmpty ? [coordinator].compactMap { $0 } : coordinators - switch await AlertHelper.confirmSaveChanges( - message: RunningWorkPrompt.saveMessage( - unsavedWorkMessage(across: targets), - action: .close, - alsoStopping: hasAnsweredForWork ? [] : work - ), - window: presentingWindow - ) { - case .save: - // Save closes too, once the save has landed. - guard await saveEveryWindowsWork(across: coordinators, fallback: coordinator) else { - WindowManager.shared.show(wasShowing, inWindowHosting: connectionId) - break + if decision == .confirmEndingTransaction { + let name = coordinator?.connection.name ?? DatabaseManager.shared.session(for: connectionId)?.connection.name ?? "" + let outcome = await TransactionEndingFlow.resolve( + connectionId: connectionId, + action: .closeConnection(name: name), + window: { reveal(connectionId: connectionId) } + ) + guard outcome != .cancelled else { + restore() + return } - registry.cancel(work) - WindowManager.shared.closeWindow(for: connectionId) - case .dontSave: - registry.cancel(work) - WindowManager.shared.closeWindow(for: connectionId) - case .cancel: - WindowManager.shared.show(wasShowing, inWindowHosting: connectionId) } + + // Stopped only once the close is certain: a Cancel, a failed save or a failed commit keeps the work running. + registry.cancel(work) + WindowManager.shared.closeWindow(for: connectionId) } // The answer can name a connection never on screen, so it is brought forward first. diff --git a/apple/TablePro/Core/Services/Infrastructure/ConnectionDisconnectAction.swift b/apple/TablePro/Core/Services/Infrastructure/ConnectionDisconnectAction.swift index 138049f5ea..caf8207e37 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ConnectionDisconnectAction.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ConnectionDisconnectAction.swift @@ -15,55 +15,50 @@ internal enum ConnectionDisconnectAction { presentingWindow: NSWindow?, registry: RunningWorkRegistry = .shared ) async { - let holding = await DatabaseManager.shared.databasesHoldingTransaction(for: connectionId) let hasUnsavedWork = MainContentCoordinator.hasUnsavedWork(forConnection: connectionId) let work = registry.items(in: .connection(connectionId)) + var namedUnsavedWork = false if let prompt = RunningWorkPrompt.make( action: .disconnect, work: work, - otherLosses: otherLosses(holdingTransaction: holding, hasUnsavedWork: hasUnsavedWork) + otherLosses: otherLosses(hasUnsavedWork: hasUnsavedWork) ) { guard await RunningWorkApproval.confirm(prompt, window: presentingWindow) else { return } - registry.cancel(work) - } else if let message = confirmationMessage(holdingTransaction: holding, hasUnsavedWork: hasUnsavedWork) { - let confirmed = await AlertHelper.confirmDestructive( - title: String(format: String(localized: "Disconnect from “%@”?"), connectionName), - message: message, - confirmButton: String(localized: "Disconnect"), - window: presentingWindow - ) - guard confirmed else { return } + namedUnsavedWork = hasUnsavedWork } + switch await TransactionEndingFlow.resolve( + connectionId: connectionId, + action: .disconnect(name: connectionName), + hasUnsavedWork: hasUnsavedWork && !namedUnsavedWork, + window: { presentingWindow } + ) { + case .cancelled: + return + case .ended: + break + case .nothingOpen: + let unsaved = MainContentCoordinator.hasUnsavedWork(forConnection: connectionId) + if !namedUnsavedWork, let message = confirmationMessage(hasUnsavedWork: unsaved) { + let confirmed = await AlertHelper.confirmDestructive( + title: String(format: String(localized: "Disconnect from “%@”?"), connectionName), + message: message, + confirmButton: String(localized: "Disconnect"), + window: presentingWindow + ) + guard confirmed else { return } + } + } + // Stopped only once every answer is in and any commit landed. + registry.cancel(work) await DatabaseManager.shared.disconnectSession(connectionId, origin: .userRequested) } // Nil means disconnect without asking: ending a session the user asked to end loses nothing. - internal static func confirmationMessage(holdingTransaction holding: [String], hasUnsavedWork: Bool) -> String? { - if let message = transactionMessage(for: holding) { - return message - } - return hasUnsavedWork ? String(localized: "Unsaved changes will be lost.") : nil + internal static func confirmationMessage(hasUnsavedWork: Bool) -> String? { + hasUnsavedWork ? String(localized: "Unsaved changes will be lost.") : nil } - internal static func otherLosses(holdingTransaction holding: [String], hasUnsavedWork: Bool) -> [String] { - var losses = RunningWorkPrompt.transactionLines(for: holding) - if hasUnsavedWork { - losses.append(String(localized: "Unsaved changes")) - } - return losses - } - - internal static func transactionMessage(for databases: [String]) -> String? { - guard let first = databases.first else { return nil } - guard databases.count > 1 else { - return String( - format: String(localized: "The database “%@” has an open transaction. Disconnecting rolls it back and discards its uncommitted changes."), - first - ) - } - return String( - format: String(localized: "These databases have open transactions: %@. Disconnecting rolls them back and discards their uncommitted changes."), - ListFormatter.localizedString(byJoining: databases.map { "“\($0)”" }) - ) + internal static func otherLosses(hasUnsavedWork: Bool) -> [String] { + hasUnsavedWork ? [String(localized: "Unsaved changes")] : [] } } diff --git a/apple/TablePro/Core/Services/Infrastructure/ConnectionScriptRunner.swift b/apple/TablePro/Core/Services/Infrastructure/ConnectionScriptRunner.swift index 8f1f1379b4..064f3f2944 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ConnectionScriptRunner.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ConnectionScriptRunner.swift @@ -6,7 +6,6 @@ import Darwin import Foundation import TableProProcess -import TableProUtilities enum ConnectionScriptOutcome: Equatable, Sendable { case succeeded diff --git a/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift b/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift index 640d218993..964e95b0e8 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift @@ -13,7 +13,7 @@ import Foundation @MainActor internal final class ConnectionWorkspace { internal let connectionId: UUID - internal let payload: EditorTabPayload? + internal private(set) var payload: EditorTabPayload? internal let autoConnect: Bool internal var payloadConnection: DatabaseConnection? @@ -304,6 +304,12 @@ internal final class ConnectionWorkspace { ) } + // The session's tabs were saved as it went down, so a reconnect restores them rather than opening the founding content again. + internal func retireFoundingPayload() { + guard let payload, payload.intent != .restoreOrDefault else { return } + self.payload = EditorTabPayload(connectionId: connectionId, intent: .restoreOrDefault) + } + /// Opens what the payload names. Held until the session exists if the connection is still /// being established. internal func open(_ payload: EditorTabPayload) { diff --git a/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionGate.swift b/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionGate.swift index 9d28a608a4..4fbd27d1f1 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionGate.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionGate.swift @@ -19,16 +19,19 @@ internal struct ExternalConnectionGate { self.prompt = prompt ?? ExternalConnectionAlertPrompt() } - /// The trust key names neither a filter nor an SSH server, so trusting a link that carries one - /// would cover whatever SQL or server a later link picks. A tunnelled loopback host is not on - /// this Mac either. + // The key names neither a filter nor a transport, so trusting a link that carries one would cover whatever SQL or + // server a later link picks. A loopback host behind any tunnel or proxy is not on this Mac either. internal func authorize( _ connection: DatabaseConnection, + isSaved: Bool = false, scopeName: String?, filter: ConnectionURLFilter? = nil ) async -> Bool { - let key = ExternalConnectionTrustKey(connection: connection, scopeName: scopeName) - let trustable = key.isLoopbackHost && !connection.sshConfig.enabled && filter == nil + let key = ExternalConnectionTrustKey(connection: connection, scopeName: scopeName, isSaved: isSaved) + let staysOnThisMac = connection.enabledTunnelKinds.isEmpty + && !connection.sshConfig.enabled + && !connection.resolvedSSHConfig.enabled + let trustable = key.isLoopbackHost && staysOnThisMac && filter == nil if trustable, trustStore.isTrusted(key) { return true } switch await prompt.prompt(for: connection, filter: filter, offerAlwaysAllow: trustable) { diff --git a/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionPrompting.swift b/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionPrompting.swift index de4a286fd2..c2d5e01583 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionPrompting.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ExternalConnectionPrompting.swift @@ -104,15 +104,15 @@ internal struct ExternalConnectionAlertPrompt: ExternalConnectionPrompting { details.append(String(format: String(localized: "Database: %@"), connection.database)) } details.append(contentsOf: sshDetails(for: connection)) + details.append(contentsOf: transportDetails(for: connection)) return details } - /// The database host of a tunnelled connection is the far end of the tunnel, usually - /// `localhost`, so listing it alone describes none of the machines the session actually - /// crosses. The alert asks the user to decide whether they trust the link; it has to name the - /// SSH server and every hop for that to mean anything. + // Behind a tunnel the database host is its far end, usually localhost, so the alert names every server the + // session crosses. A linked profile is read as it stands now, which is where the connect goes. private static func sshDetails(for connection: DatabaseConnection) -> [String] { - let ssh = connection.sshConfig + let linked = SSHProfileStorage.shared.refreshingLinkedProfile(connection).resolvedSSHConfig + let ssh = linked.enabled ? linked : connection.sshConfig guard ssh.enabled, !ssh.host.isEmpty else { return [] } let port = ssh.port ?? 22 @@ -130,6 +130,29 @@ internal struct ExternalConnectionAlertPrompt: ExternalConnectionPrompting { return details } + private static func transportDetails(for connection: DatabaseConnection) -> [String] { + connection.enabledTunnelKinds.compactMap { kind -> String? in + let target: String + switch kind { + case .ssh, .remoteFile, .remoteDatabaseSession: + return nil + case .socksProxy: + guard let proxy = connection.resolvedSOCKSProxyConfig else { return nil } + target = "\(proxy.host):\(proxy.port)" + case .tunnelCommand: + guard let command = connection.resolvedTunnelCommandConfig else { return nil } + target = command.method.displayName + case .cloudflare: + guard let tunnel = connection.resolvedCloudflareConfig else { return nil } + target = tunnel.accessHostname + case .cloudSQLProxy: + guard let proxy = connection.resolvedCloudSQLProxyConfig else { return nil } + target = proxy.instanceConnectionName + } + return String(format: String(localized: "%1$@: %2$@"), kind.displayName, target) + } + } + private func present(_ alert: NSAlert) async -> NSApplication.ModalResponse { guard let window = AlertHelper.resolveWindow(NSApp.keyWindow) else { return alert.runModal() diff --git a/apple/TablePro/Core/Services/Infrastructure/LaunchIntentRouter.swift b/apple/TablePro/Core/Services/Infrastructure/LaunchIntentRouter.swift index ffb2448528..5842ab53a5 100644 --- a/apple/TablePro/Core/Services/Infrastructure/LaunchIntentRouter.swift +++ b/apple/TablePro/Core/Services/Infrastructure/LaunchIntentRouter.swift @@ -1,8 +1,3 @@ -// -// LaunchIntentRouter.swift -// TablePro -// - import AppKit import Foundation import os @@ -75,16 +70,34 @@ internal final class LaunchIntentRouter { } } - /// No alert on refusal: the bridge reads the refusal file and tells its client, and an alert - /// would greet every client that tries while MCP is off. + // No alert: the bridge reads the refusal file and tells its client, and an alert would greet every + // client that tries while MCP is off or broken. private func startMCPServer() async { + let quits = await Self.startMCPServerLeavesNothingToDo( + manager: MCPServerManager.shared, + isIdleMachineSession: { AppActivationPolicyController.shared.isIdleMachineSession } + ) + guard quits else { return } + // A terminate from inside a main-actor job never lets the quit's reply task run. + RunLoop.main.perform { + MainActor.assumeIsolated { NSApp.terminate(nil) } + } + } + + // A launch made only to serve MCP has nothing left to do once the server is refused or fails to start. + internal static func startMCPServerLeavesNothingToDo( + manager: MCPServerManager, + isIdleMachineSession: () -> Bool + ) async -> Bool { do { - try await MCPServerManager.shared.lazyStart() + try await manager.lazyStart() } catch { - Self.logger.info("start-mcp refused: \(error.publicLogShape, privacy: .public)") - guard AppActivationPolicyController.shared.isIdleMachineSession else { return } - NSApp.terminate(nil) + logger.info("start-mcp refused: \(error.publicLogShape, privacy: .public)") + return isIdleMachineSession() } + guard case .failed = manager.state else { return false } + logger.error("start-mcp failed: the MCP server did not start") + return isIdleMachineSession() } private func installPlugin(_ url: URL) async throws { diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+AutoRefresh.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+AutoRefresh.swift new file mode 100644 index 0000000000..d560a463f7 --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+AutoRefresh.swift @@ -0,0 +1,24 @@ +// +// MainSplitViewController+AutoRefresh.swift +// TablePro +// + +import AppKit + +extension MainSplitViewController { + @objc func setTableAutoRefreshInterval(_ sender: Any?) { + guard let raw = (sender as? NSMenuItem)?.representedObject as? Int, + let interval = TableAutoRefreshInterval(rawValue: raw) else { return } + commandActions?.setTableAutoRefreshInterval(interval) + } + + // Nil off a table tab, so the submenu's checkmark never names another tab's interval. + var tableAutoRefreshInterval: TableAutoRefreshInterval? { + guard contentMode == .browse, isConnected else { return nil } + return commandActions?.tableAutoRefreshInterval + } + + var canSetTableAutoRefresh: Bool { + tableAutoRefreshInterval != nil && !(isViewLoaded && view.window?.attachedSheet != nil) + } +} diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+Connection.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+Connection.swift index efab53b296..459d789b2d 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+Connection.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+Connection.swift @@ -52,7 +52,7 @@ internal extension MainSplitViewController { transition(to: .unavailable(.notConnected)) return } - connect(connection, cancellingPrevious: false) + connect(connection, cancellingPrevious: false, adoptsSSHProfileEdits: false) } // The menu bar's second connection command does what the state offers: Connect, Reconnect or Stop. @@ -97,23 +97,21 @@ internal extension MainSplitViewController { DatabaseManager.shared.cancelRecovery(for: connectionId) } - /// Reconnects the connection named, not whichever one the window happens to be showing. - /// Clicking a disconnected connection used to redial the selected one instead, tearing down - /// the session the user was working in. - internal func reconnectWorkspace(_ connectionId: UUID) { + // The connection named, not the one on screen: redialing the selected one tore down the session in use. A person's + // Connect or Reconnect is the one connect that moves an open session to an edited SSH profile; a link passes false. + internal func reconnectWorkspace(_ connectionId: UUID, adoptsSSHProfileEdits: Bool = true) { guard let workspace = workspaces.workspace(for: connectionId), let connection = workspace.connection else { return } workspaces.select(connectionId) - connect(connection, cancellingPrevious: true) + connect(connection, cancellingPrevious: true, adoptsSSHProfileEdits: adoptsSSHProfileEdits) } - /// Shows the connection and dials only when its phase allows a manual connect. Redialing one - /// that is already connecting cancels that attempt and restarts it, prompts included. - internal func reconnectWorkspaceIfIdle(_ connectionId: UUID) { + // Dials only when the phase allows a manual connect; one already connecting restarts, prompts included. + internal func reconnectWorkspaceIfIdle(_ connectionId: UUID, adoptsSSHProfileEdits: Bool = true) { guard let workspace = workspaces.workspace(for: connectionId) else { return } workspaces.select(connectionId) guard ConnectionWindowPhaseMachine.allowsManualConnect(phase: workspace.phase) else { return } - reconnectWorkspace(connectionId) + reconnectWorkspace(connectionId, adoptsSSHProfileEdits: adoptsSSHProfileEdits) } internal func isShowingInKeyWindow(_ connectionId: UUID) -> Bool { @@ -188,7 +186,7 @@ internal extension MainSplitViewController { ConnectionRecoveryPerformer.perform(action, for: connection) } - private func connect(_ connection: DatabaseConnection, cancellingPrevious: Bool) { + private func connect(_ connection: DatabaseConnection, cancellingPrevious: Bool, adoptsSSHProfileEdits: Bool) { guard let workspace = workspaces.workspace(for: connection.id) else { return } let token = UUID() workspace.attemptToken = token @@ -206,7 +204,7 @@ internal extension MainSplitViewController { await DatabaseManager.shared.cancelEnsureConnected(connection.id) } do { - try await DatabaseManager.shared.ensureConnected(connection) + try await DatabaseManager.shared.ensureConnected(connection, adoptsSSHProfileEdits: adoptsSSHProfileEdits) self?.finishAttempt(token, for: connection.id, outcome: nil) } catch { Self.connectionLogger.error( diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+DatabaseMenuActions.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+DatabaseMenuActions.swift index ce3c3232e6..3992905063 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+DatabaseMenuActions.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+DatabaseMenuActions.swift @@ -124,6 +124,62 @@ extension MainSplitViewController { workspaces.selected?.sessionState?.coordinator.safeModeStatus } + // Turning it off leaves a transaction already held open, and the bar with it. + @objc func toggleManualCommit(_ sender: Any?) { + guard let connectionId = workspaces.selectedConnectionId, + let session = DatabaseManager.shared.session(for: connectionId), + ManualCommit.isAvailable(for: session.connection.type) else { return } + DatabaseManager.shared.setCommitMode(session.commitMode == .manual ? .automatic : .manual, for: connectionId) + } + + @objc func commitOpenTransaction(_ sender: Any?) { + guard let hold = selectedTransactionHold else { return } + let window = view.window + Task { @MainActor in + await ConnectionTransactionActions.perform(window: window) { + try await ConnectionTransactionActions.commit(hold) + } + } + } + + @objc func rollBackOpenTransaction(_ sender: Any?) { + guard let hold = selectedTransactionHold else { return } + let window = view.window + Task { @MainActor in + await ConnectionTransactionActions.perform(window: window) { + try await ConnectionTransactionActions.rollBack(hold, window: window) + } + } + } + + // The hold the selected tab's database runs on: the connection's one driver, or that database's own lane. + var selectedTransactionHold: SessionTransactionTracker.Hold? { + guard let coordinator = workspaces.selected?.sessionState?.coordinator else { return nil } + let container = coordinator.tabManager.selectedTab.flatMap { coordinator.scope(for: $0)?.database } + ?? coordinator.browseDatabaseName + return DatabaseManager.shared.sessionTransactions.hold(connectionId: coordinator.connectionId, container: container) + } + + var transactionMenuState: TransactionMenuState { + guard let coordinator = workspaces.selected?.sessionState?.coordinator, + let session = DatabaseManager.shared.session(for: coordinator.connectionId) + else { return TransactionMenuState() } + let container = coordinator.tabManager.selectedTab.flatMap { coordinator.scope(for: $0)?.database } + ?? coordinator.browseDatabaseName + let reportsState = (session.driver?.supportsTransactions ?? false) + && !DatabaseManager.shared.sessionTransactions.cannotReportState( + connectionId: coordinator.connectionId, container: container + ) + let hold = selectedTransactionHold + return TransactionMenuState( + isAvailable: ManualCommit.isAvailable(for: session.connection.type), + isManualCommitOn: DatabaseManager.shared.commitMode(for: coordinator.connectionId) == .manual, + canHoldTransactions: reportsState, + holdsTransaction: hold != nil, + canCommit: hold.map { ConnectionTransactionActions.offersCommit($0) } ?? false + ) + } + @objc func switchSessionContext(_ sender: Any?) { guard let selection = (sender as? NSMenuItem)?.representedObject as? SessionContextSelection, let coordinator = commandActions?.coordinator else { return } diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift index af0799bf23..a63ab93d2c 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+MenuValidation.swift @@ -6,6 +6,17 @@ import AppKit import TableProPluginKit +// Read only when a transaction command is validated, as the Debug menu's state is. +struct TransactionMenuState: Equatable { + // False on an engine that commits schema and maintenance statements on its own. + var isAvailable = false + var isManualCommitOn = false + // False for an engine with no transactions, or one whose live driver has only answered that it cannot say. + var canHoldTransactions = false + var holdsTransaction = false + var canCommit = false +} + // A plain value keeps `isEnabled` pure and testable, the same split `ToolbarContextResolver` uses for the toolbar. struct MenuValidationContext: Equatable { // True for any connection on screen, connected or not, so a pane that failed to dial can still be dismissed. @@ -140,6 +151,7 @@ struct MenuValidationContext: Equatable { var hasSchemaFilter = false // Filled only when a Debug menu item is validated; the menu exists only while the module is on. var routineDebug = RoutineDebugMenuState() + var transaction = TransactionMenuState() var canOpenLiveMessages = false var canDisconnect = false var isPresentingSheet = false @@ -358,10 +370,36 @@ extension MainSplitViewController: NSMenuItemValidation { case #selector(showAllSchemas(_:)): return context.isConnected && context.canFilterSchemas && context.hasSchemaFilter default: - return nil + return isTransactionCommandEnabled(selector, context: context) + } + } + + // Manual Commit stays enabled while on, so a connection that cannot hold can still turn it off. + private static func isTransactionCommandEnabled( + _ selector: Selector, + context: MenuValidationContext + ) -> Bool? { + guard transactionSelectors.contains(selector) else { return nil } + guard context.isConnected else { return false } + switch selector { + case #selector(toggleManualCommit(_:)): + return context.transaction.isAvailable + && (context.transaction.isManualCommitOn || context.transaction.canHoldTransactions) + case #selector(commitOpenTransaction(_:)): + return context.transaction.canCommit + case #selector(rollBackOpenTransaction(_:)): + return context.transaction.holdsTransaction + default: + return false } } + static let transactionSelectors: Set = [ + #selector(toggleManualCommit(_:)), + #selector(commitOpenTransaction(_:)), + #selector(rollBackOpenTransaction(_:)), + ] + // Agent mode does not draw the debug tab these act on. private static func isRoutineDebugCommandEnabled( _ selector: Selector, @@ -471,6 +509,9 @@ extension MainSplitViewController: NSMenuItemValidation { #selector(closeConnection(_:)), #selector(requestDisconnect), #selector(setSafeModeLevel(_:)), + #selector(toggleManualCommit(_:)), + #selector(commitOpenTransaction(_:)), + #selector(rollBackOpenTransaction(_:)), #selector(releaseFileLock(_:)), #selector(openContainerSwitcher(_:)), #selector(openSchemaSwitcher(_:)), @@ -539,6 +580,8 @@ extension MainSplitViewController: NSMenuItemValidation { #selector(closeAgentSession(_:)), #selector(deleteAgentSession(_:)), #selector(clearAIConversations(_:)), + #selector(commitOpenTransaction(_:)), + #selector(rollBackOpenTransaction(_:)), ] // A command this window does not own stays enabled, which keeps `performClose:` and the system's own items working. @@ -596,6 +639,9 @@ extension MainSplitViewController: NSMenuItemValidation { return context.canClearResults case #selector(saveAsFavorite(_:)): return context.canSaveAsFavorite + // The versions are on this Mac, so a tab whose connection dropped can still go back. + case #selector(revertToQueryTabVersion(_:)), #selector(browseQueryTabVersions(_:)): + return context.isQueryTab default: return nil } @@ -766,12 +812,14 @@ extension MainSplitViewController: NSMenuItemValidation { func validateMenuItem(_ menuItem: NSMenuItem) -> Bool { applyDynamicTitle(to: menuItem) guard let action = menuItem.action else { return false } + if let identifier = menuItem.identifier, Self.manualCommitFootnotes.contains(identifier) { return false } // AppKit asks this for the View menu and `validateUserInterfaceItem` for the rest, so a rule must live in both. if action == #selector(toggleSidebar(_:)) { return true } if action == #selector(toggleInspector(_:)) { return canToggleTrailingPane } if action == #selector(toggleAssistant(_:)) { return canToggleAssistant } if action == #selector(setResultView(_:)) { return canShowResultView(menuItem) } if action == #selector(setSafeModeLevel(_:)) { return canChooseSafeModeLevel(menuItem) } + if action == #selector(setTableAutoRefreshInterval(_:)) { return canSetTableAutoRefresh } if action == #selector(retryConnection) { return canReconnect } if let zoom = FontSizeZoom.editor.enablement(action, typography: AppSettingsManager.shared.typography) { return zoom @@ -792,6 +840,9 @@ extension MainSplitViewController: NSMenuItemValidation { if action == #selector(openLiveMessages(_:)) { context.canOpenLiveMessages = canOpenLiveMessages } + if Self.transactionSelectors.contains(action) { + context.transaction = transactionMenuState + } guard Self.agentSessionSelectors.contains(action) else { return context } context.agentSessionTarget = agentSessionTarget(for: menuItem)?.status return context @@ -867,6 +918,8 @@ extension MainSplitViewController: NSMenuItemValidation { setState(isCurrentResultView(menuItem) ? .on : .off, on: menuItem) case #selector(setContentModeFromMenu(_:)): setState(isCurrentContentMode(menuItem) ? .on : .off, on: menuItem) + case #selector(toggleManualCommit(_:)): + applyManualCommitState(to: menuItem) case #selector(useFlatSidebarLayout(_:)): setState(commandActions?.sidebarLayout == .flat ? .on : .off, on: menuItem) case #selector(useTreeSidebarLayout(_:)): @@ -880,6 +933,31 @@ extension MainSplitViewController: NSMenuItemValidation { } } + private func applyManualCommitState(to menuItem: NSMenuItem) { + guard !Self.manualCommitFootnotes.contains(where: { $0 == menuItem.identifier }) else { return } + let state = transactionMenuState + setState(state.isManualCommitOn ? .on : .off, on: menuItem) + let shown = Self.shownManualCommitFootnote(state, isConnected: isConnected) + for footnote in menuItem.menu?.items ?? [] { + guard let identifier = footnote.identifier, Self.manualCommitFootnotes.contains(identifier) else { continue } + let hidden = identifier != shown + if footnote.isHidden != hidden { footnote.isHidden = hidden } + } + } + + static let manualCommitFootnotes = [ + DatabaseMenuBuilder.manualCommitFootnoteIdentifier, + DatabaseMenuBuilder.manualCommitUnavailableFootnoteIdentifier, + ] + + // While on, a write is refused rather than committed, which the reporting footnote's wording does not cover. + static func shownManualCommitFootnote(_ state: TransactionMenuState, isConnected: Bool) -> NSUserInterfaceItemIdentifier? { + guard isConnected else { return nil } + guard state.isAvailable else { return DatabaseMenuBuilder.manualCommitUnavailableFootnoteIdentifier } + guard !state.canHoldTransactions, !state.isManualCommitOn else { return nil } + return DatabaseMenuBuilder.manualCommitFootnoteIdentifier + } + // The mode is in `representedObject`, so enablement needs the item, not the selector the shared table keys on. private func canShowResultView(_ menuItem: NSMenuItem) -> Bool { guard contentMode == .browse else { return false } diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+QueryTabVersions.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+QueryTabVersions.swift new file mode 100644 index 0000000000..a36806943f --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController+QueryTabVersions.swift @@ -0,0 +1,23 @@ +// +// MainSplitViewController+QueryTabVersions.swift +// TablePro +// + +import AppKit + +extension MainSplitViewController { + @objc func revertToQueryTabVersion(_ sender: Any?) { + guard let choice = (sender as? NSMenuItem)?.representedObject as? QueryTabRevertChoice else { return } + commandActions?.coordinator?.revertQueryTab(choice.tabId, toVersion: choice.versionId) + } + + @objc func browseQueryTabVersions(_ sender: Any?) { + guard let tabId = selectedQueryTabId else { return } + commandActions?.coordinator?.showVersionHistory(ofQueryTab: tabId) + } + + var selectedQueryTabId: UUID? { + guard let tab = commandActions?.coordinator?.tabManager.selectedTab, tab.tabType == .query else { return nil } + return tab.id + } +} diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift index f0cfcd9f62..c5d66aba1d 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift @@ -636,6 +636,7 @@ internal final class MainSplitViewController: NSSplitViewController { workspace.sessionState?.coordinator.teardown(endsTabSession: false) workspace.sessionState = nil workspace.session = nil + workspace.retireFoundingPayload() if let releasedCoordinator, toolbarOwner?.coordinator === releasedCoordinator { toolbarOwner?.repoint(to: nil) } diff --git a/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift b/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift index 9d14e0be65..9e4ed6c691 100644 --- a/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift +++ b/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift @@ -1,51 +1,165 @@ import AppKit +// What an approved quit or relaunch does before it goes ahead. +internal struct QuitPlan { + // Empty when nothing holds a transaction or the answer was Roll Back. + internal let commits: [SessionTransactionTracker.Hold] + // Stopped only once the commits landed, so a failed commit leaves it running. + internal let work: [RunningWork] +} + // The termination notification comes after AppKit is told yes, too late to cancel a quit. @MainActor internal enum QuitApproval { internal static func approve( registry: RunningWorkRegistry = .shared, relaunchApproved: Bool = AppRelauncher.shared.hasApprovedRunningWork - ) -> Bool { - // A relaunch asked both questions before it quit; the work it approved stopping stops now. + ) -> QuitPlan? { + // A relaunch asked every question and committed before it quit; the work it approved stopping stops now. guard !relaunchApproved else { - registry.cancel(registry.items(in: .app)) - return true + return QuitPlan(commits: [], work: registry.items(in: .app)) } - // The theme editor can still cancel, so it asks before any work is stopped. - guard ThemeEditorWindows.shared.prepareToTerminate() else { return false } + // The theme editor can still cancel, so it asks before anything else. + guard ThemeEditorWindows.shared.prepareToTerminate() else { return nil } return approvesStoppingWork(registry: registry, action: .quit) } - // One alert for what a quit or relaunch loses. Running work lists the unsaved tabs with it. + // Running work, then the open transactions, then unsaved tabs when no alert named them. Nil when any answer was Cancel. internal static func approvesStoppingWork( registry: RunningWorkRegistry, action: RunningWorkAction, - stopsWork: Bool = true - ) -> Bool { + databaseManager: DatabaseManager = .shared, + hasUnsavedTabs: @MainActor () -> Bool = { MainContentCoordinator.hasAnyUnsavedChanges() }, + confirmWork: @MainActor (RunningWorkPrompt) -> Bool = RunningWorkApproval.confirmModally, + answerTransaction: @MainActor (TransactionEndPrompt) -> AlertHelper.TransactionEndAnswer + = QuitApproval.answerTransactionModally, + confirmDiscardingTabs: @MainActor (RunningWorkAction) -> Bool = QuitApproval.approvesDiscardingUnsavedTabs + ) -> QuitPlan? { // A large document's last keys reach its tab on a delay; without them a dirty file reads as clean. EditorEventRouter.shared.flushAllPendingText() let work = registry.items(in: .app) - let hasUnsavedTabs = MainContentCoordinator.hasAnyUnsavedChanges() - let losses = hasUnsavedTabs ? [String(localized: "Unsaved edits in open tabs")] : [] - if let prompt = RunningWorkPrompt.make(action: action, work: work, otherLosses: losses) { - guard RunningWorkApproval.confirmModally(prompt) else { return false } - if stopsWork { registry.cancel(work) } - return true + let unsavedTabs = hasUnsavedTabs() + var namedUnsavedTabs = false + if let prompt = RunningWorkPrompt.make( + action: action, + work: work, + otherLosses: unsavedTabs ? [String(localized: "Unsaved edits in open tabs")] : [] + ) { + guard confirmWork(prompt) else { return nil } + namedUnsavedTabs = unsavedTabs + } + let holds = databaseManager.sessionTransactions.holds.values.sorted { lhs, rhs in + (lhs.connectionId.uuidString, lhs.key.lane ?? "") < (rhs.connectionId.uuidString, rhs.key.lane ?? "") + } + var commits: [SessionTransactionTracker.Hold] = [] + if let prompt = transactionPrompt( + action: action, + holds: holds, + hasUnsavedTabs: unsavedTabs && !namedUnsavedTabs, + databaseManager: databaseManager + ) { + switch answerTransaction(prompt) { + case .cancel: + return nil + case .rollBack: + break + case .commit: + commits = holds + } + namedUnsavedTabs = unsavedTabs + } + guard !unsavedTabs || namedUnsavedTabs || confirmDiscardingTabs(action) else { return nil } + return QuitPlan(commits: commits, work: work) + } + + // The server rolls back what a quit leaves open, so a hold is asked about like unsaved work. + internal static func openTransactionNames( + holds: [SessionTransactionTracker.Hold], + connectionName: (UUID) -> String? + ) -> [String] { + var names: [String] = [] + for hold in holds { + guard let name = connectionName(hold.connectionId), !names.contains(name) else { continue } + names.append(name) + } + return names.sorted { $0.localizedStandardCompare($1) == .orderedAscending } + } + + // Tracker holds only, so every connection the prompt names has holds to commit through. + internal static func transactionPrompt( + action: RunningWorkAction, + holds: [SessionTransactionTracker.Hold], + hasUnsavedTabs: Bool, + databaseManager: DatabaseManager = .shared + ) -> TransactionEndPrompt? { + TransactionEndPrompt.make( + action: action == .relaunch ? .relaunch : .quit, + names: openTransactionNames( + holds: holds, + connectionName: { databaseManager.session(for: $0)?.connection.name } + ), + holds: holds, + savedUnderManualCommit: holds.contains { databaseManager.commitMode(for: $0.connectionId) == .manual }, + hasUnsavedWork: hasUnsavedTabs, + effectiveLevel: { ExecutionGateProvider.effectiveSafeModeLevel(for: $0, in: databaseManager) }, + busy: { databaseManager.isLaneBusy(holding: $0) } + ) + } + + // Runs before anything a quit cannot take back. False keeps the app open: the gate was cancelled, the commit + // failed, or a transaction is still open after it (a nested one, or one opened since the answer). + internal static func commitBeforeQuitting( + _ plan: QuitPlan, + databaseManager: DatabaseManager = .shared, + commit: @MainActor ([SessionTransactionTracker.Hold]) async throws -> Bool + = { try await ConnectionTransactionActions.commitBeforeEnding($0) }, + reportFailure: @MainActor (String) -> Void = { TransactionEndingFlow.showCommitFailure($0, window: nil) } + ) async -> Bool { + guard !plan.commits.isEmpty else { return true } + do { + guard try await commit(plan.commits) else { return false } + } catch is CancellationError { + return false + } catch { + reportFailure(error.localizedDescription) + return false + } + let open = Array(databaseManager.sessionTransactions.holds.values) + guard open.isEmpty else { + let names = openTransactionNames( + holds: open, + connectionName: { databaseManager.session(for: $0)?.connection.name } + ) + reportFailure(String( + format: String(localized: "A transaction is still open on %@."), + ListFormatter.localizedString(byJoining: names.map { "“\($0)”" }) + )) + return false } - return !hasUnsavedTabs || approvesDiscardingUnsavedTabs() + return true + } + + private static func answerTransactionModally(_ prompt: TransactionEndPrompt) -> AlertHelper.TransactionEndAnswer { + // A quit can come from outside the app, and a background process has no Dock icon to reach the alert by. + AppActivationPolicyController.shared.activate(ignoringOtherApps: true) + return AlertHelper.answerTransactionEndModally(prompt) } - private static func approvesDiscardingUnsavedTabs() -> Bool { + internal static func approvesDiscardingUnsavedTabs(_ action: RunningWorkAction) -> Bool { // A quit can come from outside the app, and a background process has no Dock icon to reach the alert by. AppActivationPolicyController.shared.activate(ignoringOtherApps: true) + let relaunches = action == .relaunch let alert = NSAlert() alert.messageText = String(localized: "You have unsaved changes") - alert.informativeText = String(localized: "Some tabs have unsaved edits. Quitting will discard these changes.") + alert.informativeText = relaunches + ? String(localized: "Some tabs have unsaved edits. Relaunching will discard these changes.") + : String(localized: "Some tabs have unsaved edits. Quitting will discard these changes.") alert.alertStyle = .warning - alert.addButton(withTitle: String(localized: "Cancel")) - alert.addButton(withTitle: String(localized: "Quit Anyway")) - alert.buttons[1].hasDestructiveAction = true + AlertHelper.addCancelButton(to: alert, title: String(localized: "Cancel")) + let confirm = alert.addButton( + withTitle: relaunches ? String(localized: "Relaunch Anyway") : String(localized: "Quit Anyway") + ) + confirm.hasDestructiveAction = true return alert.runModal() == .alertSecondButtonReturn } } diff --git a/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift b/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift index 72da8fa986..f667112ce4 100644 --- a/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift +++ b/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift @@ -58,6 +58,11 @@ internal final class TabPersistenceCoordinator: ObservableObject { hasObservedTabs = true } + // A workspace founded on one table or query never read the saved set, and the gate refuses its saves until it does. + internal func readsSavedTabs(foundedWith intent: TabIntent?) -> Bool { + intent == nil || intent == .restoreOrDefault || !hasObservedTabs + } + // The gate is one session's knowledge of the disk. Left open after the connection's last window closed, a // window later opened on content alone, which skips the restore, would write its one tab over the saved set. internal func endSession() { diff --git a/apple/TablePro/Core/Services/Infrastructure/TabRouter.swift b/apple/TablePro/Core/Services/Infrastructure/TabRouter.swift index e78576bd25..4c917cc3e1 100644 --- a/apple/TablePro/Core/Services/Infrastructure/TabRouter.swift +++ b/apple/TablePro/Core/Services/Infrastructure/TabRouter.swift @@ -165,7 +165,7 @@ internal final class TabRouter { /// installed for the whole of a connect, so asking the driver restarted one in flight. if let host = existing.contentViewController as? MainSplitViewController { if host.workspaces.contains(id) { - host.reconnectWorkspaceIfIdle(id) + host.reconnectWorkspaceIfIdle(id, adoptsSSHProfileEdits: false) } else { // Dialed by a workspace of its own, so a failure lands inline in its pane, not as an alert. if transientConnection != nil { @@ -176,7 +176,7 @@ internal final class TabRouter { autoConnect: false ) if adopted?.phase != .connected { - host.reconnectWorkspace(id) + host.reconnectWorkspace(id, adoptsSSHProfileEdits: false) } } return @@ -368,7 +368,7 @@ internal final class TabRouter { } guard await externalConnectionGate.authorize( - connection, scopeName: parsed.connectionName, filter: filter + connection, isSaved: !isTransient, scopeName: parsed.connectionName, filter: filter ) else { throw TabRouterError.userCancelled } diff --git a/apple/TablePro/Core/Services/Infrastructure/TransactionEndPrompt.swift b/apple/TablePro/Core/Services/Infrastructure/TransactionEndPrompt.swift new file mode 100644 index 0000000000..e0d4b6bb73 --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/TransactionEndPrompt.swift @@ -0,0 +1,263 @@ +// +// TransactionEndPrompt.swift +// TablePro +// + +import Foundation + +// What closing, disconnecting, quitting or relaunching asks about an open transaction it would end. +internal struct TransactionEndPrompt: Equatable, Sendable { + internal enum Action: Equatable, Sendable { + case closeEntry(database: String) + case closeConnection(name: String) + case disconnect(name: String) + case quit + case relaunch + + var followsSavePrompt: Bool { + switch self { + case .closeEntry, .closeConnection: return true + case .disconnect, .quit, .relaunch: return false + } + } + } + + internal enum CommitBlock: Equatable, Sendable { + case aborted + case readOnly + case busy + // A database the live read found holding one, with no tracked hold to commit through. + case untracked + } + + internal let title: String + internal let message: String + internal let commitBlock: CommitBlock? + // The roll-back alert's confirming button, shown when Commit is not offered. + internal let confirmButton: String + + internal var offersCommit: Bool { commitBlock == nil } + + // A failed transaction can never commit, so it is named before the reasons that pass. `effectiveLevel` is the + // gate's level per connection, so the bar, the menu and this alert offer Commit by the same rule. + internal static func commitBlock( + holds: [SessionTransactionTracker.Hold], + untracked: Bool, + effectiveLevel: (UUID) -> SafeModeLevel, + busy: (SessionTransactionTracker.Hold) -> Bool + ) -> CommitBlock? { + if holds.contains(where: { $0.state == .aborted }) { return .aborted } + let refused = holds.contains { hold in + !ConnectionTransactionActions.offersCommit(hold.state, effectiveLevel: effectiveLevel(hold.connectionId)) + } + if refused { return .readOnly } + if holds.contains(where: { busy($0) }) { return .busy } + if untracked || holds.isEmpty { return .untracked } + return nil + } + + // Nil when nothing holds a transaction. `names` are databases, or connections for quit and relaunch. + internal static func make( + action: Action, + names: [String], + holds: [SessionTransactionTracker.Hold], + untracked: Bool = false, + savedUnderManualCommit: Bool = false, + hasUnsavedWork: Bool = false, + effectiveLevel: (UUID) -> SafeModeLevel, + busy: (SessionTransactionTracker.Hold) -> Bool + ) -> TransactionEndPrompt? { + guard !names.isEmpty else { return nil } + let block = commitBlock(holds: holds, untracked: untracked, effectiveLevel: effectiveLevel, busy: busy) + let several = max(names.count, holds.count) > 1 + var paragraphs: [String] + if block == nil { + paragraphs = [commitLead(action, names: names, holds: holds)] + } else { + paragraphs = [rollBackLead(action, names: names, holds: holds)] + if let reason = block.flatMap(reason(for:)) { + paragraphs.append(reason) + } + } + // Only a close asked Save first, so only there can the transaction hold edits the user just saved. + if savedUnderManualCommit, action.followsSavePrompt { + paragraphs.append(String(localized: "Edits saved with Manual Commit on are not committed yet.")) + } + if hasUnsavedWork, let line = unsavedLine(for: action) { + paragraphs.append(line) + } + return TransactionEndPrompt( + title: block == nil ? commitTitle(action, several: several) : rollBackTitle(action), + message: paragraphs.joined(separator: "\n\n"), + commitBlock: block, + confirmButton: rollBackButton(action) + ) + } + + private static func quoted(_ names: [String]) -> String { + ListFormatter.localizedString(byJoining: names.map { "“\($0)”" }) + } + + private static func commitTitle(_ action: Action, several: Bool) -> String { + switch action { + case .closeEntry(let database): + return String( + format: String(localized: "Do you want to commit the open transaction before closing the database “%@”?"), + database + ) + case .closeConnection(let name): + return String( + format: several + ? String(localized: "Do you want to commit the open transactions before closing “%@”?") + : String(localized: "Do you want to commit the open transaction before closing “%@”?"), + name + ) + case .disconnect(let name): + return String( + format: several + ? String(localized: "Do you want to commit the open transactions before disconnecting from “%@”?") + : String(localized: "Do you want to commit the open transaction before disconnecting from “%@”?"), + name + ) + case .quit: + return several + ? String(localized: "Do you want to commit the open transactions before quitting?") + : String(localized: "Do you want to commit the open transaction before quitting?") + case .relaunch: + return several + ? String(localized: "Do you want to commit the open transactions before relaunching?") + : String(localized: "Do you want to commit the open transaction before relaunching?") + } + } + + private static func commitLead(_ action: Action, names: [String], holds: [SessionTransactionTracker.Hold]) -> String { + switch action { + case .closeEntry: + return String(localized: "Changes you don’t commit are rolled back.") + case .closeConnection, .disconnect: + // One session for every database: the title names the connection, and its database can be a file path. + if names.count == 1, holds.allSatisfy({ $0.key.lane == nil }) { + return String(localized: "Changes you don’t commit are rolled back.") + } + guard names.count > 1 else { + return String( + format: String(localized: "The database “%@” has an open transaction. Changes you don’t commit are rolled back."), + names[0] + ) + } + return String( + format: String(localized: "These databases have open transactions: %@. Changes you don’t commit are rolled back."), + quoted(names) + ) + case .quit, .relaunch: + if names.count > 1 { + return String( + format: String(localized: "These connections have open transactions: %@. Changes you don’t commit are rolled back."), + quoted(names) + ) + } + return String( + format: holds.count > 1 + ? String(localized: "The connection “%@” has open transactions. Changes you don’t commit are rolled back.") + : String(localized: "The connection “%@” has an open transaction. Changes you don’t commit are rolled back."), + names[0] + ) + } + } + + private static func rollBackTitle(_ action: Action) -> String { + switch action { + case .closeEntry(let database): + return String(format: String(localized: "Close the database “%@”?"), database) + case .closeConnection(let name): + return String(format: String(localized: "Close the connection “%@”?"), name) + case .disconnect(let name): + return String(format: String(localized: "Disconnect from “%@”?"), name) + case .quit: + return String(localized: "Quit with an Open Transaction?") + case .relaunch: + return String(localized: "Relaunch with an Open Transaction?") + } + } + + private static func rollBackLead(_ action: Action, names: [String], holds: [SessionTransactionTracker.Hold]) -> String { + let several = names.count > 1 + let oneSession = !several && !holds.isEmpty && holds.allSatisfy { $0.key.lane == nil } + switch action { + case .closeConnection where oneSession: + return String(localized: "This connection has an open transaction. Closing rolls it back and discards its uncommitted changes.") + case .disconnect where oneSession: + return String( + localized: "This connection has an open transaction. Disconnecting rolls it back and discards its uncommitted changes." + ) + case .closeEntry: + return String( + localized: "This database has an open transaction. Closing it rolls the transaction back and discards its uncommitted changes." + ) + case .closeConnection: + return several + ? String( + format: String(localized: "These databases have open transactions: %@. Closing rolls them back and discards their uncommitted changes."), + quoted(names) + ) + : String( + format: String(localized: "The database “%@” has an open transaction. Closing rolls it back and discards its uncommitted changes."), + names[0] + ) + case .disconnect: + return several + ? String( + format: String(localized: "These databases have open transactions: %@. Disconnecting rolls them back and discards their uncommitted changes."), + quoted(names) + ) + : String( + format: String(localized: "The database “%@” has an open transaction. Disconnecting rolls it back and discards its uncommitted changes."), + names[0] + ) + case .quit: + return several + ? String(format: String(localized: "Uncommitted changes on %@ are rolled back when TablePro quits."), quoted(names)) + : String(format: String(localized: "Uncommitted changes on “%@” are rolled back when TablePro quits."), names[0]) + case .relaunch: + return several + ? String(format: String(localized: "Uncommitted changes on %@ are rolled back when TablePro relaunches."), quoted(names)) + : String(format: String(localized: "Uncommitted changes on “%@” are rolled back when TablePro relaunches."), names[0]) + } + } + + private static func reason(for block: CommitBlock) -> String? { + switch block { + case .aborted: + return String(localized: "A statement failed in the transaction, so it can only be rolled back.") + case .readOnly: + return String(localized: "Read-Only Safe Mode does not allow a commit.") + case .busy: + return String(localized: "A statement is still running in the transaction, so it cannot be committed now.") + case .untracked: + return nil + } + } + + // A close asks about unsaved work on its own Save alert first, so only these carry it here. + private static func unsavedLine(for action: Action) -> String? { + switch action { + case .disconnect: + return String(localized: "Unsaved changes will be lost.") + case .quit: + return String(localized: "Some tabs have unsaved edits. Quitting will discard these changes.") + case .relaunch: + return String(localized: "Some tabs have unsaved edits. Relaunching will discard these changes.") + case .closeEntry, .closeConnection: + return nil + } + } + + private static func rollBackButton(_ action: Action) -> String { + switch action { + case .closeEntry, .closeConnection: return String(localized: "Close") + case .disconnect: return String(localized: "Disconnect") + case .quit: return String(localized: "Quit Anyway") + case .relaunch: return String(localized: "Relaunch Anyway") + } + } +} diff --git a/apple/TablePro/Core/Services/Infrastructure/TransactionEndingFlow.swift b/apple/TablePro/Core/Services/Infrastructure/TransactionEndingFlow.swift new file mode 100644 index 0000000000..1dc5780dd6 --- /dev/null +++ b/apple/TablePro/Core/Services/Infrastructure/TransactionEndingFlow.swift @@ -0,0 +1,125 @@ +// +// TransactionEndingFlow.swift +// TablePro +// + +import AppKit +import Foundation + +// Close, entry close and disconnect end a session that can hold a transaction: this asks how it ends first. +@MainActor +internal enum TransactionEndingFlow { + internal enum Outcome: Equatable { + case nothingOpen + case ended + case cancelled + } + + // A database the read found holding one, and the tracked hold a commit goes through, if any. + internal struct Entry: Equatable { + let name: String + let hold: SessionTransactionTracker.Hold? + } + + internal static func resolve( + connectionId: UUID, + action: TransactionEndPrompt.Action, + among databases: Set? = nil, + hasUnsavedWork: Bool = false, + window: @MainActor () -> NSWindow?, + databaseManager: DatabaseManager = .shared, + ask: @MainActor (TransactionEndPrompt, NSWindow?) async -> AlertHelper.TransactionEndAnswer + = AlertHelper.answerTransactionEnd(_:window:), + commit: @MainActor ([SessionTransactionTracker.Hold]) async throws -> Bool + = { try await ConnectionTransactionActions.commitBeforeEnding($0) }, + reportFailure: @MainActor (String, NSWindow?) -> Void = TransactionEndingFlow.showCommitFailure(_:window:) + ) async -> Outcome { + var parent: NSWindow? + return await run( + read: { await entries(for: connectionId, among: databases, in: databaseManager) }, + prompt: { entries in + prompt(for: entries, action: action, connectionId: connectionId, hasUnsavedWork: hasUnsavedWork, in: databaseManager) + }, + ask: { prompt in + parent = window() + return await ask(prompt, parent) + }, + commit: commit, + reportFailure: { reportFailure($0, parent) } + ) + } + + // Roll Back sends nothing: ending the session rolls it back, and an explicit one would wait behind running work. + // Each answer reads again, so a transaction opened meanwhile, or one a commit left open, is asked about too. + internal static func run( + read: @MainActor () async -> [Entry], + prompt: @MainActor ([Entry]) -> TransactionEndPrompt?, + ask: @MainActor (TransactionEndPrompt) async -> AlertHelper.TransactionEndAnswer, + commit: @MainActor ([SessionTransactionTracker.Hold]) async throws -> Bool, + reportFailure: @MainActor (String) -> Void + ) async -> Outcome { + var rolledBack: Set = [] + var outcome = Outcome.nothingOpen + while true { + let entries = await read().filter { !rolledBack.contains($0.name) } + guard let question = prompt(entries) else { return outcome } + outcome = .ended + switch await ask(question) { + case .cancel: + return .cancelled + case .rollBack: + rolledBack.formUnion(entries.map { $0.name }) + case .commit: + do { + guard try await commit(entries.compactMap { $0.hold }) else { return .cancelled } + } catch is CancellationError { + return .cancelled + } catch { + reportFailure(error.localizedDescription) + return .cancelled + } + } + } + } + + // Read fresh at every question: the answer to the one before can land after work opened or ended a transaction. + internal static func entries( + for connectionId: UUID, + among databases: Set?, + in manager: DatabaseManager + ) async -> [Entry] { + let names = await manager.databasesHoldingTransaction(for: connectionId, among: databases) + let browsed = manager.session(for: connectionId)?.resolvedBrowseDatabase ?? "" + let holds = manager.sessionTransactions.holds(for: connectionId) + return names.map { name in + Entry(name: name, hold: holds.first { ($0.key.lane ?? browsed) == name }) + } + } + + internal static func showCommitFailure(_ message: String, window: NSWindow?) { + AlertHelper.showErrorSheet( + title: String(localized: "Could Not Commit the Transaction"), + message: message, + window: AlertHelper.resolveWindow(window).map(AlertOperationConfirming.frontmostSheet(of:)) + ) + } + + private static func prompt( + for entries: [Entry], + action: TransactionEndPrompt.Action, + connectionId: UUID, + hasUnsavedWork: Bool, + in manager: DatabaseManager + ) -> TransactionEndPrompt? { + TransactionEndPrompt.make( + action: action, + names: entries.map { $0.name }, + holds: entries.compactMap { $0.hold }, + untracked: entries.contains { $0.hold == nil }, + savedUnderManualCommit: manager.commitMode(for: connectionId) == .manual, + hasUnsavedWork: hasUnsavedWork, + effectiveLevel: { ExecutionGateProvider.effectiveSafeModeLevel(for: $0, in: manager) }, + busy: { manager.isLaneBusy(holding: $0) } + ) + } +} diff --git a/apple/TablePro/Core/Services/Infrastructure/WindowManager.swift b/apple/TablePro/Core/Services/Infrastructure/WindowManager.swift index 4af160575d..ed071c40e3 100644 --- a/apple/TablePro/Core/Services/Infrastructure/WindowManager.swift +++ b/apple/TablePro/Core/Services/Infrastructure/WindowManager.swift @@ -332,7 +332,7 @@ internal final class WindowManager { } bringToFront(targetHost.view.window) if DatabaseManager.shared.activeSessions[targetConnectionId]?.reportedStatus.isConnected != true { - targetHost.reconnectWorkspaceIfIdle(targetConnectionId) + targetHost.reconnectWorkspaceIfIdle(targetConnectionId, adoptsSSHProfileEdits: false) } target.queueMove(PendingTabMove(tabId: tabId, source: source) { [weak sourceHost] in sourceHost?.isShowingInKeyWindow(sourceConnectionId) ?? false diff --git a/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift b/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift index b124761aaf..99c83fd07b 100644 --- a/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift +++ b/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift @@ -75,9 +75,7 @@ internal enum WorkspaceCloseAction { await ConnectionCloseAction.close(connectionId: workspace.connectionId) return } - /// Every window hosting the connection. A tab torn off into its own window can be the - /// container's only remaining tab, and closing the entry from the window that happens to - /// answer first would leave that tab open under an entry the user just closed. + // Every window hosting the connection: a torn-off tab can be the container's only one. let hostedWorkspaces = WindowManager.shared.workspaces(for: workspace.connectionId) guard let hosted = hostedWorkspaces.first else { Self.logger.error("close has no hosted workspace container=\(workspace.container, privacy: .private(mask: .hash))") @@ -86,16 +84,9 @@ internal enum WorkspaceCloseAction { let coordinators = hostedWorkspaces.compactMap { $0.sessionState?.coordinator } let coordinator = coordinators.first - /// Where the user was before the alert. Confirming reveals the work at risk, which switches - /// the window to that connection and selects one of the tabs, and an answer that closes - /// nothing has to put all of that back: leaving the user on a connection they did not ask - /// for, with the entry still listed, is a close that reads as a switch. + // Asking reveals the work at risk, so an answer that closes nothing puts the window back here. let wasShowing = WindowManager.shared.shownConnection(besides: workspace.connectionId) - guard await confirmEndingTransaction(in: workspace) else { - Self.logger.info("close cancelled at the transaction prompt container=\(workspace.container, privacy: .private(mask: .hash))") - return - } - // Taken after the transaction prompt: the tabs stay editable while it is answered. + // Running work, then unsaved work, then the transaction, read after the save: Save under Manual Commit lands in one. let victims = committedTabs(in: workspace.container, across: coordinators) // Closing a tab stops its query, so a yes needs no cancel of its own. let running = RunningWorkRegistry.shared.items(in: .tabs(Set(victims.map(\.id)))) @@ -115,16 +106,15 @@ internal enum WorkspaceCloseAction { return } } - guard let closable = await confirm( + guard let confirmed = await confirm( victims, across: coordinators, revealing: workspace, alsoStopping: holdsUnsavedWork ? running : [] ) else { WindowManager.shared.show(wasShowing, inWindowHosting: workspace.connectionId) Self.logger.info("close cancelled at the save prompt container=\(workspace.container, privacy: .private(mask: .hash))") return } - /// A tab whose work the save could not take keeps the container open, because it is still - /// work in it. Closing the entry regardless would destroy exactly what the alert said would - /// stay, which is the promise the wording makes. + let closable = confirmed.closable + // A tab whose work the save could not take keeps the container open: the alert promised it would stay. guard closable.isSuperset(of: Set(victims.map(\.id))) else { closeTabs(victims.map(\.id).filter { closable.contains($0) }, across: coordinators) WindowManager.shared.show(wasShowing, inWindowHosting: workspace.connectionId) @@ -136,11 +126,29 @@ internal enum WorkspaceCloseAction { ) return } - /// The entry goes now, before the connection leaves the container, because leaving it is a - /// schema reload, and on an engine that can neither change database on a live connection nor - /// open a second one, a reconnect: waiting for that left the row the user just closed sitting - /// there for seconds while the window loaded somewhere else. `beginClosing` is what lets the strip drop the - /// browse cursor's own row early, and the cursor follows underneath. + // Before anything closes, so a Cancel or a failed commit leaves the entry as it was. + guard await resolveTransaction(in: workspace) else { + confirmed.restoreRevealed() + WindowManager.shared.show(wasShowing, inWindowHosting: workspace.connectionId) + Self.logger.info("close cancelled at the transaction prompt container=\(workspace.container, privacy: .private(mask: .hash))") + return + } + // One session serves every entry, so its transaction outlives the close, and an engine that cannot switch + // database inside one refuses the move: moving first means a refusal closes nothing. + let movesFirst = await holdsSessionWideTransaction(workspace.connectionId) + if movesFirst { + guard await browseAway(from: workspace, among: containers, coordinator: coordinator) else { + confirmed.restoreRevealed() + WindowManager.shared.show(wasShowing, inWindowHosting: workspace.connectionId) + Self.logger.error( + "close could not leave container=\(workspace.container, privacy: .private(mask: .hash))" + ) + return + } + } + // Otherwise the entry goes before the connection leaves the container: leaving can be a reconnect, and + // waiting for it left the closed row on the strip for seconds. `beginClosing` lets the strip drop the + // browse cursor's own row early, and the cursor follows underneath. if !victims.isEmpty { closeTabs(victims.map(\.id), across: coordinators) } @@ -158,7 +166,10 @@ internal enum WorkspaceCloseAction { """ ) - let left = await browseAway(from: workspace, among: containers, coordinator: coordinator) + var left = true + if !movesFirst { + left = await browseAway(from: workspace, among: containers, coordinator: coordinator) + } if left { leave(workspace, in: coordinators) } @@ -166,9 +177,7 @@ internal enum WorkspaceCloseAction { hostedWorkspace.endClosing() } guard left else { - /// The connection never left, so the container is open again: it is where the next tab - /// still opens, and a strip that did not list it would be lying about where the user is. - /// The driver's own error is already on screen. + // The connection never left, so the next tab still opens there; the driver's error is already on screen. for hostedWorkspace in hostedWorkspaces { hostedWorkspace.openContainer(workspace.container) } @@ -179,9 +188,7 @@ internal enum WorkspaceCloseAction { return } - /// Read again rather than reusing the list from before the switch: a table opened while the - /// reconnect ran anchors the container all over again, and the entry would come back with - /// one stray tab under it. A tab that new has nothing to lose. + // Read again: a table opened while the switch ran anchors the container anew, and has nothing to lose. let opened = coordinators.flatMap { tabs(in: workspace.container, of: $0) } if !opened.isEmpty { closeTabs(opened.map(\.id), across: coordinators) @@ -189,43 +196,27 @@ internal enum WorkspaceCloseAction { hostedWorkspace.closeContainer(workspace.container) } } - /// The database keeps its own connection and its catalog while its entry is listed; closing the - /// entry is what ends them, and the connection has already moved off it above. + // Closing the entry ends the database's own connection and catalog; the browse cursor moved off it above. DatabaseManager.shared.sessionLanes.close(database: workspace.container, for: workspace.connectionId) SchemaService.shared.forget(database: workspace.container, connectionId: workspace.connectionId) } - /// Shown, then asked, for the same reason a connection close reveals itself first: an alert - /// about work the user cannot see names tabs they have no way to look at before answering. - /// Selecting one of the victims is also what makes Save save a victim. It runs only when there - /// is something to confirm, because raising a background connection's window and moving its tab - /// selection is a side effect no one asked for when the close has nothing to lose. - /// nil when the user cancelled; otherwise the victims that may now be closed, which is every one - /// of them unless Save could not reach some. - /// Asked of the window that owns each tab. - /// - /// A tab's live grid and structure edits exist only in its own coordinator: a background - /// snapshot of a tab in another window reads as clean whatever is staged in it. Once a - /// connection can be hosted twice, a container's tabs span both windows, and putting all of - /// them to the coordinator that answered first reported the foreign ones as safe and closed - /// over the edits without a prompt. - /// - /// Cancel anywhere cancels everything, because the entry either closes or it does not. + // nil when the user cancelled; otherwise the victims that may close (all, unless Save could not reach some) and + // the windows revealed to ask, for a later Cancel to put back. Each tab is asked of the window that owns it, + // since its live edits exist only in its own coordinator, and only when it holds unsaved work: revealing a + // background window is a side effect no one asked for when nothing is lost. Cancel anywhere cancels everything. private static func confirm( _ victims: [QueryTab], across coordinators: [MainContentCoordinator], revealing workspace: WorkspaceID, alsoStopping running: [RunningWork] - ) async -> Set? { + ) async -> Confirmation? { var running = running let everything = Set(victims.map(\.id)) - guard !victims.isEmpty else { return everything } + guard !victims.isEmpty else { return Confirmation(closable: everything, revealed: []) } var closable: Set = [] - /// What each window was showing before it was brought forward to ask, the entry and its tab - /// as well as the connection. A dirty split connection reveals more than one of them, and - /// the caller can only put one back, so an abandoned close left the others switched to a - /// connection the user never chose. + // A dirty split connection reveals more than one window, and the caller can only put one back. var revealed: [RevealedWindow] = [] func restoreRevealed() { for window in revealed { @@ -259,12 +250,24 @@ internal enum WorkspaceCloseAction { closable.formUnion(resolved) } } - /// A victim no window claims is already gone, so nothing is holding work for it. + // A victim no window claims is already gone, so nothing is holding work for it. let unclaimed = victims.map(\.id).filter { id in !coordinators.contains { coordinator in coordinator.tabManager.tabs.contains { $0.id == id } } } closable.formUnion(unclaimed) - return closable + return Confirmation(closable: closable, revealed: revealed) + } + + @MainActor + private struct Confirmation { + let closable: Set + let revealed: [RevealedWindow] + + func restoreRevealed() { + for window in revealed { + window.restore() + } + } } /// A window as it was before it was brought forward to ask: the connection it showed, and that @@ -294,27 +297,27 @@ internal enum WorkspaceCloseAction { } } - /// Asked before the unsaved-work prompt, whose Save writes at once, so a Cancel here leaves - /// everything as it was. Nothing is revealed: the alert names the database, and the transaction - /// is not in any one tab. Only a database with its own connection loses a transaction on close. - private static func confirmEndingTransaction(in workspace: WorkspaceID) async -> Bool { + // Only a database with its own connection loses a transaction when its entry closes. Nothing is revealed: the + // alert names the database, and the transaction is in no one tab. + private static func resolveTransaction(in workspace: WorkspaceID) async -> Bool { let manager = DatabaseManager.shared guard let session = manager.session(for: workspace.connectionId), manager.usesDatabaseLanes(session) else { return true } - let holding = await manager.databasesHoldingTransaction( - for: workspace.connectionId, - among: [workspace.container] - ) - guard !holding.isEmpty else { return true } - return await AlertHelper.confirmDestructive( - title: String(format: String(localized: "Close the database “%@”?"), workspace.container), - message: String( - localized: "This database has an open transaction. Closing it rolls the transaction back and discards its uncommitted changes." - ), - confirmButton: String(localized: "Close"), - window: WindowManager.shared.window(for: workspace.connectionId) + let outcome = await TransactionEndingFlow.resolve( + connectionId: workspace.connectionId, + action: .closeEntry(database: workspace.container), + among: [workspace.container], + window: { WindowManager.shared.window(for: workspace.connectionId) } ) + return outcome != .cancelled + } + + private static func holdsSessionWideTransaction(_ connectionId: UUID) async -> Bool { + let manager = DatabaseManager.shared + guard let session = manager.session(for: connectionId), !manager.usesDatabaseLanes(session) else { return false } + let holding = await manager.databasesHoldingTransaction(for: connectionId) + return !holding.isEmpty } /// Leaves the container before it stops being listed, and only when it is the one being browsed. diff --git a/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift b/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift index d14aeb5aa4..f0fac9b9eb 100644 --- a/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift +++ b/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift @@ -24,7 +24,10 @@ struct MaskingPolicy: Equatable, Sendable { for rule in rules where Self.matches(rule, table: table, column: column, typeName: typeName) { return rule.style } - return nil + // Only after no rule matched as written, so a later column rule keeps beating a type rule that names no width. + guard let typeName else { return nil } + let base = Self.baseTypeName(typeName) + return rules.first { $0.target == .type && MaskingGlob.matches($0.trimmedPattern, base) }?.style } func columnMasks(table: String?, columns: [String], typeNames: [String?]) -> ColumnMasks { @@ -111,6 +114,12 @@ struct MaskingPolicy: Equatable, Sendable { return MaskingGlob.matches(tablePattern, String(table[table.index(after: bareStart)...])) } } + + // A result names `TINYINT(1)`, `BIT(8)` or `Nullable(String)` where the rule says `tinyint`, `bit` or `string`. + private static func baseTypeName(_ typeName: String) -> String { + let unwrapped = ColumnType.unwrappedTypeName(typeName) ?? typeName + return unwrapped.prefix { $0 != "(" }.trimmingCharacters(in: .whitespaces) + } } extension MaskingStyle { diff --git a/apple/TablePro/Core/Services/Operations/RunningWorkPrompt.swift b/apple/TablePro/Core/Services/Operations/RunningWorkPrompt.swift index 5d254a3e4a..e0a1b3d463 100644 --- a/apple/TablePro/Core/Services/Operations/RunningWorkPrompt.swift +++ b/apple/TablePro/Core/Services/Operations/RunningWorkPrompt.swift @@ -41,11 +41,6 @@ internal struct RunningWorkPrompt: Equatable, Sendable { return ([message, lines, consequence(of: action)] + notes(for: work)).joined(separator: "\n\n") } - // An open transaction rides along as a loss: closing or disconnecting rolls it back. - internal static func transactionLines(for databases: [String]) -> [String] { - databases.map { String(format: String(localized: "Open transaction in “%@”"), $0) } - } - internal static func line(for work: RunningWork) -> String { let subject = work.subject.trimmingCharacters(in: .whitespacesAndNewlines) let connection = work.connectionName diff --git a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService+CatalogChange.swift b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService+CatalogChange.swift index 4e47982744..00b95434c5 100644 --- a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService+CatalogChange.swift +++ b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService+CatalogChange.swift @@ -18,11 +18,13 @@ struct CatalogTreeRefreshPlan: Equatable, Sendable { var routines: Set = [] var triggers: Set = [] var types: Set = [] + var serverObjects: Set = [] var allSchemaTables: Set = [] var isEmpty: Bool { !refreshesDatabaseList && schemaLists.isEmpty && tables.isEmpty && partitions.isEmpty - && routines.isEmpty && triggers.isEmpty && types.isEmpty && allSchemaTables.isEmpty + && routines.isEmpty && triggers.isEmpty && types.isEmpty && serverObjects.isEmpty + && allSchemaTables.isEmpty } } @@ -43,6 +45,7 @@ extension DatabaseTreeMetadataService { routineKeys: routinesState.keys, triggerKeys: triggersState.keys, typeKeys: typesState.keys, + serverObjectKeys: serverObjectsState.keys, allSchemaTableKeys: allSchemaTablesState.keys ) guard !plan.isEmpty else { return } @@ -65,6 +68,7 @@ extension DatabaseTreeMetadataService { routineKeys: routinesState.keys, triggerKeys: triggersState.keys, typeKeys: typesState.keys, + serverObjectKeys: serverObjectsState.keys, allSchemaTableKeys: allSchemaTablesState.keys ) markAllSchemaTablesChanged(plan.allSchemaTables) @@ -80,6 +84,7 @@ extension DatabaseTreeMetadataService { routineKeys: some Sequence, triggerKeys: some Sequence, typeKeys: some Sequence, + serverObjectKeys: some Sequence = EmptyCollection(), allSchemaTableKeys: some Sequence = EmptyCollection() ) -> CatalogTreeRefreshPlan { func reached(_ key: ObjectsKey) -> Bool { @@ -108,6 +113,14 @@ extension DatabaseTreeMetadataService { plan.routines = objectKeys(routineKeys, for: .routines) plan.triggers = objectKeys(triggerKeys, for: .triggers) plan.types = objectKeys(typeKeys, for: .types) + // A key with no schema lists a whole database, which a change in any of its schemas reaches. + if change.kinds.contains(.serverObjects) { + plan.serverObjects = Set(serverObjectKeys.filter { key in + guard key.connectionId == change.connectionId else { return false } + guard let schema = key.schema else { return change.reaches(database: key.database) } + return change.reaches(database: key.database, schema: schema) + }) + } if !change.kinds.isDisjoint(with: [.tables, .schemas]) { plan.allSchemaTables = Set(allSchemaTableKeys.filter { key in key.connectionId == change.connectionId && change.reaches(database: key.database) diff --git a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift index f7e894e09b..bda60636ef 100644 --- a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift +++ b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift @@ -26,6 +26,14 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { let schema: String? } + // A `.database` kind, and every kind on an engine without schemas, is keyed with no schema. + struct ServerObjectsKey: Hashable, Sendable { + let connectionId: UUID + let database: String + let schema: String? + let kindId: String + } + struct PartitionsKey: Hashable, Sendable { let connectionId: UUID let database: String @@ -58,14 +66,19 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { let commit: ReferenceWritableKeyPath> let dedup: OnceTask, Value> let label: String + // What a read uses while nothing is on screen; a reread of rows already shown always goes bulk. + let workload: MetadataConnectionPool.Workload } + private typealias ListFetch = @Sendable (MetadataConnectionPool.Workload) async throws -> Value + @Published private(set) var databaseList: [UUID: MetadataLoadState<[DatabaseMetadata]>] = [:] @Published private(set) var schemaList: [DatabaseKey: MetadataLoadState<[String]>] = [:] @Published private(set) var tablesState: [ObjectsKey: MetadataLoadState<[TableInfo]>] = [:] @Published private(set) var routinesState: [ObjectsKey: MetadataLoadState<[RoutineInfo]>] = [:] @Published private(set) var triggersState: [ObjectsKey: MetadataLoadState<[TriggerInfo]>] = [:] @Published private(set) var typesState: [ObjectsKey: MetadataLoadState<[UserDefinedTypeInfo]>] = [:] + @Published private(set) var serverObjectsState: [ServerObjectsKey: MetadataLoadState<[ServerObjectInfo]>] = [:] @Published private(set) var partitionsState: [PartitionsKey: MetadataLoadState<[PartitionInfo]>] = [:] @Published private(set) var allSchemaTablesState: [DatabaseKey: MetadataLoadState] = [:] @Published private(set) var listsRevisions: [UUID: Int] = [:] @@ -76,6 +89,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { private let routinesDedup = OnceTask, [RoutineInfo]>() private let triggersDedup = OnceTask, [TriggerInfo]>() private let typesDedup = OnceTask, [UserDefinedTypeInfo]>() + private let serverObjectsDedup = OnceTask, [ServerObjectInfo]>() private let partitionsDedup = OnceTask, [PartitionInfo]>() private let allSchemaTablesDedup = OnceTask() @@ -85,6 +99,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { private var routinesCommit = ListCommitState() private var triggersCommit = ListCommitState() private var typesCommit = ListCommitState() + private var serverObjectsCommit = ListCommitState() private var partitionsCommit = ListCommitState() private var allSchemaTablesFence = CommitFence() private var allSchemaTablesFreshness = CatalogFreshness() @@ -98,27 +113,37 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { private init() {} private var schemaLists: TreeList { - TreeList(state: \.schemaList, commit: \.schemaListCommit, dedup: schemaDedup, label: "schemas") + TreeList(state: \.schemaList, commit: \.schemaListCommit, dedup: schemaDedup, label: "schemas", workload: .interactive) } private var tableLists: TreeList { - TreeList(state: \.tablesState, commit: \.tablesCommit, dedup: tablesDedup, label: "tables") + TreeList(state: \.tablesState, commit: \.tablesCommit, dedup: tablesDedup, label: "tables", workload: .interactive) } private var routineLists: TreeList { - TreeList(state: \.routinesState, commit: \.routinesCommit, dedup: routinesDedup, label: "routines") + TreeList(state: \.routinesState, commit: \.routinesCommit, dedup: routinesDedup, label: "routines", workload: .bulk) } private var triggerLists: TreeList { - TreeList(state: \.triggersState, commit: \.triggersCommit, dedup: triggersDedup, label: "triggers") + TreeList(state: \.triggersState, commit: \.triggersCommit, dedup: triggersDedup, label: "triggers", workload: .bulk) } private var typeLists: TreeList { - TreeList(state: \.typesState, commit: \.typesCommit, dedup: typesDedup, label: "types") + TreeList(state: \.typesState, commit: \.typesCommit, dedup: typesDedup, label: "types", workload: .bulk) + } + + private var serverObjectLists: TreeList { + TreeList( + state: \.serverObjectsState, + commit: \.serverObjectsCommit, + dedup: serverObjectsDedup, + label: "server objects", + workload: .bulk + ) } private var partitionLists: TreeList { - TreeList(state: \.partitionsState, commit: \.partitionsCommit, dedup: partitionsDedup, label: "partitions") + TreeList(state: \.partitionsState, commit: \.partitionsCommit, dedup: partitionsDedup, label: "partitions", workload: .interactive) } // MARK: - Reads @@ -173,6 +198,45 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { typesState[Self.objectsKey(connectionId: connectionId, database: database, schema: schema)]?.value ?? [] } + func serverObjectsLoadState( + connectionId: UUID, database: String, schema: String?, kindId: String + ) -> MetadataLoadState<[ServerObjectInfo]> { + serverObjectsState[Self.serverObjectsKey(connectionId: connectionId, database: database, schema: schema, kindId: kindId)] + ?? .idle + } + + func serverObjects(connectionId: UUID, database: String, schema: String?, kindId: String) -> [ServerObjectInfo] { + serverObjectsLoadState(connectionId: connectionId, database: database, schema: schema, kindId: kindId).value ?? [] + } + + func isServerObjectsCurrent(connectionId: UUID, database: String, schema: String?, kindId: String) -> Bool { + isConnected(connectionId) && serverObjectsCommit.freshness.isCurrent( + Self.serverObjectsKey(connectionId: connectionId, database: database, schema: schema, kindId: kindId) + ) + } + + // A container lists the kinds of its own level: a schema its `.container` kinds, a schema engine's + // database its `.database` kinds, and a database without schemas every kind. + nonisolated static func serverKinds( + declared: [SidebarServerObjectKind], + schema: String?, + listsTablesPerSchema: Bool + ) -> [SidebarServerObjectKind] { + if schema?.isEmpty == false { return declared.filter { $0.scope == .container } } + return listsTablesPerSchema ? declared.filter { $0.scope == .database } : declared + } + + func serverKinds(connectionId: UUID, database: String, schema: String?) -> [SidebarServerObjectKind] { + guard let session = DatabaseManager.shared.session(for: connectionId) else { return [] } + return Self.serverKinds( + declared: SidebarServerObjectKind.declared(by: DatabaseManager.shared.driver(for: connectionId)), + schema: schema, + listsTablesPerSchema: Self.listsTablesPerSchema( + PluginManager.shared.databaseGroupingStrategy(for: session.connection.type) + ) + ) + } + func partitionsLoadState( connectionId: UUID, database: String, schema: String?, table: String ) -> MetadataLoadState<[PartitionInfo]> { @@ -235,6 +299,12 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { .isEmpty } + func serverObjectsNeedFetch(connectionId: UUID, database: String, schema: String?) -> Bool { + guard isConnected(connectionId) else { return false } + return !serverKindsNeedingFetch(Self.objectsKey(connectionId: connectionId, database: database, schema: schema)) + .isEmpty + } + func partitionsNeedFetch(connectionId: UUID, database: String, schema: String?, table: String) -> Bool { isConnected(connectionId) && partitionsCommit.freshness.needsFetch( Self.partitionsKey(connectionId: connectionId, database: database, schema: schema, table: table) @@ -272,6 +342,10 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { typesCommit.freshness.markChanged(key) marked = true } + for key in plan.serverObjects { + serverObjectsCommit.freshness.markChanged(key) + marked = true + } guard marked else { return } listsRevisions[connectionId, default: 0] &+= 1 } @@ -282,9 +356,16 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { if routinesCommit.freshness.needsFetch(key) { kinds.insert(.routines) } if browsesTriggers(key.connectionId), triggersCommit.freshness.needsFetch(key) { kinds.insert(.triggers) } if browsesUserDefinedTypes(key.connectionId), typesCommit.freshness.needsFetch(key) { kinds.insert(.types) } + if !serverKindsNeedingFetch(key).isEmpty { kinds.insert(.serverObjects) } return kinds } + private func serverKindsNeedingFetch(_ key: ObjectsKey) -> [SidebarServerObjectKind] { + serverKinds(connectionId: key.connectionId, database: key.database, schema: key.schema).filter { + serverObjectsCommit.freshness.needsFetch(serverObjectsKey(key, kindId: $0.id)) + } + } + // MARK: - All-schema tables /// Read when next asked for, never on a catalog change itself: a COMMIT is one too, and relisting @@ -467,10 +548,14 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { } } - private func fetchDatabaseList(connectionId: UUID, databaseType: DatabaseType) async throws -> [DatabaseMetadata] { + private func fetchDatabaseList( + connectionId: UUID, + databaseType: DatabaseType, + workload: MetadataConnectionPool.Workload = .interactive + ) async throws -> [DatabaseMetadata] { let systemNames = Set(PluginManager.shared.systemDatabaseNames(for: databaseType)) return try await databaseDedup.execute(key: connectionId) { [self] in - try await withDriver(connectionId: connectionId, database: nil) { driver in + try await withDriver(connectionId: connectionId, database: nil, workload: workload) { driver in try await driver.fetchDatabases().sorted().map { DatabaseMetadata.minimal(name: $0, isSystem: systemNames.contains($0)) } @@ -532,9 +617,29 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { await self.loadUserDefinedTypes(connectionId: connectionId, database: database, schema: schema) } } + if kinds.contains(.serverObjects) { + group.addTask { await self.loadServerObjects(connectionId: connectionId, database: database, schema: schema) } + } + } + } + + // Only the kinds of this container's level, and only those a change overtook or nobody has read. + func loadServerObjects(connectionId: UUID, database: String, schema: String?) async { + guard isConnected(connectionId) else { return } + let container = Self.objectsKey(connectionId: connectionId, database: database, schema: schema) + let kinds = serverKindsNeedingFetch(container) + await withTaskGroup(of: Void.self) { group in + for kind in kinds { + let key = serverObjectsKey(container, kindId: kind.id) + group.addTask { await self.loadServerObjectList(key) } + } } } + private func loadServerObjectList(_ key: ServerObjectsKey) async { + await read(key, connectionId: key.connectionId, in: serverObjectLists, fetch: serverObjectListFetch(key)) + } + // MARK: - Refresh /// Fetches first and commits over the old list, so a refresh never empties the tree @@ -550,7 +655,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { } guard isConnected(connectionId) else { return } do { - let list = try await fetchDatabaseList(connectionId: connectionId, databaseType: databaseType) + let list = try await fetchDatabaseList(connectionId: connectionId, databaseType: databaseType, workload: .bulk) guard databaseListFence.isCurrent(token, for: connectionId) else { return } databaseList[connectionId] = .loaded(list) } catch is CancellationError { @@ -573,7 +678,18 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { async let types: Void = refreshUserDefinedTypeObjects( connectionId: connectionId, database: database, schema: schema ) - _ = await (tables, routines, triggers, types) + async let serverObjects: Void = refreshServerObjects(connectionId: connectionId, database: database, schema: schema) + _ = await (tables, routines, triggers, types, serverObjects) + } + + func refreshServerObjects(connectionId: UUID, database: String, schema: String?, kindId: String? = nil) async { + let container = Self.objectsKey(connectionId: connectionId, database: database, schema: schema) + let kinds = serverKinds(connectionId: connectionId, database: database, schema: schema) + .filter { kindId == nil || $0.id == kindId } + for kind in kinds { + let key = serverObjectsKey(container, kindId: kind.id) + await reread(key, connectionId: connectionId, in: serverObjectLists, fetch: serverObjectListFetch(key)) + } } /// A row refreshes only the fetch its kind comes from. Partitions ride with the tables, because @@ -648,7 +764,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { connectionId: UUID, in list: TreeList, token explicitToken: Int? = nil, - fetch: @Sendable @escaping () async throws -> Value + fetch: @escaping ListFetch ) async { guard isConnected(connectionId) else { return } let token = explicitToken ?? self[keyPath: list.commit].fence.token(for: key) @@ -656,15 +772,24 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { !self[keyPath: list.commit].freshness.isCurrent(key) else { return } let revision = self[keyPath: list.commit].freshness.revision(for: key) self[keyPath: list.commit].freshness.noteFetchStarted(revision, for: key) + // Rows on screen mean nobody waits on this read, so completion goes ahead of it. + let workload: MetadataConnectionPool.Workload switch self[keyPath: list.state][key] ?? .idle { - case .loaded, .loading: break - case .idle, .failed: self[keyPath: list.state][key] = .loading + case .loaded: + workload = .bulk + case .loading: + workload = list.workload + case .idle, .failed: + workload = list.workload + self[keyPath: list.state][key] = .loading } let outcome: MetadataFetchOutcome var failure: Error? do { - outcome = .fetched(try await list.dedup.execute(key: ListFetchKey(key: key, revision: revision), work: fetch)) + outcome = .fetched(try await list.dedup.execute(key: ListFetchKey(key: key, revision: revision)) { + try await fetch(workload) + }) } catch is CancellationError { outcome = .cancelled } catch { @@ -713,7 +838,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { _ key: Key, connectionId: UUID, in list: TreeList, - fetch: @Sendable @escaping () async throws -> Value + fetch: @escaping ListFetch ) async { let token = self[keyPath: list.commit].fence.supersede(key) self[keyPath: list.commit].freshness.markChanged(key) @@ -722,55 +847,65 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { await read(key, connectionId: connectionId, in: list, token: token, fetch: fetch) } - private func schemaListFetch(_ key: DatabaseKey) -> @Sendable () async throws -> [String] { - { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + private func schemaListFetch(_ key: DatabaseKey) -> ListFetch<[String]> { + { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchSchemas() } } } - private func tableListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [TableInfo] { + private func tableListFetch(_ key: ObjectsKey) -> ListFetch<[TableInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchTables(schema: schema) } } } - private func routineListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [RoutineInfo] { + private func routineListFetch(_ key: ObjectsKey) -> ListFetch<[RoutineInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchRoutines(schema: schema) } } } - private func triggerListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [TriggerInfo] { + private func triggerListFetch(_ key: ObjectsKey) -> ListFetch<[TriggerInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchAllTriggers(schema: schema) } } } - private func typeListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [UserDefinedTypeInfo] { + private func typeListFetch(_ key: ObjectsKey) -> ListFetch<[UserDefinedTypeInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchUserDefinedTypes(schema: schema) } } } - private func partitionListFetch(_ key: PartitionsKey) -> @Sendable () async throws -> [PartitionInfo] { + private func serverObjectListFetch(_ key: ServerObjectsKey) -> ListFetch<[ServerObjectInfo]> { + let schema = key.schema + let kindId = key.kindId + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in + try await driver.fetchServerObjects(ofKind: kindId, schema: schema) + } + } + } + + private func partitionListFetch(_ key: PartitionsKey) -> ListFetch<[PartitionInfo]> { let schema = key.schema let table = key.table - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchPartitionDetails(table: table, schema: schema) } } @@ -800,6 +935,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { routinesCommit.freshness.removeAll { $0.connectionId == connectionId } triggersCommit.freshness.removeAll { $0.connectionId == connectionId } typesCommit.freshness.removeAll { $0.connectionId == connectionId } + serverObjectsCommit.freshness.removeAll { $0.connectionId == connectionId } partitionsCommit.freshness.removeAll { $0.connectionId == connectionId } allSchemaTablesFreshness.removeAll { $0.connectionId == connectionId } allSchemaTablesRequested = allSchemaTablesRequested.filter { $0.key.connectionId != connectionId } @@ -812,6 +948,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { routinesState = routinesState.filter { $0.key.connectionId != connectionId } triggersState = triggersState.filter { $0.key.connectionId != connectionId } typesState = typesState.filter { $0.key.connectionId != connectionId } + serverObjectsState = serverObjectsState.filter { $0.key.connectionId != connectionId } partitionsState = partitionsState.filter { $0.key.connectionId != connectionId } } @@ -835,6 +972,9 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { triggersCommit.fence.supersede(key) typesCommit.fence.supersede(key) } + for key in serverObjectsState.keys where key.connectionId == connectionId { + serverObjectsCommit.fence.supersede(key) + } for key in connectionPartitionKeys(connectionId) { partitionsCommit.fence.supersede(key) } @@ -850,6 +990,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { await routinesDedup.cancel { $0.key.connectionId == connectionId } await triggersDedup.cancel { $0.key.connectionId == connectionId } await typesDedup.cancel { $0.key.connectionId == connectionId } + await serverObjectsDedup.cancel { $0.key.connectionId == connectionId } await partitionsDedup.cancel { $0.key.connectionId == connectionId } await allSchemaTablesDedup.cancel { $0.database.connectionId == connectionId } } @@ -863,6 +1004,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { idlePending(\.routinesState) { $0.connectionId == connectionId } idlePending(\.triggersState) { $0.connectionId == connectionId } idlePending(\.typesState) { $0.connectionId == connectionId } + idlePending(\.serverObjectsState) { $0.connectionId == connectionId } idlePending(\.partitionsState) { $0.connectionId == connectionId } idlePending(\.allSchemaTablesState) { $0.connectionId == connectionId } } @@ -922,6 +1064,17 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { return ObjectsKey(connectionId: connectionId, database: database, schema: normalized) } + private static func serverObjectsKey( + connectionId: UUID, database: String, schema: String?, kindId: String + ) -> ServerObjectsKey { + let normalized: String? = (schema?.isEmpty == true) ? nil : schema + return ServerObjectsKey(connectionId: connectionId, database: database, schema: normalized, kindId: kindId) + } + + private func serverObjectsKey(_ container: ObjectsKey, kindId: String) -> ServerObjectsKey { + ServerObjectsKey(connectionId: container.connectionId, database: container.database, schema: container.schema, kindId: kindId) + } + private static func partitionsKey( connectionId: UUID, database: String, schema: String?, table: String ) -> PartitionsKey { diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index 1b51ce9465..29dfc758d7 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -4,8 +4,28 @@ // import Foundation +import os import TableProPluginKit +// The lease a task's work runs under. That work keeps the lease until a lease it asks for returns, so such a +// request never waits for room; a task started inside keeps the exemption only while the lease is open. +@MainActor +private final class PooledLease { + let connectionId: UUID + // A lease nested in this one never takes this entry: it would queue behind the work that waits on it. + let entryId: ObjectIdentifier + var isOpen = true + + init(connectionId: UUID, entryId: ObjectIdentifier) { + self.connectionId = connectionId + self.entryId = entryId + } +} + +private enum PooledLeaseContext { + @TaskLocal static var open: [PooledLease] = [] +} + @MainActor final class MetadataConnectionPool { static let shared = MetadataConnectionPool() @@ -15,38 +35,81 @@ final class MetadataConnectionPool { case bulk } + // A scope with no schema reads wherever the connection's own settings put it, so its entries never switch. + private enum Slot: Hashable, Sendable { + case home + case followsSchema + } + + // No schema in the key: an entry moves to each lease's schema, so the schemas of one database share connections. private struct Key: Hashable, Sendable { - let scope: DatabaseScope + let connectionId: UUID + let database: String let workload: Workload + let slot: Slot + + init(scope: DatabaseScope, workload: Workload) { + connectionId = scope.connectionId + database = scope.database + self.workload = workload + slot = scope.schema == nil ? .home : .followsSchema + } + } + + private struct OpenKey: Hashable, Sendable { + let key: Key + let schema: String? + // Set for a lease inside another lease's work, so its open is its own and nobody queues on it. + let owner: UUID? + } + + private enum LeaseOutcome: Sendable { + case ran(T) + // The entry was left in a transaction and is closing, so the lease takes another. + case retired } @MainActor private final class Entry { let driver: DatabaseDriver - var lastUsed: Date - var inFlightCount: Int - var closeWhenIdle: Bool + var lastUsed = Date() + var inFlightCount = 0 + var closeWhenIdle = false + // The schema the last lease moved it to, used only to pick an entry; nil until a lease has moved it. + var schema: String? + // Only leases on one schema queue on a busy entry, so this is the schema its queue is on. + var queuedSchema: String? + // A startup command can move the search path after the driver read it, so the driver's cache is not trusted. + var forcesNextSwitch: Bool + var hasServedALease = false private var tail: Task = Task {} - init(driver: DatabaseDriver) { + init(driver: DatabaseDriver, schema: String?, forcesNextSwitch: Bool) { self.driver = driver - self.lastUsed = Date() - self.inFlightCount = 0 - self.closeWhenIdle = false + self.schema = schema + self.queuedSchema = schema + self.forcesNextSwitch = forcesNextSwitch } - /// The work runs in its own task so the next caller can queue behind it, so - /// cancelling the caller has to be forwarded explicitly or a stopped query - /// would keep running with nobody waiting on it. + // The work runs in its own task so the next caller can queue behind it, so a cancel is forwarded by hand. func runSerially( + connectionId: UUID, + prepare: @escaping @MainActor @Sendable (Entry) async throws -> Bool, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T - ) async throws -> T { + ) async throws -> LeaseOutcome { let previous = tail let driver = self.driver - let work = Task { @MainActor () async throws -> T in + let work = Task { @MainActor [self] () async throws -> LeaseOutcome in await previous.value try Task.checkCancellation() - return try await body(driver) + guard try await prepare(self) else { return .retired } + hasServedALease = true + let lease = PooledLease(connectionId: connectionId, entryId: ObjectIdentifier(self)) + defer { lease.isOpen = false } + let value = try await PooledLeaseContext.$open.withValue(PooledLeaseContext.open + [lease]) { + try await body(driver) + } + return .ran(value) } tail = Task { @MainActor in _ = try? await work.value } return try await withTaskCancellationHandler( @@ -58,27 +121,22 @@ final class MetadataConnectionPool { typealias DriverOpener = @MainActor (DatabaseScope) async throws -> DatabaseDriver - /// Why an open still in progress was taken away from the callers waiting on it. + // Why an open still in progress was taken away from the callers waiting on it. private enum Withdrawal { - /// The transport it was dialing is being rebuilt, so its callers take a connection again - /// once the replacement is in place. + // The transport it was dialing is being rebuilt, so its callers take a connection again after the replacement. case transportReplaced - /// What it was opened for is going away, so nothing may open it again. + // What it was opened for is going away, so nothing may open it again. case closed } - /// One open in progress, shared by every caller that asks for its key while it runs. It stays - /// listed until the last of those callers has taken its entry, because an entry nobody has - /// taken yet looks idle and would otherwise be the first one closed. + // Shared by every caller that asks for its key and schema while it runs. It stays listed until the last of them + // has taken its entry, because an entry nobody has taken yet looks idle and would be the first one closed. @MainActor private final class PendingOpen { - let task: Task + var task: Task? + var entry: Entry? var withdrawal: Withdrawal? var waiterCount = 0 - - init(task: Task) { - self.task = task - } } private struct TransportWaiter { @@ -87,13 +145,30 @@ final class MetadataConnectionPool { let continuation: CheckedContinuation } - private var entries: [Key: Entry] = [:] - private var pending: [Key: PendingOpen] = [:] + @MainActor + private final class RoomTicket { + let key: Key + let schema: String? + let held: Set + var continuation: CheckedContinuation? + var isWithdrawn = false + + init(key: Key, schema: String?, held: Set) { + self.key = key + self.schema = schema + self.held = held + } + } + + private var entries: [Key: [Entry]] = [:] + private var pending: [OpenKey: PendingOpen] = [:] private var transportReplacements: [UUID: Int] = [:] private var transportWaiters: [UUID: [TransportWaiter]] = [:] + private var roomQueues: [UUID: [RoomTicket]] = [:] private let openDriver: DriverOpener static let maxPerConnection = 6 private var sweeper: Task? + private static let logger = Logger(subsystem: "com.TablePro", category: "MetadataConnectionPool") /// How long a pooled connection may sit unused before it is handed back. /// @@ -105,7 +180,7 @@ final class MetadataConnectionPool { /// Ten minutes rather than something tighter, because re-taking one is a whole connect, /// measured at 1.7-5.8ms on loopback but 800-1900ms across the internet, and someone still /// working must never pay that. Ten minutes of silence is nobody still working. - static let idleTimeout: TimeInterval = 600 + nonisolated static let idleTimeout: TimeInterval = 600 /// The sweeper wakes more often than the timeout so a connection that goes idle just after a /// tick does not wait a whole second timeout, and never so often that an idle app spends the @@ -123,38 +198,47 @@ final class MetadataConnectionPool { workload: Workload = .interactive, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { - let entry = try await acquireEntry(scope: scope, workload: workload) - entry.inFlightCount += 1 - entry.lastUsed = Date() - defer { releaseEntry(entry, connectionId: scope.connectionId) } - return try await entry.runSerially(body) + let key = Key(scope: scope, workload: workload) + while true { + let entry = try await acquireEntry(forKey: key, schema: scope.schema, scope: scope) + if case .ran(let value) = try await run(body, on: entry, inKey: key, schema: scope.schema) { + return value + } + } + } + + private func run( + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T, + on entry: Entry, + inKey key: Key, + schema: String? + ) async throws -> LeaseOutcome { + defer { releaseEntry(entry, connectionId: key.connectionId) } + return try await entry.runSerially( + connectionId: key.connectionId, + prepare: { [self] entry in try await moveEntry(entry, inKey: key, to: schema) }, + body + ) } /// Closes only the leases attached to one database, which is what a rename of that database /// needs: PostgreSQL refuses `ALTER DATABASE ... RENAME` while any backend is connected to it, /// and an expanded row or a tab that ran a query there leaves one here. func closeAll(connectionId: UUID, database: String) { - closeEntries(withdrawingOpensAs: .closed) { scope in - scope.connectionId == connectionId && scope.database == database - } + closeEntries(withdrawingOpensAs: .closed) { $0 == connectionId && $1 == database } } func closeAll(connectionId: UUID) { - closeEntries(withdrawingOpensAs: .closed) { $0.connectionId == connectionId } + closeEntries(withdrawingOpensAs: .closed) { id, _ in id == connectionId } } - /// Holds a connection's pooled work while its transport is rebuilt. - /// - /// A pooled entry stands on the session's effective connection plus its own database, so it only - /// has to go when that endpoint does: a tunnel rebuilt on a new local port, or a reconnect - /// recovering a connection that stopped answering. An open already dialing is withdrawn, and once - /// it has returned the callers waiting on it take a connection again after `endTransportReplacement` - /// rather than failing a load nobody cancelled. A new caller waits for the same moment instead of - /// dialing an endpoint about to close. Replacements nest, and pooled work resumes when the last one - /// ends. Closing the connection, or one of its databases, fails the callers waiting for it. + // Holds a connection's pooled work while its transport is rebuilt: a tunnel on a new local port, or a reconnect of + // a connection that stopped answering. An open already dialing is withdrawn and its callers take a connection again + // after `endTransportReplacement`; a new caller waits for that too. Replacements nest. Closing the connection, or + // one of its databases, fails the callers waiting for it. func beginTransportReplacement(connectionId: UUID) { transportReplacements[connectionId, default: 0] += 1 - closeEntries(withdrawingOpensAs: .transportReplaced) { $0.connectionId == connectionId } + closeEntries(withdrawingOpensAs: .transportReplaced) { id, _ in id == connectionId } } /// Has to follow every `beginTransportReplacement` on every exit, cancellation included, or the @@ -172,29 +256,35 @@ final class MetadataConnectionPool { } #if DEBUG - /// Seeds a connected driver as the pooled connection for `scope`, so a test can observe - /// what runs on the pool without a plugin to open a real connection. + // Seeds a connected driver as a pooled connection already on `scope`'s schema, for a test with no plugin to open one. internal func injectEntry( _ driver: DatabaseDriver, scope: DatabaseScope, workload: Workload = .interactive, lastUsed: Date = Date() ) { - let entry = Entry(driver: driver) + let entry = Entry(driver: driver, schema: scope.schema, forcesNextSwitch: false) entry.lastUsed = lastUsed - entries[Key(scope: scope, workload: workload)] = entry + entry.hasServedALease = true + entries[Key(scope: scope, workload: workload), default: []].append(entry) } internal func pooledDriverCount(for connectionId: UUID) -> Int { - entries.keys.filter { $0.scope.connectionId == connectionId }.count + pooled(on: connectionId).count } internal func heldConnectionCount(for connectionId: UUID) -> Int { heldCount(for: connectionId) } + internal func leaseCount(for connectionId: UUID) -> Int { + pooled(on: connectionId).reduce(0) { $0 + $1.entry.inFlightCount } + } + internal func markInFlight(scope: DatabaseScope, workload: Workload = .interactive) { - entries[Key(scope: scope, workload: workload)]?.inFlightCount += 1 + let list = entries[Key(scope: scope, workload: workload)] ?? [] + let entry = list.first(where: { $0.schema == scope.schema }) ?? list.first + entry?.inFlightCount += 1 } internal var hasSweeper: Bool { @@ -219,6 +309,10 @@ final class MetadataConnectionPool { internal func transportWaiterCount(for connectionId: UUID) -> Int { transportWaiters[connectionId]?.count ?? 0 } + + internal func roomWaiterCount(for connectionId: UUID) -> Int { + roomQueues[connectionId]?.filter { $0.continuation != nil }.count ?? 0 + } #endif /// Whether the connection's transport is being rebuilt, which ends every connection dialed @@ -235,43 +329,59 @@ final class MetadataConnectionPool { return } trimIdleEntries(for: connectionId) + handRoom(on: connectionId) } - private func closeOrDeferEntry(forKey key: Key) { - guard let entry = entries.removeValue(forKey: key) else { return } + @discardableResult + private func detach(_ entry: Entry, fromKey key: Key) -> Bool { + guard var list = entries[key], let index = list.firstIndex(where: { $0 === entry }) else { return false } + list.remove(at: index) + entries[key] = list.isEmpty ? nil : list + return true + } + + private func removeEntry(_ entry: Entry, fromKey key: Key) { + guard detach(entry, fromKey: key) else { return } if entry.inFlightCount == 0 { entry.driver.disconnect() } else { entry.closeWhenIdle = true } + handRoom(on: key.connectionId) } private func closeEntries( withdrawingOpensAs withdrawal: Withdrawal, - where matches: (DatabaseScope) -> Bool + where matches: (UUID, String) -> Bool ) { - for key in pending.keys where matches(key.scope) { - guard let open = pending.removeValue(forKey: key) else { continue } + var withdrawnFrom: Set = [] + for openKey in pending.keys where matches(openKey.key.connectionId, openKey.key.database) { + guard let open = pending.removeValue(forKey: openKey) else { continue } open.withdrawal = withdrawal - open.task.cancel() + open.task?.cancel() + withdrawnFrom.insert(openKey.key.connectionId) } if withdrawal == .closed { failTransportWaiters(where: matches) + withdrawRoomTickets(where: matches) } - for key in entries.keys where matches(key.scope) { - closeOrDeferEntry(forKey: key) + for (key, list) in entries where matches(key.connectionId, key.database) { + for entry in list { + removeEntry(entry, fromKey: key) + } + } + for connectionId in withdrawnFrom { + handRoom(on: connectionId) } stopSweeperIfEmpty() } - /// A caller parked for a replacement is waiting on its connection as much as one waiting on a - /// pending open, so closing what it waits for fails it too. Left parked, it would wake when the - /// replacement ends and open a connection for a session, or a database, that has gone. - private func failTransportWaiters(where matches: (DatabaseScope) -> Bool) { + // A parked caller would otherwise wake when the replacement ends and open a connection for what has gone. + private func failTransportWaiters(where matches: (UUID, String) -> Bool) { for (connectionId, waiters) in transportWaiters { - let failing = waiters.filter { matches($0.scope) } + let failing = waiters.filter { matches($0.scope.connectionId, $0.scope.database) } guard !failing.isEmpty else { continue } - let remaining = waiters.filter { !matches($0.scope) } + let remaining = waiters.filter { !matches($0.scope.connectionId, $0.scope.database) } transportWaiters[connectionId] = remaining.isEmpty ? nil : remaining for waiter in failing { waiter.continuation.resume(throwing: CancellationError()) @@ -279,17 +389,29 @@ final class MetadataConnectionPool { } } - private func acquireEntry(scope: DatabaseScope, workload: Workload) async throws -> Entry { - let key = Key(scope: scope, workload: workload) + // In order: an idle entry on the schema, any idle entry, the busy entry on the schema, an open for the schema + // already dialing, a new entry if there is room, else a place in the room line. A lease inside another lease's + // work skips the busy entry and the open: two such leases nesting in opposite directions would wait on each other. + private func acquireEntry(forKey key: Key, schema: String?, scope: DatabaseScope) async throws -> Entry { + let held = entriesHeldByThisTask() + let openKey = OpenKey(key: key, schema: schema, owner: held.isEmpty ? nil : UUID()) while true { try await waitForTransport(for: scope) - if let entry = reusableEntry(forKey: key) { + if let entry = claimEntry(forKey: key, schema: schema, excluding: held) { return entry } - let open = try pending[key] ?? startOpen(forKey: key) - let failure = await completion(of: open, forKey: key) - /// A withdrawn open can end either way: a driver whose connect honours cancellation - /// throws, and one that finished first returns without keeping its entry. + if pending[openKey] == nil { + try await waitForRoom(forKey: key, schema: schema, held: held) + // A wait can outlast its reason: an entry came free, the schema's open started, or the transport is being rebuilt. + if pending[openKey] != nil || placeWithoutOpening(forKey: key, schema: schema, excluding: held) != nil + || isReplacingTransport(for: key.connectionId) { + continue + } + } + let open = try pending[openKey] ?? startOpen(openKey) + let failure = await completion(of: open, for: openKey) + // A withdrawn open can end either way: a driver whose connect honours cancellation throws, and one that + // finished first returns without keeping its entry. if open.withdrawal == .transportReplaced { try Task.checkCancellation() continue @@ -297,69 +419,170 @@ final class MetadataConnectionPool { if let failure { throw failure } - if let entry = entries[key] { + if let entry = open.entry, isListed(entry, inKey: key) { + claim(entry, for: schema) return entry } if open.withdrawal == .closed { throw DatabaseError.notConnected } - forgetSpentOpen(open, forKey: key) + forgetSpentOpen(open, for: openKey) + } + } + + private func entriesHeldByThisTask() -> Set { + Set(PooledLeaseContext.open.filter { $0.isOpen }.map { $0.entryId }) + } + + private func claimEntry(forKey key: Key, schema: String?, excluding held: Set) -> Entry? { + // A Mac that slept comes back with entries the sweeper never reached and sockets the server already closed. + for entry in entries[key] ?? [] where !isUsable(entry) { + removeEntry(entry, fromKey: key) } + guard let entry = placeWithoutOpening(forKey: key, schema: schema, excluding: held) else { return nil } + claim(entry, for: schema) + return entry } - /// An open whose entry went before this caller could take it has nothing left to hand out. - /// Joining it again returns at once, without a suspension, so a caller that did would spin on - /// the main actor and starve the other callers still waiting to resume from it. - private func forgetSpentOpen(_ open: PendingOpen, forKey key: Key) { - guard pending[key] === open else { return } - pending.removeValue(forKey: key) + // At most one busy entry per schema, so two schemas share a connection only while one of them is idle. + private func placeWithoutOpening( + forKey key: Key, + schema: String?, + excluding held: Set + ) -> Entry? { + let candidates = (entries[key] ?? []).filter { !held.contains(ObjectIdentifier($0)) && isUsable($0) } + let idle = candidates.filter { isIdle($0) } + if let onSchema = idle.first(where: { $0.schema == schema }) { + return onSchema + } + if let any = idle.min(by: { $0.lastUsed < $1.lastUsed }) { + return any + } + guard held.isEmpty else { return nil } + return candidates.first { $0.inFlightCount > 0 && $0.queuedSchema == schema } } - /// A cached entry is only worth reusing while it is both connected and recent. The - /// staleness half matters even with the sweeper running, because a Mac that slept comes - /// back with entries the sweeper never got to and sockets the server has long since - /// closed. The old entry is closed rather than left behind: overwriting `entries[key]` - /// with a fresh one used to leak the driver it replaced. - private func reusableEntry(forKey key: Key) -> Entry? { - guard let entry = entries[key] else { return nil } - if entry.driver.status == .connected, !entry.driver.hasLostConnection, !Self.isStale(entry.lastUsed) { - return entry + private func claim(_ entry: Entry, for schema: String?) { + entry.inFlightCount += 1 + entry.lastUsed = Date() + entry.queuedSchema = schema + } + + // Runs on the entry's queue before the lease's work, so the switch lands where the work reads. + private func moveEntry(_ entry: Entry, inKey key: Key, to schema: String?) async throws -> Bool { + guard key.slot == .followsSchema, let schema, let switchable = entry.driver as? SchemaSwitchable else { + return true + } + let unconditionally = entry.forcesNextSwitch + // Asked on every lease: Snowflake's drivers for one connection share a session, so another one may have moved it. + if !unconditionally, switchable.currentSchema == schema { + entry.schema = schema + return true + } + // A SET sent inside a transaction is undone by its rollback while the driver keeps the new name. A fresh + // entry's transaction came from the startup commands, and every replacement would carry one too. + if entry.hasServedALease { + let state = await entry.driver.sessionTransactionState() + if state == .inTransaction || state == .abortedTransaction { + Self.logger.info("Closing a pooled connection left in a transaction instead of moving its schema") + removeEntry(entry, fromKey: key) + return false + } } - closeOrDeferEntry(forKey: key) - return nil + do { + try await Self.switchSchema( + entry.driver, + to: schema, + timeoutSeconds: Self.stepTimeoutSeconds(for: key.connectionId), + unconditionally: unconditionally + ) + } catch { + // The server may have run a statement the client gave up on, so the next lease sends its own. + entry.schema = nil + entry.forcesNextSwitch = true + if !Self.isConnected(entry.driver) { + removeEntry(entry, fromKey: key) + } + throw error + } + entry.schema = schema + entry.forcesNextSwitch = false + return true + } + + private static func stepTimeoutSeconds(for connectionId: UUID) -> Double? { + guard let seconds = DatabaseManager.shared.session(for: connectionId)?.effectiveQueryTimeoutSeconds, + seconds > 0 + else { return nil } + return Double(seconds) + } + + // Joining an open that has already finished returns without a suspension, so a caller that rejoined one whose entry + // went would spin on the main actor and starve the callers still waiting to resume from it. + private func forgetSpentOpen(_ open: PendingOpen, for openKey: OpenKey) { + guard pending[openKey] === open else { return } + pending.removeValue(forKey: openKey) + handRoom(on: openKey.key.connectionId) } - private func startOpen(forKey key: Key) throws -> PendingOpen { - guard DatabaseManager.shared.session(for: key.scope.connectionId) != nil else { + private static func isConnected(_ driver: DatabaseDriver) -> Bool { + driver.status == .connected && !driver.hasLostConnection + } + + private func isUsable(_ entry: Entry) -> Bool { + Self.isConnected(entry.driver) && !Self.isStale(entry.lastUsed) + } + + private func isListed(_ entry: Entry, inKey key: Key) -> Bool { + entries[key]?.contains { $0 === entry } == true + } + + private func isAwaitingItsOpeners(_ entry: Entry) -> Bool { + pending.values.contains { $0.entry === entry } + } + + private func isIdle(_ entry: Entry) -> Bool { + entry.inFlightCount == 0 && !isAwaitingItsOpeners(entry) + } + + // A new entry opens on the connection's own default and each lease moves it, so it can serve every schema. + private func startOpen(_ openKey: OpenKey) throws -> PendingOpen { + let key = openKey.key + guard let session = DatabaseManager.shared.session(for: key.connectionId) else { throw DatabaseError.notConnected } - evictIdleIfNeeded(for: key.scope.connectionId) - let task = Task { [self] in - let entry = Entry(driver: try await openDriver(key.scope)) + evictIdleIfNeeded(for: key.connectionId) + let scope = DatabaseScope(connectionId: key.connectionId, database: key.database, schema: nil) + let forcesFirstSwitch = key.slot == .followsSchema + && DatabaseManager.hasStartupCommands(session.connection.startupCommands) + let open = PendingOpen() + open.task = Task { [self] in + let entry = Entry(driver: try await openDriver(scope), schema: nil, forcesNextSwitch: forcesFirstSwitch) if Task.isCancelled { entry.driver.disconnect() return } - entries[key] = entry + entries[key, default: []].append(entry) + open.entry = entry startSweeperIfNeeded() } - let open = PendingOpen(task: task) - pending[key] = open + pending[openKey] = open return open } - /// Waits for an open to finish, takes it off the pending list once the last caller waiting on it - /// has, and returns how it failed. - private func completion(of open: PendingOpen, forKey key: Key) async -> Error? { + // Waits for an open to finish, takes it off the pending list once its last caller has, and returns how it failed. + private func completion(of open: PendingOpen, for openKey: OpenKey) async -> Error? { open.waiterCount += 1 defer { open.waiterCount -= 1 - if open.waiterCount == 0, pending[key] === open { - pending.removeValue(forKey: key) + if open.waiterCount == 0, pending[openKey] === open { + pending.removeValue(forKey: openKey) + // A failed open frees its place, and a taken one leaves an entry that is now evictable once idle. + handRoom(on: openKey.key.connectionId) } } do { - try await open.task.value + _ = try await open.task?.value return nil } catch { return error @@ -408,6 +631,101 @@ final class MetadataConnectionPool { waiter.continuation.resume(throwing: CancellationError()) } + // First come first served, interactive ahead of bulk; a caller arriving while others wait goes behind them. + private func waitForRoom(forKey key: Key, schema: String?, held: Set) async throws { + let connectionId = key.connectionId + guard !PooledLeaseContext.open.contains(where: { $0.isOpen && $0.connectionId == connectionId }) else { return } + let ticket = RoomTicket(key: key, schema: schema, held: held) + roomQueues[connectionId, default: []].append(ticket) + defer { leaveRoomQueue(ticket) } + while !mayOpen(ticket) { + try await parkForRoom(ticket) + try Task.checkCancellation() + if ticket.isWithdrawn { throw CancellationError() } + } + } + + private func proceedsWithoutAPlace(_ ticket: RoomTicket) -> Bool { + let key = ticket.key + let joinsAnOpen = ticket.held.isEmpty && pending[OpenKey(key: key, schema: ticket.schema, owner: nil)] != nil + return joinsAnOpen + || placeWithoutOpening(forKey: key, schema: ticket.schema, excluding: ticket.held) != nil + || isReplacingTransport(for: key.connectionId) + } + + private func mayOpen(_ ticket: RoomTicket) -> Bool { + if proceedsWithoutAPlace(ticket) { return true } + let connectionId = ticket.key.connectionId + return nextInLine(on: connectionId) === ticket && hasRoom(for: ticket.key.workload, on: connectionId) + } + + private func nextInLine(on connectionId: UUID) -> RoomTicket? { + let line = roomQueues[connectionId] ?? [] + return line.first { $0.key.workload == .interactive } ?? line.first + } + + // A freed place goes only to the caller next in line. Callers that need no new place go too, and first, so an + // idle entry is reused before the next in line can evict it. + private func handRoom(on connectionId: UUID) { + let parked = (roomQueues[connectionId] ?? []).filter { $0.continuation != nil } + var woken = parked.filter { proceedsWithoutAPlace($0) } + if let next = nextInLine(on: connectionId), next.continuation != nil, + !woken.contains(where: { $0 === next }), mayOpen(next) { + woken.append(next) + } + for ticket in woken { + guard let continuation = ticket.continuation else { continue } + ticket.continuation = nil + continuation.resume() + } + } + + private func leaveRoomQueue(_ ticket: RoomTicket) { + let connectionId = ticket.key.connectionId + roomQueues[connectionId]?.removeAll { $0 === ticket } + if roomQueues[connectionId]?.isEmpty == true { + roomQueues.removeValue(forKey: connectionId) + } + handRoom(on: connectionId) + } + + private func parkForRoom(_ ticket: RoomTicket) async throws { + try await withTaskCancellationHandler( + operation: { try await parkUntilHanded(ticket) }, + onCancel: { [weak self] in + Task { @MainActor in + self?.failRoomTicket(ticket) + } + } + ) + } + + private func parkUntilHanded(_ ticket: RoomTicket) async throws { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + guard !Task.isCancelled, !ticket.isWithdrawn else { + continuation.resume(throwing: CancellationError()) + return + } + ticket.continuation = continuation + } + } + + private func failRoomTicket(_ ticket: RoomTicket) { + guard let continuation = ticket.continuation else { return } + ticket.continuation = nil + continuation.resume(throwing: CancellationError()) + } + + // Same reason as `failTransportWaiters`: a parked caller would otherwise open what was just closed. + private func withdrawRoomTickets(where matches: (UUID, String) -> Bool) { + for line in roomQueues.values { + for ticket in line where matches(ticket.key.connectionId, ticket.key.database) { + ticket.isWithdrawn = true + failRoomTicket(ticket) + } + } + } + /// Opens a second connection to the session's server, on `scope`'s database, through the /// transport the session already holds. Every tunnel manager closes its tunnel before building a /// new one, so dialing the session's effective endpoint is what keeps the other connections up. @@ -454,13 +772,14 @@ final class MetadataConnectionPool { ) do { try await driver.connect() - try await Self.prepareSession( + let startupFailures = try await Self.prepareSession( driver, queryTimeoutSeconds: queryTimeout, startupCommands: session.connection.startupCommands, connectionName: session.connection.name, timeoutSeconds: operationTimeoutSeconds ) + DatabaseManager.shared.reportStartupFailures(startupFailures, for: scope.connectionId) if let database = plan.switchDatabase { try await Self.switchDatabase(driver, to: database, timeoutSeconds: operationTimeoutSeconds) } @@ -522,14 +841,23 @@ final class MetadataConnectionPool { } } - static func switchSchema(_ driver: DatabaseDriver, to schema: String, timeoutSeconds: Double?) async throws { + static func switchSchema( + _ driver: DatabaseDriver, + to schema: String, + timeoutSeconds: Double?, + unconditionally: Bool = false + ) async throws { guard let switchable = driver as? SchemaSwitchable else { return } try await bounded( driver: driver, timeoutSeconds: timeoutSeconds, timeoutMessage: String(format: String(localized: "Switching to schema '%@' timed out."), schema) ) { - try await switchable.switchSchemaIfNeeded(to: schema) + if unconditionally { + try await switchable.switchSchema(to: schema) + } else { + try await switchable.switchSchemaIfNeeded(to: schema) + } } } @@ -543,14 +871,14 @@ final class MetadataConnectionPool { startupCommands: String?, connectionName: String, timeoutSeconds: Double? - ) async throws { + ) async throws -> [StartupCommandFailure] { try await bounded( driver: driver, timeoutSeconds: timeoutSeconds, timeoutMessage: String(localized: "Preparing the metadata connection timed out.") ) { try? await driver.applyQueryTimeout(queryTimeoutSeconds) - await DatabaseManager.shared.executeStartupCommands( + return await DatabaseManager.shared.executeStartupCommands( startupCommands, on: driver, connectionName: connectionName ) } @@ -558,18 +886,17 @@ final class MetadataConnectionPool { /// Disconnects the driver when the deadline fires so a driver call that /// ignores task cancellation still completes and the timeout can propagate. - private static func bounded( + private static func bounded( driver: DatabaseDriver, timeoutSeconds: Double?, timeoutMessage: String, - _ operation: @escaping @Sendable () async throws -> Void - ) async throws { + _ operation: @escaping @Sendable () async throws -> T + ) async throws -> T { guard let timeoutSeconds, timeoutSeconds > 0 else { - try await operation() - return + return try await operation() } do { - try await withTimeout( + return try await withTimeout( seconds: timeoutSeconds, onTimeout: { driver.disconnect() }, operation: operation @@ -603,28 +930,60 @@ final class MetadataConnectionPool { sweeper = nil } - /// Hands back every connection nobody has used for `idleTimeout`. An entry with work on it is - /// left alone whatever its age: `lastUsed` is stamped when the work starts, so a long export - /// would otherwise have its own connection closed underneath it. + // An entry with work on it stays whatever its age: `lastUsed` is stamped when the work starts, so a long export + // would otherwise have its own connection closed underneath it. internal func sweepIdleEntries(now: Date = Date()) { - for (key, entry) in entries where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { - entries.removeValue(forKey: key) - entry.driver.disconnect() + var sweptFrom: Set = [] + for (key, list) in entries { + for entry in list where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { + detach(entry, fromKey: key) + entry.driver.disconnect() + sweptFrom.insert(key.connectionId) + } + } + for connectionId in sweptFrom { + handRoom(on: connectionId) } stopSweeperIfEmpty() } - /// Makes room for one more before an open. The caller never waits for room, so a burst of opens - /// can still run past the limit while its work is in flight; `trimIdleEntries` hands the extra - /// back as that work ends. + // A caller with no room waits in `waitForRoom`, so only a lease taken inside another one's work opens past the limit. private func evictIdleIfNeeded(for connectionId: UUID) { guard heldCount(for: connectionId) >= Self.maxPerConnection else { return } evictLeastRecentlyUsedIdleEntry(for: connectionId) } - /// Keeps a connection at the limit once its work ends. Closing only one idle entry per open, as - /// the pool used to, left a burst's high-water mark standing until the idle sweep: a tree Refresh - /// across many databases held every connection it opened for ten minutes (#3103). + // Bulk never takes the last place, so a completion read or a sidebar expand never waits behind a burst of bulk work. + private func hasRoom(for workload: Workload, on connectionId: UUID) -> Bool { + let placeIsFree = heldCount(for: connectionId) < Self.maxPerConnection + || leastRecentlyUsedIdleEntry(for: connectionId) != nil + guard workload == .bulk else { return placeIsFree } + return placeIsFree && busyCount(for: connectionId) < Self.maxPerConnection - 1 + } + + private func pooled(on connectionId: UUID) -> [(key: Key, entry: Entry)] { + var found: [(key: Key, entry: Entry)] = [] + for (key, list) in entries where key.connectionId == connectionId { + found += list.map { (key: key, entry: $0) } + } + return found + } + + // An open still dialing holds a place. One that landed holds it through its entry, and counts once. + private func openingCount(for connectionId: UUID) -> Int { + pending.filter { openKey, open in + guard openKey.key.connectionId == connectionId else { return false } + guard let landed = open.entry else { return true } + return !isListed(landed, inKey: openKey.key) + }.count + } + + private func busyCount(for connectionId: UUID) -> Int { + pooled(on: connectionId).filter { !isIdle($0.entry) }.count + openingCount(for: connectionId) + } + + // Closing one idle entry per open, as the pool used to, left a burst's high-water mark standing until the idle + // sweep: a tree Refresh across many databases held every connection it opened for ten minutes (#3103). private func trimIdleEntries(for connectionId: UUID) { while heldCount(for: connectionId) > Self.maxPerConnection { guard evictLeastRecentlyUsedIdleEntry(for: connectionId) else { break } @@ -632,24 +991,22 @@ final class MetadataConnectionPool { stopSweeperIfEmpty() } - /// Every key the connection holds a connection for or is opening one for. Counted by key, so an - /// open that has finished but is still listed until its callers take the entry counts once. private func heldCount(for connectionId: UUID) -> Int { - var keys = Set(entries.keys.filter { $0.scope.connectionId == connectionId }) - keys.formUnion(pending.keys.filter { $0.scope.connectionId == connectionId }) - return keys.count + pooled(on: connectionId).count + openingCount(for: connectionId) } - /// An entry whose open is still listed has callers that have not taken it yet, so it is not idle - /// however it looks. @discardableResult private func evictLeastRecentlyUsedIdleEntry(for connectionId: UUID) -> Bool { - let idle = entries.filter { key, entry in - key.scope.connectionId == connectionId && entry.inFlightCount == 0 && pending[key] == nil - } - guard let oldest = idle.min(by: { $0.value.lastUsed < $1.value.lastUsed }) else { return false } - entries.removeValue(forKey: oldest.key) - oldest.value.driver.disconnect() + guard let victim = leastRecentlyUsedIdleEntry(for: connectionId), + detach(victim.entry, fromKey: victim.key) + else { return false } + victim.entry.driver.disconnect() return true } + + private func leastRecentlyUsedIdleEntry(for connectionId: UUID) -> (key: Key, entry: Entry)? { + pooled(on: connectionId) + .filter { isIdle($0.entry) } + .min { $0.entry.lastUsed < $1.entry.lastUsed } + } } diff --git a/apple/TablePro/Core/Services/Query/QueryExecutor.swift b/apple/TablePro/Core/Services/Query/QueryExecutor.swift index f67b7650aa..5233d4dc07 100644 --- a/apple/TablePro/Core/Services/Query/QueryExecutor.swift +++ b/apple/TablePro/Core/Services/Query/QueryExecutor.swift @@ -110,13 +110,18 @@ final class QueryExecutor { sql: String, parameters: [Any?]? = nil, rowCap: Int?, + safeModeIsReadOnly: Bool = false, capturingOutputInto failureOutput: ServerOutputBox? ) async throws -> QueryFetchResult { guard let failureOutput else { - return try await executeQuery(driver: driver, sql: sql, parameters: parameters, rowCap: rowCap) + return try await executeQuery( + driver: driver, sql: sql, parameters: parameters, rowCap: rowCap, safeModeIsReadOnly: safeModeIsReadOnly + ) } return try await ServerOutputCapture.running(on: driver, failureOutput: failureOutput) { - try await executeQuery(driver: driver, sql: sql, parameters: parameters, rowCap: rowCap) + try await executeQuery( + driver: driver, sql: sql, parameters: parameters, rowCap: rowCap, safeModeIsReadOnly: safeModeIsReadOnly + ) } } @@ -124,20 +129,24 @@ final class QueryExecutor { driver: DatabaseDriver, sql: String, parameters: [Any?]? = nil, - rowCap: Int? + rowCap: Int?, + safeModeIsReadOnly: Bool = false ) async throws -> QueryFetchResult { + let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) if let parameters { return try await Self.fetchQueryDataParameterized( driver: driver, sql: sql, parameters: parameters, - rowCap: rowCap + rowCap: rowCap, + context: context ) } return try await Self.fetchQueryData( driver: driver, sql: sql, - rowCap: rowCap + rowCap: rowCap, + context: context ) } @@ -150,11 +159,18 @@ final class QueryExecutor { nonisolated static func fetchBoundedQueryData( driver: DatabaseDriver, sql: String, - rowCap: Int? + rowCap: Int?, + context: PluginStatementContext? = nil ) async throws -> QueryFetchResult? { guard let rowCap, rowCap > 0 else { return nil } let start = CFAbsoluteTimeGetCurrent() - guard let result = try await driver.executeBoundedQuery(query: sql, rowCap: rowCap) else { + let bounded: QueryResult? + if let context { + bounded = try await driver.executeBoundedQuery(query: sql, rowCap: rowCap, context: context) + } else { + bounded = try await driver.executeBoundedQuery(query: sql, rowCap: rowCap) + } + guard let result = bounded else { return nil } let elapsed = CFAbsoluteTimeGetCurrent() - start @@ -197,17 +213,24 @@ final class QueryExecutor { ) } + // A non-nil context is Read-Only's, which the bounded read and its fallback both carry. nonisolated static func fetchQueryData( driver: DatabaseDriver, sql: String, - rowCap: Int? + rowCap: Int?, + context: PluginStatementContext? = nil ) async throws -> QueryFetchResult { - if let bounded = try await fetchBoundedQueryData(driver: driver, sql: sql, rowCap: rowCap) { + if let bounded = try await fetchBoundedQueryData(driver: driver, sql: sql, rowCap: rowCap, context: context) { return bounded } let start = CFAbsoluteTimeGetCurrent() queryExecutorLog.info("[executeUserQuery] sql=\(sql.prefix(100), privacy: .private) rowCap=\(rowCap?.description ?? "nil")") - let result = try await driver.executeUserQuery(query: sql, rowCap: rowCap, parameters: nil) + let result: QueryResult + if let context { + result = try await driver.executeUserQuery(query: sql, rowCap: rowCap, parameters: nil, context: context) + } else { + result = try await driver.executeUserQuery(query: sql, rowCap: rowCap, parameters: nil) + } let elapsed = CFAbsoluteTimeGetCurrent() - start queryExecutorLog.info("[executeUserQuery] rows=\(result.rows.count) truncated=\(result.isTruncated) driverTime=\(String(format: "%.3f", result.executionTime))s totalTime=\(String(format: "%.3f", elapsed))s") return QueryFetchResult( @@ -231,11 +254,17 @@ final class QueryExecutor { driver: DatabaseDriver, sql: String, parameters: [Any?], - rowCap: Int? + rowCap: Int?, + context: PluginStatementContext? = nil ) async throws -> QueryFetchResult { let start = CFAbsoluteTimeGetCurrent() queryExecutorLog.info("[executeUserQueryParameterized] sql=\(sql.prefix(100), privacy: .private) rowCap=\(rowCap?.description ?? "nil") params=\(parameters.count)") - let result = try await driver.executeUserQuery(query: sql, rowCap: rowCap, parameters: parameters) + let result: QueryResult + if let context { + result = try await driver.executeUserQuery(query: sql, rowCap: rowCap, parameters: parameters, context: context) + } else { + result = try await driver.executeUserQuery(query: sql, rowCap: rowCap, parameters: parameters) + } let elapsed = CFAbsoluteTimeGetCurrent() - start queryExecutorLog.info("[executeUserQueryParameterized] rows=\(result.rows.count) truncated=\(result.isTruncated) driverTime=\(String(format: "%.3f", result.executionTime))s totalTime=\(String(format: "%.3f", elapsed))s") return QueryFetchResult( diff --git a/apple/TablePro/Core/Services/Query/QuerySqlParser.swift b/apple/TablePro/Core/Services/Query/QuerySqlParser.swift index 13b77e1805..76a0d66045 100644 --- a/apple/TablePro/Core/Services/Query/QuerySqlParser.swift +++ b/apple/TablePro/Core/Services/Query/QuerySqlParser.swift @@ -121,3 +121,14 @@ enum QuerySqlParser { return EnumValueParser.parseMySQLEnumOrSet(from: "ENUM(\(valuesString))") } } + +// What `extractTableName` reads from the connection and its session, held as a value so a parse can run off the main actor. +struct TableNameReading: Equatable, Sendable { + let dialect: SqlDialect + let readings: SQLLexicalReadings + let browseSchema: String? + + func tableName(in sql: String) -> String? { + QuerySqlParser.extractTableName(from: sql, dialect: dialect, readings: readings, browseSchema: browseSchema) + } +} diff --git a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift index 66f5ebf5e4..3f03741ab2 100644 --- a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift +++ b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift @@ -49,10 +49,12 @@ final class RowOperationsManager { func addNewRow(tableRows: inout TableRows, uuidKeyForms: [Int: UUIDCellForm] = [:]) -> AddNewRowResult? { var newRowValues: [PluginCellValue] = [] var absentColumns: Set = [] + var defaultColumns: Set = [] var prefilled: Set = [] for (index, column) in tableRows.columns.enumerated() { if tableRows.generatedColumns.contains(column) || tableRows.serverAssignsValue(forColumn: column) { - newRowValues.append(.text("__DEFAULT__")) + newRowValues.append(.columnDefault) + defaultColumns.insert(index) } else if let form = uuidKeyForms[index] { newRowValues.append(Self.freshUUID(form)) prefilled.insert(index) @@ -62,19 +64,60 @@ final class RowOperationsManager { } } return appendInsertedRow( - values: newRowValues, absentColumns: absentColumns, prefilledColumns: prefilled, to: &tableRows + values: newRowValues, + absentColumns: absentColumns, + expressionColumns: defaultColumns, + prefilledColumns: prefilled, + to: &tableRows ) } - // A pending source row keeps its staged SQL, so a copy of now() is now() again, not the text. func duplicateRow( sourceRowIndex: Int, tableRows: inout TableRows, uuidKeyForms: [Int: UUIDCellForm] = [:] ) -> AddNewRowResult? { - guard sourceRowIndex >= 0, sourceRowIndex < tableRows.count else { return nil } + duplicateRows(sourceRowIndices: [sourceRowIndex], tableRows: &tableRows, uuidKeyForms: uuidKeyForms).rows.first + } + + struct DuplicateRowsResult { + let rows: [AddNewRowResult] + let delta: Delta + } + + // Every copy is one undo step, as a paste of many rows is. + func duplicateRows( + sourceRowIndices: [Int], + tableRows: inout TableRows, + uuidKeyForms: [Int: UUIDCellForm] = [:] + ) -> DuplicateRowsResult { + let firstIndex = tableRows.count + var results: [AddNewRowResult] = [] + var recorded: [DataChangeManager.InsertedRow] = [] + for sourceRowIndex in sourceRowIndices where sourceRowIndex >= 0 && sourceRowIndex < firstIndex { + let copy = duplicatedRow(of: tableRows.rows[sourceRowIndex], in: tableRows, uuidKeyForms: uuidKeyForms) + let rowID = RowID.inserted(UUID()) + let storageIndex = tableRows.count + let delta = tableRows.appendInsertedRow(id: rowID, values: copy.values, absentColumns: copy.absentColumns) + recorded.append(DataChangeManager.InsertedRow( + location: InsertedRowLocation(rowID: rowID, storageIndex: storageIndex), + values: copy.values, absentColumns: copy.absentColumns, expressionColumns: copy.expressionColumns + )) + results.append(AddNewRowResult( + rowID: rowID, values: copy.values, delta: delta, prefilledColumns: copy.prefilledColumns + )) + } + changeManager.recordBatchRowInsertion(rows: recorded) + let delta: Delta = results.isEmpty ? .none : .rowsInserted(IndexSet(integersIn: firstIndex.. RowCopy { var newValues = Array(source.values) var absentColumns = source.absentColumns var expressionColumns = changeManager.expressionColumns(forRow: source.id).filter { $0 < newValues.count } @@ -86,25 +129,29 @@ final class RowOperationsManager { + Array(tableRows.columnIdentity.keys) for resetColumn in resetColumns { if let index = tableRows.columns.firstIndex(of: resetColumn), index < newValues.count { + absentColumns.remove(index) if let form = uuidKeyForms[index] { newValues[index] = Self.freshUUID(form) prefilled.insert(index) + expressionColumns.remove(index) } else { - newValues[index] = .text("__DEFAULT__") + newValues[index] = .columnDefault + expressionColumns.insert(index) } - absentColumns.remove(index) - expressionColumns.remove(index) } } - return appendInsertedRow( - values: newValues, - absentColumns: absentColumns, - expressionColumns: expressionColumns, - prefilledColumns: prefilled, - to: &tableRows + return RowCopy( + values: newValues, absentColumns: absentColumns, expressionColumns: expressionColumns, prefilledColumns: prefilled ) } + private struct RowCopy { + let values: [PluginCellValue] + let absentColumns: Set + let expressionColumns: Set + let prefilledColumns: Set + } + private static func freshUUID(_ form: UUIDCellForm) -> PluginCellValue { form.value(for: UUIDValueGenerator.shared.make(.v4)) } @@ -345,7 +392,9 @@ final class RowOperationsManager { let parsedRows = Self.reconcileStructuredRows( payload, schema: schema, keepsMissingFields: keepsMissingFields ) - return insertParsedRows(parsedRows, uuidKeyForms: uuidKeyForms, into: &tableRows) + return insertParsedRows( + parsedRows, primaryKeyColumns: primaryKeyColumns, uuidKeyForms: uuidKeyForms, into: &tableRows + ) } guard let clipboardText = clipboardProvider.readText() else { @@ -353,9 +402,14 @@ final class RowOperationsManager { } if parser == nil, let cells = clipboardProvider.readGridCells(), cells.text == clipboardText { - let parsedRows = Self.rows(fromCopiedCells: cells, schema: schema, tableRows: tableRows) + guard let parsedRows = Self.readingCopiedHex( + Self.rows(fromCopiedCells: cells, schema: schema, tableRows: tableRows), tableRows: tableRows + ) else { + return PasteRowsResult(pastedRows: [], delta: .none) + } return insertParsedRows( keepsMissingFields ? parsedRows.map(Self.nullsAsMissingFields) : parsedRows, + primaryKeyColumns: primaryKeyColumns, uuidKeyForms: uuidKeyForms, into: &tableRows ) @@ -366,10 +420,14 @@ final class RowOperationsManager { switch parseResult { case .success(let parsedRows): - let unfilled = Self.unfilledCellValues(columns: columns, tableRows: tableRows) + let unfilled = Self.unfilledCellValues(columns: columns, primaryKeyColumns: primaryKeyColumns, tableRows: tableRows) let filled = parsedRows.map { Self.fillingBlankCells($0, with: unfilled) } + guard let filled = Self.readingCopiedHex(filled, tableRows: tableRows) else { + return PasteRowsResult(pastedRows: [], delta: .none) + } return insertParsedRows( keepsMissingFields ? filled.map(Self.nullsAsMissingFields) : filled, + primaryKeyColumns: primaryKeyColumns, uuidKeyForms: uuidKeyForms, into: &tableRows ) @@ -393,7 +451,7 @@ final class RowOperationsManager { let sourceExpressions = payload.expressionCells?[index] ?? [] var absentColumns: Set = [] var expressionColumns: Set = [] - var values: [PluginCellValue] = sourceForDestination.enumerated().map { destination, sourceIndex in + let values: [PluginCellValue] = sourceForDestination.enumerated().map { destination, sourceIndex in guard let sourceIndex, sourceIndex < row.count else { if keepsMissingFields { absentColumns.insert(destination) } return .null @@ -405,12 +463,6 @@ final class RowOperationsManager { return row[sourceIndex] } - if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = .text("__DEFAULT__") - absentColumns.remove(pkIndex) - expressionColumns.remove(pkIndex) - } - return ParsedRow( values: values, sourceLineNumber: index + 1, @@ -429,7 +481,7 @@ final class RowOperationsManager { cells.fields.enumerated().map { index, fields in let marked = cells.expressionFields?[index] ?? [] var expressionColumns: Set = [] - var values: [PluginCellValue] = (0.. [ParsedRow]? { + var read: [ParsedRow] = [] + read.reserveCapacity(rows.count) + for row in rows { + var values = row.values + for index in values.indices where !row.expressionColumns.contains(index) { + guard case .text(let text) = values[index], BlobFormattingService.hasHexPrefix(text), + tableRows.editingColumnType(at: index)?.isBlobType == true else { continue } + guard let bytes = BlobFormattingService.shared.bytes(fromPrefixedHex: text) else { + logger.warning("Paste refused: a binary column's field is not hex after 0x") + return nil + } + values[index] = .bytes(bytes) + } + read.append(ParsedRow( + values: values, + sourceLineNumber: row.sourceLineNumber, + absentColumns: row.absentColumns, + expressionColumns: row.expressionColumns, + blankColumns: row.blankColumns + )) + } + return read + } + // A blank cell of pasted text was left unfilled, so it takes what a new row's cell does: the server's value where - // the column has a default, else NULL. A NOT NULL text column with no default takes the empty string it can hold. - private static func unfilledCellValues(columns: [String], tableRows: TableRows) -> [PluginCellValue] { - columns.map { name in + // the column has a default, else NULL. A NOT NULL text column with no default takes the empty string it can hold, + // except a key, which an empty string would not name. + private static func unfilledCellValues( + columns: [String], + primaryKeyColumns: [String], + tableRows: TableRows + ) -> (values: [PluginCellValue], defaults: Set) { + var defaults: Set = [] + let values = columns.enumerated().map { position, name -> PluginCellValue in guard let index = tableRows.columns.firstIndex(of: name) else { return .null } if tableRows.generatedColumns.contains(name) || tableRows.serverAssignsValue(forColumn: name) { - return .text("__DEFAULT__") + defaults.insert(position) + return .columnDefault } - if tableRows.refusesNull(columnAt: index), tableRows.editingColumnType(at: index)?.hasEmptyValue == true { + if tableRows.refusesNull(columnAt: index), tableRows.editingColumnType(at: index)?.hasEmptyValue == true, + !primaryKeyColumns.contains(name) { return .text("") } return .null } + return (values, defaults) } - private static func fillingBlankCells(_ row: ParsedRow, with unfilled: [PluginCellValue]) -> ParsedRow { + private static func fillingBlankCells( + _ row: ParsedRow, + with unfilled: (values: [PluginCellValue], defaults: Set) + ) -> ParsedRow { guard !row.blankColumns.isEmpty else { return row } var values = row.values - for index in row.blankColumns where index < values.count && index < unfilled.count { - values[index] = unfilled[index] + var expressionColumns = row.expressionColumns + for index in row.blankColumns where index < values.count && index < unfilled.values.count { + values[index] = unfilled.values[index] + if unfilled.defaults.contains(index) { expressionColumns.insert(index) } } var filled = ParsedRow(values: values, sourceLineNumber: row.sourceLineNumber) filled.absentColumns = row.absentColumns - filled.expressionColumns = row.expressionColumns + filled.expressionColumns = expressionColumns return filled } @@ -491,6 +580,36 @@ final class RowOperationsManager { return destination.indices.map { $0 < source.count ? $0 : nil } } + // Left out of the INSERT, the server gives the key a value of its own: an identity, a default that calls a function + // such as nextval(), SQLite's rowid, or a key the driver takes no value for, as MongoDB's _id. A constant default + // would put every pasted row under the same key. + private func serverFillsKey(_ column: String, primaryKeyColumns: [String], in tableRows: TableRows) -> Bool { + guard primaryKeyColumns.contains(column) else { return false } + if tableRows.columnIdentity[column] != nil || !changeManager.isColumnWritable(column) { return true } + if let defaultValue = tableRows.columnDefaults[column] ?? nil, Self.givesEachRowItsOwnValue(defaultValue) { + return true + } + guard let databaseType = changeManager.databaseType else { return false } + return SQLiteRowidAlias.isAlias( + column: column, + declaredType: tableRows.columnDeclaredTypes[column], + primaryKey: primaryKeyColumns, + family: SQLTypeFamily.of(databaseType) + ) + } + + // A sequence, or a function other than the clock. SQL Server wraps a default in parentheses, `((1))` or `(newid())`, + // and PostgreSQL casts it, `'a'::text`. + private static func givesEachRowItsOwnValue(_ defaultValue: String) -> Bool { + var text = defaultValue.trimmingCharacters(in: .whitespaces) + while text.hasPrefix("("), text.hasSuffix(")") { + text = String(text.dropFirst().dropLast()).trimmingCharacters(in: .whitespaces) + } + let upper = text.uppercased() + if upper.contains("NEXTVAL") || upper.contains("NEXT VALUE FOR") { return true } + return CrossEngineDefaultValue.strippingCast(text).contains("(") && !CrossEngineDefaultValue.readsTheClock(text) + } + static func detectParser(for text: String) -> RowDataParser { var containsTab = false var containsComma = false @@ -509,9 +628,11 @@ final class RowOperationsManager { return containsComma ? CSVRowParser() : TSVRowParser() } - // Every paste path resets the key to __DEFAULT__ first; an unfilled UUID key then takes a fresh UUID per row. + // A key the server fills goes back to its default, a UUID key it does not fill takes a fresh UUID per row, and any + // other key keeps the pasted value. private func insertParsedRows( _ parsedRows: [ParsedRow], + primaryKeyColumns: [String], uuidKeyForms: [Int: UUIDCellForm], into tableRows: inout TableRows ) -> PasteRowsResult { @@ -524,6 +645,7 @@ final class RowOperationsManager { // Pasted server-owned values skip the cell-edit check and the generator drops them, so the grid showed unsaved values. let serverOwned = tableRows.columns.enumerated().filter { _, name in tableRows.generatedColumns.contains(name) || tableRows.columnIdentity[name] != nil + || serverFillsKey(name, primaryKeyColumns: primaryKeyColumns, in: tableRows) }.map(\.offset) for parsedRow in parsedRows { @@ -531,11 +653,11 @@ final class RowOperationsManager { var absentColumns = parsedRow.absentColumns var expressionColumns = parsedRow.expressionColumns.filter { $0 < rowValues.count } for index in serverOwned where index < rowValues.count { - rowValues[index] = .text("__DEFAULT__") + rowValues[index] = .columnDefault absentColumns.remove(index) - expressionColumns.remove(index) + expressionColumns.insert(index) } - for (index, form) in uuidKeyForms where index < rowValues.count && rowValues[index] == .text("__DEFAULT__") { + for (index, form) in uuidKeyForms where index < rowValues.count { rowValues[index] = Self.freshUUID(form) absentColumns.remove(index) expressionColumns.remove(index) diff --git a/apple/TablePro/Core/Services/Query/RowParser.swift b/apple/TablePro/Core/Services/Query/RowParser.swift index cdd13c09a6..3284ad0c7c 100644 --- a/apple/TablePro/Core/Services/Query/RowParser.swift +++ b/apple/TablePro/Core/Services/Query/RowParser.swift @@ -19,8 +19,7 @@ protocol RowDataParser { func parse(_ text: String, schema: TableSchema) -> Result<[ParsedRow], RowParseError> } -// One line's fields fitted to the table: a short line's missing fields are blank cells, extra fields are dropped, -// and the key is left to the server. +// One line's fields fitted to the table: a short line's missing fields are blank cells, extra fields are dropped. private func fittedRow( _ fields: [String], lineNumber: Int, @@ -37,10 +36,6 @@ private func fittedRow( blankColumns.insert(values.count) values.append(nil) } - if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = "__DEFAULT__" - blankColumns.remove(pkIndex) - } return ParsedRow( values: values.map(PluginCellValue.fromOptional), sourceLineNumber: lineNumber, @@ -233,11 +228,12 @@ struct CSVRowParser: RowDataParser { return matchCount > schema.columnCount / 2 } + // RFC 4180: spaces are part of a field. Only a blank or NULL field reads as no value. private func normalizeValue(_ rawValue: String) -> String? { let trimmed = rawValue.trimmingCharacters(in: .whitespaces) if trimmed.isEmpty || trimmed.uppercased() == "NULL" { return nil } - return trimmed + return rawValue } } diff --git a/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift b/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift index 1b86aeb4b8..fbe27536cb 100644 --- a/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift +++ b/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift @@ -1,11 +1,3 @@ -// -// SchemaProviderRegistry.swift -// TablePro -// -// Manages shared SQLSchemaProvider instances, one per DatabaseScope. -// Ref-counted per connection with grace period removal to avoid redundant schema loads. -// - import Foundation import os import TableProSQLGrammar @@ -106,6 +98,32 @@ final class SchemaProviderRegistry: CatalogChangeTarget { try await metadataProvider.withMetadataDriver(scope: scope) { driver in try await driver.sampleFieldPaths(table: table, limit: limit) } + }, + fetchForeignKeys: { table, schema in + try await metadataProvider.withMetadataDriver(scope: scope) { driver -> [ForeignKeyInfo] in + let type = driver.connection.type + guard type.supportsForeignKeys else { return [] } + let keys = try await driver.fetchForeignKeys(table: table, schema: schema) + // A key into another schema would pair with a same-named table of this one. On an engine with no + // schema layer a qualifier names a database. With no schema named, the keys come from the + // driver's current one, and a scope with no schema would keep keys into every schema. + let slot = await MainActor.run { EngineNamespaceSlot(databaseType: type) } + let origin = slot == .database + ? DatabaseScope(connectionId: scope.connectionId, database: schema ?? scope.database, schema: nil) + : DatabaseScope( + connectionId: scope.connectionId, + database: scope.database, + schema: schema ?? scope.schema ?? (driver as? SchemaSwitchable)?.currentSchema + ) + return keys.filter { key in + ForeignKeyTargetScope.pointsWithin( + origin, + referencedDatabase: key.referencedDatabase, + referencedSchema: key.referencedSchema, + slot: slot + ) + } + } } ) let provider = SQLSchemaProvider(metadataSource: source) diff --git a/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift b/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift index 384a97a6b7..af9f368966 100644 --- a/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift +++ b/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift @@ -22,7 +22,8 @@ final class SchemaRefreshService { private struct InFlightRefresh { let id: UUID - let task: Task + // Whether the refresh committed a fresh tables list. + let task: Task } nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "SchemaRefreshService") @@ -62,21 +63,23 @@ final class SchemaRefreshService { SchemaForeignKeyStore.shared.invalidate(connectionId: connection.id) let key = RefreshKey(connectionId: connection.id, database: database) if let existing = inFlight[key] { - await existing.task.value + _ = await existing.task.value return } let entry = InFlightRefresh(id: UUID(), task: Task { @MainActor [weak self] in - guard let self else { return } - await self.performRefresh(connection: connection) + guard let self else { return false } + return await self.performRefresh(connection: connection) }) inFlight[key] = entry - await entry.task.value + _ = await entry.task.value finish(entry, for: key) } - /// Never joins a load that began before the write, which would hand back the old catalog. The - /// entry is replaced before anything awaits, so every later plain refresh joins this one instead. - func refreshAfterWrite(connection: DatabaseConnection) async { + // Never joins a load that began before the write, which would hand back the old catalog. The entry + // is replaced before anything awaits, so every later plain refresh joins this one instead. Returns + // whether a fresh tables list was committed: false when the load failed or another refresh replaced it. + @discardableResult + func refreshAfterWrite(connection: DatabaseConnection) async -> Bool { let connectionId = connection.id SchemaForeignKeyStore.shared.invalidate(connectionId: connectionId) for key in Array(inFlight.keys) where key.connectionId == connectionId { @@ -84,13 +87,14 @@ final class SchemaRefreshService { } let key = RefreshKey(connectionId: connectionId, database: nil) let entry = InFlightRefresh(id: UUID(), task: Task { @MainActor [weak self] in - guard let self else { return } + guard let self else { return false } await self.schemaService.prepareForReload(connectionId: connectionId) - await self.performRefresh(connection: connection) + return await self.performRefresh(connection: connection) }) inFlight[key] = entry - await entry.task.value + let committed = await entry.task.value finish(entry, for: key) + return committed } private func finish(_ entry: InFlightRefresh, for key: RefreshKey) { @@ -103,7 +107,7 @@ final class SchemaRefreshService { key.connectionId == connectionId ? entry.task : nil } for task in tasks { - await task.value + _ = await task.value } } @@ -277,8 +281,9 @@ final class SchemaRefreshService { await syncAutocompleteProvider(connectionId: connectionId) } - private func performRefresh(connection: DatabaseConnection) async { + private func performRefresh(connection: DatabaseConnection) async -> Bool { let connectionId = connection.id + var committed = false if pluginManager.databaseGroupingStrategy(for: connection.type) == .hierarchicalSchema { await schemaService.prepareForReload(connectionId: connectionId) @@ -290,11 +295,11 @@ final class SchemaRefreshService { throw DatabaseError.notConnected } let awaitedSchemas = schemasAwaitingJudgement(in: scope) - try await metadataDriverProvider.withMetadataDriver( + committed = try await metadataDriverProvider.withMetadataDriver( scope: scope, workload: .bulk ) { [schemaService] driver in - await schemaService.reload( + let didLoad = await schemaService.reload( connectionId: connectionId, driver: driver, connection: connection, @@ -305,9 +310,10 @@ final class SchemaRefreshService { fetchingNow: awaitedSchemas, driver: driver ) + return didLoad } } catch is CancellationError { - return + return false } catch { Self.logger.warning( "[schema] refresh failed connId=\(connectionId, privacy: .public) error=\(error.publicLogShape, privacy: .public)" @@ -320,6 +326,7 @@ final class SchemaRefreshService { } await syncAutocompleteProvider(connectionId: connectionId) + return committed } /// The schemas judged against the refreshed catalog as soon as it settles: the browsed one, and diff --git a/apple/TablePro/Core/Services/Query/SchemaService+ServerObjects.swift b/apple/TablePro/Core/Services/Query/SchemaService+ServerObjects.swift new file mode 100644 index 0000000000..e8493eb51a --- /dev/null +++ b/apple/TablePro/Core/Services/Query/SchemaService+ServerObjects.swift @@ -0,0 +1,60 @@ +// +// SchemaService+ServerObjects.swift +// TablePro +// + +import Foundation + +extension SchemaService { + func serverObjects(for connectionId: UUID, kindId: String) -> [ServerObjectInfo] { + serverObjectsLoadState(for: connectionId, kindId: kindId).value ?? [] + } + + func serverObjectsLoadState(for connectionId: UUID, kindId: String) -> MetadataLoadState<[ServerObjectInfo]> { + sideObjects[shownKey(connectionId)]?.serverObjects[kindId] ?? .idle + } + + func serverObjectStates(for connectionId: UUID) -> [String: MetadataLoadState<[ServerObjectInfo]>] { + sideObjects[shownKey(connectionId)]?.serverObjects ?? [:] + } + + func serverObjects(for connectionId: UUID, schema: String, kindId: String) -> [ServerObjectInfo] { + serverObjectsLoadState(for: connectionId, schema: schema, kindId: kindId).value ?? [] + } + + func serverObjectsByKind(for connectionId: UUID, schema: String) -> [String: [ServerObjectInfo]] { + (perSchemaSideObjects[catalogKey(connectionId, schema: schema)]?.serverObjects ?? [:]).compactMapValues(\.value) + } + + func serverObjectsLoadState( + for connectionId: UUID, + schema: String, + kindId: String + ) -> MetadataLoadState<[ServerObjectInfo]> { + perSchemaSideObjects[catalogKey(connectionId, schema: schema)]?.serverObjects[kindId] ?? .idle + } + + @discardableResult + func reloadServerObjects( + connectionId: UUID, + driver: DatabaseDriver, + scope: DatabaseScope?, + kindId: String + ) async -> Bool { + let key = reloadKey(connectionId, scope: scope, driver: driver) + updateSideObjects(key) { $0.serverObjects[kindId] = ($0.serverObjects[kindId] ?? .idle).enteringLoad } + bumpGeneration(connectionId) + let outcome = await Self.fetchObjectsSafely( + key: ServerObjectLoadKey(load: LoadKey(connectionId: connectionId, scope: scope), kindId: kindId), + connectionId: connectionId, + label: "server objects", + dedup: serverObjectsDedup, + fetch: { try await driver.fetchServerObjects(ofKind: kindId, schema: nil) } + ) + updateKeptSideObjects(key) { + $0.serverObjects[kindId] = ($0.serverObjects[kindId] ?? .idle).settled(by: outcome, discardingValue: false) + } + bumpGeneration(connectionId) + return outcome.didFetch + } +} diff --git a/apple/TablePro/Core/Services/Query/SchemaService.swift b/apple/TablePro/Core/Services/Query/SchemaService.swift index 5cf3d3befd..003faf91e9 100644 --- a/apple/TablePro/Core/Services/Query/SchemaService.swift +++ b/apple/TablePro/Core/Services/Query/SchemaService.swift @@ -18,6 +18,8 @@ final class SchemaService: ObservableObject { var routines: MetadataLoadState<[RoutineInfo]> = .idle var triggers: MetadataLoadState<[TriggerInfo]> = .idle var userDefinedTypes: MetadataLoadState<[UserDefinedTypeInfo]> = .idle + // Keyed by the declared kind's id. + var serverObjects: [String: MetadataLoadState<[ServerObjectInfo]>] = [:] } @Published private(set) var states: [CatalogKey: SchemaState] = [:] @@ -51,7 +53,7 @@ final class SchemaService: ObservableObject { generations[connectionId] ?? 0 } - private func bumpGeneration(_ connectionId: UUID) { + func bumpGeneration(_ connectionId: UUID) { generations[connectionId, default: 0] &+= 1 } @@ -64,6 +66,18 @@ final class SchemaService: ObservableObject { private let perSchemaRoutinesDedup = OnceTask() private let perSchemaTriggersDedup = OnceTask() private let perSchemaTypesDedup = OnceTask() + let serverObjectsDedup = OnceTask() + private let perSchemaServerObjectsDedup = OnceTask() + + struct ServerObjectLoadKey: Hashable, Sendable { + let load: LoadKey + let kindId: String + } + + struct ServerObjectSchemaFetchKey: Hashable, Sendable { + let fetch: SchemaFetchKey + let kindId: String + } /// A schema is named inside a database, and an engine that changes database on a live /// connection reaches a `PUBLIC` in every one of them. @@ -94,11 +108,15 @@ final class SchemaService: ObservableObject { private struct SideKinds { let triggers: Bool let types: Bool + let serverKinds: [SidebarServerObjectKind] - init(_ type: DatabaseType) { + init(_ type: DatabaseType, serverKinds: [SidebarServerObjectKind] = []) { triggers = type.supportsDatabaseTriggerBrowse types = type.supportsUserDefinedTypeBrowse + self.serverKinds = serverKinds } + + var serverKindIds: [String] { serverKinds.map(\.id) } } private struct RefreshWaiter { @@ -138,7 +156,7 @@ final class SchemaService: ObservableObject { !unrefreshedCatalogs.contains(shownKey(connectionId)) } - private func shownKey(_ connectionId: UUID) -> CatalogKey { + func shownKey(_ connectionId: UUID) -> CatalogKey { shownCatalogs[connectionId]?.key ?? CatalogKey(connectionId: connectionId, database: "", schema: nil) } @@ -337,7 +355,7 @@ final class SchemaService: ObservableObject { /// Every per-schema read answers for the database `schemas(for:)` was listed from, so the /// schema rows and the objects under them describe one database even while a switch settles. - private func catalogKey(_ connectionId: UUID, schema: String) -> SchemaKey { + func catalogKey(_ connectionId: UUID, schema: String) -> SchemaKey { SchemaKey(connectionId: connectionId, database: catalogDatabase(connectionId), schema: schema) } @@ -380,7 +398,8 @@ final class SchemaService: ObservableObject { let key = catalogKey(connectionId, schema: schema) guard hasLoadedContent(key) else { return false } let side = perSchemaSideObjects[key] ?? SideObjects() - return [side.routines.erased, side.triggers.erased, side.userDefinedTypes.erased] + return ([side.routines.erased, side.triggers.erased, side.userDefinedTypes.erased] + + side.serverObjects.values.map(\.erased)) .allSatisfy { $0 == .loaded || $0 == .idle } } @@ -551,7 +570,10 @@ final class SchemaService: ObservableObject { nextLoadGeneration += 1 let generation = nextLoadGeneration schemaLoadGenerations[key] = generation - let kinds = SideKinds(driver.connection.type) + let kinds = SideKinds( + driver.connection.type, + serverKinds: SidebarServerObjectKind.declared(by: driver).filter { $0.scope == .container } + ) if !hasLoadedContent(key) { setPerSchemaState(.loading, key: key) @@ -588,6 +610,16 @@ final class SchemaService: ObservableObject { fetch: { try await driver.fetchUserDefinedTypes(schema: schema) } ) : nil + async let serverObjectsTask = fetchServerObjectsSafely( + kinds: kinds.serverKindIds, + keys: Dictionary(uniqueKeysWithValues: kinds.serverKindIds.map { + ($0, ServerObjectSchemaFetchKey(fetch: fetchKey, kindId: $0)) + }), + connectionId: connectionId, + dedup: perSchemaServerObjectsDedup, + schema: schema, + driver: driver + ) let tablesOutcome: MetadataFetchOutcome<[TableInfo]> do { @@ -606,10 +638,14 @@ final class SchemaService: ObservableObject { let routinesOutcome = await routinesTask let triggersOutcome = await triggersTask let typesOutcome = await typesTask + let serverObjectsOutcomes = await serverObjectsTask guard schemaLoadGenerations[key] == generation else { return } schemaLoadGenerations.removeValue(forKey: key) updateSchemaSideObjects(key) { side in - side = Self.settled(side, routines: routinesOutcome, triggers: triggersOutcome, types: typesOutcome) + side = Self.settled( + side, routines: routinesOutcome, triggers: triggersOutcome, types: typesOutcome, + serverObjects: serverObjectsOutcomes + ) } bumpGeneration(connectionId) } @@ -643,9 +679,10 @@ final class SchemaService: ObservableObject { await perSchemaRoutinesDedup.cancel(where: shouldCancel) await perSchemaTriggersDedup.cancel(where: shouldCancel) await perSchemaTypesDedup.cancel(where: shouldCancel) + await perSchemaServerObjectsDedup.cancel { shouldCancel($0.fetch) } } - private func updateSideObjects(_ key: CatalogKey, _ change: (inout SideObjects) -> Void) { + func updateSideObjects(_ key: CatalogKey, _ change: (inout SideObjects) -> Void) { var side = sideObjects[key] ?? SideObjects() change(&side) sideObjects[key] = side @@ -653,14 +690,14 @@ final class SchemaService: ObservableObject { /// The catalog of the scope the reload reads, not of whatever is shown when it starts: a switch /// while it waits for a connection would file one database's objects under another. - private func reloadKey(_ connectionId: UUID, scope: DatabaseScope?, driver: DatabaseDriver) -> CatalogKey { + func reloadKey(_ connectionId: UUID, scope: DatabaseScope?, driver: DatabaseDriver) -> CatalogKey { guard let scope else { return shownKey(connectionId) } return catalogKey(for: scope, type: driver.connection.type) } /// A reload that lands after `forget` dropped its catalog writes nothing, rather than bringing /// back objects of a database whose entry closed. - private func updateKeptSideObjects(_ key: CatalogKey, _ change: (inout SideObjects) -> Void) { + func updateKeptSideObjects(_ key: CatalogKey, _ change: (inout SideObjects) -> Void) { guard var side = sideObjects[key] else { return } change(&side) sideObjects[key] = side @@ -676,10 +713,11 @@ final class SchemaService: ObservableObject { _ key: CatalogKey, routines: MetadataFetchOutcome<[RoutineInfo]>, triggers: MetadataFetchOutcome<[TriggerInfo]>?, - types: MetadataFetchOutcome<[UserDefinedTypeInfo]>? + types: MetadataFetchOutcome<[UserDefinedTypeInfo]>?, + serverObjects: [String: MetadataFetchOutcome<[ServerObjectInfo]>] = [:] ) { updateSideObjects(key) { side in - side = Self.settled(side, routines: routines, triggers: triggers, types: types) + side = Self.settled(side, routines: routines, triggers: triggers, types: types, serverObjects: serverObjects) } } @@ -698,6 +736,7 @@ final class SchemaService: ObservableObject { side.routines = side.routines.settled(by: .cancelled, discardingValue: false) side.triggers = side.triggers.settled(by: .cancelled, discardingValue: false) side.userDefinedTypes = side.userDefinedTypes.settled(by: .cancelled, discardingValue: false) + side.serverObjects = side.serverObjects.mapValues { $0.settled(by: .cancelled, discardingValue: false) } } bumpGeneration(key.connectionId) } @@ -707,6 +746,9 @@ final class SchemaService: ObservableObject { next.routines = side.routines.enteringLoad if kinds.triggers { next.triggers = side.triggers.enteringLoad } if kinds.types { next.userDefinedTypes = side.userDefinedTypes.enteringLoad } + for id in kinds.serverKindIds { + next.serverObjects[id] = (side.serverObjects[id] ?? .idle).enteringLoad + } return next } @@ -715,7 +757,8 @@ final class SchemaService: ObservableObject { _ side: SideObjects, routines: MetadataFetchOutcome<[RoutineInfo]>, triggers: MetadataFetchOutcome<[TriggerInfo]>?, - types: MetadataFetchOutcome<[UserDefinedTypeInfo]>? + types: MetadataFetchOutcome<[UserDefinedTypeInfo]>?, + serverObjects: [String: MetadataFetchOutcome<[ServerObjectInfo]>] = [:] ) -> SideObjects { var next = side next.routines = side.routines.settled(by: routines, discardingValue: false) @@ -725,6 +768,9 @@ final class SchemaService: ObservableObject { if let types { next.userDefinedTypes = side.userDefinedTypes.settled(by: types, discardingValue: false) } + for (id, outcome) in serverObjects { + next.serverObjects[id] = (side.serverObjects[id] ?? .idle).settled(by: outcome, discardingValue: false) + } return next } @@ -739,16 +785,19 @@ final class SchemaService: ObservableObject { && isCatalogCurrent(for: connectionId): return case .idle, .loading, .failed, .loaded: - await runLoad(connectionId: connectionId, driver: driver, connection: connection, scope: scope) + _ = await runLoad(connectionId: connectionId, driver: driver, connection: connection, scope: scope) } } + // Returns whether a fresh tables list was committed, so a caller can tell a reload from a failed one. + // A hierarchical engine loads its tables per schema later, so its reload always returns true. + @discardableResult func reload( connectionId: UUID, driver: DatabaseDriver, connection: DatabaseConnection, scope: DatabaseScope? = nil - ) async { + ) async -> Bool { await runLoad(connectionId: connectionId, driver: driver, connection: connection, scope: scope) } @@ -818,6 +867,7 @@ final class SchemaService: ObservableObject { await routinesDedup.cancel { $0.connectionId == connectionId } await triggersDedup.cancel { $0.connectionId == connectionId } await typesDedup.cancel { $0.connectionId == connectionId } + await serverObjectsDedup.cancel { $0.load.connectionId == connectionId } await schemasDedup.cancel { $0.connectionId == connectionId } await cancelSchemaLoads { $0.schemaKey.connectionId == connectionId } } @@ -858,7 +908,7 @@ final class SchemaService: ObservableObject { await prepareForReload(connectionId: connectionId) let connection = session.connection do { - try await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { [self] driver in + _ = try await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { [self] driver in await reload( connectionId: connectionId, driver: driver, @@ -885,13 +935,17 @@ final class SchemaService: ObservableObject { } } if settleTablesFailed(key, message: message) { - let kinds = type.map(SideKinds.init) + let kinds = type.map { SideKinds($0) } + let serverKindIds = SidebarServerObjectKind.declared(by: DatabaseManager.shared.driver(for: connectionId)).map(\.id) updateSideObjects(key) { side in side = Self.settled( side, routines: .failed(message), triggers: kinds?.triggers == true ? .failed(message) : nil, - types: kinds?.types == true ? .failed(message) : nil + types: kinds?.types == true ? .failed(message) : nil, + serverObjects: Dictionary(uniqueKeysWithValues: serverKindIds.map { + ($0, MetadataFetchOutcome<[ServerObjectInfo]>.failed(message)) + }) ) } } else if !moved { @@ -914,7 +968,7 @@ final class SchemaService: ObservableObject { driver: DatabaseDriver, connection: DatabaseConnection, scope: DatabaseScope? - ) async { + ) async -> Bool { let type = connection.type let key = scope.map { catalogKey(for: $0, type: type) } ?? shownKey(connectionId) let generation = beginLoadGeneration(for: key) @@ -924,7 +978,12 @@ final class SchemaService: ObservableObject { if !hasLoadedContent(key) { states[key] = .loading } - let kinds = SideKinds(type) + let grouping = PluginManager.shared.databaseGroupingStrategy(for: type) + // A hierarchical engine lists its declared kinds per schema, in `runSchemaLoad`. + let kinds = SideKinds( + type, + serverKinds: grouping == .hierarchicalSchema ? [] : SidebarServerObjectKind.declared(by: driver) + ) updateSideObjects(key) { $0 = Self.enteringLoad($0, kinds: kinds) } bumpGeneration(connectionId) @@ -938,7 +997,6 @@ final class SchemaService: ObservableObject { } let loadKey = LoadKey(connectionId: connectionId, scope: scope) - let grouping = PluginManager.shared.databaseGroupingStrategy(for: type) if grouping == .hierarchicalSchema { await runHierarchicalLoad( key: key, @@ -948,7 +1006,7 @@ final class SchemaService: ObservableObject { generation: generation, type: type ) - return + return true } async let tablesTask: [TableInfo] = loadDedup.execute(key: loadKey) { @@ -979,6 +1037,16 @@ final class SchemaService: ObservableObject { fetch: { try await driver.fetchUserDefinedTypes(schema: nil) } ) : nil + async let serverObjectsTask = fetchServerObjectsSafely( + kinds: kinds.serverKindIds, + keys: Dictionary(uniqueKeysWithValues: kinds.serverKindIds.map { + ($0, ServerObjectLoadKey(load: loadKey, kindId: $0)) + }), + connectionId: connectionId, + dedup: serverObjectsDedup, + schema: nil, + driver: driver + ) async let schemasTask: [String]? = supportsSchemas || listsAttached ? Self.fetchSchemasSafely( key: loadKey, @@ -990,24 +1058,26 @@ final class SchemaService: ObservableObject { /// Tables are committed the moment they arrive and the other kinds settle behind them, so a /// failed tables fetch still settles routines, triggers and types instead of leaving each /// section on a spinner no load is coming to replace. + var committed = false do { let tables = try await tablesTask guard isCurrentLoadGeneration(generation, for: key, phase: "tables-loaded") else { - return + return false } adoptShown(key, scope: scope, generation: generation, type: type) states[key] = .loaded(tables) unrefreshedCatalogs.remove(key) bumpGeneration(connectionId) + committed = true } catch is CancellationError { abandonLoad(key, generation: generation) - return + return false } catch { guard isCurrentLoadGeneration(generation, for: key, phase: "tables-failed") else { if loadGenerations[key] == nil, case .loading = states[key] { states[key] = .idle } - return + return false } Self.logger.warning( "[schema] load failed connId=\(connectionId, privacy: .public) error=\(error.publicLogShape, privacy: .public)" @@ -1021,14 +1091,18 @@ final class SchemaService: ObservableObject { let routinesOutcome = await routinesTask let triggersOutcome = await triggersTask let typesOutcome = await typesTask + let serverObjectsOutcomes = await serverObjectsTask guard isCurrentLoadGeneration(generation, for: key, phase: "side-objects-loaded") else { - return + return committed } - commitSideObjects(key, routines: routinesOutcome, triggers: triggersOutcome, types: typesOutcome) + commitSideObjects( + key, routines: routinesOutcome, triggers: triggersOutcome, types: typesOutcome, + serverObjects: serverObjectsOutcomes + ) if let loadedSchemas = await schemasTask { guard isCurrentLoadGeneration(generation, for: key, phase: "schemas-loaded") else { - return + return committed } if listsAttached { attachedSchemasInOrder[key] = loadedSchemas @@ -1037,6 +1111,7 @@ final class SchemaService: ObservableObject { } } bumpGeneration(connectionId) + return committed } private func runHierarchicalLoad( @@ -1198,11 +1273,44 @@ final class SchemaService: ObservableObject { } } + // One fetch per declared kind, side by side; each settles its own group. + private func fetchServerObjectsSafely( + kinds: [String], + keys: [String: Key], + connectionId: UUID, + dedup: OnceTask, + schema: String?, + driver: DatabaseDriver + ) async -> [String: MetadataFetchOutcome<[ServerObjectInfo]>] { + guard !kinds.isEmpty else { return [:] } + return await withTaskGroup(of: (String, MetadataFetchOutcome<[ServerObjectInfo]>)?.self) { group in + for kindId in kinds { + guard let key = keys[kindId] else { continue } + group.addTask { + let outcome = await Self.fetchObjectsSafely( + key: key, + connectionId: connectionId, + label: "server objects", + dedup: dedup, + fetch: { try await driver.fetchServerObjects(ofKind: kindId, schema: schema) } + ) + return (kindId, outcome) + } + } + var outcomes: [String: MetadataFetchOutcome<[ServerObjectInfo]>] = [:] + for await result in group { + guard let result else { continue } + outcomes[result.0] = result.1 + } + return outcomes + } + } + /// A failure comes back as a failure, never as an empty list. An empty list made a refresh that /// failed indistinguishable from a database with no routines, and the caller committed it over /// the loaded one: a single dropped connection emptied the sidebar's procedures and functions /// while the refresh reported success, with nothing scheduled to put them back. - private static func fetchObjectsSafely( + static func fetchObjectsSafely( key: Key, connectionId: UUID, label: String, diff --git a/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift b/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift index 7470c95bb8..4d031fd612 100644 --- a/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift +++ b/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift @@ -61,11 +61,8 @@ actor SelectionSummaryCalculator { } private struct Tally { - private static let defaultMarkerLength = PluginCellValue.defaultMarkerText.utf8.count - let emptyTextIsEmpty: Bool - let insertedRowIDs: Set - let modifiedCells: [RowID: Set] + let defaultedCells: [RowID: Set] var valueCount = 0 var emptyCount = 0 var notANumberCount = 0 @@ -73,8 +70,7 @@ private struct Tally { init(input: SelectionSummaryInput) { emptyTextIsEmpty = input.policy.emptyTextIsEmpty - insertedRowIDs = input.insertedRowIDs - modifiedCells = input.modifiedCells + defaultedCells = input.defaultedCells } mutating func add(_ value: PluginCellValue, column: Int, rule: SelectionSummaryColumnRule, rowID: RowID) { @@ -108,13 +104,9 @@ private struct Tally { return numbers.add(text.trimmingCharacters(in: .whitespaces)) } - // The default marker is the grid's placeholder only where an edit put it; elsewhere it is text the table holds. private func isEmpty(_ text: String, column: Int, rowID: RowID) -> Bool { if text.isEmpty { return emptyTextIsEmpty } - guard text.utf8.count == Self.defaultMarkerLength, text == PluginCellValue.defaultMarkerText else { - return false - } - return insertedRowIDs.contains(rowID) || modifiedCells[rowID]?.contains(column) == true + return defaultedCells[rowID]?.contains(column) == true } func summary(coversWholeColumn: Bool) -> SelectionSummary { diff --git a/apple/TablePro/Core/Services/SQL/FileSystemEventWatcher.swift b/apple/TablePro/Core/Services/SQL/FileSystemEventWatcher.swift new file mode 100644 index 0000000000..022d0d7e48 --- /dev/null +++ b/apple/TablePro/Core/Services/SQL/FileSystemEventWatcher.swift @@ -0,0 +1,152 @@ +// +// FileSystemEventWatcher.swift +// TablePro +// + +import CoreServices +import Foundation +import os + +internal struct FileSystemEvent: Equatable, Sendable { + let path: String + let flags: FSEventStreamEventFlags + + func has(_ flag: Int) -> Bool { + flags & FSEventStreamEventFlags(flag) != 0 + } +} + +@MainActor +internal final class FileSystemEventWatcher { + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "FileSystemEventWatcher") + + var onChange: (() -> Void)? + + private let latency: CFTimeInterval + private let reportsFileEvents: Bool + private let accepts: @Sendable ([FileSystemEvent]) -> Bool + private var watchedPaths: Set = [] + private var stream: EventStream? + + // `accepts` runs on the event queue, so a batch it refuses never wakes the main actor. + init( + latency: CFTimeInterval = 0.5, + reportsFileEvents: Bool = false, + accepts: @escaping @Sendable ([FileSystemEvent]) -> Bool = { _ in true } + ) { + self.latency = latency + self.reportsFileEvents = reportsFileEvents + self.accepts = accepts + } + + func watch(_ paths: Set) { + guard paths != watchedPaths else { return } + cancel() + watchedPaths = paths + guard !paths.isEmpty else { return } + + var flags = kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot | kFSEventStreamCreateFlagUseCFTypes + if reportsFileEvents { + flags |= kFSEventStreamCreateFlagFileEvents + } + let sink = EventSink(watcher: self, accepts: accepts) + guard let created = EventStream( + paths: paths, + latency: latency, + flags: FSEventStreamCreateFlags(flags), + sink: sink + ) else { + Self.logger.error("Failed to start a file system event stream") + return + } + stream = created + } + + func cancel() { + watchedPaths = [] + // A batch already on its way to the main actor finds no watcher and is dropped. + stream?.sink.watcher = nil + stream = nil + } +} + +// Retained by the stream through its context, so a callback never reads freed memory, and it holds +// the watcher weakly, so a released watcher is never called. +@MainActor +private final class EventSink { + weak var watcher: FileSystemEventWatcher? + nonisolated let accepts: @Sendable ([FileSystemEvent]) -> Bool + + init(watcher: FileSystemEventWatcher, accepts: @escaping @Sendable ([FileSystemEvent]) -> Bool) { + self.watcher = watcher + self.accepts = accepts + } + + func deliver() { + watcher?.onChange?() + } +} + +// Stops and releases the stream when it goes away, whether the watcher cancelled it or was released. +private final class EventStream { + let sink: EventSink + private let stream: FSEventStreamRef + + init?(paths: Set, latency: CFTimeInterval, flags: FSEventStreamCreateFlags, sink: EventSink) { + var context = FSEventStreamContext( + version: 0, + info: Unmanaged.passUnretained(sink).toOpaque(), + retain: EventStream.retainSink, + release: EventStream.releaseSink, + copyDescription: nil + ) + guard let stream = FSEventStreamCreate( + kCFAllocatorDefault, + EventStream.callback, + &context, + Array(paths) as CFArray, + FSEventStreamEventId(kFSEventStreamEventIdSinceNow), + latency, + flags + ) else { + return nil + } + FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) + guard FSEventStreamStart(stream) else { + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + return nil + } + self.sink = sink + self.stream = stream + } + + deinit { + FSEventStreamStop(stream) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } + + nonisolated private static let retainSink: CFAllocatorRetainCallBack = { info in + guard let info else { return nil } + _ = Unmanaged.fromOpaque(info).retain() + return info + } + + nonisolated private static let releaseSink: CFAllocatorReleaseCallBack = { info in + guard let info else { return } + Unmanaged.fromOpaque(info).release() + } + + nonisolated private static let callback: FSEventStreamCallback = { _, info, count, eventPaths, eventFlags, _ in + guard let info else { return } + let sink = Unmanaged.fromOpaque(info).takeUnretainedValue() + let paths = (Unmanaged.fromOpaque(eventPaths).takeUnretainedValue() as NSArray) as? [String] ?? [] + let flags = UnsafeBufferPointer(start: eventFlags, count: count) + let events = zip(paths, flags).map { FileSystemEvent(path: $0, flags: $1) } + guard sink.accepts(events) else { return } + Task { @MainActor in + sink.deliver() + } + } +} diff --git a/apple/TablePro/Core/Services/SQL/GitRepositoryWatcher.swift b/apple/TablePro/Core/Services/SQL/GitRepositoryWatcher.swift deleted file mode 100644 index 3c2cb7be52..0000000000 --- a/apple/TablePro/Core/Services/SQL/GitRepositoryWatcher.swift +++ /dev/null @@ -1,65 +0,0 @@ -// -// GitRepositoryWatcher.swift -// TablePro -// - -import CoreServices -import Foundation -import os - -@MainActor -internal final class GitRepositoryWatcher { - nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "GitRepositoryWatcher") - - var onChange: (() -> Void)? - - private var eventStream: FSEventStreamRef? - private var watchedPaths: Set = [] - - nonisolated private static let eventCallback: FSEventStreamCallback = { _, info, _, _, _, _ in - guard let info else { return } - let watcher = Unmanaged.fromOpaque(info).takeUnretainedValue() - Task { @MainActor in - watcher.onChange?() - } - } - - func watch(_ paths: Set) { - guard paths != watchedPaths else { return } - cancel() - watchedPaths = paths - guard !paths.isEmpty else { return } - - var context = FSEventStreamContext( - version: 0, - info: Unmanaged.passUnretained(self).toOpaque(), - retain: nil, - release: nil, - copyDescription: nil - ) - guard let stream = FSEventStreamCreate( - kCFAllocatorDefault, - Self.eventCallback, - &context, - Array(paths) as CFArray, - FSEventStreamEventId(kFSEventStreamEventIdSinceNow), - 0.5, - UInt32(kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot) - ) else { - Self.logger.error("Failed to create the Git repository event stream") - return - } - FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) - FSEventStreamStart(stream) - eventStream = stream - } - - func cancel() { - watchedPaths = [] - guard let stream = eventStream else { return } - FSEventStreamStop(stream) - FSEventStreamInvalidate(stream) - FSEventStreamRelease(stream) - eventStream = nil - } -} diff --git a/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift b/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift index dec2fb4a1d..21eddbd047 100644 --- a/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift +++ b/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift @@ -49,9 +49,10 @@ internal final class LinkedFolderGitStatusStore: ObservableObject { private let clientFactory: @Sendable () -> GitClient? private let foldersProvider: @MainActor () -> [LinkedSQLFolder] - private let repositoryWatcher = GitRepositoryWatcher() + private let repositoryWatcher = FileSystemEventWatcher() private var cancellables: Set = [] private var refreshTask: Task? + private var repositorySearch: Task? private var hasStarted = false init( @@ -68,13 +69,35 @@ internal final class LinkedFolderGitStatusStore: ObservableObject { AppEvents.shared.linkedSQLFoldersDidUpdate .sink { [weak self] _ in self?.scheduleRefresh() } .store(in: &cancellables) + // A folder in a repository is followed by the folder and `.git` watchers. One that is not can join a + // repository made in a parent folder, which neither watcher sees, so only those folders are asked again. NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification) - .sink { [weak self] _ in self?.scheduleRefresh() } + .sink { [weak self] _ in self?.lookForNewRepositories() } .store(in: &cancellables) repositoryWatcher.onChange = { [weak self] in self?.scheduleRefresh() } scheduleRefresh(after: .zero) } + @discardableResult + func lookForNewRepositories() -> Task? { + let outside = foldersProvider().filter { $0.isEnabled && snapshots[$0.id] == nil } + guard !outside.isEmpty, repositorySearch == nil else { return nil } + let client = clientFactory() + let search = Task { [weak self] in + defer { self?.repositorySearch = nil } + guard let client else { return } + for folder in outside { + let repository = try? await client.repositoryInfo(in: folder.expandedURL) + if repository != nil { + self?.scheduleRefresh() + return + } + } + } + repositorySearch = search + return search + } + func state(for favorite: LinkedSQLFavorite) -> LinkedFileGitState? { snapshots[favorite.folderId]?.state(forRelativePath: favorite.relativePath) } diff --git a/apple/TablePro/Core/Services/SQL/LinkedSQLFolderScanner.swift b/apple/TablePro/Core/Services/SQL/LinkedSQLFolderScanner.swift new file mode 100644 index 0000000000..ede46d884c --- /dev/null +++ b/apple/TablePro/Core/Services/SQL/LinkedSQLFolderScanner.swift @@ -0,0 +1,107 @@ +// +// LinkedSQLFolderScanner.swift +// TablePro +// + +import Foundation +import TableProUtilities + +internal struct LinkedSQLFolderScanner: Sendable { + typealias HeaderLoader = @Sendable (URL) -> FileTextLoader.LoadedText? + + private let index: LinkedSQLIndex + private let loadHeader: HeaderLoader + + init( + index: LinkedSQLIndex = .shared, + loadHeader: @escaping HeaderLoader = { FileTextLoader.loadHeader($0) } + ) { + self.index = index + self.loadHeader = loadHeader + } + + // Returns whether the folder's rows changed. A file whose size and modification time match its + // row keeps the row without reading the file. + @concurrent + func scan(folderId: UUID, at folderURL: URL) async -> Bool { + guard FileManager.default.fileExists(atPath: folderURL.path) else { + return await index.removeFolder(folderId: folderId) + } + let stored = await index.storedFiles(folderId: folderId) + let files = indexedFiles(in: folderURL, stored: stored) + return await index.synchronize(folderId: folderId, files: files) + } + + // Returns whether any folder was dropped from the index. + @concurrent + func pruneFolders(keeping known: Set) async -> Bool { + var pruned = false + let stale = await index.allFolderIds().subtracting(known) + for id in stale { + if await index.removeFolder(folderId: id) { + pruned = true + } + } + return pruned + } + + private func indexedFiles( + in folderURL: URL, + stored: [String: LinkedSQLIndex.IndexedFile] + ) -> [LinkedSQLIndex.IndexedFile] { + let keys: [URLResourceKey] = [.isRegularFileKey, .contentModificationDateKey, .fileSizeKey] + guard let enumerator = FileManager.default.enumerator( + at: folderURL, + includingPropertiesForKeys: keys, + options: [.skipsHiddenFiles] + ) else { + return [] + } + + var files: [LinkedSQLIndex.IndexedFile] = [] + for case let url as URL in enumerator { + guard SQLFileService.supportedExtensions.contains(url.pathExtension.lowercased()), + let values = try? url.resourceValues(forKeys: Set(keys)), + values.isRegularFile == true, + let relativePath = Self.relativePath(of: url, in: folderURL) else { + continue + } + let mtime = (values.contentModificationDate ?? Date()).timeIntervalSince1970 + let fileSize = Int64(values.fileSize ?? 0) + if let known = stored[relativePath], known.mtime == mtime, known.fileSize == fileSize { + files.append(known) + } else { + files.append(readFile(at: url, relativePath: relativePath, mtime: mtime, fileSize: fileSize)) + } + } + return files + } + + private func readFile( + at url: URL, + relativePath: String, + mtime: TimeInterval, + fileSize: Int64 + ) -> LinkedSQLIndex.IndexedFile { + let header = loadHeader(url) + let metadata = header.map { SQLFrontmatter.parse($0.content) } ?? SQLFrontmatter.Metadata() + let baseName = (url.lastPathComponent as NSString).deletingPathExtension + let name = metadata.name?.trimmingCharacters(in: .whitespaces) + return LinkedSQLIndex.IndexedFile( + relativePath: relativePath, + name: name.flatMap { $0.isEmpty ? nil : $0 } ?? baseName, + keyword: metadata.keyword, + description: metadata.description, + mtime: mtime, + fileSize: fileSize, + encodingName: (header?.encoding ?? .utf8).ianaName + ) + } + + private static func relativePath(of url: URL, in base: URL) -> String? { + let urlPath = url.standardizedFileURL.path + let basePath = base.standardizedFileURL.path + guard urlPath.hasPrefix(basePath + "/") else { return nil } + return String(urlPath.dropFirst(basePath.count + 1)) + } +} diff --git a/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift b/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift index ba46a5a216..3aba27af14 100644 --- a/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift +++ b/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift @@ -7,48 +7,86 @@ import Combine import CoreServices import Foundation import os -import TableProUtilities @MainActor internal final class SQLFolderWatcher: ObservableObject { static let shared = SQLFolderWatcher() - nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFolderWatcher") @Published private(set) var lastScanCompletedAt: Date? - private var eventStream: FSEventStreamRef? + private let eventWatcher = FileSystemEventWatcher( + latency: 1.0, + reportsFileEvents: true, + accepts: { events in + if SQLFolderWatcher.touchesGitState(events) { + SQLFolderWatcher.gitStateTouched.withLock { $0 = true } + return true + } + return SQLFolderWatcher.shouldRescan(after: events) + } + ) private var debounceTask: Task? private var hasStarted = false - nonisolated private static let eventCallback: FSEventStreamCallback = { _, info, _, eventPaths, _, _ in - guard let info else { return } - let paths = (Unmanaged.fromOpaque(eventPaths).takeUnretainedValue() as NSArray) as? [String] ?? [] - guard !touchesOnlyRepositoryMetadata(paths) else { return } - let watcher = Unmanaged.fromOpaque(info).takeUnretainedValue() - Task { @MainActor in - watcher.scheduleDebouncedRescan() - } - } + // Flags after which the paths in the batch no longer tell what changed. + nonisolated private static let rescanFlags = FSEventStreamEventFlags( + kFSEventStreamEventFlagMustScanSubDirs + | kFSEventStreamEventFlagRootChanged + | kFSEventStreamEventFlagUserDropped + | kFSEventStreamEventFlagKernelDropped + | kFSEventStreamEventFlagEventIdsWrapped + ) nonisolated static func touchesOnlyRepositoryMetadata(_ paths: [String]) -> Bool { guard !paths.isEmpty else { return false } - return paths.allSatisfy { path in - path.split(separator: "/").contains(".git") + return paths.allSatisfy(isRepositoryMetadata) + } + + // A batch is skipped only when every path is inside `.git` or is a file that cannot be a linked query. + nonisolated static func shouldRescan(after events: [FileSystemEvent]) -> Bool { + guard !events.isEmpty else { return true } + if events.contains(where: { $0.flags & rescanFlags != 0 }) { + return true + } + return !events.allSatisfy { event in + if isRepositoryMetadata(event.path) { + return true + } + guard event.has(kFSEventStreamEventFlagItemIsFile) else { return false } + let pathExtension = (event.path as NSString).pathExtension.lowercased() + return !SQLFileService.supportedExtensions.contains(pathExtension) + } + } + + // A repository created or removed here, or an ignore rule edited, changes Git status without touching + // a query, so the update is sent even when no row changed. + nonisolated private static let gitStateTouched = OSAllocatedUnfairLock(initialState: false) + + nonisolated static func touchesGitState(_ events: [FileSystemEvent]) -> Bool { + events.contains { event in + let name = (event.path as NSString).lastPathComponent + return name == ".git" || name == ".gitignore" } } - private init() {} + nonisolated private static func isRepositoryMetadata(_ path: String) -> Bool { + path.split(separator: "/").contains(".git") + } + + private init() { + eventWatcher.onChange = { [weak self] in self?.scheduleDebouncedRescan() } + } func start() { guard !hasStarted else { return } hasStarted = true let folders = LinkedSQLFolderStorage.shared.loadFolders().filter(\.isEnabled) scheduleFullRescan(folders: folders) - setupEventStream(for: folders) + eventWatcher.watch(Set(folders.map(\.expandedURL.path))) } func stop() { - cancelEventStream() + eventWatcher.cancel() debounceTask?.cancel() debounceTask = nil hasStarted = false @@ -59,60 +97,12 @@ internal final class SQLFolderWatcher: ObservableObject { start() } - // MARK: - Event stream - - private func setupEventStream(for folders: [LinkedSQLFolder]) { - cancelEventStream() - guard !folders.isEmpty else { return } - - let paths = folders.map(\.expandedURL.path) as CFArray - var context = FSEventStreamContext( - version: 0, - info: Unmanaged.passUnretained(self).toOpaque(), - retain: nil, - release: nil, - copyDescription: nil - ) - - let flags = UInt32( - kFSEventStreamCreateFlagFileEvents - | kFSEventStreamCreateFlagNoDefer - | kFSEventStreamCreateFlagWatchRoot - | kFSEventStreamCreateFlagUseCFTypes - ) - - guard let stream = FSEventStreamCreate( - kCFAllocatorDefault, - Self.eventCallback, - &context, - paths, - FSEventStreamEventId(kFSEventStreamEventIdSinceNow), - 1.0, - flags - ) else { - Self.logger.error("Failed to create FSEventStream") - return - } - - FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) - FSEventStreamStart(stream) - eventStream = stream - } - - private func cancelEventStream() { - guard let stream = eventStream else { return } - FSEventStreamStop(stream) - FSEventStreamInvalidate(stream) - FSEventStreamRelease(stream) - eventStream = nil - } - // MARK: - Scan scheduling private func scheduleFullRescan(folders: [LinkedSQLFolder]) { debounceTask?.cancel() debounceTask = Task { @MainActor [weak self] in - await Self.rescan(folders: folders) + _ = await Self.rescan(folders: folders) self?.lastScanCompletedAt = Date() AppEvents.shared.linkedSQLFoldersDidUpdate.send(nil) } @@ -127,100 +117,31 @@ internal final class SQLFolderWatcher: ObservableObject { return } let folders = LinkedSQLFolderStorage.shared.loadFolders().filter(\.isEnabled) - await Self.rescan(folders: folders) + let changed = await Self.rescan(folders: folders) + let gitStateTouched = Self.gitStateTouched.withLock { touched in + defer { touched = false } + return touched + } self?.lastScanCompletedAt = Date() + guard changed || gitStateTouched else { return } AppEvents.shared.linkedSQLFoldersDidUpdate.send(nil) } } - private static func rescan(folders: [LinkedSQLFolder]) async { - await Task.detached(priority: .utility) { - for folder in folders { - await scanFolder(folder) + // Returns whether any folder's rows changed or a folder was dropped from the index. + @concurrent + nonisolated private static func rescan(folders: [LinkedSQLFolder]) async -> Bool { + let scanner = LinkedSQLFolderScanner() + var changed = false + for folder in folders { + if await scanner.scan(folderId: folder.id, at: folder.expandedURL) { + changed = true } - let allKnownIds = Set(LinkedSQLFolderStorage.shared.loadFolders().map(\.id)) - await pruneRemovedFolders(stillKnownIds: allKnownIds) - }.value - } - - // MARK: - Per-folder scan (background) - - private static func scanFolder(_ folder: LinkedSQLFolder) async { - let folderURL = folder.expandedURL - let fileManager = FileManager.default - guard fileManager.fileExists(atPath: folderURL.path) else { - await LinkedSQLIndex.shared.removeFolder(folderId: folder.id) - return } - - var indexed: [LinkedSQLIndex.IndexedFile] = [] - - enumerateSQLFileURLs(in: folderURL, fileManager: fileManager) { url in - let resourceValues = try? url.resourceValues(forKeys: [ - .isRegularFileKey, .contentModificationDateKey, .fileSizeKey - ]) - guard resourceValues?.isRegularFile == true else { return } - let mtime = resourceValues?.contentModificationDate ?? Date() - let fileSize = Int64(resourceValues?.fileSize ?? 0) - - guard let relativePath = relativePathFor(url: url, base: folderURL) else { return } - let header = FileTextLoader.loadHeader(url) - let metadata = header.map { SQLFrontmatter.parse($0.content) } ?? SQLFrontmatter.Metadata() - let encoding = header?.encoding ?? .utf8 - - let baseName = (url.lastPathComponent as NSString).deletingPathExtension - let displayName = metadata.name?.trimmingCharacters(in: .whitespaces).nonEmpty - ?? baseName - - indexed.append(LinkedSQLIndex.IndexedFile( - relativePath: relativePath, - name: displayName, - keyword: metadata.keyword, - description: metadata.description, - mtime: mtime, - fileSize: fileSize, - encoding: encoding - )) + let known = Set(LinkedSQLFolderStorage.shared.loadFolders().map(\.id)) + if await scanner.pruneFolders(keeping: known) { + changed = true } - - await LinkedSQLIndex.shared.replaceAll(folderId: folder.id, files: indexed, folderURL: folderURL) + return changed } - - private static func enumerateSQLFileURLs( - in folderURL: URL, - fileManager: FileManager, - handle: (URL) -> Void - ) { - guard let enumerator = fileManager.enumerator( - at: folderURL, - includingPropertiesForKeys: [.isRegularFileKey, .contentModificationDateKey, .fileSizeKey], - options: [.skipsHiddenFiles] - ) else { - return - } - - for case let url as URL in enumerator { - guard SQLFileService.supportedExtensions.contains(url.pathExtension.lowercased()) else { continue } - handle(url) - } - } - - private static func pruneRemovedFolders(stillKnownIds: Set) async { - let indexedIds = await LinkedSQLIndex.shared.allFolderIds() - let stale = indexedIds.subtracting(stillKnownIds) - for id in stale { - await LinkedSQLIndex.shared.removeFolder(folderId: id) - } - } - - private static func relativePathFor(url: URL, base: URL) -> String? { - let urlPath = url.standardizedFileURL.path - let basePath = base.standardizedFileURL.path - guard urlPath.hasPrefix(basePath + "/") else { return nil } - return String(urlPath.dropFirst(basePath.count + 1)) - } -} - -private extension String { - var nonEmpty: String? { isEmpty ? nil : self } } diff --git a/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift b/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift index 47d4e2da17..0065a4e3d0 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift @@ -16,17 +16,17 @@ nonisolated enum PayloadRowPlanner { dropsEmptyPrimaryKey: Bool = true ) throws -> [[(column: String, value: String?)]] { guard !columns.isEmpty else { throw IntentDataError.noColumns(table) } - let columnNames = Set(columns.map(\.name)) + let columnNames = columns.map(\.name) let primaryKeys = Set(columns.filter(\.isPrimaryKey).map(\.name)) return try rows.compactMap { row in - let unknown = row.keys.filter { !columnNames.contains($0) } - guard unknown.isEmpty else { throw IntentDataError.unknownColumns(unknown.sorted(), table) } + let matched = keysByColumn(row.keys, columns: columnNames) + guard matched.unknown.isEmpty else { throw IntentDataError.unknownColumns(matched.unknown.sorted(), table) } var values: [(column: String, value: String?)] = [] for column in columns { guard !column.isServerOwned else { continue } - guard let value = row.value(for: column.name) else { continue } + guard let key = matched.byColumn[column.name], let value = row.value(for: key) else { continue } if dropsEmptyPrimaryKey, primaryKeys.contains(column.name), value.isEmptyOrNull { continue } values.append((column: column.name, value: value.sqlValue)) } @@ -34,4 +34,46 @@ nonisolated enum PayloadRowPlanner { return values } } + + // As the import sheet matches a header: its exact spelling claims a column first, then a case-insensitive match. + // There is no sheet to map a header by hand, so SQL Server's `[name]` also finds `name`. + static func keysByColumn(_ keys: [String], columns: [String]) -> (byColumn: [String: String], unknown: [String]) { + var byColumn: [String: String] = [:] + var loose: [String] = [] + let spelled = Set(columns) + for key in keys.sorted() { + if spelled.contains(key) { + byColumn[key] = key + } else { + loose.append(key) + } + } + var unknown: [String] = [] + for key in loose { + let claimed = byColumn + guard let column = matchingColumn(for: key, in: columns, unclaimed: { claimed[$0] == nil }) else { + unknown.append(key) + continue + } + byColumn[column] = key + } + return (byColumn, unknown) + } + + private static func matchingColumn(for key: String, in columns: [String], unclaimed: (String) -> Bool) -> String? { + for name in [key, unbracketed(key)].compactMap({ $0 }) { + if columns.contains(name), unclaimed(name) { return name } + let folded = columns.filter { $0.caseInsensitiveCompare(name) == .orderedSame && unclaimed($0) } + if folded.count == 1 { return folded[0] } + } + return nil + } + + // `]` inside the brackets is doubled, so a lone one means the key is not a bracketed name. + private static func unbracketed(_ key: String) -> String? { + guard key.count > 2, key.hasPrefix("["), key.hasSuffix("]") else { return nil } + let inner = String(key.dropFirst().dropLast()) + guard !inner.replacingOccurrences(of: "]]", with: "", options: .literal).contains("]") else { return nil } + return inner.replacingOccurrences(of: "]]", with: "]", options: .literal) + } } diff --git a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift index 99fd7d2c56..95736a027e 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift @@ -11,15 +11,29 @@ nonisolated enum RowPayload { let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw IntentDataError.emptyPayload } - let rows = trimmed.hasPrefix("{") || trimmed.hasPrefix("[") - ? try parseJSON(trimmed) - : try parseCSV(raw) + let rows = try sniffedRows(raw: raw, trimmed: trimmed) guard !rows.isEmpty else { throw IntentDataError.emptyPayload } guard rows.count <= maxRows else { throw IntentDataError.tooManyRows(maxRows) } return rows } + // A CSV header can open with `[` or `{`, as SQL Server's `[id],[name]` does, so only text that parses as JSON is + // JSON. Text that is neither keeps the JSON error, which names what broke in a payload meant as JSON. So does JSON + // that still splits into even CSV records, one object per line with a trailing comma: its headers hold `":`. + private static func sniffedRows(raw: String, trimmed: String) throws -> [PayloadRow] { + guard trimmed.hasPrefix("{") || trimmed.hasPrefix("[") else { return try parseCSV(raw) } + do { + return try parseJSON(trimmed) + } catch IntentDataError.malformedPayload(let reason) { + guard let rows = try? parseCSV(raw), !rows.isEmpty, + !rows.contains(where: { row in row.keys.contains { $0.contains("\":") } }) else { + throw IntentDataError.malformedPayload(reason) + } + return rows + } + } + static func parseSingle(data: String?, file: IntentFile?) async throws -> [PayloadRow] { let rows = try await parse(data: data, file: file) guard rows.count == 1 else { throw IntentDataError.expectedSingleRow } diff --git a/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift b/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift index 1baee5a518..9d189e0fae 100644 --- a/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift +++ b/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift @@ -157,10 +157,11 @@ internal enum ShortcutsRowWriter { let rowID = RowID.inserted(UUID()) let byColumn = Dictionary(values.map { ($0.column, $0.value) }, uniquingKeysWith: { first, _ in first }) insertedRowData[rowID] = names.map { name in - guard let value = byColumn[name] else { return .text(PluginCellValue.defaultMarkerText) } + guard let value = byColumn[name] else { return .columnDefault } return value.map(PluginCellValue.text) ?? .null } - changes.append(RowChange(rowID: rowID, type: .insert, sequence: index)) + let defaulted = Set(names.indices.filter { byColumn[names[$0]] == nil }) + changes.append(RowChange(rowID: rowID, type: .insert, expressionColumns: defaulted, sequence: index)) } let factory = RowChangeStatementFactory( @@ -195,7 +196,8 @@ internal enum ShortcutsRowWriter { private static func refuseHeldTransaction(on driver: DatabaseDriver, ownsConnection: Bool) async throws { let owner = WriteTransactionOwner.resolve( supportsTransactions: driver.supportsTransactions, - sessionState: await driver.heldSessionTransactionState() + sessionState: await driver.heldSessionTransactionState(), + commitMode: .automatic ) guard owner == .session else { return } guard ownsConnection else { diff --git a/apple/TablePro/Core/Storage/AIChatStorage.swift b/apple/TablePro/Core/Storage/AIChatStorage.swift index 5420eb2f35..0e09c5245d 100644 --- a/apple/TablePro/Core/Storage/AIChatStorage.swift +++ b/apple/TablePro/Core/Storage/AIChatStorage.swift @@ -2,13 +2,10 @@ // AIChatStorage.swift // TablePro // -// File-based persistence for AI chat conversations. -// import Foundation import os -/// Manages persistent storage of AI chat conversations as individual JSON files actor AIChatStorage { static let shared = AIChatStorage() @@ -35,7 +32,6 @@ actor AIChatStorage { .appendingPathComponent("ai_chats", isDirectory: true) directory = dir - // Create directory inline since actor init is nonisolated do { try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) try FileManager.default.setAttributes( @@ -49,23 +45,16 @@ actor AIChatStorage { // MARK: - Public Methods - /// Maximum encoded size for a single conversation file (500 KB) private static let maxFileSize = 500_000 - /// Maximum number of messages to keep after trimming - private static let trimmedMessageCount = 50 + // Past `maxFileSize` only the last turns are written, so a longer transcript may be on disk in part. + static let turnsKeptWhenTrimmed = 50 - /// Save a conversation to disk func save(_ conversation: AIConversation) { Self.write(conversation, into: directory) } - /// Writes on the calling thread, for the one caller that cannot wait for an actor hop. - /// - /// `applicationWillTerminate` runs to the end of the main run loop turn and the process goes; - /// a hop booked there is never scheduled, so the last turns of a reply were lost while the - /// session record on disk still named the conversation they belonged to. The body touches no - /// actor state beyond `directory`, which is a `let`. + // A hop booked in applicationWillTerminate is never scheduled. Touches nothing but the `let` directory. nonisolated func saveSynchronously(_ conversation: AIConversation) { Self.write(conversation, into: directory) } @@ -80,7 +69,7 @@ actor AIChatStorage { let originalSize = data.count let originalCount = conversation.messages.count var trimmed = conversation - trimmed.messages = Array(trimmed.messages.suffix(Self.trimmedMessageCount)) + trimmed.messages = Array(trimmed.messages.suffix(Self.turnsKeptWhenTrimmed)) let dropped = originalCount - trimmed.messages.count data = try Self.encoder.encode(trimmed) Self.logger.warning( @@ -98,51 +87,48 @@ actor AIChatStorage { } } - /// One connection's conversations, newest first. - /// - /// A conversation stored before conversations carried a connection id has no answer, so it is - /// shown on every connection rather than lost. - func loadAll(connectionId: UUID?) -> [AIConversation] { - guard let connectionId else { return loadAll() } - return loadAll().filter { $0.connectionId == nil || $0.connectionId == connectionId } - } - - /// One conversation by id, without reading the rest of the directory. - func load(id: UUID) -> AIConversation? { - let fileURL = directory.appendingPathComponent("\(id.uuidString).json") - guard let data = try? Data(contentsOf: fileURL) else { return nil } - return try? Self.decoder.decode(AIConversation.self, from: data) + // A conversation stored before conversations carried a connection id is shown everywhere, not lost. + static func isShown(conversationOf owner: UUID?, on connectionId: UUID?) -> Bool { + connectionId == nil || owner == nil || owner == connectionId } - /// Load all conversations, sorted by updatedAt descending - func loadAll() -> [AIConversation] { + // The history list needs titles only, so no stored conversation's messages are held. + func loadSummaries(connectionId: UUID?) -> [AIConversation] { + let files: [URL] do { - let files = try FileManager.default.contentsOfDirectory( + files = try FileManager.default.contentsOfDirectory( at: directory, includingPropertiesForKeys: nil, options: .skipsHiddenFiles ) - - let conversations: [AIConversation] = files - .filter { $0.pathExtension == "json" } - .compactMap { fileURL in - do { - let data = try Data(contentsOf: fileURL) - return try Self.decoder.decode(AIConversation.self, from: data) - } catch { - Self.logger.error("Failed to load conversation from \(fileURL.lastPathComponent): \(error.localizedDescription)") - return nil - } - } - - return conversations.sorted { $0.updatedAt > $1.updatedAt } } catch { Self.logger.error("Failed to list conversations: \(error.localizedDescription)") return [] } + + let summaries: [AIConversation] = files + .filter { $0.pathExtension == "json" } + .compactMap { fileURL in + do { + let data = try Data(contentsOf: fileURL) + let header = try Self.decoder.decode(ConversationHeader.self, from: data) + guard Self.isShown(conversationOf: header.connectionId, on: connectionId) else { return nil } + return header.summary + } catch { + Self.logger.error("Failed to load conversation from \(fileURL.lastPathComponent): \(error.localizedDescription)") + return nil + } + } + + return summaries.sorted { $0.updatedAt > $1.updatedAt } + } + + func load(id: UUID) -> AIConversation? { + let fileURL = directory.appendingPathComponent("\(id.uuidString).json") + guard let data = try? Data(contentsOf: fileURL) else { return nil } + return try? Self.decoder.decode(AIConversation.self, from: data) } - /// Delete a conversation by ID func delete(_ id: UUID) { let fileURL = directory.appendingPathComponent("\(id.uuidString).json") @@ -153,7 +139,6 @@ actor AIChatStorage { } } - /// Delete all conversations func deleteAll() { do { let files = try FileManager.default.contentsOfDirectory( @@ -168,4 +153,27 @@ actor AIChatStorage { Self.logger.error("Failed to delete all conversations: \(error.localizedDescription)") } } + + // Decodes the same keys as `AIConversation` and the same defaults, without `messages`. + private struct ConversationHeader: Decodable { + let id: UUID + let title: String? + let createdAt: Date + let updatedAt: Date + let connectionId: UUID? + let connectionName: String? + let schemaVersion: Int? + + var summary: AIConversation { + AIConversation( + id: id, + title: title ?? "", + createdAt: createdAt, + updatedAt: updatedAt, + connectionId: connectionId, + connectionName: connectionName, + schemaVersion: max(schemaVersion ?? 0, AIConversation.currentSchemaVersion) + ) + } + } } diff --git a/apple/TablePro/Core/Storage/AppSettingsManager.swift b/apple/TablePro/Core/Storage/AppSettingsManager.swift index 47b3a24e27..f444064e16 100644 --- a/apple/TablePro/Core/Storage/AppSettingsManager.swift +++ b/apple/TablePro/Core/Storage/AppSettingsManager.swift @@ -62,8 +62,7 @@ final class AppSettingsManager: ObservableObject { formatter.options(for: editor) } - /// Device-local: it is deliberately absent from `AppSettingsCategory.synced`, so a zoom press - /// never pushes an iCloud record and a Mac on an older build cannot reset it. + // Device-local, absent from `AppSettingsCategory.synced`: a zoom press never pushes a record an older build could reset. @Published var typography: TypographySettings { didSet { guard typography != oldValue else { return } @@ -107,6 +106,15 @@ final class AppSettingsManager: ObservableObject { } } + @Published var valueColors: ValueColorSettings { + didSet { + guard valueColors != oldValue else { return } + storage.saveValueColors(valueColors) + appEvents.valueColorSettingsChanged.send(()) + syncTracker.markDirty(.settings, id: AppSettingsCategory.valueColors) + } + } + @Published var history: HistorySettings { didSet { guard !isValidating else { return } @@ -151,9 +159,14 @@ final class AppSettingsManager: ObservableObject { } } + // Every writer, a reset from another pane included, has to reach the Keychain flag and the coordinator. @Published var sync: SyncSettings { didSet { storage.saveSync(sync) + passwordSyncDefaults.set(sync.syncsPasswords, forKey: KeychainHelper.passwordSyncEnabledKey) + if oldValue.enabled != sync.enabled { + switchSync(sync.enabled) + } } } @@ -161,6 +174,7 @@ final class AppSettingsManager: ObservableObject { didSet { guard !isValidating else { return } var validated = mcp + validated.port = mcp.validatedPort validated.maxRowLimit = mcp.validatedMaxRowLimit validated.defaultRowLimit = mcp.validatedDefaultRowLimit validated.queryTimeoutSeconds = mcp.validatedQueryTimeoutSeconds @@ -192,6 +206,8 @@ final class AppSettingsManager: ObservableObject { private let mcpServerManager: MCPServerManager private let copilotService: CopilotService private let connectionListPreferences: ConnectionListPreferences + private let passwordSyncDefaults: UserDefaults + private let switchSync: @MainActor (Bool) -> Void private var isValidating = false init( @@ -203,7 +219,16 @@ final class AppSettingsManager: ObservableObject { queryHistoryManager: QueryHistoryManager = .shared, mcpServerManager: MCPServerManager = .shared, copilotService: CopilotService = .shared, - connectionListPreferences: ConnectionListPreferences = .shared + connectionListPreferences: ConnectionListPreferences = .shared, + passwordSyncDefaults: UserDefaults = AppStorageEnvironment.shared.defaults, + switchSync: @escaping @MainActor (Bool) -> Void = { enabled in + // Read at call time, so building the shared settings never builds the coordinator. + if enabled { + SyncCoordinator.shared.enableSync() + } else { + SyncCoordinator.shared.disableSync() + } + } ) { self.storage = storage self.themeEngine = themeEngine @@ -214,6 +239,8 @@ final class AppSettingsManager: ObservableObject { self.mcpServerManager = mcpServerManager self.copilotService = copilotService self.connectionListPreferences = connectionListPreferences + self.passwordSyncDefaults = passwordSyncDefaults + self.switchSync = switchSync self.general = storage.loadGeneral() self.appearance = storage.loadAppearance() @@ -221,6 +248,7 @@ final class AppSettingsManager: ObservableObject { self.editor = storage.loadEditor() self.formatter = storage.loadFormatter() self.dataGrid = storage.loadDataGrid() + self.valueColors = storage.loadValueColors() self.history = storage.loadHistory() self.tabs = storage.loadTabs() self.keyboard = storage.loadKeyboard() @@ -242,10 +270,7 @@ final class AppSettingsManager: ObservableObject { dateFormattingService.updateFormat(dataGrid.dateFormat) } - /// Auto-pick the first configured provider as active when nothing is selected. - /// Avoids a "AI suddenly stopped working" upgrade UX when older settings JSON - /// (with multiple providers and no activeProviderID concept) is loaded. - /// Internal so `@testable` tests can exercise it directly. + // Settings saved before there was an active provider would otherwise read as AI turned off. internal static func migrateAI(_ settings: AISettings) -> AISettings { guard settings.activeProviderID == nil, let first = settings.providers.first else { return settings @@ -261,7 +286,7 @@ final class AppSettingsManager: ObservableObject { await queryHistoryManager.applySettingsChange() } - /// Resets Sparkle's update preferences here too: the reset alert promises every section. + // Resets Sparkle's update preferences too: the reset alert promises every section. func resetToDefaults() { general = .default appearance = .default @@ -269,10 +294,11 @@ final class AppSettingsManager: ObservableObject { editor = .default formatter = .default dataGrid = .default + valueColors = .default history = .default tabs = .default keyboard = .default - ai = .default + ai = ai.resetKeepingProviders sync = .default mcp = .default typography = .default diff --git a/apple/TablePro/Core/Storage/AppSettingsStorage.swift b/apple/TablePro/Core/Storage/AppSettingsStorage.swift index 42809b4038..2b78d39e5b 100644 --- a/apple/TablePro/Core/Storage/AppSettingsStorage.swift +++ b/apple/TablePro/Core/Storage/AppSettingsStorage.swift @@ -9,7 +9,6 @@ import Foundation import os -/// Persistent storage for app settings final class AppSettingsStorage: Sendable { static let shared = AppSettingsStorage() private static let logger = Logger(subsystem: "com.TablePro", category: "AppSettingsStorage") @@ -27,6 +26,7 @@ final class AppSettingsStorage: Sendable { static let editor = "com.TablePro.settings.editor" static let formatter = "com.TablePro.settings.formatter" static let dataGrid = "com.TablePro.settings.dataGrid" + static let valueColors = "com.TablePro.settings.valueColors" static let history = "com.TablePro.settings.history" static let tabs = "com.TablePro.settings.tabs" static let keyboard = "com.TablePro.settings.keyboard" @@ -47,11 +47,7 @@ final class AppSettingsStorage: Sendable { ] } - /// The storage environment's defaults, not `.standard`. A UI test runs against a per-sandbox - /// suite, and reading the standard domain here put every setting, the onboarding flag among - /// them, in the machine's real preferences: a developer who had finished onboarding never saw - /// it under test, a fresh runner saw it until the first case skipped it, and every case after - /// that inherited the skip. + // Not `.standard`: a UI test runs against a per-sandbox suite, and the standard domain leaked the onboarding flag between runs. init(userDefaults: UserDefaults = AppStorageEnvironment.shared.defaults) { self.defaults = userDefaults } @@ -127,8 +123,7 @@ final class AppSettingsStorage: Sendable { } } - /// The first launch with no typography setting lifts the fonts out of the old theme files once, - /// so an upgrade does not return everyone to System Mono 13. + // The first launch with no typography setting lifts the fonts out of the old theme files, so an upgrade keeps them. func loadTypography( systemIsDark: Bool, legacyThemesDirectory: URL = ThemeCatalog.defaultUserDirectory @@ -186,6 +181,16 @@ final class AppSettingsStorage: Sendable { save(settings, key: Keys.dataGrid) } + // MARK: - Value Color Settings + + func loadValueColors() -> ValueColorSettings { + load(key: Keys.valueColors, default: .default) + } + + func saveValueColors(_ settings: ValueColorSettings) { + save(settings, key: Keys.valueColors) + } + // MARK: - History Settings func loadHistory() -> HistorySettings { @@ -295,7 +300,6 @@ final class AppSettingsStorage: Sendable { // MARK: - Reset - /// Reset all settings to defaults func resetToDefaults() { saveGeneral(.default) saveAppearance(.default) @@ -303,10 +307,11 @@ final class AppSettingsStorage: Sendable { saveEditor(.default) saveFormatter(.default) saveDataGrid(.default) + saveValueColors(.default) saveHistory(.default) saveTabs(.default) saveKeyboard(.default) - saveAI(.default) + saveAI(loadAI().resetKeepingProviders) saveSync(.default) saveMCP(.default) saveTypography(.default) diff --git a/apple/TablePro/Core/Storage/ConnectionApprovalStore.swift b/apple/TablePro/Core/Storage/ConnectionApprovalStore.swift index 6c89dfc86a..cc4147e58a 100644 --- a/apple/TablePro/Core/Storage/ConnectionApprovalStore.swift +++ b/apple/TablePro/Core/Storage/ConnectionApprovalStore.swift @@ -30,23 +30,26 @@ internal final class ConnectionApprovalStore { private let kind: Kind private let defaults: UserDefaults private let savedConnections: @MainActor () -> [DatabaseConnection]? + private let dialedDestination: @MainActor (DatabaseConnection) -> ConnectionDestination internal init( kind: Kind, defaults: UserDefaults = AppStorageEnvironment.shared.defaults, savedConnections: @escaping @MainActor () -> [DatabaseConnection]? = ConnectionApprovalStore - .trustedSavedConnections + .trustedSavedConnections, + dialedDestination: @escaping @MainActor (DatabaseConnection) -> ConnectionDestination = { $0.dialedDestination } ) { self.kind = kind self.defaults = defaults self.savedConnections = savedConnections + self.dialedDestination = dialedDestination } internal func isApproved(_ connection: DatabaseConnection) -> Bool { let items = kind.items(in: connection) guard !items.isEmpty else { return true } migrateIfNeeded() - let approved = approvals()[connection.id.uuidString]?[connection.destination.digest] ?? [] + let approved = approvals()[connection.id.uuidString]?[dialedDestination(connection).digest] ?? [] return items.isSubset(of: approved) } @@ -100,7 +103,7 @@ internal final class ConnectionApprovalStore { private func record(_ connection: DatabaseConnection, in all: inout Approvals) -> Bool { let items = kind.items(in: connection) guard !items.isEmpty else { return false } - let digest = connection.destination.digest + let digest = dialedDestination(connection).digest let approved = Set(all[connection.id.uuidString]?[digest] ?? []).union(items) all[connection.id.uuidString, default: [:]][digest] = approved.sorted() return true diff --git a/apple/TablePro/Core/Storage/ConnectionLocalState.swift b/apple/TablePro/Core/Storage/ConnectionLocalState.swift index aa36566e31..a39458b3ca 100644 --- a/apple/TablePro/Core/Storage/ConnectionLocalState.swift +++ b/apple/TablePro/Core/Storage/ConnectionLocalState.swift @@ -6,18 +6,12 @@ import Foundation import os -/// Everything a deleted connection leaves behind on this device, cleaned up in one place. -/// -/// The list used to be written out at each of the three delete sites, which is how they drifted: -/// the remote-deletion path cleared two stores where the local one cleared nine, and no site had -/// ever removed a single `SidebarPersistenceKey`. A store added here is cleaned up everywhere. +// Everything a deleted connection leaves on this device, in one place: three delete sites each kept a list and drifted. @MainActor internal enum ConnectionLocalState { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "ConnectionLocalState") - /// Who deleted the connection. A local delete leaves tombstones so the other devices follow; - /// a remote delete must not, or it pushes back a deletion the sender already made, and neither - /// may the local delete of a connection kept off iCloud. + // Only a local delete leaves tombstones; a remote one would push back the sender's own deletion. internal enum Origin { case local case localOnly @@ -89,27 +83,14 @@ internal enum ConnectionLocalState { } } - /// The two stores that can only be reached with `await`, so `purge` fires them and does not - /// wait. They belong here for the reason everything else does: written out at the delete sites, - /// the query history clear reached the two local ones and never the remote one, so a connection - /// deleted on another Mac left every statement it had ever run, with its literals, in the - /// history on this one. - /// - /// Separate from `purge` so a test can await what `purge` cannot. - /// - /// `origin` splits the SQL favorites the way `purgeFavorites` splits the table ones, and for the - /// same reason. `SyncCoordinator.applyRemoteChanges` suppresses the change tracker only for the - /// length of its own synchronous body, and this runs from a `Task` that starts after that body - /// has returned and the suppression has been reset, so a remote delete really did write - /// tombstones and push the sender's own deletion back at it. - /// - /// Query history takes no origin: it is device-local and never synced, so a remote delete - /// should forget this device's copy and has no tombstone to leave either way. + // Separate from `purge` so a test can await it. Saved queries split by origin themselves: this runs after + // `applyRemoteChanges` stopped suppressing the tracker. Query history and tab versions are device-local. internal static func purgeAsyncStores( _ connectionIds: Set, origin: Origin, sqlFavorites: SQLFavoriteManager = .shared, queryHistory: QueryHistoryManager = .shared, + queryTabVersions: QueryTabVersionStorage = .shared, syncTracker: SyncChangeTracker = .shared ) async { for connectionId in connectionIds { @@ -128,16 +109,15 @@ internal enum ConnectionLocalState { "Query history for a deleted connection could not be cleared: \(connectionId, privacy: .public)" ) } + if await !queryTabVersions.removeVersions(forConnection: connectionId) { + logger.error( + "Tab versions for a deleted connection could not be cleared: \(connectionId, privacy: .public)" + ) + } } } - /// The trailing pane's two keys, which are written straight onto the defaults object rather - /// than through a store with a `remove(for:)` of its own, so `purge`'s list of stores never - /// reached them. A deleted connection left the surface it was last showing and its inspector's - /// view mode behind, and a connection later given the same id inherited both. - /// - /// Separate from `purge` for the reason `purgeAsyncStores` is: `purge` reaches nine shared - /// singletons and a test cannot call it, while this takes the one thing it writes to. + // Written straight onto the defaults, with no store of their own; separate so a test can pass its own defaults. internal static func purgeTrailingPaneKeys( _ connectionIds: Set, defaults: UserDefaults = AppStorageEnvironment.shared.defaults @@ -148,8 +128,7 @@ internal enum ConnectionLocalState { } } - /// The in-memory registries go first. A live `SharedSidebarState` for this connection rewrites - /// its own defaults keys on the next mutation, so removing the keys under it achieves nothing. + // First: a live `SharedSidebarState` rewrites its defaults keys on its next mutation. private static func purgeLiveState(_ connectionId: UUID) { SharedSidebarState.removeConnection(connectionId) SidebarViewModel.removeConnection(connectionId) diff --git a/apple/TablePro/Core/Storage/ConnectionStorage.swift b/apple/TablePro/Core/Storage/ConnectionStorage.swift index f1f9cfbd49..80ef70f90b 100644 --- a/apple/TablePro/Core/Storage/ConnectionStorage.swift +++ b/apple/TablePro/Core/Storage/ConnectionStorage.swift @@ -29,6 +29,8 @@ final class ConnectionStorage { private var cachedConnections: [DatabaseConnection]? private(set) var lastLoadFailed = false + private(set) var fileReadCount = 0 + private(set) var fileWriteCount = 0 /// Whether the file on disk is the one TablePro last wrote. False once it has been edited by /// something else, which is the signal to refuse to run a connection's password source. @@ -91,6 +93,7 @@ final class ConnectionStorage { func loadConnections() -> [DatabaseConnection] { if let cached = cachedConnections { return cached } + fileReadCount += 1 guard let storedConnections = file.load() else { lastLoadFailed = true return [] @@ -139,6 +142,7 @@ final class ConnectionStorage { @discardableResult private func persist(_ connections: [DatabaseConnection], byUser: Bool = true) -> Bool { guard file.save(connections.map { StoredConnection(from: $0) }, byUser: byUser) else { return false } + fileWriteCount += 1 cachedConnections = nil return true } @@ -440,22 +444,11 @@ final class ConnectionStorage { if connection.participatesInSync { syncTracker.markDeleted(.connection, id: connection.id.uuidString) } - deletePassword(for: connection.id) - deleteSSHPassword(for: connection.id) - deleteKeyPassphrase(for: connection.id) - deleteSSLClientKeyPassphrase(for: connection.id) - deleteTOTPSecret(for: connection.id) - deleteCloudflareTokenId(for: connection.id) - deleteCloudflareTokenSecret(for: connection.id) - deleteCloudSQLProxyServiceAccountKey(for: connection.id) - deleteSOCKSProxyPassword(for: connection.id) - - let secureFieldIds = Self.secureFieldIds(for: connection.type) - deleteAllPluginSecureFields(for: connection.id, fieldIds: secureFieldIds) - + let origin: ConnectionLocalState.Origin = connection.participatesInSync ? .local : .localOnly + deleteSecrets(of: [connection], origin: origin) ConnectionLocalState.purge( connectionIds: [connection.id], - origin: connection.participatesInSync ? .local : .localOnly, + origin: origin, appSettings: appSettingsProvider(), syncTracker: syncTracker ) @@ -474,20 +467,9 @@ final class ConnectionStorage { for conn in connectionsToDelete where conn.participatesInSync { syncTracker.markDeleted(.connection, id: conn.id.uuidString) } - for conn in connectionsToDelete { - deletePassword(for: conn.id) - deleteSSHPassword(for: conn.id) - deleteKeyPassphrase(for: conn.id) - deleteSSLClientKeyPassphrase(for: conn.id) - deleteTOTPSecret(for: conn.id) - deleteCloudflareTokenId(for: conn.id) - deleteCloudflareTokenSecret(for: conn.id) - deleteCloudSQLProxyServiceAccountKey(for: conn.id) - deleteSOCKSProxyPassword(for: conn.id) - let fields = Self.secureFieldIds(for: conn.type) - deleteAllPluginSecureFields(for: conn.id, fieldIds: fields) - } let syncedIds = Set(connectionsToDelete.filter(\.participatesInSync).map(\.id)) + deleteSecrets(of: connectionsToDelete.filter { syncedIds.contains($0.id) }, origin: .local) + deleteSecrets(of: connectionsToDelete.filter { !syncedIds.contains($0.id) }, origin: .localOnly) ConnectionLocalState.purge( connectionIds: syncedIds, origin: .local, @@ -807,6 +789,7 @@ final class ConnectionStorage { "cloudflaretokenid", "cloudflaretokensecret", "cloudsqlproxyserviceaccountkey", "socksproxypassword" ] private static let localOnlySecretsMigratedKey = "com.TablePro.localOnlySecretsKeptOnDevice" + private static let pendingSyncedCopyRemovalsKey = "com.TablePro.localOnlySyncedCopiesPendingRemoval" private func secretKeys(for connection: DatabaseConnection) -> [String] { let id = connection.id.uuidString @@ -818,6 +801,21 @@ final class ConnectionStorage { + fieldIds.map { "com.TablePro.plugin.\($0).\(id)" } } + // Removing an iCloud Keychain item removes it from every device, so only a delete the other devices follow does. + // One from another device leaves it to that device. A Local only delete cannot tell whether another device still + // syncs the row, so it keeps it. + func deleteSecrets(of connections: [DatabaseConnection], origin: ConnectionLocalState.Origin) { + for connection in connections { + for key in secretKeys(for: connection) { + if origin.leavesTombstones { + deleteSecret(forKey: key, connectionId: connection.id) + } else { + keychain.delete(forKey: key, deviceOnly: true) + } + } + } + } + private func keepsSecretsOnDevice(_ connectionId: UUID) -> Bool { loadConnection(id: connectionId).map { !$0.participatesInSync } ?? false } @@ -857,16 +855,27 @@ final class ConnectionStorage { staging?.edits.deletedKeys.append(key) } - // Rewrites each stored secret into the scope the connection now has. Going device-only keeps the copy iCloud - // Keychain holds: removing it would remove it from every device. False when a secret could not be read or written. + // Rewrites each stored secret into the scope the connection now has. removingSyncedCopy then removes the iCloud + // Keychain copy, from every device, but only of a secret this Mac now holds itself. False when a secret could not + // be read or written. @discardableResult - private func rewriteSecrets(of connection: DatabaseConnection, readingOwnItemFirst: Bool, deviceOnly: Bool) -> Bool { + private func rewriteSecrets( + of connection: DatabaseConnection, + readingOwnItemFirst: Bool, + deviceOnly: Bool, + removingSyncedCopy: Bool = false + ) -> Bool { var complete = true for key in secretKeys(for: connection) { switch keychain.readStringResult(forKey: key, deviceOnly: readingOwnItemFirst) { case .found(let value): - if !keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) { + guard keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) else { complete = false + continue + } + if removingSyncedCopy, !keychain.removeSyncedCopy(forKey: key) { + Self.logger.warning("A Local only secret kept its iCloud Keychain copy: \(key, privacy: .public)") + notePendingSyncedCopyRemoval(key, connectionId: connection.id) } case .notFound: break @@ -877,7 +886,8 @@ final class ConnectionStorage { return complete } - // Each direction reads the value the connection was using, so the toggle never changes a secret. + // Each direction reads the value the connection was using, so the toggle never changes a secret. Becoming Local + // only also takes the secrets off iCloud Keychain, so the other devices ask for them again. private func moveSecretsWhoseScopeChanged(from previous: [DatabaseConnection], to current: [DatabaseConnection]) { let wasSynced = Dictionary( previous.map { ($0.id, $0.participatesInSync) }, @@ -885,19 +895,48 @@ final class ConnectionStorage { ) for connection in current { guard let synced = wasSynced[connection.id], synced != connection.participatesInSync else { continue } - rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced) + rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced, removingSyncedCopy: synced) + } + } + + private func notePendingSyncedCopyRemoval(_ key: String, connectionId: UUID) { + var pending = defaults.dictionary(forKey: Self.pendingSyncedCopyRemovalsKey) as? [String: String] ?? [:] + pending[key] = connectionId.uuidString + defaults.set(pending, forKey: Self.pendingSyncedCopyRemovalsKey) + } + + // A removal the user was promised and the keychain refused is tried again at launch, while the connection is + // still Local only. One that is back in sync, or gone, wants its synced copy kept or already removed it. + func removePendingSyncedCopies() { + guard let pending = defaults.dictionary(forKey: Self.pendingSyncedCopyRemovalsKey) as? [String: String], + !pending.isEmpty else { return } + let connections = loadConnections() + guard !lastLoadFailed else { return } + let byId = Dictionary(connections.map { ($0.id.uuidString, $0) }, uniquingKeysWith: { first, _ in first }) + var remaining: [String: String] = [:] + for (key, idString) in pending { + guard let connection = byId[idString], !connection.participatesInSync else { continue } + if !keychain.removeSyncedCopy(forKey: key) { + remaining[key] = idString + } + } + if remaining.isEmpty { + defaults.removeObject(forKey: Self.pendingSyncedCopyRemovalsKey) + } else { + defaults.set(remaining, forKey: Self.pendingSyncedCopyRemovalsKey) } } - // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. + // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. Only + // copied: the user saw no note saying the other devices lose them. func keepLocalOnlySecretsOnDeviceIfNeeded() { + removePendingSyncedCopies() guard !defaults.bool(forKey: Self.localOnlySecretsMigratedKey) else { return } let connections = loadConnections() guard !lastLoadFailed else { return } var complete = true for connection in connections where !connection.participatesInSync { - // A plugin that is not registered yet has secure fields this pass cannot name. - let knowsFields = PluginMetadataRegistry.shared.snapshot(for: connection.type) != nil + let knowsFields = Self.knowsSecureFields(of: connection.type) if !rewriteSecrets(of: connection, readingOwnItemFirst: true, deviceOnly: true) || !knowsFields { complete = false } @@ -1043,29 +1082,63 @@ final class ConnectionStorage { PluginManager.shared.secureConnectionFieldIds(for: databaseType) } - func migratePluginSecureFieldsIfNeeded() { - let migrationKey = "com.TablePro.pluginSecureFieldsMigrated" - guard !defaults.bool(forKey: migrationKey) else { return } - defer { defaults.set(true, forKey: migrationKey) } + // A plugin that is not registered yet has secure fields nothing here can name. + private static func knowsSecureFields(of databaseType: DatabaseType) -> Bool { + PluginMetadataRegistry.shared.snapshot(for: databaseType) != nil + } + + // Not the key older builds read: they set it on every exit, including runs that moved nothing. + private static let pluginSecureFieldsMovedKey = "com.TablePro.pluginSecureFieldsMovedToKeychain" + // Done only once every connection's secure fields could be named and each value reached the Keychain. A field + // leaves the record only after its own write, and a file edited outside TablePro moves nothing. + func migratePluginSecureFieldsIfNeeded() { + guard !defaults.bool(forKey: Self.pluginSecureFieldsMovedKey) else { return } var connections = loadConnections() - var changed = false + guard !lastLoadFailed, storeIsTrusted else { + Self.logger.notice("Plugin secure fields not moved: connections.json is unreadable or changed outside TablePro; retrying next launch") + return + } + var complete = true + var changed = false for index in connections.indices { - let secureFieldIds = Self.secureFieldIds(for: connections[index].type) - for fieldId in secureFieldIds { - if let value = connections[index].additionalFields[fieldId], !value.isEmpty { - savePluginSecureField(value, fieldId: fieldId, for: connections[index].id) - connections[index].additionalFields.removeValue(forKey: fieldId) - changed = true + let connection = connections[index] + if !Self.knowsSecureFields(of: connection.type) { + complete = false + } + for fieldId in Self.secureFieldIds(for: connection.type) { + guard let value = connection.additionalFields[fieldId], !value.isEmpty else { continue } + guard holdPluginSecureFieldInKeychain(value, fieldId: fieldId, for: connection.id) else { + complete = false + continue } + connections[index].additionalFields.removeValue(forKey: fieldId) + changed = true } } - if changed { - if !saveConnections(connections) { - Self.logger.error("Failed to persist plugin secure field migration; will retry on next launch") - } + if changed, !saveConnections(connections, byUser: false) { + complete = false + } + if complete { + defaults.set(true, forKey: Self.pluginSecureFieldsMovedKey) + } else { + Self.logger.notice("Not every plugin secure field has moved to the Keychain yet; retrying next launch") + } + } + + // A stored item wins: with password sync on it can be a newer value another device saved, which the plain-text + // copy here would overwrite everywhere. The plain-text value is written only where no item exists. + private func holdPluginSecureFieldInKeychain(_ value: String, fieldId: String, for connectionId: UUID) -> Bool { + let key = "com.TablePro.plugin.\(fieldId).\(connectionId.uuidString)" + switch readSecret(forKey: key, connectionId: connectionId) { + case .found(let stored) where !stored.isEmpty: + return true + case .found, .notFound: + return savePluginSecureField(value, fieldId: fieldId, for: connectionId) + case .locked, .userCancelled, .authFailed, .error: + return false } } diff --git a/apple/TablePro/Core/Storage/DamagedStoreNotice.swift b/apple/TablePro/Core/Storage/DamagedStoreNotice.swift new file mode 100644 index 0000000000..d358d74d8b --- /dev/null +++ b/apple/TablePro/Core/Storage/DamagedStoreNotice.swift @@ -0,0 +1,47 @@ +import AppKit + +enum DamagedStoreNotice { + static func queryHistorySetAside(_ setAside: SetAsideDamagedStore) { + let message = String( + format: String( + localized: """ + TablePro could not read its query history, so it moved the file aside as “%@” and started a new one. \ + The copy stays until you delete it or clear all of your history. + """ + ), + setAside.setAsideURL.lastPathComponent + ) + Task { @MainActor in + present(title: String(localized: "Query History Was Damaged"), message: message, setAside: setAside) + } + } + + static func savedQueriesSetAside(_ setAside: SetAsideDamagedStore) { + let message = String( + format: String( + localized: """ + TablePro could not read its saved queries, so it moved the file aside as “%@” and started an empty one. \ + Nothing in that file was deleted. + """ + ), + setAside.setAsideURL.lastPathComponent + ) + Task { @MainActor in + present(title: String(localized: "Saved Queries Were Damaged"), message: message, setAside: setAside) + } + } + + @MainActor + private static func present(title: String, message: String, setAside: SetAsideDamagedStore) { + let alert = NSAlert() + alert.messageText = title + alert.informativeText = message + alert.alertStyle = .warning + alert.addButton(withTitle: String(localized: "OK")) + alert.addButton(withTitle: String(localized: "Show in Finder")) + AlertHelper.present(alert, in: nil) { response in + guard response == .alertSecondButtonReturn else { return } + NSWorkspace.shared.activateFileViewerSelecting([setAside.setAsideURL]) + } + } +} diff --git a/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift b/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift index 857199fdb0..51969fa7b5 100644 --- a/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift +++ b/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift @@ -56,12 +56,18 @@ final class IntegrityStampedFileStore { self.integrity = integrity } - /// Nil when the file exists and could not be decoded. An absent file is an empty list, which is - /// a fresh install rather than a failure. + // Only an absent file is an empty list, a fresh install; one that cannot be read or decoded is nil, never saved over. func load() -> [Element]? { - guard let data = try? Data(contentsOf: fileURL) else { + let data: Data + do { + data = try Data(contentsOf: fileURL) + } catch let error as CocoaError where error.code == .fileReadNoSuchFile { isTrusted = true return [] + } catch { + logger.error("Failed to read \(self.label, privacy: .public): \(error.publicLogShape, privacy: .public)") + isTrusted = false + return nil } switch integrity.verify(data, fileURL: fileURL) { diff --git a/apple/TablePro/Core/Storage/KeychainHelper.swift b/apple/TablePro/Core/Storage/KeychainHelper.swift index eba7a3b291..f62a632250 100644 --- a/apple/TablePro/Core/Storage/KeychainHelper.swift +++ b/apple/TablePro/Core/Storage/KeychainHelper.swift @@ -35,6 +35,11 @@ protocol KeychainStoring: Sendable { // deviceOnly reads this Mac's own item before one iCloud Keychain delivered. func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult func delete(forKey key: String) + // deviceOnly removes this Mac's own item and leaves the one iCloud Keychain holds, which every device shares. + func delete(forKey key: String, deviceOnly: Bool) + // Removes the iCloud Keychain copy, which removes it from every device. Refused while this Mac has no item of its own. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool } extension KeychainStoring { @@ -42,10 +47,20 @@ extension KeychainStoring { readStringResult(forKey: key) } + func delete(forKey key: String, deviceOnly: Bool) { + delete(forKey: key) + } + @discardableResult func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { writeString(value, forKey: key) } + + // A store with one item per key has no synced copy apart from the one it holds. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + true + } } struct KeychainCopiedItem: Sendable { @@ -230,8 +245,14 @@ final class KeychainHelper: KeychainStoring { } func delete(forKey key: String) { + delete(forKey: key, deviceOnly: false) + } + + func delete(forKey key: String, deviceOnly: Bool) { var query = baseQuery(forKey: key) - query[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny + if !deviceOnly { + query[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny + } let status = items.delete(query) if status != errSecSuccess, status != errSecItemNotFound { @@ -239,6 +260,21 @@ final class KeychainHelper: KeychainStoring { } } + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + let own = items.copyItem(itemQuery(forKey: key, synchronizable: false)) + guard own.status == errSecSuccess, own.data != nil else { + log(status: own.status, operation: "find the local copy before removing the synced one", key: key) + return false + } + let status = items.delete(itemQuery(forKey: key, synchronizable: true)) + guard status == errSecSuccess || status == errSecItemNotFound else { + log(status: status, operation: "remove the synced copy", key: key) + return false + } + return true + } + // MARK: - String Convenience @discardableResult diff --git a/apple/TablePro/Core/Storage/LinkedSQLIndex.swift b/apple/TablePro/Core/Storage/LinkedSQLIndex.swift index 78ea3b7998..79b78740b2 100644 --- a/apple/TablePro/Core/Storage/LinkedSQLIndex.swift +++ b/apple/TablePro/Core/Storage/LinkedSQLIndex.swift @@ -6,7 +6,6 @@ import Foundation import os import SQLite3 -import TableProUtilities internal actor LinkedSQLIndex { static let shared = LinkedSQLIndex() @@ -62,11 +61,15 @@ internal actor LinkedSQLIndex { let dir = databaseURL.deletingLastPathComponent() try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - let dbPath = databaseURL.path(percentEncoded: false) - if sqlite3_open(dbPath, &dbHandle.pointer) != SQLITE_OK { + guard case .opened(let pointer, let setAside) = LocalSQLiteStoreOpener.open(at: databaseURL) else { Self.logger.error("Error opening linked SQL index database") return } + dbHandle.pointer = pointer + // Only a cache of the linked files, which the next folder scan rebuilds, so no damaged copy is kept. + if setAside != nil { + LocalSQLiteStoreOpener.removeSetAsideCopies(of: databaseURL) + } execute("PRAGMA journal_mode=WAL;") execute("PRAGMA synchronous=NORMAL;") @@ -128,55 +131,120 @@ internal actor LinkedSQLIndex { // MARK: - Mutations - func replaceAll(folderId: UUID, files: [IndexedFile], folderURL: URL) { - guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { return } + // Writes only what differs from the stored rows, in one transaction, and returns whether any row changed. + @discardableResult + func synchronize(folderId: UUID, files: [IndexedFile]) -> Bool { + let stored = storedFiles(folderId: folderId) + let listed = Dictionary(files.map { ($0.relativePath, $0) }, uniquingKeysWith: { _, last in last }) + let removed = stored.keys.filter { listed[$0] == nil }.sorted() + let changed = listed.values.filter { stored[$0.relativePath] != $0 }.sorted { $0.relativePath < $1.relativePath } + guard !removed.isEmpty || !changed.isEmpty else { return false } + + guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { + Self.logger.error("Could not begin a linked SQL index write: \(String(cString: sqlite3_errmsg(self.db)))") + return false + } + guard deleteRows(folderId: folderId, relativePaths: removed), upsertRows(folderId: folderId, files: changed) else { + sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) + return false + } + guard sqlite3_exec(db, "COMMIT;", nil, nil, nil) == SQLITE_OK else { + Self.logger.error("Could not commit a linked SQL index write: \(String(cString: sqlite3_errmsg(self.db)))") + sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) + return false + } + return true + } - let deleteSQL = "DELETE FROM linked_sql_files WHERE folder_id = ?;" - var deleteStatement: OpaquePointer? - let folderIdString = folderId.uuidString - if sqlite3_prepare_v2(db, deleteSQL, -1, &deleteStatement, nil) == SQLITE_OK { - sqlite3_bind_text(deleteStatement, 1, folderIdString, -1, Self.transient) - sqlite3_step(deleteStatement) + private func deleteRows(folderId: UUID, relativePaths: [String]) -> Bool { + guard !relativePaths.isEmpty else { return true } + let sql = "DELETE FROM linked_sql_files WHERE folder_id = ? AND relative_path = ?;" + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } + for relativePath in relativePaths { + sqlite3_reset(statement) + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + sqlite3_bind_text(statement, 2, relativePath, -1, Self.transient) + guard sqlite3_step(statement) == SQLITE_DONE else { + Self.logger.error("Failed to delete a linked file: \(String(cString: sqlite3_errmsg(self.db)))") + return false + } } - sqlite3_finalize(deleteStatement) + return true + } - let insertSQL = """ + private func upsertRows(folderId: UUID, files: [IndexedFile]) -> Bool { + guard !files.isEmpty else { return true } + let sql = """ INSERT INTO linked_sql_files (folder_id, relative_path, name, keyword, description, mtime, file_size, encoding) - VALUES (?, ?, ?, ?, ?, ?, ?, ?); + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(folder_id, relative_path) DO UPDATE SET + name = excluded.name, + keyword = excluded.keyword, + description = excluded.description, + mtime = excluded.mtime, + file_size = excluded.file_size, + encoding = excluded.encoding; """ - var insertStatement: OpaquePointer? - guard sqlite3_prepare_v2(db, insertSQL, -1, &insertStatement, nil) == SQLITE_OK else { - sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) - return - } - defer { sqlite3_finalize(insertStatement) } - + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } for file in files { - sqlite3_reset(insertStatement) - sqlite3_bind_text(insertStatement, 1, folderIdString, -1, Self.transient) - sqlite3_bind_text(insertStatement, 2, file.relativePath, -1, Self.transient) - sqlite3_bind_text(insertStatement, 3, file.name, -1, Self.transient) - if let keyword = file.keyword { - sqlite3_bind_text(insertStatement, 4, keyword, -1, Self.transient) - } else { - sqlite3_bind_null(insertStatement, 4) - } - if let description = file.description { - sqlite3_bind_text(insertStatement, 5, description, -1, Self.transient) - } else { - sqlite3_bind_null(insertStatement, 5) + sqlite3_reset(statement) + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + sqlite3_bind_text(statement, 2, file.relativePath, -1, Self.transient) + sqlite3_bind_text(statement, 3, file.name, -1, Self.transient) + bindOptionalText(statement, 4, file.keyword) + bindOptionalText(statement, 5, file.description) + sqlite3_bind_double(statement, 6, file.mtime) + sqlite3_bind_int64(statement, 7, file.fileSize) + sqlite3_bind_text(statement, 8, file.encodingName, -1, Self.transient) + guard sqlite3_step(statement) == SQLITE_DONE else { + Self.logger.error("Failed to write a linked file: \(String(cString: sqlite3_errmsg(self.db)))") + return false } - sqlite3_bind_double(insertStatement, 6, file.mtime.timeIntervalSince1970) - sqlite3_bind_int64(insertStatement, 7, file.fileSize) - sqlite3_bind_text(insertStatement, 8, file.encoding.ianaName, -1, Self.transient) + } + return true + } - if sqlite3_step(insertStatement) != SQLITE_DONE { - Self.logger.error("Failed to insert linked file: \(String(cString: sqlite3_errmsg(self.db)))") - } + private func bindOptionalText(_ statement: OpaquePointer?, _ index: Int32, _ value: String?) { + if let value { + sqlite3_bind_text(statement, index, value, -1, Self.transient) + } else { + sqlite3_bind_null(statement, index) } + } + + func storedFiles(folderId: UUID) -> [String: IndexedFile] { + let sql = """ + SELECT relative_path, name, keyword, description, mtime, file_size, encoding + FROM linked_sql_files + WHERE folder_id = ?; + """ + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return [:] } + defer { sqlite3_finalize(statement) } + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) - sqlite3_exec(db, "COMMIT;", nil, nil, nil) + var files: [String: IndexedFile] = [:] + while sqlite3_step(statement) == SQLITE_ROW { + guard let relativePath = sqlite3_column_text(statement, 0).map({ String(cString: $0) }), + let name = sqlite3_column_text(statement, 1).map({ String(cString: $0) }) else { + continue + } + files[relativePath] = IndexedFile( + relativePath: relativePath, + name: name, + keyword: sqlite3_column_text(statement, 2).map { String(cString: $0) }, + description: sqlite3_column_text(statement, 3).map { String(cString: $0) }, + mtime: sqlite3_column_double(statement, 4), + fileSize: sqlite3_column_int64(statement, 5), + encodingName: sqlite3_column_text(statement, 6).map { String(cString: $0) } ?? "utf-8" + ) + } + return files } func allFolderIds() -> Set { @@ -195,14 +263,16 @@ internal actor LinkedSQLIndex { return ids } - func removeFolder(folderId: UUID) { + // Returns whether the folder had any rows. + @discardableResult + func removeFolder(folderId: UUID) -> Bool { let sql = "DELETE FROM linked_sql_files WHERE folder_id = ?;" var statement: OpaquePointer? - if sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK { - sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) - sqlite3_step(statement) - } - sqlite3_finalize(statement) + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + guard sqlite3_step(statement) == SQLITE_DONE else { return false } + return sqlite3_changes(db) > 0 } // MARK: - Queries @@ -282,13 +352,14 @@ internal actor LinkedSQLIndex { } internal extension LinkedSQLIndex { - struct IndexedFile { + struct IndexedFile: Equatable, Sendable { let relativePath: String let name: String let keyword: String? let description: String? - let mtime: Date + // Seconds since 1970 as the column stores them, so a row read back compares exactly. + let mtime: TimeInterval let fileSize: Int64 - let encoding: String.Encoding + let encodingName: String } } diff --git a/apple/TablePro/Core/Storage/LocalSQLiteStoreOpener.swift b/apple/TablePro/Core/Storage/LocalSQLiteStoreOpener.swift new file mode 100644 index 0000000000..7f151d8782 --- /dev/null +++ b/apple/TablePro/Core/Storage/LocalSQLiteStoreOpener.swift @@ -0,0 +1,169 @@ +import Foundation +import os +import SQLite3 + +struct SetAsideDamagedStore: Sendable, Equatable { + let storeURL: URL + let setAsideURL: URL +} + +enum LocalSQLiteStoreOpening { + case opened(OpaquePointer, setAside: SetAsideDamagedStore?) + case unavailable +} + +// sqlite3_open succeeds on a damaged file, so quick_check runs before anything reads or writes it. +enum LocalSQLiteStoreOpener { + private static let logger = Logger(subsystem: "com.TablePro", category: "LocalSQLiteStore") + private static let busyTimeoutMilliseconds: Int32 = 3_000 + + private enum Verdict { + case healthy(OpaquePointer) + case damaged(String) + case unavailable(String) + } + + static func open(at url: URL, now: Date = Date()) -> LocalSQLiteStoreOpening { + let path = url.path(percentEncoded: false) + switch openChecked(path: path) { + case .healthy(let handle): + return .opened(handle, setAside: nil) + case .unavailable(let reason): + logger.error("Local store could not be opened: \(reason, privacy: .public)") + return .unavailable + case .damaged(let reason): + logger.fault("Local store is damaged: \(reason, privacy: .public)") + guard let moved = setAside(url, now: now) else { return .unavailable } + guard case .healthy(let handle) = openChecked(path: path) else { + logger.error("A fresh store could not be opened after setting the damaged one aside") + return .unavailable + } + return .opened(handle, setAside: moved) + } + } + + private static func isDamage(_ resultCode: Int32) -> Bool { + let primary = resultCode & 0xFF + return primary == SQLITE_CORRUPT || primary == SQLITE_NOTADB + } + + static func setAsideURL(for url: URL, at date: Date, fileManager: FileManager = .default) -> URL { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = .current + formatter.dateFormat = "yyyyMMdd-HHmmss" + let stamp = formatter.string(from: date) + + let directory = url.deletingLastPathComponent() + let stem = url.deletingPathExtension().lastPathComponent + let pathExtension = url.pathExtension + var attempt = 0 + while true { + let marker = attempt == 0 ? stamp : "\(stamp)-\(attempt)" + let name = pathExtension.isEmpty + ? "\(stem).damaged-\(marker)" + : "\(stem).damaged-\(marker).\(pathExtension)" + let candidate = directory.appendingPathComponent(name) + let candidatePath = candidate.path(percentEncoded: false) + let taken = ["", "-wal", "-shm"].contains { fileManager.fileExists(atPath: candidatePath + $0) } + if !taken { return candidate } + attempt += 1 + } + } + + // Every copy setAsideURL(for:) ever named for this store, with its -wal and -shm. + @discardableResult + static func removeSetAsideCopies(of url: URL, fileManager: FileManager = .default) -> Bool { + let directory = url.deletingLastPathComponent() + let prefix = url.deletingPathExtension().lastPathComponent + ".damaged-" + guard let names = try? fileManager.contentsOfDirectory(atPath: directory.path(percentEncoded: false)) else { + return false + } + var removedAll = true + for name in names where name.hasPrefix(prefix) { + do { + try fileManager.removeItem(at: directory.appendingPathComponent(name)) + } catch { + logger.error("Could not delete a set-aside store: \(error.publicLogShape, privacy: .public)") + removedAll = false + } + } + return removedAll + } + + private static func openChecked(path: String) -> Verdict { + var handle: OpaquePointer? + let openResult = sqlite3_open(path, &handle) + guard openResult == SQLITE_OK, let handle else { + let reason = String(cString: sqlite3_errstr(openResult)) + sqlite3_close_v2(handle) + return isDamage(openResult) ? .damaged(reason) : .unavailable(reason) + } + sqlite3_busy_timeout(handle, busyTimeoutMilliseconds) + + let verdict = quickCheck(handle) + if case .healthy = verdict { + return verdict + } + sqlite3_close_v2(handle) + return verdict + } + + private static func quickCheck(_ handle: OpaquePointer) -> Verdict { + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + + let prepareResult = sqlite3_prepare_v2(handle, "PRAGMA quick_check(1);", -1, &statement, nil) + guard prepareResult == SQLITE_OK else { + let reason = String(cString: sqlite3_errmsg(handle)) + return isDamage(prepareResult) ? .damaged(reason) : .unavailable(reason) + } + + let stepResult = sqlite3_step(statement) + guard stepResult == SQLITE_ROW else { + let reason = String(cString: sqlite3_errmsg(handle)) + return isDamage(stepResult) ? .damaged(reason) : .unavailable(reason) + } + let report = sqlite3_column_text(statement, 0).map { String(cString: $0) } ?? "" + return report == "ok" ? .healthy(handle) : .damaged(report) + } + + // The -wal moves first: SQLite deletes a -wal it finds beside an empty database. + private static func setAside(_ url: URL, now: Date) -> SetAsideDamagedStore? { + let fileManager = FileManager.default + let path = url.path(percentEncoded: false) + let destination = setAsideURL(for: url, at: now, fileManager: fileManager) + let destinationPath = destination.path(percentEncoded: false) + + let walPath = path + "-wal" + let walDestinationPath = destinationPath + "-wal" + let movedWal = fileManager.fileExists(atPath: walPath) + if movedWal { + do { + try fileManager.moveItem(atPath: walPath, toPath: walDestinationPath) + } catch { + logger.error("Could not set aside a damaged store's log: \(error.publicLogShape, privacy: .public)") + return nil + } + } + + do { + try fileManager.moveItem(atPath: path, toPath: destinationPath) + } catch { + logger.error("Could not set aside a damaged store: \(error.publicLogShape, privacy: .public)") + if movedWal { + try? fileManager.moveItem(atPath: walDestinationPath, toPath: walPath) + } + return nil + } + + let shmPath = path + "-shm" + if fileManager.fileExists(atPath: shmPath), + (try? fileManager.moveItem(atPath: shmPath, toPath: destinationPath + "-shm")) == nil { + try? fileManager.removeItem(atPath: shmPath) + } + + logger.fault("Set a damaged store aside as \(destination.lastPathComponent, privacy: .private(mask: .hash))") + return SetAsideDamagedStore(storeURL: url, setAsideURL: destination) + } +} diff --git a/apple/TablePro/Core/Storage/MCPApprovalStore.swift b/apple/TablePro/Core/Storage/MCPApprovalStore.swift index ead86232cd..4b4c7c4e8d 100644 --- a/apple/TablePro/Core/Storage/MCPApprovalStore.swift +++ b/apple/TablePro/Core/Storage/MCPApprovalStore.swift @@ -1,8 +1,3 @@ -// -// MCPApprovalStore.swift -// TablePro -// - import Foundation import os @@ -108,43 +103,3 @@ actor MCPApprovalStore: MCPApprovalStoring { defaults.set(data, forKey: key) } } - -/// Keeps grants for its own lifetime and never writes them anywhere. For tests, and for any path -/// that must not leave a decision on disk. -actor MCPInMemoryApprovalStore: MCPApprovalStoring { - private var storedGrants: [MCPConnectionGrant] = [] - private var storedDenials: [MCPConnectionDenial] = [] - - init() {} - - func grants() -> [MCPConnectionGrant] { - storedGrants - } - - func record(_ grant: MCPConnectionGrant) { - storedGrants.removeAll { $0.id == grant.id } - storedGrants.append(grant) - } - - func revoke(subject: String, connectionId: UUID) { - storedGrants.removeAll { $0.subject == subject && $0.connectionId == connectionId } - } - - func revokeAll(subject: String) { - storedGrants.removeAll { $0.subject == subject } - } - - func revokeEverything() { - storedGrants.removeAll() - } - - func denial(subject: String, connectionId: UUID, now: Date) -> Date? { - storedDenials.removeAll { $0.expiresAt <= now } - return storedDenials.first { $0.subject == subject && $0.connectionId == connectionId }?.expiresAt - } - - func record(_ denial: MCPConnectionDenial) { - storedDenials.removeAll { $0.subject == denial.subject && $0.connectionId == denial.connectionId } - storedDenials.append(denial) - } -} diff --git a/apple/TablePro/Core/Storage/QueryHistoryManager.swift b/apple/TablePro/Core/Storage/QueryHistoryManager.swift index 1842118b1d..3ea0d54397 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryManager.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryManager.swift @@ -3,7 +3,9 @@ import Foundation import TableProPluginKit final class QueryHistoryManager: QueryHistoryRecording, QueryHistoryReading, QueryPlanSnapshotReading, Sendable { - static let shared = QueryHistoryManager() + static let shared = QueryHistoryManager( + storage: QueryHistoryStorage(onSetAside: { DamagedStoreNotice.queryHistorySetAside($0) }) + ) // Not private: QueryHistoryManager+Rewind reaches it from another file. internal let storage: QueryHistoryStorage diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage+SetAside.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage+SetAside.swift new file mode 100644 index 0000000000..76150b1f7b --- /dev/null +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage+SetAside.swift @@ -0,0 +1,14 @@ +import Foundation + +extension QueryHistoryStorage { + // A set-aside copy holds rows of every connection and source, so only a clear of all of them reaches it. + static func coversEveryRow(_ filter: QueryHistoryFilter) -> Bool { + filter.scope.connectionId == nil + && filter.sources == Set(QueryHistorySource.allCases) + && filter.outcome == .any + && (filter.searchText ?? "").isEmpty + && filter.since == nil + && filter.until == nil + && filter.allowedConnectionIds == nil + } +} diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift index 14a8966b21..7394509bfc 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift @@ -37,13 +37,16 @@ actor QueryHistoryStorage { private let databaseURL: URL private let removeDatabaseOnDeinit: Bool + private let onSetAside: @Sendable (SetAsideDamagedStore) -> Void init( databaseURL: URL = QueryHistoryStorage.defaultDatabaseURL(), - removeDatabaseOnDeinit: Bool = false + removeDatabaseOnDeinit: Bool = false, + onSetAside: @escaping @Sendable (SetAsideDamagedStore) -> Void = { _ in } ) { self.databaseURL = databaseURL self.removeDatabaseOnDeinit = removeDatabaseOnDeinit + self.onSetAside = onSetAside } static func defaultDatabaseURL() -> URL { @@ -77,25 +80,29 @@ actor QueryHistoryStorage { let dbPath = databaseURL.path(percentEncoded: false) protectDatabaseFiles(at: dbPath) - guard sqlite3_open(dbPath, &dbHandle.pointer) == SQLITE_OK else { + guard case .opened(let pointer, let setAside) = LocalSQLiteStoreOpener.open(at: databaseURL) else { Self.logger.error("Failed to open query history database at \(dbPath, privacy: .private(mask: .hash))") - if let pointer = dbHandle.pointer { - sqlite3_close_v2(pointer) - } - dbHandle.pointer = nil return } + dbHandle.pointer = pointer + if let setAside { + onSetAside(setAside) + } execute("PRAGMA journal_mode=WAL;") execute("PRAGMA synchronous=NORMAL;") - /// Off by default in SQLite, and `plan_snapshots.history_id` needs it: without it the - /// `ON DELETE SET NULL` never fires and a pruned history row leaves a plan pointing at a - /// row that no longer exists. + // plan_snapshots.history_id's ON DELETE SET NULL only fires with foreign keys on. execute("PRAGMA foreign_keys=ON;") sqlite3_busy_timeout(db, 3_000) createTables() - migrateIfNeeded() + guard migrateIfNeeded() else { + // A half-migrated schema reads as an empty history, so the store stays closed instead. + Self.logger.error("Query history migration failed; the store stays closed") + sqlite3_close_v2(pointer) + dbHandle.pointer = nil + return + } createFingerprintIndex() createPlanSnapshotStorage() createRewindSnapshotStorage() @@ -154,23 +161,31 @@ actor QueryHistoryStorage { createIndexes() } - private func createFtsTriggers() { - execute(""" - CREATE TRIGGER IF NOT EXISTS history_ai AFTER INSERT ON history BEGIN - INSERT INTO history_fts(rowid, query) VALUES (new.rowid, new.query); - END; - """) - execute(""" - CREATE TRIGGER IF NOT EXISTS history_ad AFTER DELETE ON history BEGIN - INSERT INTO history_fts(history_fts, rowid, query) VALUES('delete', old.rowid, old.query); - END; - """) + @discardableResult + private func createFtsTriggers() -> Bool { + let results = [ + execute(""" + CREATE TRIGGER IF NOT EXISTS history_ai AFTER INSERT ON history BEGIN + INSERT INTO history_fts(rowid, query) VALUES (new.rowid, new.query); + END; + """), + execute(""" + CREATE TRIGGER IF NOT EXISTS history_ad AFTER DELETE ON history BEGIN + INSERT INTO history_fts(history_fts, rowid, query) VALUES('delete', old.rowid, old.query); + END; + """), + ] + return !results.contains(false) } - private func createIndexes() { - execute("CREATE INDEX IF NOT EXISTS idx_history_connection_executed ON history(connection_id, executed_at DESC);") - execute("CREATE INDEX IF NOT EXISTS idx_history_executed_at ON history(executed_at DESC);") - execute("CREATE INDEX IF NOT EXISTS idx_history_source ON history(source);") + @discardableResult + private func createIndexes() -> Bool { + let results = [ + execute("CREATE INDEX IF NOT EXISTS idx_history_connection_executed ON history(connection_id, executed_at DESC);"), + execute("CREATE INDEX IF NOT EXISTS idx_history_executed_at ON history(executed_at DESC);"), + execute("CREATE INDEX IF NOT EXISTS idx_history_source ON history(source);"), + ] + return !results.contains(false) } /// Kept out of `createIndexes` because that runs before the migration, when a database written @@ -195,47 +210,26 @@ actor QueryHistoryStorage { // MARK: - Migration - /// `createTables` already builds the current shape, so a fresh database needs no migration at - /// all. Only a table left over from an older release is missing `source`, and the rebuild below - /// copies it forward by name, so every earlier version reaches v3 through the same path. - private func migrateIfNeeded() { - guard db != nil else { return } - - if hasColumn("source", inTable: "history") == false { - migrateToVersion3() - } - migrateToVersion4() - migrateToVersion5() - // Stamped only once the column it describes is actually there. Stamping regardless would - // claim a schema a failed ALTER never produced, and `record`'s insert would then fail to - // prepare against the old column count, silently recording nothing. - if hasColumn("fingerprint_hash", inTable: "history") { - setUserVersion(hasColumn("server_time", inTable: "history") ? 5 : 4) - } - } + private func migrateIfNeeded() -> Bool { + guard db != nil, let hasSource = hasColumn("source", inTable: "history") else { return false } - /// Both columns are nullable and deliberately not backfilled. A row written before the split - /// existed has no honest value to put in either, and `COALESCE` down to `execution_time` reads - /// it exactly as this release's predecessor did. - private func migrateToVersion5() { - if hasColumn("first_row_time", inTable: "history") == false { - execute("ALTER TABLE history ADD COLUMN first_row_time REAL;") + if !hasSource { + guard migrateToVersion3() else { return false } } - if hasColumn("server_time", inTable: "history") == false { - execute("ALTER TABLE history ADD COLUMN server_time REAL;") + guard addColumnIfMissing("fingerprint_hash", definition: "INTEGER NOT NULL DEFAULT 0"), + addColumnIfMissing("first_row_time", definition: "REAL"), + addColumnIfMissing("server_time", definition: "REAL") + else { + return false } + setUserVersion(5) + return true } - /// Grouping a statement by its shape needs the shape stored, because deriving it on every read - /// costs about sixty times what reading a stored column does. The column is added first and - /// filled second so an app killed between the two reopens with an unfilled column and finishes - /// the backfill, rather than with no column and a version number claiming otherwise. - /// - /// A row the v3 rebuild carried forward arrives here with the column's `0` default, so both the - /// upgrade-from-v3 path and the upgrade-from-v2 path are covered by the same backfill. - private func migrateToVersion4() { - guard hasColumn("fingerprint_hash", inTable: "history") == false else { return } - execute("ALTER TABLE history ADD COLUMN fingerprint_hash INTEGER NOT NULL DEFAULT 0;") + // first_row_time and server_time stay NULL for older rows: COALESCE falls back to execution_time. + private func addColumnIfMissing(_ column: String, definition: String) -> Bool { + guard let present = hasColumn(column, inTable: "history") else { return false } + return present || execute("ALTER TABLE history ADD COLUMN \(column) \(definition);") } /// Filling the column is deliberately not part of `init`. @@ -307,69 +301,73 @@ actor QueryHistoryStorage { return pending } - /// Renaming the table takes its triggers with it and dropping it destroys them, so between the - /// rename and `createFtsTriggers` the search index has no way to stay in step. Everything runs - /// in one transaction, and the version is only stamped once that transaction commits, so an - /// app killed midway reopens as the old schema and migrates again instead of stranding a - /// half-built index behind a version number that says the work is done. - private func migrateToVersion3() { - let legacyRows = readLegacyRowsForBackfill() - - beginTransaction() - execute("ALTER TABLE history RENAME TO history_legacy;") - execute(Self.historyTableSql) - execute(""" - INSERT INTO history ( - id, query, connection_id, database_name, database_type, schema_name, - source, statement_type, executed_at, execution_time, row_count, - was_successful, error_message - ) - SELECT - id, query, connection_id, database_name, '', NULL, - 'editor', 'other', executed_at, execution_time, row_count, - was_successful, error_message - FROM history_legacy; - """) - execute("DROP TABLE history_legacy;") - - execute("DROP TRIGGER IF EXISTS history_ai;") - execute("DROP TRIGGER IF EXISTS history_ad;") - execute("DROP TRIGGER IF EXISTS history_au;") - execute("DROP TABLE IF EXISTS history_fts;") - execute(Self.ftsTableSql) - createFtsTriggers() - execute("INSERT INTO history_fts(history_fts) VALUES('rebuild');") - - execute("DROP INDEX IF EXISTS idx_history_connection;") - createIndexes() - - applyStatementTypes(legacyRows) - setUserVersion(3) - - commitTransaction() + // One transaction, stamped inside it: any failed step rolls back to the legacy table intact. + private func migrateToVersion3() -> Bool { + guard let legacyRows = readLegacyRowsForBackfill(), beginTransaction() else { return false } + + let migrated = execute("ALTER TABLE history RENAME TO history_legacy;") + && execute(Self.historyTableSql) + && execute(""" + INSERT INTO history ( + id, query, connection_id, database_name, database_type, schema_name, + source, statement_type, executed_at, execution_time, row_count, + was_successful, error_message + ) + SELECT + id, query, connection_id, database_name, '', NULL, + 'editor', 'other', executed_at, execution_time, row_count, + was_successful, error_message + FROM history_legacy; + """) + && execute("DROP TABLE history_legacy;") + && execute("DROP TRIGGER IF EXISTS history_ai;") + && execute("DROP TRIGGER IF EXISTS history_ad;") + && execute("DROP TRIGGER IF EXISTS history_au;") + && execute("DROP TABLE IF EXISTS history_fts;") + && execute(Self.ftsTableSql) + && createFtsTriggers() + && execute("INSERT INTO history_fts(history_fts) VALUES('rebuild');") + && execute("DROP INDEX IF EXISTS idx_history_connection;") + && createIndexes() + && applyStatementTypes(legacyRows) + && setUserVersion(3) + + guard migrated else { + Self.logger.error("Query history migration to version 3 failed and was rolled back") + rollbackTransaction() + return false + } + return commitTransaction() } - private func readLegacyRowsForBackfill() -> [(id: String, statementType: String)] { + private func readLegacyRowsForBackfill() -> [(id: String, statementType: String)]? { var pending: [(id: String, statementType: String)] = [] var statement: OpaquePointer? defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2(db, "SELECT id, query FROM history;", -1, &statement, nil) == SQLITE_OK else { logSqliteError(context: "prepare backfill read") - return pending + return nil } - while sqlite3_step(statement) == SQLITE_ROW { - guard let id = sqlite3_column_text(statement, 0).map({ String(cString: $0) }), - let query = sqlite3_column_text(statement, 1).map({ String(cString: $0) }) - else { continue } - // A legacy row has no engine; standard SQL comments are enough to find a label's keyword. - pending.append((id, QueryHistoryStatementType.classify(query, grammar: .ansi).rawValue)) + while true { + switch sqlite3_step(statement) { + case SQLITE_ROW: + guard let id = sqlite3_column_text(statement, 0).map({ String(cString: $0) }), + let query = sqlite3_column_text(statement, 1).map({ String(cString: $0) }) + else { continue } + // A legacy row has no engine; standard SQL comments are enough to find a label's keyword. + pending.append((id, QueryHistoryStatementType.classify(query, grammar: .ansi).rawValue)) + case SQLITE_DONE: + return pending + default: + logSqliteError(context: "backfill read") + return nil + } } - return pending } - private func applyStatementTypes(_ rows: [(id: String, statementType: String)]) { - guard !rows.isEmpty else { return } + private func applyStatementTypes(_ rows: [(id: String, statementType: String)]) -> Bool { + guard !rows.isEmpty else { return true } var statement: OpaquePointer? defer { sqlite3_finalize(statement) } @@ -381,33 +379,42 @@ actor QueryHistoryStorage { nil ) == SQLITE_OK else { logSqliteError(context: "prepare backfill write") - return + return false } for row in rows { QueryHistorySqlBinding.text(row.statementType).bind(to: statement, at: 1) QueryHistorySqlBinding.text(row.id).bind(to: statement, at: 2) - if sqlite3_step(statement) != SQLITE_DONE { + guard sqlite3_step(statement) == SQLITE_DONE else { logSqliteError(context: "backfill statement type") + return false } sqlite3_reset(statement) sqlite3_clear_bindings(statement) } + return true } - private func hasColumn(_ column: String, inTable table: String) -> Bool { - guard db != nil else { return false } + // nil when the schema cannot be read: a guess of false would rebuild a current table as legacy. + private func hasColumn(_ column: String, inTable table: String) -> Bool? { + guard db != nil else { return nil } var statement: OpaquePointer? defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2(db, "PRAGMA table_info(\(table))", -1, &statement, nil) == SQLITE_OK else { - return false + return nil } - while sqlite3_step(statement) == SQLITE_ROW { - if let name = sqlite3_column_text(statement, 1), String(cString: name) == column { - return true + while true { + switch sqlite3_step(statement) { + case SQLITE_ROW: + if let name = sqlite3_column_text(statement, 1), String(cString: name) == column { + return true + } + case SQLITE_DONE: + return false + default: + return nil } } - return false } private func userVersion() -> Int32 { @@ -422,25 +429,29 @@ actor QueryHistoryStorage { return sqlite3_column_int(statement, 0) } - private func setUserVersion(_ version: Int32) { + @discardableResult + private func setUserVersion(_ version: Int32) -> Bool { execute("PRAGMA user_version = \(version);") } // MARK: - Statement Helpers - func execute(_ sql: String) { - guard let db else { return } + @discardableResult + func execute(_ sql: String) -> Bool { + guard let db else { return false } var statement: OpaquePointer? defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { logSqliteError(context: "prepare") - return + return false } let result = sqlite3_step(statement) - if result != SQLITE_DONE, result != SQLITE_ROW { + guard result == SQLITE_DONE || result == SQLITE_ROW else { logSqliteError(context: "step") + return false } + return true } func logSqliteError(context: String) { @@ -448,19 +459,30 @@ actor QueryHistoryStorage { Self.logger.error("Query history SQL \(context, privacy: .public) failed: \(String(cString: message), privacy: .private)") } - private func beginTransaction() { - guard let db else { return } - if sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) != SQLITE_OK { + @discardableResult + private func beginTransaction() -> Bool { + guard let db else { return false } + guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { logSqliteError(context: "begin") + return false } + return true } - private func commitTransaction() { - guard let db else { return } - if sqlite3_exec(db, "COMMIT;", nil, nil, nil) != SQLITE_OK { + @discardableResult + private func commitTransaction() -> Bool { + guard let db else { return false } + guard sqlite3_exec(db, "COMMIT;", nil, nil, nil) == SQLITE_OK else { logSqliteError(context: "commit") - sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) + rollbackTransaction() + return false } + return true + } + + private func rollbackTransaction() { + guard let db else { return } + sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) } // MARK: - Writes @@ -1101,6 +1123,9 @@ actor QueryHistoryStorage { logSqliteError(context: "clear") return false } + if Self.coversEveryRow(filter), !LocalSQLiteStoreOpener.removeSetAsideCopies(of: databaseURL) { + Self.logger.fault("History was cleared, but a damaged copy set aside earlier could not be deleted") + } return true } diff --git a/apple/TablePro/Core/Storage/QueryTabVersionStorage.swift b/apple/TablePro/Core/Storage/QueryTabVersionStorage.swift new file mode 100644 index 0000000000..2bf78efbc2 --- /dev/null +++ b/apple/TablePro/Core/Storage/QueryTabVersionStorage.swift @@ -0,0 +1,433 @@ +// +// QueryTabVersionStorage.swift +// TablePro +// + +import Combine +import Foundation +import os +import SQLite3 + +internal enum QueryTabVersionReason: Int, Sendable, CaseIterable { + case run = 0 + case idle = 1 + case beforeReplace = 2 + case beforeRestore = 3 +} + +internal struct QueryTabVersionSummary: Sendable, Equatable, Identifiable { + let id: Int64 + let reason: QueryTabVersionReason + let savedAt: Date +} + +internal struct QueryTabVersion: Sendable, Equatable { + let summary: QueryTabVersionSummary + let query: String +} + +// Device-local and never synced. Rows outlive a closed tab, since Reopen Closed Tab brings the same tab id back. +internal actor QueryTabVersionStorage { + static let shared = QueryTabVersionStorage() + static let retainedVersionCount = 50 + static let maximumRetentionDays = 30 + static let idleCoalescingWindow: TimeInterval = 5 * 60 + + private static let logger = Logger(subsystem: "com.TablePro", category: "QueryTabVersionStorage") + private static let secondsPerDay: TimeInterval = 86_400 + + private struct DatabaseHandle: @unchecked Sendable { + var pointer: OpaquePointer? + } + + private struct NewestRow { + let id: Int64 + let reason: QueryTabVersionReason? + let idleSince: Date? + let hasSameText: Bool + } + + private var dbHandle = DatabaseHandle() + private var isPrepared = false + private let databaseURL: URL + private let removeDatabaseOnDeinit: Bool + private let now: @Sendable () -> Date + private let historyMaxDays: @Sendable () -> Int + private let announce: @Sendable (UUID?) -> Void + private let reader: QueryTabVersionReader + + private var db: OpaquePointer? { + if !isPrepared { + isPrepared = true + setupDatabase() + } + return dbHandle.pointer + } + + init( + databaseURL: URL = QueryTabVersionStorage.defaultDatabaseURL(), + now: @escaping @Sendable () -> Date = Date.init, + historyMaxDays: @escaping @Sendable () -> Int = { AppSettingsStorage.shared.loadHistory().maxDays }, + announce: @escaping @Sendable (UUID?) -> Void = { tabId in + Task { @MainActor in AppEvents.shared.queryTabVersionsDidUpdate.send(tabId) } + }, + removeDatabaseOnDeinit: Bool = false + ) { + self.databaseURL = databaseURL + self.now = now + self.historyMaxDays = historyMaxDays + self.announce = announce + self.removeDatabaseOnDeinit = removeDatabaseOnDeinit + reader = QueryTabVersionReader(databaseURL: databaseURL) + } + + static func defaultDatabaseURL() -> URL { + let directory = AppStorageEnvironment.shared.applicationSupportRoot.appendingPathComponent("TablePro") + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory.appendingPathComponent("query_tab_versions.db") + } + + deinit { + if let pointer = dbHandle.pointer { + sqlite3_close_v2(pointer) + } + reader.close() + if removeDatabaseOnDeinit { + let path = databaseURL.path(percentEncoded: false) + try? FileManager.default.removeItem(atPath: path) + for suffix in ["-wal", "-shm"] { + try? FileManager.default.removeItem(atPath: path + suffix) + } + } + } + + // A shorter Query History window applies here too: this store keeps text the user deleted. + static func retentionDays(historyMaxDays: Int) -> Int { + historyMaxDays > 0 ? min(maximumRetentionDays, historyMaxDays) : maximumRetentionDays + } + + static func isRecordable(_ text: String) -> Bool { + (text as NSString).length <= TabQueryContent.maxPersistableQuerySize + && !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + nonisolated var retentionDays: Int { + Self.retentionDays(historyMaxDays: historyMaxDays()) + } + + nonisolated func summaries( + tabId: UUID, + limit: Int = QueryTabVersionStorage.retainedVersionCount + ) -> [QueryTabVersionSummary] { + reader.summaries(tabId: tabId, savedSince: cutoff(before: now()), limit: limit) + } + + nonisolated func version(id: Int64, tabId: UUID) -> QueryTabVersion? { + reader.version(id: id, tabId: tabId, savedSince: cutoff(before: now())) + } + + @discardableResult + func record(tabId: UUID, connectionId: UUID, text: String, reason: QueryTabVersionReason) -> Bool { + guard Self.isRecordable(text), let db else { return false } + let savedAt = now() + let keptSince = cutoff(before: savedAt) + let newest = newestRow(tabId: tabId, comparedWith: text, savedSince: keptSince, db: db) + if let newest, newest.hasSameText { + // The idle row already holds this text; sealing it keeps the next pause from overwriting it. + guard reason != .idle, newest.reason == .idle else { return false } + return sealRow(newest.id, tabId: tabId, reason: reason, db: db) + } + + let written: Bool + if reason == .idle, let newest, newest.reason == .idle, let idleSince = newest.idleSince, + savedAt.timeIntervalSince(idleSince) < Self.idleCoalescingWindow { + written = replaceRow(newest.id, connectionId: connectionId, text: text, savedAt: savedAt, db: db) + } else { + written = insertRow( + tabId: tabId, + connectionId: connectionId, + text: text, + reason: reason, + savedAt: savedAt, + db: db + ) + } + guard written else { + Self.logger.error("A query tab version could not be written") + return false + } + trim(tabId: tabId, savedBefore: keptSince, db: db) + announce(tabId) + return true + } + + @discardableResult + func removeVersions(forConnection connectionId: UUID) -> Bool { + guard let db else { return false } + let removed = run("DELETE FROM tab_versions WHERE connection_id = ?;", db: db) { statement in + Self.bind(connectionId.uuidString, to: statement, at: 1) + } + if removed { truncateLog(db) } + if removed { announce(nil) } + return removed + } + + @discardableResult + func removeAll() -> Bool { + guard let db else { return false } + let removed = run("DELETE FROM tab_versions;", db: db) { _ in } + if removed { truncateLog(db) } + if removed { announce(nil) } + return removed + } + + // A checkpoint alone leaves deleted text in the -wal file until its frames are reused. + private func truncateLog(_ db: OpaquePointer) { + _ = run("PRAGMA wal_checkpoint(TRUNCATE);", db: db) { _ in } + } + + // MARK: - Setup + + private static let schemaStatements = [ + "PRAGMA journal_mode=WAL;", + "PRAGMA synchronous=NORMAL;", + // Clear History is a privacy command, so deleted text is overwritten rather than left in free pages. + "PRAGMA secure_delete=ON;", + """ + CREATE TABLE IF NOT EXISTS tab_versions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tab_id TEXT NOT NULL, + connection_id TEXT NOT NULL, + query TEXT NOT NULL, + reason INTEGER NOT NULL, + saved_at REAL NOT NULL, + idle_since REAL + ); + """, + "CREATE INDEX IF NOT EXISTS idx_tab_versions_tab ON tab_versions(tab_id, id);", + "CREATE INDEX IF NOT EXISTS idx_tab_versions_connection ON tab_versions(connection_id);", + "CREATE INDEX IF NOT EXISTS idx_tab_versions_saved ON tab_versions(saved_at);", + ] + + private func setupDatabase() { + try? FileManager.default.createDirectory( + at: databaseURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + guard case .opened(let pointer, let setAside) = LocalSQLiteStoreOpener.open(at: databaseURL, now: now()) else { + Self.logger.error("The query tab versions database could not be opened") + return + } + dbHandle.pointer = pointer + // The reader may hold the file a damaged store was moved to. + reader.close() + // Snapshots of text the tabs still hold, so a damaged copy is not worth keeping. + if setAside != nil { + LocalSQLiteStoreOpener.removeSetAsideCopies(of: databaseURL) + } + for statement in Self.schemaStatements { + _ = run(statement, db: pointer) { _ in } + } + trim(tabId: nil, savedBefore: cutoff(before: now()), db: pointer) + } + + // MARK: - Writes + + private func newestRow(tabId: UUID, comparedWith text: String, savedSince cutoff: Date, db: OpaquePointer) -> NewestRow? { + let sql = """ + SELECT id, reason, idle_since, query = ? FROM tab_versions + WHERE tab_id = ? AND saved_at >= ? ORDER BY id DESC LIMIT 1; + """ + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return nil } + Self.bind(text, to: statement, at: 1) + Self.bind(tabId.uuidString, to: statement, at: 2) + sqlite3_bind_double(statement, 3, cutoff.timeIntervalSince1970) + guard sqlite3_step(statement) == SQLITE_ROW else { return nil } + let idleSince = sqlite3_column_type(statement, 2) == SQLITE_NULL + ? nil + : Date(timeIntervalSince1970: sqlite3_column_double(statement, 2)) + return NewestRow( + id: sqlite3_column_int64(statement, 0), + reason: QueryTabVersionReason(rawValue: Int(sqlite3_column_int(statement, 1))), + idleSince: idleSince, + hasSameText: sqlite3_column_int(statement, 3) != 0 + ) + } + + // The coalescing window counts from the first snapshot of the run of idle pauses, so steady typing still leaves a + // version every few minutes. + private func replaceRow(_ id: Int64, connectionId: UUID, text: String, savedAt: Date, db: OpaquePointer) -> Bool { + run("UPDATE tab_versions SET query = ?, saved_at = ?, connection_id = ? WHERE id = ?;", db: db) { statement in + Self.bind(text, to: statement, at: 1) + sqlite3_bind_double(statement, 2, savedAt.timeIntervalSince1970) + Self.bind(connectionId.uuidString, to: statement, at: 3) + sqlite3_bind_int64(statement, 4, id) + } + } + + private func sealRow(_ id: Int64, tabId: UUID, reason: QueryTabVersionReason, db: OpaquePointer) -> Bool { + let sealed = run("UPDATE tab_versions SET reason = ?, idle_since = NULL WHERE id = ?;", db: db) { statement in + sqlite3_bind_int(statement, 1, Int32(reason.rawValue)) + sqlite3_bind_int64(statement, 2, id) + } + if sealed { announce(tabId) } + return sealed + } + + private func insertRow( + tabId: UUID, + connectionId: UUID, + text: String, + reason: QueryTabVersionReason, + savedAt: Date, + db: OpaquePointer + ) -> Bool { + let sql = """ + INSERT INTO tab_versions (tab_id, connection_id, query, reason, saved_at, idle_since) + VALUES (?, ?, ?, ?, ?, ?); + """ + return run(sql, db: db) { statement in + Self.bind(tabId.uuidString, to: statement, at: 1) + Self.bind(connectionId.uuidString, to: statement, at: 2) + Self.bind(text, to: statement, at: 3) + sqlite3_bind_int(statement, 4, Int32(reason.rawValue)) + sqlite3_bind_double(statement, 5, savedAt.timeIntervalSince1970) + if reason == .idle { + sqlite3_bind_double(statement, 6, savedAt.timeIntervalSince1970) + } else { + sqlite3_bind_null(statement, 6) + } + } + } + + private func trim(tabId: UUID?, savedBefore cutoff: Date, db: OpaquePointer) { + _ = run("DELETE FROM tab_versions WHERE saved_at < ?;", db: db) { statement in + sqlite3_bind_double(statement, 1, cutoff.timeIntervalSince1970) + } + guard let tabId else { return } + let sql = """ + DELETE FROM tab_versions WHERE tab_id = ?1 AND id NOT IN ( + SELECT id FROM tab_versions WHERE tab_id = ?1 ORDER BY id DESC LIMIT \(Self.retainedVersionCount) + ); + """ + _ = run(sql, db: db) { statement in + Self.bind(tabId.uuidString, to: statement, at: 1) + } + } + + // MARK: - Helpers + + nonisolated private func cutoff(before date: Date) -> Date { + date.addingTimeInterval(-TimeInterval(retentionDays) * Self.secondsPerDay) + } + + private func run(_ sql: String, db: OpaquePointer, bind: (OpaquePointer?) -> Void) -> Bool { + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } + bind(statement) + let result = sqlite3_step(statement) + return result == SQLITE_DONE || result == SQLITE_ROW + } + + fileprivate static func bind(_ text: String, to statement: OpaquePointer?, at index: Int32) { + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + sqlite3_bind_text(statement, index, text, -1, transient) + } +} + +// Menus fill synchronously, so reads take their own connection instead of a hop to the actor. +internal final class QueryTabVersionReader: @unchecked Sendable { + private let lock = NSLock() + private let databaseURL: URL + private var handle: OpaquePointer? + + init(databaseURL: URL) { + self.databaseURL = databaseURL + } + + deinit { + if let handle { + sqlite3_close_v2(handle) + } + } + + func close() { + lock.withLock { + if let handle { + sqlite3_close_v2(handle) + } + handle = nil + } + } + + func summaries(tabId: UUID, savedSince cutoff: Date, limit: Int) -> [QueryTabVersionSummary] { + let sql = """ + SELECT id, reason, saved_at FROM tab_versions + WHERE tab_id = ? AND saved_at >= ? ORDER BY id DESC LIMIT ?; + """ + return lock.withLock { + guard let db = openIfNeeded() else { return [] } + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return [] } + QueryTabVersionStorage.bind(tabId.uuidString, to: statement, at: 1) + sqlite3_bind_double(statement, 2, cutoff.timeIntervalSince1970) + sqlite3_bind_int(statement, 3, Int32(clamping: limit)) + var rows: [QueryTabVersionSummary] = [] + while sqlite3_step(statement) == SQLITE_ROW { + if let summary = Self.summary(from: statement) { + rows.append(summary) + } + } + return rows + } + } + + func version(id: Int64, tabId: UUID, savedSince cutoff: Date) -> QueryTabVersion? { + let sql = """ + SELECT id, reason, saved_at, query FROM tab_versions + WHERE id = ? AND tab_id = ? AND saved_at >= ? LIMIT 1; + """ + return lock.withLock { + guard let db = openIfNeeded() else { return nil } + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return nil } + sqlite3_bind_int64(statement, 1, id) + QueryTabVersionStorage.bind(tabId.uuidString, to: statement, at: 2) + sqlite3_bind_double(statement, 3, cutoff.timeIntervalSince1970) + guard sqlite3_step(statement) == SQLITE_ROW, + let summary = Self.summary(from: statement), + let text = sqlite3_column_text(statement, 3) else { return nil } + return QueryTabVersion(summary: summary, query: String(cString: text)) + } + } + + // Opened without create: before the first write there is nothing to read. + private func openIfNeeded() -> OpaquePointer? { + if let handle { return handle } + let path = databaseURL.path(percentEncoded: false) + guard FileManager.default.fileExists(atPath: path) else { return nil } + var opened: OpaquePointer? + guard sqlite3_open_v2(path, &opened, SQLITE_OPEN_READWRITE, nil) == SQLITE_OK, let opened else { + sqlite3_close_v2(opened) + return nil + } + sqlite3_busy_timeout(opened, 1_000) + sqlite3_exec(opened, "PRAGMA query_only = 1;", nil, nil, nil) + handle = opened + return opened + } + + private static func summary(from statement: OpaquePointer?) -> QueryTabVersionSummary? { + guard let reason = QueryTabVersionReason(rawValue: Int(sqlite3_column_int(statement, 1))) else { return nil } + return QueryTabVersionSummary( + id: sqlite3_column_int64(statement, 0), + reason: reason, + savedAt: Date(timeIntervalSince1970: sqlite3_column_double(statement, 2)) + ) + } +} diff --git a/apple/TablePro/Core/Storage/SQLFavoriteManager.swift b/apple/TablePro/Core/Storage/SQLFavoriteManager.swift index a0ed19d3ff..fca41a492b 100644 --- a/apple/TablePro/Core/Storage/SQLFavoriteManager.swift +++ b/apple/TablePro/Core/Storage/SQLFavoriteManager.swift @@ -11,7 +11,9 @@ import TableProSyncTransport import TableProUtilities internal final class SQLFavoriteManager: @unchecked Sendable { - static let shared = SQLFavoriteManager() + static let shared = SQLFavoriteManager( + storage: SQLFavoriteStorage(onSetAside: { DamagedStoreNotice.savedQueriesSetAside($0) }) + ) private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFavoriteManager") private let storage: SQLFavoriteStorage @@ -146,11 +148,19 @@ internal final class SQLFavoriteManager: @unchecked Sendable { } func favoritesForSync() async -> [SQLFavorite]? { - await storage.readAllFavorites() + await storage.readAllFavoritesForSync() } func foldersForSync() async -> [SQLFavoriteFolder]? { - await storage.readAllFolders() + await storage.readAllFoldersForSync() + } + + func isAwaitingSyncRefill() async -> Bool { + await storage.isAwaitingSyncRefill() + } + + func finishSyncRefill() async { + await storage.finishSyncRefill() } func ledgerEntries() async -> [SavedQueryLedger.Entry]? { @@ -271,7 +281,8 @@ internal final class SQLFavoriteManager: @unchecked Sendable { ) async -> RemoteApplyOutcome { guard !batch.isEmpty else { return .skipped } return await operations.run { [self] in - let admitted = admitting(batch, echoGuard: echoGuard) + let refilling = await storage.isAwaitingSyncRefill() + let admitted = admitting(batch, echoGuard: echoGuard, refilling: refilling) guard !admitted.isEmpty else { return .skipped } guard await applyRemoteFavoriteDeletions(admitted.deletedFavoriteIds), await applyRemoteFolders(admitted.folders), @@ -285,9 +296,13 @@ internal final class SQLFavoriteManager: @unchecked Sendable { } @MainActor - private func admitting(_ batch: RemoteSQLFavoriteBatch, echoGuard: SyncEchoGuard?) -> RemoteSQLFavoriteBatch { - let deletedFavoriteIds = syncTracker.tombstonedIds(for: .favorite) - let deletedFolderIds = syncTracker.tombstonedIds(for: .favoriteFolder) + private func admitting( + _ batch: RemoteSQLFavoriteBatch, + echoGuard: SyncEchoGuard?, + refilling: Bool + ) -> RemoteSQLFavoriteBatch { + let deletedFavoriteIds = locallyDeletedIds(.favorite, refilling: refilling) + let deletedFolderIds = locallyDeletedIds(.favoriteFolder, refilling: refilling) var admitted = batch admitted.favorites = batch.favorites.filter { favorite in let id = favorite.id.uuidString @@ -302,6 +317,14 @@ internal final class SQLFavoriteManager: @unchecked Sendable { return admitted } + // While a replaced store refills, a record deleted and then edited again here comes back: + // the edit is the later intent, and pushing the deletion would remove it on every device. + @MainActor + private func locallyDeletedIds(_ type: SyncRecordType, refilling: Bool) -> Set { + let tombstoned = syncTracker.tombstonedIds(for: type) + return refilling ? tombstoned.subtracting(syncTracker.dirtyRecords(for: type)) : tombstoned + } + @MainActor private func applyRemoteFavoriteDeletions(_ ids: Set) async -> Bool { guard !ids.isEmpty else { return true } diff --git a/apple/TablePro/Core/Storage/SQLFavoriteStorage+Migration.swift b/apple/TablePro/Core/Storage/SQLFavoriteStorage+Migration.swift new file mode 100644 index 0000000000..0bbc68c7c8 --- /dev/null +++ b/apple/TablePro/Core/Storage/SQLFavoriteStorage+Migration.swift @@ -0,0 +1,59 @@ +import Foundation + +extension SQLFavoriteStorage { + // Run in one transaction by migrateIfNeeded; the version stamp is the last step. + static let version2MigrationStatements: [String] = [ + "ALTER TABLE favorites RENAME TO favorites_old", + """ + CREATE TABLE IF NOT EXISTS favorites ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, query TEXT NOT NULL, + keyword TEXT, folder_id TEXT, connection_id TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL + ) + """, + """ + INSERT INTO favorites SELECT id, name, query, keyword, folder_id, connection_id, + sort_order, created_at, updated_at FROM favorites_old + """, + "DROP TABLE favorites_old", + "ALTER TABLE folders RENAME TO folders_old", + """ + CREATE TABLE IF NOT EXISTS folders ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, parent_id TEXT, connection_id TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL + ) + """, + """ + INSERT INTO folders SELECT id, name, parent_id, connection_id, + sort_order, created_at, updated_at FROM folders_old + """, + "DROP TABLE folders_old", + "CREATE INDEX IF NOT EXISTS idx_favorites_connection ON favorites(connection_id);", + "CREATE INDEX IF NOT EXISTS idx_favorites_folder ON favorites(folder_id);", + "CREATE INDEX IF NOT EXISTS idx_favorites_keyword ON favorites(keyword);", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_favorites_keyword_scope ON favorites(keyword, connection_id) WHERE keyword IS NOT NULL;", + "CREATE INDEX IF NOT EXISTS idx_folders_connection ON folders(connection_id);", + "CREATE INDEX IF NOT EXISTS idx_folders_parent ON folders(parent_id);", + "DROP TRIGGER IF EXISTS favorites_ai;", + "DROP TRIGGER IF EXISTS favorites_ad;", + "DROP TRIGGER IF EXISTS favorites_au;", + """ + CREATE TRIGGER IF NOT EXISTS favorites_ai AFTER INSERT ON favorites BEGIN + INSERT INTO favorites_fts(rowid, name, query, keyword) VALUES (new.rowid, new.name, new.query, new.keyword); + END; + """, + """ + CREATE TRIGGER IF NOT EXISTS favorites_ad AFTER DELETE ON favorites BEGIN + INSERT INTO favorites_fts(favorites_fts, rowid, name, query, keyword) VALUES('delete', old.rowid, old.name, old.query, old.keyword); + END; + """, + """ + CREATE TRIGGER IF NOT EXISTS favorites_au AFTER UPDATE ON favorites BEGIN + INSERT INTO favorites_fts(favorites_fts, rowid, name, query, keyword) VALUES('delete', old.rowid, old.name, old.query, old.keyword); + INSERT INTO favorites_fts(rowid, name, query, keyword) VALUES (new.rowid, new.name, new.query, new.keyword); + END; + """, + "INSERT INTO favorites_fts(favorites_fts) VALUES('rebuild');", + "PRAGMA user_version = 2;", + ] +} diff --git a/apple/TablePro/Core/Storage/SQLFavoriteStorage+SyncRefill.swift b/apple/TablePro/Core/Storage/SQLFavoriteStorage+SyncRefill.swift new file mode 100644 index 0000000000..a64c61ff33 --- /dev/null +++ b/apple/TablePro/Core/Storage/SQLFavoriteStorage+SyncRefill.swift @@ -0,0 +1,53 @@ +import Foundation +import os +import SQLite3 + +// A store that replaced a damaged one is not the user's library for sync until a full fetch refills it. +extension SQLFavoriteStorage { + private static let syncRefillLogger = Logger(subsystem: "com.TablePro", category: "SQLFavoriteSyncRefill") + + func isAwaitingSyncRefill() -> Bool { + _ = db + return syncRefillPending + } + + func readAllFavoritesForSync() -> [SQLFavorite]? { + isAwaitingSyncRefill() ? nil : readAllFavorites() + } + + func readAllFoldersForSync() -> [SQLFavoriteFolder]? { + isAwaitingSyncRefill() ? nil : readAllFolders() + } + + func finishSyncRefill() { + guard isAwaitingSyncRefill() else { return } + guard run("DROP TABLE IF EXISTS sync_refill;") else { + Self.syncRefillLogger.error("The saved queries refill marker could not be cleared; sync keeps waiting") + return + } + syncRefillPending = false + } + + func loadSyncRefillState(afterSetAside setAside: Bool) { + guard setAside else { + syncRefillPending = hasSyncRefillMarker() + return + } + // Pending for this session even when the marker cannot be written. + syncRefillPending = true + let marked = run("CREATE TABLE IF NOT EXISTS sync_refill (id INTEGER PRIMARY KEY CHECK (id = 1));") + && run("INSERT OR IGNORE INTO sync_refill (id) VALUES (1);") + if !marked { + Self.syncRefillLogger.fault("The saved queries refill marker could not be written") + } + } + + private func hasSyncRefillMarker() -> Bool { + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, "SELECT 1 FROM sync_refill LIMIT 1;", -1, &statement, nil) == SQLITE_OK else { + return false + } + return sqlite3_step(statement) == SQLITE_ROW + } +} diff --git a/apple/TablePro/Core/Storage/SQLFavoriteStorage.swift b/apple/TablePro/Core/Storage/SQLFavoriteStorage.swift index 9350f4e701..da0246b1b4 100644 --- a/apple/TablePro/Core/Storage/SQLFavoriteStorage.swift +++ b/apple/TablePro/Core/Storage/SQLFavoriteStorage.swift @@ -26,13 +26,17 @@ internal actor SQLFavoriteStorage { private let databaseURL: URL private let removeDatabaseOnDeinit: Bool + private let onSetAside: @Sendable (SetAsideDamagedStore) -> Void + var syncRefillPending = false init( databaseURL: URL = SQLFavoriteStorage.defaultDatabaseURL(), - removeDatabaseOnDeinit: Bool = false + removeDatabaseOnDeinit: Bool = false, + onSetAside: @escaping @Sendable (SetAsideDamagedStore) -> Void = { _ in } ) { self.databaseURL = databaseURL self.removeDatabaseOnDeinit = removeDatabaseOnDeinit + self.onSetAside = onSetAside } static func defaultDatabaseURL() -> URL { @@ -61,16 +65,14 @@ internal actor SQLFavoriteStorage { let dir = databaseURL.deletingLastPathComponent() try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - let dbPath = databaseURL.path(percentEncoded: false) - - var pointer: OpaquePointer? - let openResult = sqlite3_open(dbPath, &pointer) - guard openResult == SQLITE_OK else { - Self.logger.error("Error opening database: \(String(cString: sqlite3_errstr(openResult)), privacy: .public)") - sqlite3_close_v2(pointer) + guard case .opened(let pointer, let setAside) = LocalSQLiteStoreOpener.open(at: databaseURL) else { + Self.logger.error("The saved queries database could not be opened") return } dbHandle.pointer = pointer + if let setAside { + onSetAside(setAside) + } execute("PRAGMA journal_mode=WAL;") execute("PRAGMA synchronous=NORMAL;") @@ -78,12 +80,16 @@ internal actor SQLFavoriteStorage { createTables() migrateIfNeeded() Self.versionSchemaStatements.forEach { execute($0) } + loadSyncRefillState(afterSetAside: setAside != nil) } // MARK: - Schema Migration private func migrateIfNeeded() { - let currentVersion = getUserVersion() + guard let currentVersion = getUserVersion() else { + Self.logger.error("The saved queries schema version could not be read, so no migration ran") + return + } if currentVersion < 1 { setUserVersion(2) @@ -91,73 +97,24 @@ internal actor SQLFavoriteStorage { } if currentVersion < 2 { - execute("ALTER TABLE favorites RENAME TO favorites_old") - execute(""" - CREATE TABLE IF NOT EXISTS favorites ( - id TEXT PRIMARY KEY, name TEXT NOT NULL, query TEXT NOT NULL, - keyword TEXT, folder_id TEXT, connection_id TEXT, - sort_order INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL - ) - """) - execute(""" - INSERT INTO favorites SELECT id, name, query, keyword, folder_id, connection_id, - sort_order, created_at, updated_at FROM favorites_old - """) - execute("DROP TABLE favorites_old") - - execute("ALTER TABLE folders RENAME TO folders_old") - execute(""" - CREATE TABLE IF NOT EXISTS folders ( - id TEXT PRIMARY KEY, name TEXT NOT NULL, parent_id TEXT, connection_id TEXT, - sort_order INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL - ) - """) - execute(""" - INSERT INTO folders SELECT id, name, parent_id, connection_id, - sort_order, created_at, updated_at FROM folders_old - """) - execute("DROP TABLE folders_old") - - execute("CREATE INDEX IF NOT EXISTS idx_favorites_connection ON favorites(connection_id);") - execute("CREATE INDEX IF NOT EXISTS idx_favorites_folder ON favorites(folder_id);") - execute("CREATE INDEX IF NOT EXISTS idx_favorites_keyword ON favorites(keyword);") - execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_favorites_keyword_scope ON favorites(keyword, connection_id) WHERE keyword IS NOT NULL;") - execute("CREATE INDEX IF NOT EXISTS idx_folders_connection ON folders(connection_id);") - execute("CREATE INDEX IF NOT EXISTS idx_folders_parent ON folders(parent_id);") - - execute("DROP TRIGGER IF EXISTS favorites_ai;") - execute("DROP TRIGGER IF EXISTS favorites_ad;") - execute("DROP TRIGGER IF EXISTS favorites_au;") - execute(""" - CREATE TRIGGER IF NOT EXISTS favorites_ai AFTER INSERT ON favorites BEGIN - INSERT INTO favorites_fts(rowid, name, query, keyword) VALUES (new.rowid, new.name, new.query, new.keyword); - END; - """) - execute(""" - CREATE TRIGGER IF NOT EXISTS favorites_ad AFTER DELETE ON favorites BEGIN - INSERT INTO favorites_fts(favorites_fts, rowid, name, query, keyword) VALUES('delete', old.rowid, old.name, old.query, old.keyword); - END; - """) - execute(""" - CREATE TRIGGER IF NOT EXISTS favorites_au AFTER UPDATE ON favorites BEGIN - INSERT INTO favorites_fts(favorites_fts, rowid, name, query, keyword) VALUES('delete', old.rowid, old.name, old.query, old.keyword); - INSERT INTO favorites_fts(rowid, name, query, keyword) VALUES (new.rowid, new.name, new.query, new.keyword); - END; - """) - - execute("INSERT INTO favorites_fts(favorites_fts) VALUES('rebuild');") - - setUserVersion(2) + // Stamped inside the transaction, so a failed step rolls back to the old tables intact. + let migrated = inTransaction { () -> Bool? in + let applied = Self.version2MigrationStatements.allSatisfy { execute($0) } + return applied ? true : nil + } + if migrated == nil { + Self.logger.error("The saved queries migration to version 2 failed and was rolled back") + } } } - private func getUserVersion() -> Int32 { + private func getUserVersion() -> Int32? { var statement: OpaquePointer? defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &statement, nil) == SQLITE_OK, sqlite3_step(statement) == SQLITE_ROW else { - return 0 + return nil } return sqlite3_column_int(statement, 0) } @@ -242,18 +199,21 @@ internal actor SQLFavoriteStorage { // MARK: - Helper Methods - private func execute(_ sql: String) { + @discardableResult + private func execute(_ sql: String) -> Bool { var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } let prepareResult = sqlite3_prepare_v2(db, sql, -1, &statement, nil) - if prepareResult == SQLITE_OK { - let stepResult = sqlite3_step(statement) - if stepResult != SQLITE_DONE && stepResult != SQLITE_ROW { - Self.logger.error("sqlite3_step failed (\(stepResult)): \(String(cString: sqlite3_errmsg(self.db)))") - } - } else { + guard prepareResult == SQLITE_OK else { Self.logger.error("sqlite3_prepare_v2 failed (\(prepareResult)): \(String(cString: sqlite3_errmsg(self.db)))") + return false + } + let stepResult = sqlite3_step(statement) + guard stepResult == SQLITE_DONE || stepResult == SQLITE_ROW else { + Self.logger.error("sqlite3_step failed (\(stepResult)): \(String(cString: sqlite3_errmsg(self.db)))") + return false } - sqlite3_finalize(statement) + return true } // MARK: - Favorite Operations diff --git a/apple/TablePro/Core/Storage/TabDiskActor.swift b/apple/TablePro/Core/Storage/TabDiskActor.swift index a91cc40df4..cfe4a44a76 100644 --- a/apple/TablePro/Core/Storage/TabDiskActor.swift +++ b/apple/TablePro/Core/Storage/TabDiskActor.swift @@ -2,11 +2,8 @@ // TabDiskActor.swift // TablePro // -// Thread-safe actor for tab state persistence. -// Replaces TabStateStorage with actor-based serialization -// to eliminate data races on concurrent file writes. -// +import CryptoKit import Foundation import os @@ -52,9 +49,15 @@ private struct LossyTab: Decodable { } } +// What this process last wrote or read for one connection, so an unchanged save leaves the disk alone. +internal struct TabDiskWrittenState { + var stateDigest: SHA256.Digest? + var sidecars = TabQueryOverflowStore.WrittenSidecars() +} + private final class TabDiskConnectionWriteGate: @unchecked Sendable { private let generation = OSAllocatedUnfairLock(initialState: UInt64.zero) - private let operation = OSAllocatedUnfairLock(initialState: ()) + private let operation = OSAllocatedUnfairLock(initialState: TabDiskWrittenState()) func issueToken() -> UInt64 { generation.withLock { value in @@ -65,17 +68,17 @@ private final class TabDiskConnectionWriteGate: @unchecked Sendable { func performIfCurrent( token: UInt64, - operation work: () throws -> T + operation work: (inout TabDiskWrittenState) throws -> T ) rethrows -> T? { - try operation.withLockUnchecked { _ in + try operation.withLockUnchecked { written in guard generation.withLock({ $0 == token }) else { return nil } - return try work() + return try work(&written) } } - func perform(_ work: () throws -> T) rethrows -> T { - try operation.withLockUnchecked { _ in - try work() + func perform(_ work: (inout TabDiskWrittenState) throws -> T) rethrows -> T { + try operation.withLockUnchecked { written in + try work(&written) } } } @@ -116,7 +119,7 @@ internal actor TabDiskActor { private init() { let directory = Self.resolvedTabStateDirectory() tabStateDirectory = directory - encoder = JSONEncoder() + encoder = Self.makeEncoder() decoder = JSONDecoder() do { @@ -137,26 +140,62 @@ internal actor TabDiskActor { lastActiveSchema: String? = nil, writeToken: UInt64 ) throws -> Bool { - let result = try Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { - let state = TabDiskState( - tabs: TabQueryOverflowStore.externalize( - tabs, - connectionId: connectionId, - tabStateDirectory: tabStateDirectory - ), + let result = try Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + try Self.write( + tabs: tabs, selectedTabId: selectedTabId, lastActiveDatabase: lastActiveDatabase, - lastActiveSchema: lastActiveSchema + lastActiveSchema: lastActiveSchema, + connectionId: connectionId, + directory: tabStateDirectory, + encoder: encoder, + written: &written ) - let data = try encoder.encode(state) - let fileURL = tabStateFileURL(for: connectionId) - try data.write(to: fileURL, options: .atomic) } return result != nil } + // Sorted keys, so a dictionary rebuilt in another order encodes to the same bytes and the unchanged check holds. + nonisolated private static func makeEncoder() -> JSONEncoder { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return encoder + } + + // Every save funnels here: the 30 s timer, the draft and tab-switch saves, and quit. Most find nothing new. + nonisolated private static func write( + tabs: [PersistedTab], + selectedTabId: UUID?, + lastActiveDatabase: String?, + lastActiveSchema: String?, + connectionId: UUID, + directory: URL, + encoder: JSONEncoder, + written: inout TabDiskWrittenState + ) throws { + let state = TabDiskState( + tabs: TabQueryOverflowStore.externalize( + tabs, + connectionId: connectionId, + tabStateDirectory: directory, + written: &written.sidecars + ), + selectedTabId: selectedTabId, + lastActiveDatabase: lastActiveDatabase, + lastActiveSchema: lastActiveSchema + ) + let data = try encoder.encode(state) + let digest = SHA256.hash(data: data) + let fileURL = directory.appendingPathComponent("\(connectionId.uuidString).json") + if digest == written.stateDigest, FileManager.default.fileExists(atPath: fileURL.path) { + return + } + try data.write(to: fileURL, options: .atomic) + written.stateDigest = digest + } + internal func load(connectionId: UUID) -> TabDiskState? { - Self.writeGate(for: connectionId).perform { + Self.writeGate(for: connectionId).perform { written in let fileURL = tabStateFileURL(for: connectionId) guard FileManager.default.fileExists(atPath: fileURL.path) else { @@ -166,13 +205,20 @@ internal actor TabDiskActor { do { let data = try Data(contentsOf: fileURL) let state = try decoder.decode(TabDiskState.self, from: data) + written.stateDigest = SHA256.hash(data: data) return TabDiskState( - tabs: TabQueryOverflowStore.inline(state.tabs, tabStateDirectory: tabStateDirectory), + tabs: TabQueryOverflowStore.inline( + state.tabs, + tabStateDirectory: tabStateDirectory, + written: &written.sidecars + ), selectedTabId: state.selectedTabId, lastActiveDatabase: state.lastActiveDatabase, lastActiveSchema: state.lastActiveSchema ) } catch { + // The file no longer holds what this process wrote, so the next save must write it. + written = TabDiskWrittenState() Self.logger.error("Failed to load tab state for \(connectionId): \(error.localizedDescription)") return nil } @@ -180,7 +226,8 @@ internal actor TabDiskActor { } internal func clear(connectionId: UUID, writeToken: UInt64) -> Bool { - let result = Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { + let result = Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + written = TabDiskWrittenState() TabQueryOverflowStore.removeAll(connectionId: connectionId, tabStateDirectory: tabStateDirectory) let fileURL = tabStateFileURL(for: connectionId) @@ -227,21 +274,18 @@ internal actor TabDiskActor { let writeToken = issueWriteToken(for: connectionId) let directory = resolvedTabStateDirectory() do { - _ = try writeGate(for: connectionId).performIfCurrent(token: writeToken) { - let state = TabDiskState( - tabs: TabQueryOverflowStore.externalize( - tabs, - connectionId: connectionId, - tabStateDirectory: directory - ), + _ = try writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try write( + tabs: tabs, selectedTabId: selectedTabId, lastActiveDatabase: lastActiveDatabase, - lastActiveSchema: lastActiveSchema + lastActiveSchema: lastActiveSchema, + connectionId: connectionId, + directory: directory, + encoder: makeEncoder(), + written: &written ) - let data = try JSONEncoder().encode(state) - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - let fileURL = tabStateFileURL(for: connectionId) - try data.write(to: fileURL, options: .atomic) } } catch { logger.fault("saveSync failed for \(connectionId): \(error.localizedDescription)") @@ -250,7 +294,8 @@ internal actor TabDiskActor { nonisolated internal static func clearSync(connectionId: UUID) { let writeToken = issueWriteToken(for: connectionId) - _ = writeGate(for: connectionId).performIfCurrent(token: writeToken) { + _ = writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + written = TabDiskWrittenState() TabQueryOverflowStore.removeAll( connectionId: connectionId, tabStateDirectory: resolvedTabStateDirectory() diff --git a/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift b/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift index b3977473f2..fe2a2fe0a6 100644 --- a/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift +++ b/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift @@ -2,17 +2,21 @@ // TabQueryOverflowStore.swift // TablePro // -// Keeps a query too large to sit inside the tab-state JSON in a sidecar file -// beside it, so restoring a tab returns the text instead of an empty editor. -// Mirrors the sidecar `RecentlyClosedTabStore` already uses for the same limit. -// +import CryptoKit import Foundation import os +// A query too large for the tab-state JSON lives in a sidecar beside it, as in `RecentlyClosedTabStore`. internal enum TabQueryOverflowStore { private static let logger = Logger(subsystem: "com.TablePro", category: "TabDiskActor") + // The sidecars one connection has on disk from this process, so an unchanged query is not rewritten. + internal struct WrittenSidecars { + var digests: [String: SHA256.Digest] = [:] + var keptByLastPrune: Set? + } + static func directory(inside tabStateDirectory: URL) -> URL { tabStateDirectory.appendingPathComponent("Overflow", isDirectory: true) } @@ -21,34 +25,46 @@ internal enum TabQueryOverflowStore { "\(connectionId.uuidString)_\(tabId.uuidString).sql" } - /// Moves any oversized query out to a sidecar and blanks the inline field. - /// Sidecars this connection no longer references are removed in the same pass, - /// so a closed tab does not leave its text behind forever. + // Sidecars the connection no longer references are removed in the same pass, so a closed tab leaves no text behind. static func externalize( _ tabs: [PersistedTab], connectionId: UUID, - tabStateDirectory: URL + tabStateDirectory: URL, + written: inout WrittenSidecars ) -> [PersistedTab] { let overflowDirectory = directory(inside: tabStateDirectory) - var written: Set = [] + var kept: Set = [] let processed = tabs.map { tab -> PersistedTab in guard (tab.query as NSString).length > TabQueryContent.maxPersistableQuerySize else { return tab } let name = fileName(connectionId: connectionId, tabId: tab.id) - guard write(tab.query, named: name, in: overflowDirectory) else { return tab } - written.insert(name) + let digest = digest(of: tab.query) + let path = overflowDirectory.appendingPathComponent(name).path + if written.digests[name] != digest || !FileManager.default.fileExists(atPath: path) { + written.digests[name] = nil + guard write(tab.query, named: name, in: overflowDirectory) else { return tab } + written.digests[name] = digest + } + kept.insert(name) var copy = tab copy.query = "" copy.overflowFileName = name return copy } - pruneOrphans(connectionId: connectionId, keeping: written, in: overflowDirectory) + written.digests = written.digests.filter { kept.contains($0.key) } + if written.keptByLastPrune != kept { + pruneOrphans(connectionId: connectionId, keeping: kept, in: overflowDirectory) + written.keptByLastPrune = kept + } return processed } - /// Reads sidecar text back into the tabs it belongs to. - static func inline(_ tabs: [PersistedTab], tabStateDirectory: URL) -> [PersistedTab] { + static func inline( + _ tabs: [PersistedTab], + tabStateDirectory: URL, + written: inout WrittenSidecars + ) -> [PersistedTab] { let overflowDirectory = directory(inside: tabStateDirectory) return tabs.map { tab -> PersistedTab in guard let name = tab.overflowFileName else { return tab } @@ -57,12 +73,20 @@ internal enum TabQueryOverflowStore { logger.error("Missing overflow sidecar \(name, privacy: .public)") return tab } + written.digests[name] = digest(of: text) var copy = tab copy.query = text return copy } } + private static func digest(of query: String) -> SHA256.Digest { + if let digest = query.utf8.withContiguousStorageIfAvailable({ SHA256.hash(data: UnsafeRawBufferPointer($0)) }) { + return digest + } + return SHA256.hash(data: Data(query.utf8)) + } + static func removeAll(connectionId: UUID, tabStateDirectory: URL) { pruneOrphans(connectionId: connectionId, keeping: [], in: directory(inside: tabStateDirectory)) } diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DirtyMarking.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DirtyMarking.swift new file mode 100644 index 0000000000..e9b9580b57 --- /dev/null +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DirtyMarking.swift @@ -0,0 +1,75 @@ +import Foundation +import os +import TableProSyncTransport + +extension SyncCoordinator { + /// Marks existing SQL favorites and folders dirty. Separate from `markAllLocalDataDirty` + /// because the favorite store is an actor and must be read asynchronously. + func markSQLFavoritesDirty() async { + let favorites = await services.sqlFavoriteManager.fetchFavorites() + changeTracker.markDirty(.favorite, ids: favorites.map { $0.id.uuidString }) + + let folders = await services.sqlFavoriteManager.fetchFolders() + changeTracker.markDirty(.favoriteFolder, ids: folders.map { $0.id.uuidString }) + } + + /// Marks every synced record dirty so the first sync after enabling pushes the lot. + /// + /// Every type is marked as one batch. Marking record by record posted a change notification per + /// record, and an account with a few hundred saved column layouts built a chain of hundreds of + /// tasks each waiting on its predecessor until the app stopped responding to the switch that + /// started it. + func markAllLocalDataDirty() { + let connections = services.connectionStorage.loadConnections() + changeTracker.markDirty( + .connection, + ids: connections.filter(\.participatesInSync).map { $0.id.uuidString } + ) + + let groups = services.groupStorage.loadGroups() + changeTracker.markDirty(.group, ids: groups.map { $0.id.uuidString }) + + let tags = services.tagStorage.loadTags() + changeTracker.markDirty(.tag, ids: tags.map { $0.id.uuidString }) + + let sshProfiles = services.sshProfileStorage.loadProfiles() + changeTracker.markDirty(.sshProfile, ids: sshProfiles.map { $0.id.uuidString }) + + let credentialProfiles = services.credentialProfileStorage.loadProfiles() + changeTracker.markDirty(.credentialProfile, ids: credentialProfiles.map { $0.id.uuidString }) + + let favoriteTables = services.favoriteTablesStorage.loadFavorites() + changeTracker.markDirty( + .tableFavorite, + ids: favoriteTables.map { FavoriteTablesStorage.syncId(for: $0) } + ) + + let favoriteDatabases = services.favoriteDatabasesStorage.loadFavorites() + changeTracker.markDirty( + .favoriteDatabase, + ids: favoriteDatabases.map { FavoriteDatabasesStorage.syncId(for: $0) } + ) + + let tableFolderIds = tableFolderSyncIds() + changeTracker.markDirty(.tableFolder, ids: tableFolderIds.folders) + changeTracker.markDirty(.tableFolderItem, ids: tableFolderIds.items) + + let settingsCategories = AppSettingsCategory.synced + [CustomSlashCommandStorage.syncCategory] + let columnLayoutCategories = columnLayouts().customizedStorageKeys() + .map { FileColumnLayoutPersister.syncCategory(for: $0) } + let themeCategories = themeCatalog.syncCategories + changeTracker.markDirty(.settings, ids: settingsCategories + columnLayoutCategories + themeCategories) + + let summary = [ + "connections=\(connections.count)", + "groups=\(groups.count)", + "tags=\(tags.count)", + "sshProfiles=\(sshProfiles.count)", + "favoriteTables=\(favoriteTables.count)", + "tableFolders=\(tableFolderIds.folders.count)", + "settings=\(AppSettingsCategory.synced.count + 1)", + "themes=\(themeCategories.count)" + ].joined(separator: ", ") + Self.logger.info("Marked all local data dirty: \(summary, privacy: .public)") + } +} diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift index 31ee8e5537..f5be2f2cac 100644 --- a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift @@ -37,8 +37,9 @@ extension SyncCoordinator { pushed: Set ) async -> [CKRecord.ID] { let fetchedCreations = result.changedRecords.compactMap(recordCreationDate) - guard result.unreadRecordIDs.isEmpty, !fetchedCreations.isEmpty else { return [] } - let fetchedNames = Set(result.changedRecords.map(\.recordID.recordName)) + guard !fetchedCreations.isEmpty else { return [] } + // A record the fetch listed but could not read is still in the zone, so it counts as listed. + let fetchedNames = Set((result.changedRecords.map(\.recordID) + result.unreadRecordIDs).map(\.recordName)) let boundary = syncBoundary(settings: settings) var candidates: [SyncFullFetchDeletions.Candidate] = [] for (type, ids) in await localRecordIds() diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Push.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Push.swift new file mode 100644 index 0000000000..ce893efdb7 --- /dev/null +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Push.swift @@ -0,0 +1,23 @@ +import Foundation +import TableProSyncTransport + +extension SyncCoordinator { + // CloudKit registers the device for its own notices, so the token goes nowhere. + func pushRegistrationDidFinish(registered: Bool) { + pushChannel.registrationDidFinish(registered: registered) + } + + func handleRemoteNotification(_ userInfo: [AnyHashable: Any]) { + guard syncStatus.isEnabled, SyncPushSubscription.announcesChange(userInfo) else { return } + pushChannel.noticeArrived() + requestSync(.remoteChange) + } + + var isPushLive: Bool { + pushChannel.isLive + } + + var activationHoldInterval: Duration { + isPushLive ? Self.pushFallbackInterval : Self.activationInterval + } +} diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift index e042c7bd28..47207d218b 100644 --- a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+RemoteDeletions.swift @@ -98,12 +98,14 @@ extension SyncCoordinator { var connections = services.connectionStorage.loadConnections() let deletedIds = ids.subtracting(connections.filter(\.localOnly).map(\.id)) guard !deletedIds.isEmpty else { return true } + let removed = connections.filter { deletedIds.contains($0.id) } connections.removeAll { deletedIds.contains($0.id) } guard services.connectionStorage.saveConnections(connections, byUser: false) else { Self.logger.error("Failed to apply remote connection deletions: persistence error") return false } changeTracker.discardDirty(.connection, ids: deletedIds.map(\.uuidString)) + services.connectionStorage.deleteSecrets(of: removed, origin: .remote) ConnectionLocalState.purge( connectionIds: deletedIds, origin: .remote, diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+SavedQueriesRefill.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+SavedQueriesRefill.swift new file mode 100644 index 0000000000..c462235b79 --- /dev/null +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+SavedQueriesRefill.swift @@ -0,0 +1,29 @@ +import CloudKit +import Foundation +import os +import TableProSyncTransport + +// A replaced saved queries store refills only from a full fetch: a delta never resends what this Mac pulled. +extension SyncCoordinator { + private static let refillLogger = Logger(subsystem: "com.TablePro", category: "SyncSavedQueriesRefill") + + func clearTokenForSavedQueriesRefill(settings: SyncSettings) async { + guard settings.syncSQLFavorites, await services.sqlFavoriteManager.isAwaitingSyncRefill() else { return } + metadataStorage.saveToken(nil) + Self.refillLogger.info("Saved queries were set aside as damaged: cleared sync token for full fetch") + } + + // A saved query the fetch could not read never reached the store, so the refill waits for the next full fetch. + func finishSavedQueriesRefill(after result: PullResult, settings: SyncSettings) async { + guard settings.syncSQLFavorites else { return } + let leftSavedQueriesUnread = result.unreadRecordIDs.contains { recordID in + guard let type = SyncRecordMapper.parse(recordName: recordID.recordName)?.type else { return false } + return type == .favorite || type == .favoriteFolder + } + guard !leftSavedQueriesUnread else { + Self.refillLogger.info("A full fetch left saved queries unread: the refill waits for the next one") + return + } + await services.sqlFavoriteManager.finishSyncRefill() + } +} diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift index ef318b30cb..5f058b5d0a 100644 --- a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+Settings.swift @@ -22,6 +22,7 @@ extension SyncCoordinator { case AppSettingsCategory.ai: return try encoder.encode(storage.loadAI()) case AppSettingsCategory.notifications: return try encoder.encode(storage.loadNotifications()) case AppSettingsCategory.formatter: return try encoder.encode(storage.loadFormatter()) + case AppSettingsCategory.valueColors: return try encoder.encode(storage.loadValueColors()) case CustomSlashCommandStorage.syncCategory: return try encoder.encode(CustomSlashCommandStorage.shared.commands) case let category where category.hasPrefix(FileColumnLayoutPersister.syncCategoryPrefix): @@ -57,6 +58,8 @@ extension SyncCoordinator { manager.notifications = try decoder.decode(NotificationSettings.self, from: data) case AppSettingsCategory.formatter: manager.formatter = try decoder.decode(SQLFormatterSettings.self, from: data) + case AppSettingsCategory.valueColors: + manager.valueColors = try decoder.decode(ValueColorSettings.self, from: data) case CustomSlashCommandStorage.syncCategory: CustomSlashCommandStorage.shared.applyRemote(try decoder.decode([CustomSlashCommand].self, from: data)) case let category where category.hasPrefix(FileColumnLayoutPersister.syncCategoryPrefix): @@ -82,9 +85,17 @@ extension SyncCoordinator { // Fetches the zone once for a formatter record this Mac skipped while on an older build. static let formatterContentKind = "\(SyncRecordType.settings.rawValue)/\(AppSettingsCategory.formatter).v1" + // Fetches the zone once for a value colors record this Mac skipped while on an older build. + static let valueColorsContentKind = "\(SyncRecordType.settings.rawValue)/\(AppSettingsCategory.valueColors).v1" + static var readableContent: Set { Set(SyncRecordType.verifiedInProduction.map(\.rawValue)) - .union([Self.themeContentKind, Self.formatterContentKind, ConnectionSyncField.sshRouteContentKind]) + .union([ + Self.themeContentKind, + Self.formatterContentKind, + Self.valueColorsContentKind, + ConnectionSyncField.sshRouteContentKind, + ]) } // Only what this Mac applies: a pull skips a category that is off and still moves the token past it. @@ -96,6 +107,7 @@ extension SyncCoordinator { if settings.syncs(.settings) { content.insert(Self.themeContentKind) content.insert(Self.formatterContentKind) + content.insert(Self.valueColorsContentKind) } return content } diff --git a/apple/TablePro/Core/Sync/SyncCoordinator.swift b/apple/TablePro/Core/Sync/SyncCoordinator.swift index 3455061e16..5db2c92cf3 100644 --- a/apple/TablePro/Core/Sync/SyncCoordinator.swift +++ b/apple/TablePro/Core/Sync/SyncCoordinator.swift @@ -62,6 +62,19 @@ final class SyncCoordinator: ObservableObject { /// suspended across the network can tell whether its outcome is still the current answer. private var statusGeneration = 0 + // An activation asks only for what other Macs sent, so one download per interval answers every activation in it. + nonisolated static let activationInterval: Duration = .seconds(60) + private let activationClock: any Clock + private var activationCooldown: Task? + private var activationHeldForCooldown = false + private var downloadIsAheadInRun = false + + // While CloudKit's notices reach this Mac they bring other devices' changes, so an activation only covers a lost one. + nonisolated static let pushFallbackInterval: Duration = .seconds(15 * 60) + let pushChannel: SyncPushChannel + // Notices received when the last successful download began; one after that may be what a later download brings. + private var noticesAtLastDownload = 0 + init( services: AppServices = .live, recordCache: SyncRecordCache = SyncCoordinator.makeRecordCache(), @@ -70,9 +83,13 @@ final class SyncCoordinator: ObservableObject { columnLayouts: @escaping @autoclosure () -> FileColumnLayoutPersister = .shared, themeCatalog: ThemeCatalog = .shared, licenseGate: (@MainActor () -> DisableReason?)? = nil, - recordCreationDate: @escaping (CKRecord) -> Date? = { $0.creationDate } + recordCreationDate: @escaping (CKRecord) -> Date? = { $0.creationDate }, + activationClock: any Clock = ContinuousClock(), + pushRegistrar: SyncPushRegistrar = .live ) { self.services = services + self.activationClock = activationClock + self.pushChannel = SyncPushChannel(registrar: pushRegistrar, metadata: services.syncMetadataStorage) self.makeTransport = transport self.makeNetworkMonitor = makeNetworkMonitor self.recordCreationDate = recordCreationDate @@ -99,6 +116,7 @@ final class SyncCoordinator: ObservableObject { deinit { debounceTask?.cancel() retryTask?.cancel() + activationCooldown?.cancel() } // MARK: - Lifecycle @@ -136,6 +154,13 @@ final class SyncCoordinator: ObservableObject { Self.logger.info("Sync skipped: not allowed for \(String(describing: trigger), privacy: .public)") return } + if trigger == .activation, holdsActivationBack() { + return + } + // The download still ahead in this run brings what the notice announced. + if trigger == .remoteChange, isRunning, downloadIsAheadInRun { + return + } guard !isRunning else { if !pendingTriggers.contains(trigger) { pendingTriggers.append(trigger) @@ -163,7 +188,7 @@ final class SyncCoordinator: ObservableObject { if previousError == nil || trigger == .userRequest { syncStatus = .syncing } - let result = await accountThenPushThenPull(admission) + let result = await accountThenPushThenPull(admission, startedBy: trigger) isRunning = false finish(result, previousError: previousError, from: generation) if uploadAgainRequested { @@ -173,8 +198,43 @@ final class SyncCoordinator: ObservableObject { await runPendingTrigger() } + // A run whose download has not started brings what the activation asks for. After a download, activations + // wait for the interval to end and then run once, so a change made on another Mac still arrives. + private func holdsActivationBack() -> Bool { + if isRunning { + return downloadIsAheadInRun + } + guard activationCooldown != nil else { return false } + activationHeldForCooldown = true + return true + } + + var isHoldingActivations: Bool { + activationCooldown != nil + } + + private func startActivationCooldown() { + activationCooldown?.cancel() + activationHeldForCooldown = false + let clock = activationClock + let interval = activationHoldInterval + activationCooldown = Task { [weak self] in + do { + try await clock.sleep(for: interval) + } catch { + return + } + // A newer download may have replaced this hold after the sleep ended. + guard !Task.isCancelled, let self else { return } + activationCooldown = nil + guard activationHeldForCooldown else { return } + activationHeldForCooldown = false + await sync(.activation) + } + } + internal func runSyncCycle(_ admission: SyncAdmission = .full) async -> SyncError? { - await accountThenPushThenPull(admission).failure?.error + await accountThenPushThenPull(admission, startedBy: .userRequest).failure?.error } private struct CycleResult { @@ -182,8 +242,10 @@ final class SyncCoordinator: ObservableObject { var failure: SyncStepFailure? } - private func accountThenPushThenPull(_ requested: SyncAdmission) async -> CycleResult { + private func accountThenPushThenPull(_ requested: SyncAdmission, startedBy trigger: SyncTrigger) async -> CycleResult { var admission = requested + downloadIsAheadInRun = true + defer { downloadIsAheadInRun = false } switch await confirmAccount() { case .unavailable(let failure): return CycleResult(admission: admission, failure: failure) @@ -193,6 +255,7 @@ final class SyncCoordinator: ObservableObject { admission = .full } } + await pushChannel.saveSubscriptionIfNeeded { try await transport.saveSubscription() } /// Only a run that may upload creates the zone. A download looks for it, and finding it /// again is how this Mac learns another device brought deleted data back. @@ -207,7 +270,17 @@ final class SyncCoordinator: ObservableObject { } let push = admission == .full ? await performPush() : PushReport() - let pullFailure = await performPull(echoGuard: push.echoGuard) + let noticesAtThisDownload = pushChannel.noticesReceived + let pull = await performPull(echoGuard: push.echoGuard) + let pullFailure = pull.failure + if pullFailure == nil { + if trigger == .activation, pull.broughtRemoteChanges, + pushChannel.activationFoundUnannouncedChanges(noticesSince: noticesAtLastDownload) { + Self.logger.notice("An activation brought changes no CloudKit notice announced, so activations download once a minute again") + } + noticesAtLastDownload = noticesAtThisDownload + startActivationCooldown() + } let zone = metadataStorage.zoneState let download = zone.reconciled(downloadFailure: pullFailure) @@ -222,6 +295,7 @@ final class SyncCoordinator: ObservableObject { recordCache.removeAll() metadataStorage.deferredDeletions = [] metadataStorage.saveToken(nil) + pushChannel.forgetSubscription() } metadataStorage.zoneState = nextZone return CycleResult(admission: admission, failure: failure) @@ -357,8 +431,13 @@ final class SyncCoordinator: ObservableObject { guard !pendingTriggers.isEmpty else { return } let error = syncStatus.error /// The network coming back mid-run matters only if the run ended unable to reach iCloud. - let triggers = pendingTriggers.filter { $0 != .networkRestored || error == .offline } + var triggers = pendingTriggers.filter { $0 != .networkRestored || error == .offline } pendingTriggers = [] + // Taken out before the pick, so a held activation never stands in for an edit queued behind it. + if activationCooldown != nil, triggers.contains(.activation) { + triggers.removeAll { $0 == .activation } + activationHeldForCooldown = true + } let attempt = nextAttempt let throttle = throttledUntil let reach: (SyncTrigger) -> Int = { trigger in @@ -378,6 +457,7 @@ final class SyncCoordinator: ObservableObject { // Clear token to force a full fetch on first sync after enabling metadataStorage.saveToken(nil) + pushChannel.forgetSubscription() metadataStorage.deferredDeletions = [] /// Turning sync on is the person's choice to upload, a removed zone included. metadataStorage.zoneState = .unknown @@ -406,6 +486,7 @@ final class SyncCoordinator: ObservableObject { metadataStorage.zoneState = .unknown metadataStorage.saveToken(nil) metadataStorage.deferredDeletions = [] + pushChannel.forgetSubscription() recordCache.removeAll() markAllLocalDataDirty() await markSQLFavoritesDirty() @@ -416,76 +497,6 @@ final class SyncCoordinator: ObservableObject { await sync(.userRequest) } - /// Marks existing SQL favorites and folders dirty. Separate from `markAllLocalDataDirty` - /// because the favorite store is an actor and must be read asynchronously. - private func markSQLFavoritesDirty() async { - let favorites = await services.sqlFavoriteManager.fetchFavorites() - changeTracker.markDirty(.favorite, ids: favorites.map { $0.id.uuidString }) - - let folders = await services.sqlFavoriteManager.fetchFolders() - changeTracker.markDirty(.favoriteFolder, ids: folders.map { $0.id.uuidString }) - } - - /// Marks every synced record dirty so the first sync after enabling pushes the lot. - /// - /// Every type is marked as one batch. Marking record by record posted a change notification per - /// record, and an account with a few hundred saved column layouts built a chain of hundreds of - /// tasks each waiting on its predecessor until the app stopped responding to the switch that - /// started it. - private func markAllLocalDataDirty() { - let connections = services.connectionStorage.loadConnections() - changeTracker.markDirty( - .connection, - ids: connections.filter(\.participatesInSync).map { $0.id.uuidString } - ) - - let groups = services.groupStorage.loadGroups() - changeTracker.markDirty(.group, ids: groups.map { $0.id.uuidString }) - - let tags = services.tagStorage.loadTags() - changeTracker.markDirty(.tag, ids: tags.map { $0.id.uuidString }) - - let sshProfiles = services.sshProfileStorage.loadProfiles() - changeTracker.markDirty(.sshProfile, ids: sshProfiles.map { $0.id.uuidString }) - - let credentialProfiles = services.credentialProfileStorage.loadProfiles() - changeTracker.markDirty(.credentialProfile, ids: credentialProfiles.map { $0.id.uuidString }) - - let favoriteTables = services.favoriteTablesStorage.loadFavorites() - changeTracker.markDirty( - .tableFavorite, - ids: favoriteTables.map { FavoriteTablesStorage.syncId(for: $0) } - ) - - let favoriteDatabases = services.favoriteDatabasesStorage.loadFavorites() - changeTracker.markDirty( - .favoriteDatabase, - ids: favoriteDatabases.map { FavoriteDatabasesStorage.syncId(for: $0) } - ) - - let tableFolderIds = tableFolderSyncIds() - changeTracker.markDirty(.tableFolder, ids: tableFolderIds.folders) - changeTracker.markDirty(.tableFolderItem, ids: tableFolderIds.items) - - let settingsCategories = AppSettingsCategory.synced + [CustomSlashCommandStorage.syncCategory] - let columnLayoutCategories = columnLayouts().customizedStorageKeys() - .map { FileColumnLayoutPersister.syncCategory(for: $0) } - let themeCategories = themeCatalog.syncCategories - changeTracker.markDirty(.settings, ids: settingsCategories + columnLayoutCategories + themeCategories) - - let summary = [ - "connections=\(connections.count)", - "groups=\(groups.count)", - "tags=\(tags.count)", - "sshProfiles=\(sshProfiles.count)", - "favoriteTables=\(favoriteTables.count)", - "tableFolders=\(tableFolderIds.folders.count)", - "settings=\(AppSettingsCategory.synced.count + 1)", - "themes=\(themeCategories.count)" - ].joined(separator: ", ") - Self.logger.info("Marked all local data dirty: \(summary, privacy: .public)") - } - /// Called when user disables sync in settings func disableSync() { decide(.disabled(.userDisabled)) @@ -526,11 +537,15 @@ final class SyncCoordinator: ObservableObject { arm() return } + pushChannel.unregister() triggers = nil resetRetryState() pendingTriggers = [] debounceTask?.cancel() debounceTask = nil + activationCooldown?.cancel() + activationCooldown = nil + activationHeldForCooldown = false } // Exhaustive, so a paying customer the app could not reach the server about is never told to buy. @@ -680,34 +695,44 @@ final class SyncCoordinator: ObservableObject { Self.logger.info("Reading record types the last pull skipped: cleared sync token for full fetch") } + private struct PullReport { + var failure: SyncStepFailure? + // A delta that carried records this run did not upload, which another device sent. + var broughtRemoteChanges = false + } + /// Reports a failed download instead of passing over it: a run whose pull failed did not bring /// this Mac up to date, and must not say Synced or move Last Synced. - private func performPull(echoGuard: SyncEchoGuard?) async -> SyncStepFailure? { + private func performPull(echoGuard: SyncEchoGuard?) async -> PullReport { // Applied with the categories recorded here, so one turned off during the download is still read once. let settings = services.appSettingsStorage.loadSync() adoptReadableContent(for: settings) + await clearTokenForSavedQueriesRefill(settings: settings) let token = metadataStorage.loadToken() let tokenStatus = token == nil ? "nil (full fetch)" : "present (delta)" Self.logger.info("Pull starting, token: \(tokenStatus)") + downloadIsAheadInRun = false do { let result = try await transport.pull(since: token) - return await applyPull(result, isFullFetch: token == nil, settings: settings, echoGuard: echoGuard) - ? nil : .pullNotSaved + guard await applyPull(result, isFullFetch: token == nil, settings: settings, echoGuard: echoGuard) else { + return PullReport(failure: .pullNotSaved) + } + return PullReport(broughtRemoteChanges: token != nil && result.carriesRemoteChanges(besides: echoGuard)) } catch let error where Self.isTokenExpired(error) { Self.logger.warning("Change token expired, clearing and retrying with full fetch") metadataStorage.saveToken(nil) do { let result = try await transport.pull(since: nil) return await applyPull(result, isFullFetch: true, settings: settings, echoGuard: echoGuard) - ? nil : .pullNotSaved + ? PullReport() : PullReport(failure: .pullNotSaved) } catch { Self.logger.error("Full fetch after token expiry failed: \(error.localizedDescription)") - return SyncStepFailure(error) + return PullReport(failure: SyncStepFailure(error)) } } catch { Self.logger.error("Pull failed: \(error.localizedDescription)") - return SyncStepFailure(error) + return PullReport(failure: SyncStepFailure(error)) } } @@ -722,6 +747,7 @@ final class SyncCoordinator: ObservableObject { } if isFullFetch { seedLocalThemesIfPending(after: result) + await finishSavedQueriesRefill(after: result, settings: settings) } return true } @@ -861,6 +887,17 @@ final class SyncCoordinator: ObservableObject { var tableFavorites: [FavoriteTablesStorage.FavoriteEntry] = [] let databaseFavoriteTombstoneIds = Set(metadataStorage.tombstones(for: .favoriteDatabase).map(\.id)) var tableFolderRecords: [CKRecord] = [] + var connectionBatch: RemoteConnectionBatch? + + func persistConnectionBatch() { + guard let batch = connectionBatch else { return } + connectionBatch = nil + switch persistRemoteConnections(batch) { + case .applied: actualConnectionChanges = true + case .failed: persistenceFailed = true + case .skipped: break + } + } for record in changedRecords { if let echoGuard, echoGuard.withholds(record.recordID, tracker: changeTracker) { @@ -868,13 +905,13 @@ final class SyncCoordinator: ObservableObject { continue } guard let type = SyncRecordType(rawValue: record.recordType), settings.syncs(type) else { continue } + // A record whose apply can read or write the library sees every connection record before it. + if !Self.leavesConnectionLibraryAlone(type) { + persistConnectionBatch() + } switch type { case .connection: - switch applyRemoteConnection(record, tombstoneIds: connectionTombstoneIds) { - case .applied: actualConnectionChanges = true - case .failed: persistenceFailed = true - case .skipped: break - } + stageRemoteConnection(record, tombstoneIds: connectionTombstoneIds, into: &connectionBatch) case .group: switch applyRemoteGroup(record, tombstoneIds: groupTombstoneIds) { case .applied: groupsOrTagsChanged = true @@ -911,6 +948,7 @@ final class SyncCoordinator: ObservableObject { break } } + persistConnectionBatch() applyRemoteTableFolderRecords(tableFolderRecords) var effects = applyRemoteDeletions( @@ -982,51 +1020,81 @@ final class SyncCoordinator: ObservableObject { } } - private func applyRemoteConnection(_ record: CKRecord, tombstoneIds: Set) -> RemoteApplyOutcome { + // Groups and tags write only their own store, and the rest are collected and applied after the loop. + private static func leavesConnectionLibraryAlone(_ type: SyncRecordType) -> Bool { + switch type { + case .connection, .group, .tag, .tableFavorite, .tableFolder, .tableFolderItem, .favorite, .favoriteFolder: + return true + case .settings, .sshProfile, .credentialProfile, .favoriteDatabase: + return false + } + } + + // Applied to one list and written once, where each record used to read, verify, decode, encode and write the + // whole library. + private func stageRemoteConnection( + _ record: CKRecord, + tombstoneIds: Set, + into pending: inout RemoteConnectionBatch? + ) { let remoteConnection: DatabaseConnection do { remoteConnection = try SyncRecordMapper.toConnection(record) } catch { Self.logger.error("Skipping remote connection \(record.recordID.recordName, privacy: .public): \(error.publicLogShape, privacy: .public)") - return .skipped + return } if tombstoneIds.contains(remoteConnection.id.uuidString) { - return .skipped + return + } + + let batch: RemoteConnectionBatch + if let existing = pending { + batch = existing + } else { + batch = RemoteConnectionBatch(connections: services.connectionStorage.loadConnections()) + pending = batch } - var connections = services.connectionStorage.loadConnections() var incoming = remoteConnection - let index = connections.firstIndex(where: { $0.id == remoteConnection.id }) + let index = batch.index(of: remoteConnection.id) if let index { - guard !connections[index].localOnly else { return .skipped } + guard !batch.connections[index].localOnly else { return } if changeTracker.dirtyRecords(for: .connection).contains(remoteConnection.id.uuidString) { guard let reconciled = mergeLocalEdits( into: record, - localConnection: connections[index] + localConnection: batch.connections[index] ) else { - return .skipped + return } incoming = reconciled } } // A record an older build pushed can still carry a connection string's secret in plain text. let kept = services.connectionStorage.keepingMongoURISecretsInKeychain(incoming) - let movedSecrets = kept.additionalFields != incoming.additionalFields + if kept.additionalFields != incoming.additionalFields { + batch.cleanedIds.insert(remoteConnection.id.uuidString) + } if let index { - connections[index] = kept.adoptingDeviceLocalState(from: connections[index]) + batch.replace(at: index, with: kept.adoptingDeviceLocalState(from: batch.connections[index])) } else { var inserted = kept inserted.origin = .synced - connections.append(inserted) + batch.append(inserted) } - guard services.connectionStorage.saveConnections(connections, byUser: false) else { - Self.logger.error("Failed to apply remote connection update: persistence error for \(remoteConnection.id, privacy: .public)") + batch.stagedCount += 1 + } + + private func persistRemoteConnections(_ batch: RemoteConnectionBatch) -> RemoteApplyOutcome { + guard batch.stagedCount > 0 else { return .skipped } + guard services.connectionStorage.saveConnections(batch.connections, byUser: false) else { + Self.logger.error( + "Failed to apply \(batch.stagedCount, privacy: .public) remote connection update(s): persistence error" + ) return .failed } - if movedSecrets { - connectionsCleanedOnPull.insert(remoteConnection.id.uuidString) - } + connectionsCleanedOnPull.formUnion(batch.cleanedIds) return .applied } @@ -1189,6 +1257,7 @@ final class SyncCoordinator: ObservableObject { localChanges: observeLocalChanges(), networkMonitor: observeNetwork() ) + pushChannel.register() /// Empty local storage (a fresh install or a wipe) needs a full fetch: a delta returns nothing. if services.connectionStorage.loadConnections().isEmpty { @@ -1266,6 +1335,35 @@ final class SyncCoordinator: ObservableObject { } } +// A reference, so staging a record changes the list in place instead of copying it. +private final class RemoteConnectionBatch { + private(set) var connections: [DatabaseConnection] + var cleanedIds: Set = [] + var stagedCount = 0 + private var indexById: [UUID: Int] + + init(connections: [DatabaseConnection]) { + self.connections = connections + indexById = Dictionary( + connections.enumerated().map { ($0.element.id, $0.offset) }, + uniquingKeysWith: { first, _ in first } + ) + } + + func index(of id: UUID) -> Int? { + indexById[id] + } + + func replace(at index: Int, with connection: DatabaseConnection) { + connections[index] = connection + } + + func append(_ connection: DatabaseConnection) { + indexById[connection.id] = connections.count + connections.append(connection) + } +} + /// What starts a run while sync is on. Releasing it detaches every observer; the network monitor /// stops in its own deinit. private final class SyncTriggers { diff --git a/apple/TablePro/Core/Sync/SyncEchoGuard.swift b/apple/TablePro/Core/Sync/SyncEchoGuard.swift index 0353d7d78b..9a8f6405bb 100644 --- a/apple/TablePro/Core/Sync/SyncEchoGuard.swift +++ b/apple/TablePro/Core/Sync/SyncEchoGuard.swift @@ -41,3 +41,13 @@ struct SyncEchoGuard: Sendable { return tracker.hasEdit(identity, since: snapshot) } } + +extension PullResult { + // Records this run did not upload, which another device sent. + func carriesRemoteChanges(besides echoGuard: SyncEchoGuard?) -> Bool { + let uploaded = echoGuard?.pushedRecordIDs ?? [] + let deleted = echoGuard.map { Set($0.deletedRecords.keys) } ?? [] + return changedRecords.contains { !uploaded.contains($0.recordID) } + || deletedRecordIDs.contains { !deleted.contains($0) } + } +} diff --git a/apple/TablePro/Core/Sync/SyncPushRegistrar+AppKit.swift b/apple/TablePro/Core/Sync/SyncPushRegistrar+AppKit.swift new file mode 100644 index 0000000000..f525e419a1 --- /dev/null +++ b/apple/TablePro/Core/Sync/SyncPushRegistrar+AppKit.swift @@ -0,0 +1,9 @@ +import AppKit +import TableProSyncTransport + +extension SyncPushRegistrar { + static let live = SyncPushRegistrar( + register: { NSApplication.shared.registerForRemoteNotifications() }, + unregister: { NSApplication.shared.unregisterForRemoteNotifications() } + ) +} diff --git a/apple/TablePro/Core/Sync/SyncTransport.swift b/apple/TablePro/Core/Sync/SyncTransport.swift index 0192c05266..f475ebc901 100644 --- a/apple/TablePro/Core/Sync/SyncTransport.swift +++ b/apple/TablePro/Core/Sync/SyncTransport.swift @@ -7,6 +7,7 @@ protocol SyncTransport: Sendable { func accountStatus() async throws -> CKAccountStatus func currentAccountId() async throws -> String func ensureZoneExists() async throws + func saveSubscription() async throws func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome func pull(since token: CKServerChangeToken?) async throws -> PullResult func fetch(recordIDs: [CKRecord.ID]) async throws -> [CKRecord.ID: CKRecord] diff --git a/apple/TablePro/Core/Updates/AppRelauncher.swift b/apple/TablePro/Core/Updates/AppRelauncher.swift index 9fed1e282b..8da45182c0 100644 --- a/apple/TablePro/Core/Updates/AppRelauncher.swift +++ b/apple/TablePro/Core/Updates/AppRelauncher.swift @@ -17,30 +17,37 @@ final class AppRelauncher { } private let defaults: UserDefaults - private let approveStoppingWork: @MainActor () -> Bool + private let approveStoppingWork: @MainActor () -> QuitPlan? + private let commitBeforeQuitting: @MainActor (QuitPlan) async -> Bool private let reviewUnsavedDocuments: @MainActor (@escaping @MainActor (Bool) -> Void) -> Void private let installPendingUpdate: @MainActor () -> Bool private let terminate: @MainActor () -> Void + private let performOutsideTask: @MainActor (@escaping @MainActor () -> Void) -> Void private let reopenAfterExit: @MainActor (_ pid: pid_t, _ bundleURL: URL) throws -> Void private var route: Route? + private(set) var pendingCommit: Task? init( defaults: UserDefaults = AppStorageEnvironment.shared.defaults, - approveStoppingWork: @escaping @MainActor () -> Bool = { + approveStoppingWork: @escaping @MainActor () -> QuitPlan? = { // Everything that can still refuse asks here, before Sparkle or the quit takes over. - ThemeEditorWindows.shared.prepareToTerminate() - && QuitApproval.approvesStoppingWork(registry: .shared, action: .relaunch, stopsWork: false) + guard ThemeEditorWindows.shared.prepareToTerminate() else { return nil } + return QuitApproval.approvesStoppingWork(registry: .shared, action: .relaunch) }, + commitBeforeQuitting: @escaping @MainActor (QuitPlan) async -> Bool = { await QuitApproval.commitBeforeQuitting($0) }, reviewUnsavedDocuments: @escaping @MainActor (@escaping @MainActor (Bool) -> Void) -> Void = UnsavedDocumentReview.run, installPendingUpdate: @escaping @MainActor () -> Bool = { SoftwareUpdater.shared.installAndRelaunch() }, terminate: @escaping @MainActor () -> Void = { NSApp.terminate(nil) }, + performOutsideTask: @escaping @MainActor (@escaping @MainActor () -> Void) -> Void = AppRelauncher.performOnNextRunLoopTurn, reopenAfterExit: @escaping @MainActor (_ pid: pid_t, _ bundleURL: URL) throws -> Void = AppRelauncher.reopenAfterExit(pid:bundleURL:) ) { self.defaults = defaults self.approveStoppingWork = approveStoppingWork + self.commitBeforeQuitting = commitBeforeQuitting self.reviewUnsavedDocuments = reviewUnsavedDocuments self.installPendingUpdate = installPendingUpdate self.terminate = terminate + self.performOutsideTask = performOutsideTask self.reopenAfterExit = reopenAfterExit } @@ -51,12 +58,36 @@ final class AppRelauncher { // Documents are reviewed first: AppKit cancels a quit there before the app delegate hears of it. func relaunch() { + // A commit still running is this same relaunch. + guard pendingCommit == nil else { return } reviewUnsavedDocuments { [weak self] proceed in - guard let self, proceed, self.approveStoppingWork() else { return } - self.route = .sparkleInstall - guard !self.installPendingUpdate() else { return } - self.route = .reopenAfterExit - self.terminate() + guard let self, proceed, let plan = self.approveStoppingWork() else { return } + guard !plan.commits.isEmpty else { + self.quitToRelaunch() + return + } + // The commit runs before Sparkle or the quit takes over, since the quit it starts asks nothing. + self.pendingCommit = Task { [weak self] in + guard let self else { return } + let committed = await self.commitBeforeQuitting(plan) + self.pendingCommit = nil + guard committed else { return } + // A terminate from inside a main-actor job never lets the quit's reply task run. + self.performOutsideTask { [weak self] in self?.quitToRelaunch() } + } + } + } + + private func quitToRelaunch() { + route = .sparkleInstall + guard !installPendingUpdate() else { return } + route = .reopenAfterExit + terminate() + } + + static func performOnNextRunLoopTurn(_ body: @escaping @MainActor () -> Void) { + RunLoop.main.perform { + MainActor.assumeIsolated(body) } } diff --git a/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift b/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift index 1a7be6130b..578cfc531f 100644 --- a/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift +++ b/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift @@ -432,14 +432,20 @@ final class PrincipalChangeManager: ObservableObject { // MARK: - Output + // A staged drop keeps the principal's other edits so undoing the drop brings them back, but only the DROP runs. func pendingChanges() -> [PrincipalChange] { var changes: [PrincipalChange] = pendingCreates.map { .create($0) } changes += pendingAlters.compactMap { ref, definition in - guard let original = principals.first(where: { $0.ref == ref }) else { return nil } + guard pendingDrops[ref] == nil, + let original = principals.first(where: { $0.ref == ref }) + else { return nil } return .alter(old: Self.definition(from: original), new: definition) } - changes += pendingPasswords.map { .setPassword(ref: $0.key, password: $0.value) } + changes += pendingPasswords.compactMap { ref, password in + guard pendingDrops[ref] == nil else { return nil } + return .setPassword(ref: ref, password: password) + } changes += grantChangeSets().map { .modifyGrants($0) } changes += pendingDrops.map { .drop(ref: $0.key, options: $0.value) } return changes @@ -531,11 +537,12 @@ final class PrincipalChangeManager: ObservableObject { } private func recomputeChangeCount() { - let grantChanges = grantDeltas.values.reduce(0) { $0 + $1.count } + let isLive = { (ref: PluginPrincipalRef) in self.pendingDrops[ref] == nil } + let grantChanges = grantDeltas.reduce(0) { isLive($1.key) ? $0 + $1.value.count : $0 } changeCount = pendingCreates.count + pendingDrops.count - + pendingPasswords.count - + pendingAlters.count + + pendingPasswords.keys.count(where: isLive) + + pendingAlters.keys.count(where: isLive) + grantChanges } diff --git a/apple/TablePro/Core/Utilities/Connection/ExternalConnectionTrustKey.swift b/apple/TablePro/Core/Utilities/Connection/ExternalConnectionTrustKey.swift index 415bc37b6f..464b9decf3 100644 --- a/apple/TablePro/Core/Utilities/Connection/ExternalConnectionTrustKey.swift +++ b/apple/TablePro/Core/Utilities/Connection/ExternalConnectionTrustKey.swift @@ -14,6 +14,8 @@ internal struct ExternalConnectionTrustKey: Hashable, Codable, Sendable { internal let scopeName: String // Optional so entries saved before sockets still decode. Trust covers this exact path only. internal let socketPath: String? + // The saved connection a link opened, nil for one built from the link alone, so trust never crosses between them. + internal let savedConnectionId: UUID? internal init( databaseType: String, @@ -21,10 +23,12 @@ internal struct ExternalConnectionTrustKey: Hashable, Codable, Sendable { database: String, username: String, scopeName: String, - socketPath: String? = nil + socketPath: String? = nil, + savedConnectionId: UUID? = nil ) { let socket = socketPath?.trimmingCharacters(in: .whitespaces) self.socketPath = socket?.isEmpty == false ? socket : nil + self.savedConnectionId = savedConnectionId self.databaseType = databaseType.lowercased() self.host = self.socketPath == nil ? host.trimmingCharacters(in: .whitespaces).lowercased() : "localhost" self.database = database @@ -32,14 +36,15 @@ internal struct ExternalConnectionTrustKey: Hashable, Codable, Sendable { self.scopeName = scopeName.trimmingCharacters(in: .whitespaces) } - internal init(connection: DatabaseConnection, scopeName: String?) { + internal init(connection: DatabaseConnection, scopeName: String?, isSaved: Bool = false) { self.init( databaseType: connection.type.rawValue, host: connection.host, database: connection.database, username: connection.username, scopeName: scopeName ?? "", - socketPath: connection.localSocketFile + socketPath: connection.localSocketFile, + savedConnectionId: isSaved ? connection.id : nil ) } diff --git a/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift b/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift index d6749cb279..fd1e34cc15 100644 --- a/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift +++ b/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift @@ -81,12 +81,23 @@ enum PasswordSourceResolver { } } - // A test reports what the source returns right now, so it neither reads nor fills the reuse cache. - // A Vault role test may open the browser to sign in, and revokes the lease it was given at once. - static func test(_ source: PasswordSource, for context: ConnectionScriptContext) async throws -> String { + // A command test reports what the command prints right now, so it neither reads nor fills the reuse cache. + // A Vault role test may open the browser to sign in. It shows the user `leaseOwner` signs in as, and leaves the + // lease for that owner's next connect, so a database slow to accept a new user has it ready by then. With no + // owner, as for an unsaved connection, nothing would reuse it, so it is revoked at once. + static func test(_ source: PasswordSource, for context: ConnectionScriptContext, leaseOwner: UUID?) async throws -> String { if case .vaultDatabaseRole(let role) = source { - let signIn = try await VaultDatabaseCredentials.shared.signIn(role, interactive: true) - signIn.lease?.release() + let signIn = try await VaultDatabaseCredentials.shared.signIn( + role, + owner: leaseOwner ?? UUID(), + for: nil, + interactive: true + ) + if leaseOwner == nil { + signIn.lease?.end() + } else { + signIn.lease?.release() + } return signIn.username.map { "\($0) / \(signIn.password)" } ?? signIn.password } return try await resolve(source.withoutReuse, for: context) diff --git a/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift b/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift index 5161eb1e38..a805e5be70 100644 --- a/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift +++ b/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift @@ -189,11 +189,11 @@ enum QueryStatementScanner { } } - private static func located(_ command: RedisCommandSplitter.Command) -> SQLStatementScanner.LocatedStatement { + fileprivate static func located(_ command: RedisCommandSplitter.Command) -> SQLStatementScanner.LocatedStatement { SQLStatementScanner.LocatedStatement(wholeText: command.text, offset: command.range.location) } - private static func located( + fileprivate static func located( _ statement: JavaScriptStatementScanner.Statement ) -> SQLStatementScanner.LocatedStatement { SQLStatementScanner.LocatedStatement( @@ -203,3 +203,51 @@ enum QueryStatementScanner { ) } } + +// A JavaScript or Redis document's statements, read once per text, so a caret move finds its statement by search and +// answers what `QueryStatementScanner.locatedStatementAtCursor` would. A SQL document uses `SQLStatementCursorIndex`. +struct QueryStatementCaretTable { + private let model: QueryStatementModel + private let statements: [SQLStatementScanner.LocatedStatement] + // The last caret offset each statement owns, in document order. + private let reaches: [Int] + + init?(text: String, model: QueryStatementModel) { + self.model = model + switch model { + case .sql: + return nil + case .javascript: + let found = JavaScriptStatementScanner.executableStatements(in: text) + statements = found.map(QueryStatementScanner.located) + reaches = found.map { NSMaxRange($0.range) } + case .redis: + let found = RedisCommandSplitter.commandsWithCaretReach(in: text) + statements = found.map { QueryStatementScanner.located($0.command) } + reaches = found.map { $0.reach } + } + } + + func statement(atCursor cursor: Int) -> SQLStatementScanner.LocatedStatement { + var low = 0 + var high = reaches.count + while low < high { + let middle = (low + high) / 2 + if reaches[middle] < cursor { + low = middle + 1 + } else { + high = middle + } + } + let nothing = SQLStatementScanner.LocatedStatement(sql: "", offset: 0, hasContent: false) + switch model { + case .redis: + return low < statements.count ? statements[low] : nothing + case .javascript, .sql: + // Off every statement, the caret takes the one before it, or the first. + guard !statements.isEmpty else { return nothing } + if low < statements.count, statements[low].offset <= cursor { return statements[low] } + return statements[max(0, low - 1)] + } + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/CsvRowConverter.swift b/apple/TablePro/Core/Utilities/SQL/CsvRowConverter.swift index e611a514d3..9fde311c93 100644 --- a/apple/TablePro/Core/Utilities/SQL/CsvRowConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/CsvRowConverter.swift @@ -46,10 +46,20 @@ internal struct CsvRowConverter { // An unquoted field holds no quote, so wrapping it for its edge spaces needs no escaping. static func encode(_ value: String) -> String { - let field = PluginRowWriters.csvField(value, options: fieldOptions) + encode(value, options: fieldOptions) + } + + // For a file the user opens in a spreadsheet, where a field another program wrote must not run as a formula. + static func encodeForSpreadsheet(_ value: String) -> String { + encode(value, options: spreadsheetFieldOptions) + } + + private static func encode(_ value: String, options: PluginCsvWriteOptions) -> String { + let field = PluginRowWriters.csvField(value, options: options) guard field.unicodeScalars.first != "\"", value.hasPrefix(" ") || value.hasSuffix(" ") else { return field } return "\"\(field)\"" } private static let fieldOptions = PluginCsvWriteOptions(sanitizesFormulas: false) + private static let spreadsheetFieldOptions = PluginCsvWriteOptions(sanitizesFormulas: true) } diff --git a/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift b/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift index 869817eb65..a9463a1cb4 100644 --- a/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift +++ b/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift @@ -15,12 +15,14 @@ internal enum DisplayedResultReader { internal let skippedDeletedCount: Int } - // Empty selectedDisplayIndices means every displayed row. + // Empty selectedDisplayIndices means every displayed row. A cell in defaultedCells has no value until the server + // gives it one, so it reads as NULL. internal static func read( tableRows: TableRows, displayIDs: [RowID]?, selectedDisplayIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columns projection: VisibleColumnProjection ) -> Output { let positions: [Int] @@ -38,7 +40,7 @@ internal enum DisplayedResultReader { skippedDeleted += 1 return nil } - return projection.values(Self.values(of: row)) + return projection.values(Self.values(of: row, defaulted: defaultedCells[row.id] ?? [])) } return Output( @@ -50,12 +52,15 @@ internal enum DisplayedResultReader { } // A value the page left on the server says so, rather than reading as NULL. - private static func values(of row: Row) -> [PluginCellValue] { - guard row.hasUnloadedValues else { return Array(row.values) } + private static func values(of row: Row, defaulted: Set) -> [PluginCellValue] { + guard row.hasUnloadedValues || !defaulted.isEmpty else { return Array(row.values) } var values = Array(row.values) for (column, byteCount) in row.unloadedColumns where values.indices.contains(column) { values[column] = .text(UnloadedValueText.notLoaded(byteCount: byteCount)) } + for column in defaulted where values.indices.contains(column) { + values[column] = .null + } return values } } diff --git a/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift b/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift index aeb4e15d98..0c93948d3a 100644 --- a/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift @@ -20,6 +20,12 @@ internal struct InClauseConverter { // The session's escapes, for when no driver escaper is passed. internal var stringEscapes: SQLStringEscapes = .standard + // SQL Server has no TRUE or FALSE: `flag IN (TRUE)` reads TRUE as a column name. + internal var booleanLiteralStyle: SQLDialectDescriptor.BooleanLiteralStyle = .truefalse + + // Picks the binary literal's spelling; without one it is the standard X'...'. + internal var databaseType: DatabaseType? + private static let maxRows = 50_000 func generateInClause(rows: [[PluginCellValue]]) -> String { @@ -39,21 +45,28 @@ internal struct InClauseConverter { private func format(cell: PluginCellValue, type: ColumnType) -> String? { switch cell { - case .null, .bytes: + case .null: return nil + case .bytes(let data): + return binaryLiteral(data) case .text(let value): return formatScalar(value, type: type) } } + private func binaryLiteral(_ data: Data) -> String { + databaseType.flatMap { CompareSQLLiteral.binaryLiteral(for: data, databaseType: $0) } ?? "X'\(data.uppercaseHex())'" + } + private func formatScalar(_ value: String, type: ColumnType) -> String { if type.isBooleanType { guard let synonym = ColumnTypeSQLQuoting.booleanSynonym(for: value) else { return quoted(value) } + let isNumeric = booleanLiteralStyle == .numeric switch synonym { case .isTrue: - return "TRUE" + return isNumeric ? "1" : "TRUE" case .isFalse: - return "FALSE" + return isNumeric ? "0" : "FALSE" @unknown default: return quoted(value) } diff --git a/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift b/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift index 949a2f53c7..e34247fef2 100644 --- a/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift +++ b/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift @@ -21,8 +21,11 @@ internal enum InvisibleCharacterRemover { let end = min(NSMaxRange(scope), text.length) var replacements: [TextReplacement] = [] var index = max(scope.location, 0) + // Literals are found in the text as cleaned so far, since that is what runs: PostgreSQL reads `SELECTE` as + // one name, but the cleaned `SELECT E'...'` holds an escape string whose characters stay. + let cleanedText = skippingLiteralsAndComments ? NSMutableString(string: text as String) : nil while index < end { - if skippingLiteralsAndComments, let skipTo = SQLNonCodeSpan.end(at: index, in: text, grammar: grammar) { + if let cleanedText, let skipTo = SQLNonCodeSpan.end(at: index, in: cleanedText, grammar: grammar) { index = skipTo continue } @@ -32,6 +35,10 @@ internal enum InvisibleCharacterRemover { } let cleaned = replacement(for: classified, lineEnding: lineEnding) replacements.append(TextReplacement(range: classified.range, string: cleaned)) + if let cleanedText { + let filler = standIn(for: cleaned, replacing: classified.range, in: cleanedText) + cleanedText.replaceCharacters(in: classified.range, with: filler) + } index = NSMaxRange(classified.range) } return replacements @@ -51,6 +58,21 @@ internal enum InvisibleCharacterRemover { return offset + delta } + // Keeps offsets: the unit the cleaned text has before the next one, the replacement's last or, for a removed + // character, the one already before it. + private static func standIn(for cleaned: String, replacing range: NSRange, in cleanedText: NSString) -> String { + let replacement = cleaned as NSString + let unit: UInt16 + if replacement.length > 0 { + unit = replacement.character(at: replacement.length - 1) + } else if range.location > 0 { + unit = cleanedText.character(at: range.location - 1) + } else { + unit = SqlLexer.space + } + return String(utf16CodeUnits: Array(repeating: unit, count: range.length), count: range.length) + } + private static func replacement(for classified: ClassifiedSpecialCharacter, lineEnding: String) -> String { let value = classified.scalar.value if lineBreaks.contains(value) { diff --git a/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift b/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift index efc41e9320..ddfa831950 100644 --- a/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift @@ -40,7 +40,7 @@ internal struct JsonRowConverter { } let cell = row[colIdx] - if cell.isNull || cell.isDefaultMarker { + if cell.isNull { result.append("null") appendPropertySuffix(to: &result, colIdx: colIdx) continue @@ -116,7 +116,6 @@ internal struct JsonRowConverter { return quotedEscaped(value) } - /// Checks whether a string conforms to JSON number grammar (RFC 8259 §6) private func isValidJsonNumber(_ value: String) -> Bool { let scalars = value.unicodeScalars var iter = scalars.makeIterator() @@ -173,7 +172,8 @@ internal struct JsonRowConverter { case "false", "0", "no", "off": return "false" default: - return quotedEscaped(value) + // MySQL's BOOLEAN is TINYINT(1), which also stores 2 or -1. + return formatInteger(value) } } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+CommandDataStatements.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+CommandDataStatements.swift new file mode 100644 index 0000000000..1df96aabb6 --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+CommandDataStatements.swift @@ -0,0 +1,144 @@ +// +// QueryClassifier+CommandDataStatements.swift +// TablePro +// + +import Foundation +import TableProMemcachedCore +import TableProRedisCore +import TableProSQLGrammar + +extension QueryClassifier { + // Redis 8.10.3 `COMMAND`: the commands whose second word names the subcommand that runs. + private static let redisContainerCommands: Set = [ + "ACL", "BACKUP", "CLIENT", "CLUSTER", "COMMAND", "CONFIG", "FT.CONFIG", "FT.CURSOR", "FUNCTION", "HIMPORT", + "HOTKEYS", "LATENCY", "MEMORY", "MODULE", "OBJECT", "PUBSUB", "SCRIPT", "SLOWLOG", "XGROUP", "XINFO", + "_FT.CONFIG", "_FT.DEBUG", + ] + + static func redisDataCommands(_ text: String) -> [DataCommand]? { + RedisCommandSplitter.commands(in: text).map { redisDataCommand(StatementBlank.trimming($0.text)) } + } + + // Named past the prefixes the driver unwraps, as `CONTAINER|SUB` for a container. The server folds ASCII only, so + // a word only Unicode case folding reads as a listed name is one the server does not know. + private static func redisDataCommand(_ command: String) -> DataCommand { + guard let arguments = RedisArgumentCodec.split(command) else { return .refused } + if redisCommandClassification(command).tier == .safe { return .read } + let decoded = arguments.map { String(bytes: $0, encoding: .utf8) } + let words = decoded[redisCommandPastPrefixes(decoded.map { $0 ?? "" }).startIndex...] + guard let first = words.first, let verb = first.map(redisASCIIUppercased) else { return .refused } + guard redisContainerCommands.contains(verb) else { return .named(verb) } + guard let second = words.dropFirst().first, let subcommand = second else { return .refused } + return .named(verb + "|" + redisASCIIUppercased(subcommand)) + } + + private static func redisASCIIUppercased(_ word: String) -> String { + var scalars = String.UnicodeScalarView() + for scalar in word.unicodeScalars { + let isLowercaseLetter = scalar.value >= 0x61 && scalar.value <= 0x7A + scalars.append(isLowercaseLetter ? Unicode.Scalar(UInt8(scalar.value - 0x20)) : scalar) + } + return String(scalars) + } + + // The driver runs only what its parser reads. + static func memcachedDataCommands(_ text: String, readings: SQLLexicalReadings) -> [DataCommand]? { + var commands: [DataCommand] = [] + for statement in readings.all.flatMap({ statements(of: text, grammar: $0) }) { + guard let command = try? MemcachedCommandParser.parse(statement) else { return nil } + let isRead = memcachedClassification(statement).tier == .safe + commands.append(isRead ? .read : .named(memcachedName(of: command))) + } + return commands + } + + private static func memcachedName(of command: MemcachedCommand) -> String { + switch command { + case .get(_, let withCAS): return withCAS ? "gets" : "get" + case .getAndTouch(_, _, let withCAS): return withCAS ? "gats" : "gat" + case .store(let verb, _, _, _, _): return verb.rawValue + case .compareAndSwap: return "cas" + case .touch: return "touch" + case .delete: return "delete" + case .increment: return "incr" + case .decrement: return "decr" + case .flushAll: return "flush_all" + case .stats(let arguments): return (["stats"] + arguments).joined(separator: " ") + case .version: return "version" + case .metadump: return "lru_crawler metadump" + } + } + + static func etcdDataCommands(_ text: String, readings: SQLLexicalReadings) -> [DataCommand]? { + var commands: [DataCommand] = [] + for statement in readings.all.flatMap({ statements(of: text, grammar: $0) }) { + guard let operation = try? EtcdCommandParser.parse(statement) else { return nil } + if etcdClassification(statement).tier == .safe { + commands.append(.read) + } else { + commands.append(etcdName(of: operation).map(DataCommand.named) ?? .refused) + } + } + return commands + } + + private static func etcdName(of operation: EtcdOperation) -> String? { + switch operation { + case .get: return "get" + case .put: return "put" + case .create: return "create" + case .move: return "mv" + case .del: return "del" + case .watch: return "watch" + case .leaseGrant: return "lease grant" + case .leaseRevoke: return "lease revoke" + case .leaseTimetolive: return "lease timetolive" + case .leaseList: return "lease list" + case .leaseKeepAlive: return "lease keep-alive" + case .memberList: return "member list" + case .endpointStatus: return "endpoint status" + case .endpointHealth: return "endpoint health" + case .compaction: return "compaction" + case .authEnable: return "auth enable" + case .authDisable: return "auth disable" + case .userAdd: return "user add" + case .userDelete: return "user delete" + case .userList: return "user list" + case .roleAdd: return "role add" + case .roleDelete: return "role delete" + case .roleList: return "role list" + case .userGrantRole: return "user grant-role" + case .userRevokeRole: return "user revoke-role" + case .unknown: return nil + } + } + + // The whole text, as the driver parses it: `;` is a blank in KafkaQL, so a second statement is a syntax error. + static func kafkaDataCommands(_ text: String) -> [DataCommand]? { + guard let statement = try? KafkaQL.parse(text) else { return nil } + switch statement { + // Reads by the driver's own rule, `KafkaStatement.refusal`. The SQL tier reads CONSUME as a write. + case .consume, .showTopics, .showBrokers, .showGroups, .showCluster, .describeGroup, .describeTopic: + return [.read] + case .produce(let query): + return [kafkaNamesServiceTopic(query.topic) ? .refused : .named("PRODUCE")] + case .dropTopic: + return [.named("DROP TOPIC")] + } + } + + // The broker refuses a produce only into its own `__` topics. Schema Registry keeps its schemas in `_schemas` + // (`kafkastore.topic`), and Confluent names every internal topic with a leading `_`. Kafka's + // `ReplicationPolicy.isInternalTopic` (4.4) adds a leading `.` and MirrorMaker 2's `.internal` topics, its Connect + // stores among them. A Connect worker's stores have no default name; these are the ones Kafka ships in + // `config/connect-distributed.properties`. Kafka Streams names a store's log and a repartition topic with these + // suffixes. The broker does not say which topics a worker, a registry or an application uses. + private static let kafkaConnectTopics: Set = ["connect-configs", "connect-offsets", "connect-status"] + + static func kafkaNamesServiceTopic(_ topic: String) -> Bool { + if topic.hasPrefix("_") || topic.hasPrefix(".") || kafkaConnectTopics.contains(topic) { return true } + if topic.hasSuffix("-changelog") || topic.hasSuffix("-repartition") { return true } + return (topic.hasPrefix("mm2") && topic.hasSuffix(".internal")) || topic.hasSuffix(".checkpoints.internal") + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Cypher.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Cypher.swift index e8a1307f6c..7eefed9d0a 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Cypher.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Cypher.swift @@ -73,6 +73,135 @@ extension QueryClassifier { "dbms.components", "dbms.showcurrentuser", "dbms.procedures", "dbms.functions", "apoc.meta." ] + // Cypher Manual, clauses index (Cypher 25, Neo4j 2026.04): the clauses a query opens with. Administration opens + // with any other word, including the ones a later server adds. + static let cypherHeadClauses: Set = [ + "MATCH", "OPTIONAL", "RETURN", "WITH", "UNWIND", "FOR", "CALL", "MERGE", "CREATE", "INSERT", "FOREACH", + "FINISH", "LET", "FILTER", "WHEN", "SHOW" + ] + + // Clauses upper case, procedures lower case as `cypherReadProcedures` compares them. + static func cypherDataCommands(_ text: String) -> [DataCommand]? { + var commands: [DataCommand] = [] + for code in cypherReadings(of: text) { + for statement in code.split(separator: ";") where statement.contains(where: { !$0.isWhitespace }) { + guard let named = cypherDataCommands(statement: Array(statement)) else { return [.refused] } + commands += named + } + } + guard commands.isEmpty else { return commands } + return cypherClassification(text).tier == .safe ? [.read] : [.refused] + } + + private static func cypherDataCommands(statement code: [Character]) -> [DataCommand]? { + guard let head = cypherHeadClause(in: code), cypherHeadClauses.contains(head) else { return nil } + let words = cypherClauseWords(in: code) + var commands: [DataCommand] = [] + for (position, word) in words.enumerated() { + let previous = position > 0 ? words[position - 1].text : nil + switch word.text { + case "DROP", "DENY", "REVOKE", "GRANT", "LOAD": + return nil + case "CREATE" where previous == "ON": + // MERGE's `ON CREATE SET`. + guard cypherNextWord(in: code, after: word.end) == "SET" else { return nil } + case "CREATE": + // `CREATE INDEX`, `CREATE USER`, `CREATE OR REPLACE DATABASE` name an object where a pattern goes. + guard cypherCreatesPattern(in: code, after: word.end) else { return nil } + commands.append(.named("CREATE")) + case "DELETE": + commands.append(.named(previous == "DETACH" ? "DETACH DELETE" : "DELETE")) + case "MERGE", "SET", "REMOVE", "FOREACH", "INSERT": + commands.append(.named(word.text)) + case "CALL": + guard let procedure = cypherProcedureName(in: code, after: word.end) else { continue } + guard !procedure.isEmpty, !procedure.split(separator: ".").contains("_q") else { return nil } + if !cypherReadProcedures.contains(where: procedure.hasPrefix) { + commands.append(.named(procedure)) + } + case let admin where cypherAdminObjects[admin] != nil: + if let next = cypherNextWord(in: code, after: word.end), cypherAdminObjects[admin]?.contains(next) == true { + return nil + } + default: + continue + } + } + return commands + } + + // The first clause after the prefixes a query may carry: `CYPHER` with its version and options, `EXPLAIN` or + // `PROFILE`, and `USE` with its graph. + private static func cypherHeadClause(in code: [Character]) -> String? { + var index = 0 + while let word = cypherWord(in: code, from: &index) { + switch word { + case "CYPHER": + cypherSkipQueryOptions(in: code, from: &index) + case "EXPLAIN", "PROFILE": + continue + case "USE": + cypherSkipGraph(in: code, from: &index) + default: + return word + } + } + return nil + } + + private static func cypherWord(in code: [Character], from index: inout Int) -> String? { + while index < code.count, code[index].isWhitespace { index += 1 } + let start = index + while index < code.count, code[index].isLetter || code[index].isNumber || code[index] == "_" { index += 1 } + return start < index ? String(code[start ..< index]).uppercased() : nil + } + + // `CYPHER 25 runtime=slotted`: a version, then `name=value` options. + private static func cypherSkipQueryOptions(in code: [Character], from index: inout Int) { + while index < code.count, code[index].isWhitespace { index += 1 } + while index < code.count, code[index].isNumber || code[index] == "." { index += 1 } + while true { + var probe = index + guard cypherWord(in: code, from: &probe) != nil else { return } + while probe < code.count, code[probe].isWhitespace { probe += 1 } + guard probe < code.count, code[probe] == "=" else { return } + probe += 1 + guard cypherWord(in: code, from: &probe) != nil else { return } + index = probe + } + } + + // A graph name, dotted or quoted, or a call such as `graph.byName('x')`. + private static func cypherSkipGraph(in code: [Character], from index: inout Int) { + while index < code.count, code[index].isWhitespace { index += 1 } + while index < code.count, code[index].isLetter || code[index].isNumber || code[index] == "_" || code[index] == "." { + index += 1 + } + var probe = index + while probe < code.count, code[probe].isWhitespace { probe += 1 } + guard probe < code.count, code[probe] == "(" else { return } + var depth = 0 + while probe < code.count { + if code[probe] == "(" { depth += 1 } + if code[probe] == ")" { depth -= 1 } + probe += 1 + if depth == 0 { break } + } + index = probe + } + + // A pattern, or a path name assigned one: `CREATE (n)`, `CREATE p = (a)-[:R]->(b)`. + private static func cypherCreatesPattern(in code: [Character], after end: Int) -> Bool { + guard let start = code[end...].firstIndex(where: { !$0.isWhitespace }) else { return false } + if code[start] == "(" { return true } + let name = code[start...].prefix { $0.isLetter || $0.isNumber || $0 == "_" } + guard !name.isEmpty, + let equals = code[(start + name.count)...].firstIndex(where: { !$0.isWhitespace }), + code[equals] == "=" + else { return false } + return equals + 1 == code.count || !"=~>".contains(code[equals + 1]) + } + // JavaCC's unicode-escape rule: a backslash after an even run of backslashes, one or more `u`, four hex digits. static func cypherDecodingUnicodeEscapes(_ text: String) -> String { let units = Array(text.utf16) diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DataStatements.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DataStatements.swift new file mode 100644 index 0000000000..37798a7f54 --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DataStatements.swift @@ -0,0 +1,93 @@ +// +// QueryClassifier+DataStatements.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +// One command as the assistant's Edit mode reads it. +enum DataCommand: Equatable, Sendable { + // The gate's own tier reads it as a read. + case read + // Runs only when the engine's curated snapshot lists the name in `dataCommands`. + case named(String) + // Changes the engine's own state whatever its name, such as a write into a system collection. + case refused +} + +// Edit runs reads and changes to data, on every engine the rule SQL follows. The lists are allowlists, since each server +// release adds commands: a schema, privilege, session, transaction or server command, and any command a reader cannot +// name, is refused. +extension QueryClassifier { + static func isDataStatement(_ sql: String, databaseType: DatabaseType) -> Bool { + if let answer = isSQLDataStatement(sql, databaseType: databaseType) { return answer } + let snapshot = PluginMetadataRegistry.shared.snapshot(for: databaseType) + let listed = snapshot?.capabilities.dataCommands ?? [] + if databaseType == .surrealdb { + return SQLStatementClassifier.isDataStatement( + sql, + databaseTypeId: databaseType.rawValue, + readings: databaseType.lexicalReadings, + rowChanges: listed + ) + } + guard let commands = dataCommands(of: sql, databaseType: databaseType, snapshot: snapshot) else { return false } + return commands.allSatisfy { command in + switch command { + case .read: return true + case .named(let name): return listed.contains(name) + case .refused: return false + } + } + } + + // Each command the text runs, read as the engine's driver reads it. Nil when the text cannot be read. + static func dataCommands( + of sql: String, + databaseType: DatabaseType, + snapshot: PluginMetadataSnapshot? + ) -> [DataCommand]? { + switch databaseType { + case .redis: + return redisDataCommands(sql) + case .memcached: + return memcachedDataCommands(sql, readings: databaseType.lexicalReadings) + case .etcd: + return etcdDataCommands(sql, readings: databaseType.lexicalReadings) + case .kafka: + return kafkaDataCommands(sql) + case .mongodb: + return mongoDataCommands(sql, systemDatabases: Set(snapshot?.schema.systemDatabaseNames ?? [])) + case .neo4j: + return cypherDataCommands(sql) + case .elasticsearch: + return elasticsearchDataCommands(sql) + case .typesense: + return typesenseDataCommands(sql) + case .qdrant: + return qdrantDataCommands(sql) + case .weaviate: + return weaviateDataCommands(sql) + case .firestore, .firebaseRealtimeDatabase: + return firebaseDataCommands(sql, databaseType: databaseType) + case .bigtable: + return bigtableDataCommands(sql) + default: + // An engine no reader knows runs only what the gate reads as a read. + return classify(sql, databaseType: databaseType).tier == .safe ? [.read] : nil + } + } + + // A write into the scope's database or schema when it is one the engine keeps its own state in. A table named + // without a schema lands there. Engines curated with `dataCommands` name theirs in the snapshot. + static func writesSystemScope(database: String?, schema: String?, databaseType: DatabaseType) -> Bool { + if SQLStatementClassifier.isSystemScope(database: database, schema: schema, databaseTypeId: databaseType.rawValue) { + return true + } + guard let snapshot = PluginMetadataRegistry.shared.snapshot(for: databaseType), + snapshot.capabilities.dataCommands != nil, let database + else { return false } + return snapshot.schema.systemDatabaseNames.contains(database) + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift index f6d41acbcf..e3cfa1a2d9 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+DynamoDB.swift @@ -34,6 +34,27 @@ extension QueryClassifier { return (carried + following).contains { isDangerousQuery($0, databaseType: .dynamodb) } } + // Nil for PartiQL. A request reads or changes items, or defines a table, its TTL, its backups or its tags. The + // PartiQL it carries and the text after its body are read by the SQL rule. + static func dynamoDBIsDataStatement(_ statement: String, grammar: SQLLexicalGrammar) -> Bool? { + guard let request = DynamoDBRequestStatement(statement, grammar: grammar) else { return nil } + guard let action = request.action else { return false } + switch action { + case .createTable, .updateTable, .deleteTable, .updateTimeToLive, .updateContinuousBackups, .tagResource, + .untagResource: + return false + case .executeStatement, .executeTransaction, .batchExecuteStatement: + guard let carried = request.partiQLStatements, carried.count == request.partiQLEntryCount, + carried.allSatisfy({ isSQLDataStatement($0, databaseType: .dynamodb) == true }) + else { return false } + case .browse, .scan, .query, .getItem, .batchGetItem, .transactGetItems, .describeTable, .listTables, + .describeTimeToLive, .describeContinuousBackups, .listTagsOfResource, .describeLimits, .putItem, + .updateItem, .deleteItem, .batchWriteItem, .transactWriteItems: + break + } + return request.hasOnlyReadWindowAfterBody || isSQLDataStatement(request.trailingText, databaseType: .dynamodb) == true + } + private static func dynamoDBTier(of request: DynamoDBRequestStatement) -> QueryTier { guard let action = request.action else { return .destructive } guard let body = request.body, body.isObject else { return worstTier(of: action) } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Flux.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Flux.swift index c194e6cc24..a491062c49 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Flux.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Flux.swift @@ -5,6 +5,11 @@ import TableProSQLGrammar // The SQL reading tiers a Flux read as an unknown statement, a write, so it would refuse every Flux read on a read-only // connection. Each statement is read as the driver receives it, through the driver's own routing and Flux reading. extension QueryClassifier { + // Flux reads, and writes points with `to()`; it defines no bucket or user. + static func fluxIsDataStatement(_ statement: String) -> Bool? { + InfluxStatement.isFlux(statement) ? true : nil + } + static func fluxClassification(_ statement: String) -> QueryClassification? { guard InfluxStatement.isFlux(statement) else { return nil } let reading = FluxScriptReading(InfluxStatement.fluxBody(of: statement)) diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift index ee6ca9bb43..f729530930 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+Memcached.swift @@ -4,33 +4,54 @@ // import Foundation +import TableProMemcachedCore import TableProSQLGrammar extension QueryClassifier { private static let memcachedReadCommands: Set = ["get", "gets", "version"] - private static let memcachedDestructiveCommands: Set = ["delete", "flush_all"] private static let memcachedReadingStats: Set = ["items", "slabs", "settings", "conns", "sizes", "cachedump"] - /// Only the `stats` forms that read are safe: `stats reset` clears the counters and `stats detail on` changes - /// what the server records. `gat` reads a value and moves its expiry, so it writes. The driver strips quotes - /// before it sends a word, so they are stripped here too. + // Read with the driver's own parser, which decodes `\xHH` and `\c` in a quoted word, so `"flush\x5fall"` is the + // `flush_all` it runs. Only the `stats` forms that read are safe: `stats reset` clears the counters and `stats detail + // on` changes what the server records. `gat` reads a value and moves its expiry, so it writes. `delete` removes one + // key, a write as a keyed DELETE is, and `flush_all` empties the cache. static func memcachedClassification(_ trimmed: String) -> QueryClassification { if trimmed.hasPrefix("MEMCACHED_BROWSE:") || trimmed.hasPrefix("MEMCACHED_REFUSAL:") { return .safe } + let write = QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + guard let command = try? MemcachedCommandParser.parse(trimmed) else { return memcachedUnparsedClassification(trimmed) } + switch command { + case .get, .version, .metadump: + return .safe + case .stats(let arguments): + return memcachedStatsRead(arguments.map { $0.lowercased() }) ? .safe : write + case .flushAll: + return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) + case .getAndTouch, .store, .compareAndSwap, .touch, .delete, .increment, .decrement: + return write + } + } + + private static func memcachedStatsRead(_ arguments: [String]) -> Bool { + guard let subcommand = arguments.first else { return true } + return memcachedReadingStats.contains(subcommand) || (subcommand == "detail" && arguments.dropFirst().first == "dump") + } + + // The driver refuses text its parser cannot read; it is tiered by its words all the same. + private static func memcachedUnparsedClassification(_ trimmed: String) -> QueryClassification { let words = trimmed.split(whereSeparator: \.isWhitespace).map { $0.lowercased().filter { $0 != "\"" && $0 != "'" } } guard let command = words.first else { return .safe } - let subcommand = words.dropFirst().first switch command { case "stats": - guard let subcommand, !memcachedReadingStats.contains(subcommand) else { return .safe } - if subcommand == "detail", words.dropFirst(2).first == "dump" { return .safe } - return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) - case "lru_crawler" where subcommand == "metadump": + return memcachedStatsRead(Array(words.dropFirst())) + ? .safe + : QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) + case "lru_crawler" where words.dropFirst().first == "metadump": return .safe default: if memcachedReadCommands.contains(command) { return .safe } - let tier: QueryTier = memcachedDestructiveCommands.contains(command) ? .destructive : .write + let tier: QueryTier = command == "flush_all" ? .destructive : .write return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: false) } } diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+MongoDB.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+MongoDB.swift index ea42892c39..6f0493795a 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+MongoDB.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+MongoDB.swift @@ -20,6 +20,46 @@ extension QueryClassifier { // Neo4j is here so its driver opens a proven read in Bolt's read mode, where the server refuses a write. static let enginesWithProvenReads: Set = [.mongodb, .redis, .etcd, .sapHana, .neo4j] + // Only text the grammar accepts can be named: anything else may reach any method once JavaScript runs it. + static func mongoDataCommands(_ text: String, systemDatabases: Set) -> [DataCommand]? { + let text = StatementBlank.trimming(text) + let located = JavaScriptStatementScanner.locatedStatements(in: text) + let statements = located.count > 1 ? located.map(\.text) : [text] + let systemDatabases = Set(systemDatabases.map { $0.lowercased() }) + var commands: [DataCommand] = [] + for statement in statements where !MongoStatementParser.isTrivia(statement) { + guard let parsed = MongoStatementParser.parse( + statement, shadowsCollection: MongoCollectionAccessor.isShadowedByDatabaseMember + ) else { return nil } + // `$out` and `$merge` write to a collection the pipeline names, and server-side JavaScript to any. + guard !mongoClassification(statement).reachesFilesystemOrExecutesCode else { return [.refused] } + commands += parsed.calls.map { mongoDataCommand($0, systemDatabases: systemDatabases) } + } + return commands.isEmpty ? [.read] : commands + } + + private static func mongoDataCommand(_ call: MongoStatement.Call, systemDatabases: Set) -> DataCommand { + // `use` moves the driver's database for every later statement. + if call.receiver == .shell, call.name == "use" { return .refused } + if call.isRead { return .read } + let name: String + switch call.receiver { + case .collection: + name = "db.collection.\(call.name)" + case .database where call.name == "runCommand": + // A read the command form sends; the code check above already refused `$out`, `$merge` and `$where`. + if let command = call.commandName, MongoCommandCatalog.readsOnly(command) { return .read } + guard let command = call.commandName, call.collection != nil else { return .refused } + name = command + default: + // `adminCommand` and the database helpers create, kill or administer: none of them is a data change. + return .refused + } + if let database = call.database, systemDatabases.contains(database.lowercased()) { return .refused } + if let collection = call.collection, collection.lowercased().hasPrefix("system.") { return .refused } + return .named(name) + } + private static func queryTier(of tier: MongoStatementTier) -> QueryTier { switch tier { case .read: return .safe diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier+RequestDataStatements.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+RequestDataStatements.swift new file mode 100644 index 0000000000..3e9c6360a8 --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier+RequestDataStatements.swift @@ -0,0 +1,256 @@ +// +// QueryClassifier+RequestDataStatements.swift +// TablePro +// + +import Foundation +import TableProFirebaseCore +import TableProQdrantCore +import TableProSQLGrammar +import TableProWeaviateCore + +// A request engine's change is named by its method and the route template of the engine's API reference, such as +// `PUT /{target}/_doc/{id}`. Each driver reads the whole text as one request, so each reader here does too. +extension QueryClassifier { + static func elasticsearchDataCommands(_ text: String) -> [DataCommand]? { + let request = text.trimmingCharacters(in: .whitespacesAndNewlines) + var lines = request.components(separatedBy: "\n") + let header = lines.removeFirst().trimmingCharacters(in: .whitespaces) + let parts = header.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: true) + guard let declared = parts.first.map({ $0.uppercased() }), elasticsearchMethods.contains(declared) else { + return nil + } + var path = parts.count == 2 ? parts[1].trimmingCharacters(in: .whitespaces) : "/" + if !path.hasPrefix("/") { path = "/" + path } + // From here on the URL parser reads the path, scalar by scalar. + let route = String(path.unicodeScalars.prefix(while: { $0 != "?" && $0 != "#" })) + if elasticsearchClassification(request).tier == .safe, !elasticsearchRunsServerCommand(route) { + return [.read] + } + guard let segments = elasticsearchSegments(of: route) else { return nil } + let body = lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) + // The driver sends a GET or HEAD that carries a body as a POST. + let method = !body.isEmpty && (declared == "GET" || declared == "HEAD") ? "POST" : declared + let api = segments.firstIndex { $0.unicodeScalars.first == "_" && $0 != "_all" } ?? segments.count + let rest = path.unicodeScalars.dropFirst(route.unicodeScalars.count) + let query = rest.first == "?" ? String(rest.dropFirst().prefix(while: { $0 != "#" })) : "" + if segments[.. [DataCommand]? { + let request = text.trimmingCharacters(in: .whitespacesAndNewlines) + let header = request.components(separatedBy: "\n")[0].trimmingCharacters(in: .whitespaces) + let parts = header.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: true) + guard let method = parts.first.map({ $0.uppercased() }), typesenseMethods.contains(method) else { return nil } + var path = parts.count == 2 ? parts[1].trimmingCharacters(in: .whitespaces) : "/" + if !path.hasPrefix("/") { path = "/" + path } + // The driver refuses a path with whitespace before its query string. + guard !path.prefix(while: { $0 != "?" }).contains(where: \.isWhitespace) else { return nil } + if typesenseClassification(request).tier == .safe { return [.read] } + let routePath = String(path.unicodeScalars.prefix(while: { $0 != "?" && $0 != "#" })) + guard let route = typesenseRoute(of: routePath) else { return nil } + return [.named(method + " /" + typesenseTemplate(route).joined(separator: "/"))] + } + + static func qdrantDataCommands(_ text: String) -> [DataCommand]? { + let request = text.trimmingCharacters(in: .whitespacesAndNewlines) + // The driver answers it with a ping. + if request.lowercased() == "select 1" { return [.read] } + guard let parsed = QdrantConsoleParser.parse(request) else { return nil } + if qdrantClassification(request).tier == .safe { return [.read] } + guard var route = QdrantPathEncoding.segments(of: parsed.path) else { return nil } + if route.count >= 2, route[0] == "collections" { route[1] = "{collection_name}" } + return [.named(parsed.method + " /" + route.joined(separator: "/"))] + } + + static func weaviateDataCommands(_ text: String) -> [DataCommand]? { + let request = text.trimmingCharacters(in: .whitespacesAndNewlines) + if weaviateClassification(request).tier == .safe { return [.read] } + guard let parsed = WeaviateConsoleParser.parse(request), let segments = weaviateSegments(of: parsed.path) else { + return nil + } + return [.named(parsed.method + " /" + weaviateTemplate(segments).joined(separator: "/"))] + } + + static func firebaseDataCommands(_ text: String, databaseType: DatabaseType) -> [DataCommand]? { + let request = text.trimmingCharacters(in: .whitespacesAndNewlines) + if firebaseClassification(request, databaseType: databaseType).tier == .safe { return [.read] } + guard case .request(let parsed)? = try? FirebaseStatement.parse(request) else { return nil } + let segments = parsed.path.unicodeScalars.split(separator: "/").map(String.init) + if databaseType == .firebaseRealtimeDatabase { + return [realtimeDatabaseCommand(parsed, segments: segments)] + } + return [firestoreCommand(parsed, segments: segments)] + } + + static func bigtableDataCommands(_ text: String) -> [DataCommand]? { + let request = text.trimmingCharacters(in: .whitespacesAndNewlines) + if bigtableClassification(request).tier == .safe { return [.read] } + let verb = String(request.unicodeScalars.prefix(while: { !$0.properties.isWhitespace })).uppercased() + return bigtableChanges.contains(verb) ? [.named(verb)] : nil + } + + private static let elasticsearchMethods: Set = ["GET", "POST", "PUT", "DELETE", "HEAD"] + // Each takes a GET as well, and changes index state. + private static let elasticsearchServerCommands: Set = ["_flush", "_refresh", "_reload_search_analyzers"] + private static let elasticsearchBulkActions: Set = ["index", "create", "update", "delete"] + private static let typesenseMethods: Set = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"] + static let realtimeDatabaseMetadataKeys: Set = [".value", ".priority", ".sv"] + private static let bigtableChanges: Set = ["INSERT", "PUT", "DELETE"] + + private static func elasticsearchRunsServerCommand(_ route: String) -> Bool { + route.unicodeScalars.split(separator: "/").contains { scalars in + let segment = String(scalars) + let name = segment.removingPercentEncoding ?? segment + return name == "." || name == ".." || elasticsearchServerCommands.contains(name) + } + } + + // Nil when the URL the driver builds could reach another route: a dot segment resolves away, an empty one can + // name another host, and a space, `%` or `\` is encoded or decoded on the way. + private static func elasticsearchSegments(of route: String) -> [String]? { + guard !route.unicodeScalars.contains(where: { $0.properties.isWhitespace || $0 == "%" || $0 == "\\" }) else { + return nil + } + return requestSegments(of: route) + } + + private static func elasticsearchTemplate(_ segments: [String], api: Int) -> [String] { + var template: [String] = Array(repeating: "{target}", count: api) + segments[api...] + if api + 1 < segments.count, ["_doc", "_create", "_update"].contains(segments[api]) { + template[api + 1] = "{id}" + } + return template + } + + // A comma list of index names. A name starting with `.` is a system or hidden index (Elasticsearch 8.19, API + // conventions): `.security-7`, `.watches`, `.kibana`. `+`, `-` and `<` lead an expression, `cluster:` a remote one. + private static func elasticsearchNamesSystemIndex(_ expression: String) -> Bool { + expression.unicodeScalars.split(separator: ",").contains { name in + name.split(separator: ":", omittingEmptySubsequences: false).contains { part in + part.first { $0 != "+" && $0 != "-" && $0 != "<" } == "." + } + } + } + + private static func elasticsearchExpandsToHidden(_ query: String) -> Bool { + for pair in query.unicodeScalars.split(separator: "&") { + // The server decodes `+` in a query string as a space. + let item = pair.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false) + .map { String($0).replacingOccurrences(of: "+", with: " ") } + guard let name = item[0].removingPercentEncoding else { return true } + guard name.trimmingCharacters(in: .whitespaces).lowercased() == "expand_wildcards" else { continue } + guard item.count == 2, let value = item[1].removingPercentEncoding else { return true } + let states = value.lowercased().split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + if states.contains(where: { $0 == "hidden" || $0 == "all" }) { return true } + } + return false + } + + // The `_index` of each action line. A source line follows every action but `delete`; a blank line is refused, since + // the server and this reading could pair the lines differently around it. + private static func elasticsearchBulkIndices(_ body: String) -> [String]? { + var indices: [String] = [] + var expectsSource = false + for line in body.components(separatedBy: "\n").map({ $0.trimmingCharacters(in: .whitespacesAndNewlines) }) { + guard !line.isEmpty else { return nil } + if expectsSource { + expectsSource = false + continue + } + guard let action = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any], + action.count == 1, let entry = action.first, elasticsearchBulkActions.contains(entry.key), + let metadata = entry.value as? [String: Any] + else { return nil } + if let index = metadata["_index"] { + guard let name = index as? String else { return nil } + indices.append(name) + } + expectsSource = entry.key != "delete" + } + return expectsSource ? nil : indices + } + + private static func typesenseTemplate(_ route: [String]) -> [String] { + var template = route.map { $0.lowercased() } + guard route.count >= 2, route[0] == "COLLECTIONS" else { return template } + template[1] = "{collection}" + if route.count == 4, route[2] == "DOCUMENTS", !["IMPORT", "EXPORT", "SEARCH"].contains(route[3]) { + template[3] = "{id}" + } + return template + } + + // Read as `WeaviatePathEncoding.resolve` sends it, which keeps a dot segment as written. + static func weaviateSegments(of path: String) -> [String]? { + guard let encoded = URLComponents(string: path)?.percentEncodedPath, + let segments = requestSegments(of: encoded) + else { return nil } + var decoded: [String] = [] + for segment in segments { + guard let text = segment.removingPercentEncoding, !text.isEmpty, text != ".", text != "..", + !text.contains("/"), !text.contains("\\") + else { return nil } + decoded.append(text) + } + return decoded + } + + private static func weaviateTemplate(_ segments: [String]) -> [String] { + guard segments.count >= 3, segments[0] == "v1", segments[1] == "objects" else { return segments } + switch segments.count { + case 3: + return ["v1", "objects", "{id}"] + case 4: + return ["v1", "objects", "{className}", "{id}"] + case 5 where segments[3] == "references": + return ["v1", "objects", "{id}", "references", "{propertyName}"] + case 6 where segments[4] == "references": + return ["v1", "objects", "{className}", "{id}", "references", "{propertyName}"] + default: + return segments + } + } + + // A key cannot hold `.`, so a segment that does names the server's own node: `/.settings/rules` holds the security + // rules. A PATCH names each child it writes by a path of its own. + private static func realtimeDatabaseCommand(_ request: FirebaseRequest, segments: [String]) -> DataCommand { + if segments.contains(where: { $0.unicodeScalars.contains(".") }) { return .refused } + if request.method == .patch { + guard let body = request.body, + let members = try? JSONSerialization.jsonObject(with: Data(body.utf8)) as? [String: Any] + else { return .refused } + let reachesServerNode = members.keys.contains { key in + key.unicodeScalars.split(separator: "/").contains { segment in + segment.contains(".") && !realtimeDatabaseMetadataKeys.contains(String(segment)) + } + } + if reachesServerNode { return .refused } + } + return .named(request.method.rawValue + " /{path}") + } + + // Firestore refuses `.` and `..` as IDs, and the driver sends them as written, where the front end may resolve them. + private static func firestoreCommand(_ request: FirebaseRequest, segments: [String]) -> DataCommand { + if segments.contains(where: { $0 == "." || $0 == ".." }) { return .refused } + guard !segments.isEmpty else { return .named(request.method.rawValue + " /") } + return .named(request.method.rawValue + (segments.count.isMultiple(of: 2) ? " /{document}" : " /{collection}")) + } + + // The segments after the leading `/`, one trailing `/` allowed. Nil on an empty or dot segment. + private static func requestSegments(of route: String) -> [String]? { + var segments = route.unicodeScalars.split(separator: "/", omittingEmptySubsequences: false).map(String.init) + guard segments.first == "" else { return nil } + segments.removeFirst() + if segments.last == "" { segments.removeLast() } + guard !segments.contains(where: { $0.isEmpty || $0 == "." || $0 == ".." }) else { return nil } + return segments + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index c326a505df..479213519e 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -62,6 +62,36 @@ enum QueryClassifier { } } + // Nil on an engine with a query language of its own, Cypher, KafkaQL or a shell: which of its commands define, + // grant or set the session is not a SQL question. + static func isSQLDataStatement(_ sql: String, databaseType: DatabaseType) -> Bool? { + let readings = databaseType.lexicalReadings + guard (PluginMetadataRegistry.shared.snapshot(for: databaseType)?.editorLanguage ?? .sql) == .sql, + documentStoreClassification(sql, databaseType: databaseType, readings: readings) == nil, + redisClassification(sql, databaseType: databaseType) == nil + else { return nil } + let leading = StatementBlank.trimming(SQLLeadingComments.stripping(sql, grammar: readings.execution)) + guard beancountClassification(leading, databaseType: databaseType) == nil else { return nil } + return SQLStatementClassifier.isDataStatement( + sql, + databaseTypeId: databaseType.rawValue, + readings: readings, + request: requestClassification(for: databaseType), + requestIsData: requestDataStatement(for: databaseType) + ) + } + + private static func requestDataStatement(for databaseType: DatabaseType) -> SQLStatementClassifier.RequestDataStatement? { + switch databaseType { + case .dynamodb: + return { statement, grammar in dynamoDBIsDataStatement(statement, grammar: grammar) } + case .influxdb: + return { statement, _ in fluxIsDataStatement(statement) } + default: + return nil + } + } + static func isWriteQuery(_ sql: String, databaseType: DatabaseType) -> Bool { let readings = databaseType.lexicalReadings return classify(sql, databaseType: databaseType, readings: readings) { $0.tier != .safe }.tier != .safe @@ -77,7 +107,7 @@ enum QueryClassifier { // A MongoDB statement is JavaScript, where `delete d.x` removes a field from a local object. guard databaseType != .redis, databaseType != .mongodb else { return false } if databaseType == .neo4j { return cypherDeletesUnfiltered(sql) } - // A request engine's own reading already tiers each delete it can run as destructive. + // A request engine's own reading already tiers each delete of more than one item as destructive. guard documentStoreClassification(sql, databaseType: databaseType, readings: readings) == nil else { return false } @@ -102,16 +132,21 @@ enum QueryClassifier { isMultiStatement(sql, databaseType: databaseType, readings: databaseType.lexicalReadings) } - /// Whether `sql` runs more than one statement: it holds several, a `GO n` line runs its one statement n times, or it - /// ends inside a block it opened, which an engine that reads the opener as a name runs as several statements. A - /// caller that sends one statement once cannot honour the count, and dropping it runs the statement once. + /// Whether `sql` runs more than one statement: it holds several, a `GO n` line runs its one statement n times, it + /// ends inside a block it opened, which an engine that reads the opener as a name runs as several statements, or a + /// T-SQL block in it holds several. A caller that sends one statement once cannot honour the count, and dropping it + /// runs the statement once. static func isMultiStatement(_ sql: String, databaseType: DatabaseType, readings: SQLLexicalReadings) -> Bool { let model = QueryStatementModel.forDatabaseType(databaseType) guard let lexings = readings.lexings(of: sql) else { return true } return lexings.contains { lexing in let batches = QueryBatchPlanner.batches(in: lexing.text, model: model, grammar: lexing.grammar) - return batches.flatMap(\.statements).count > 1 || batches.contains { $0.repeatCount > 1 } + let statements = batches.flatMap(\.statements) + return statements.count > 1 || batches.contains { $0.repeatCount > 1 } || SQLStatementScanner.endsInsideOpenBlock(lexing.text, grammar: lexing.grammar) + || statements.contains { statement in + SQLUnterminatedStatements.blockHoldsSeveralStatements(statement.sql, grammar: lexing.grammar) + } } } @@ -154,7 +189,7 @@ enum QueryClassifier { private static let explainPrefixes: [String] = ["EXPLAIN", "ANALYZE"] } -private extension QueryClassifier { +extension QueryClassifier { static let redisDestructiveCommands: Set = [ "FLUSHDB", "FLUSHALL", "SHUTDOWN", "MIGRATE", "SWAPDB", "REPLICAOF", "SLAVEOF", "CLUSTER", "FAILOVER", "RESET" @@ -180,7 +215,7 @@ private extension QueryClassifier { } } - private static func redisCommandClassification(_ command: String) -> QueryClassification { + static func redisCommandClassification(_ command: String) -> QueryClassification { guard let arguments = RedisArgumentCodec.split(command) else { return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) } @@ -278,14 +313,16 @@ private extension QueryClassifier { } } - /// The APIs a POST may reach and still only read. `POST /orders/_mapping` changes the mapping. + // The APIs a POST may reach and still only read (REST API spec 8.19). `POST /orders/_mapping` changes the mapping. static let elasticsearchReadEndpoints: Set = [ - "_search", "_count", "_msearch", "_explain", "_analyze", "_field_caps", "_validate", "_render", "_sql" + "_search", "_count", "_msearch", "_explain", "_analyze", "_field_caps", "_validate", "_render", "_sql", + "_mget", "_query", "_knn_search", "_termvectors", "_mtermvectors", "_terms_enum", "_search_shards", + "_rank_eval", "_mvt" ] - /// Read the way `ElasticsearchConsoleParser` reads a console request: the method and path on the - /// first line, the rest a body. The driver sends a GET or HEAD that carries a body as a POST, so - /// `GET /orders/_delete_by_query` with a query under it deletes, and it is classified as that POST. + // Read the way `ElasticsearchConsoleParser` reads a console request: the method and path on the first line, the + // rest a body. The driver sends a GET or HEAD that carries a body as a POST, so `GET /orders/_delete_by_query` with + // a query under it deletes, and it is classified as that POST. static func elasticsearchClassification(_ sql: String) -> QueryClassification { let request = sql.trimmingCharacters(in: .whitespacesAndNewlines) var lines = request.components(separatedBy: "\n") @@ -294,6 +331,7 @@ private extension QueryClassifier { let parts = header.split(separator: " ", maxSplits: 1, omittingEmptySubsequences: true) let declared = parts.first.map { $0.uppercased() } ?? "" let method = hasBody && (declared == "GET" || declared == "HEAD") ? "POST" : declared + let path = parts.count == 2 ? parts[1].trimmingCharacters(in: .whitespaces) : "" let upper = request.uppercased() let touchesUnsafeSurface = upper.contains("_SCRIPTS") || upper.contains("_PAINLESS_EXECUTE") @@ -302,21 +340,38 @@ private extension QueryClassifier { case "GET", "HEAD": tier = .safe case "POST": - let path = parts.count == 2 ? parts[1].trimmingCharacters(in: .whitespaces) : "" if path.lowercased().contains("_delete_by_query") { tier = .destructive } else { let readsOnly = elasticsearchEndpoint(of: path).map(elasticsearchReadEndpoints.contains) ?? false - tier = readsOnly ? .safe : .write + tier = readsOnly && !elasticsearchKeepsResults(path) ? .safe : .write } case "DELETE": - tier = .destructive + tier = elasticsearchDeletesOneDocument(path) ? .write : .destructive default: tier = .write } return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) } + // `//_doc/` on one concrete index. A name cannot start with `_`, `-` or `+`, one starting with `.` is a + // system index, and the characters below make a pattern, a list, date math or a remote target. An id starting with + // `_` stays destructive, as older servers read `///_query` as a delete by query. + static func elasticsearchDeletesOneDocument(_ path: String) -> Bool { + let rooted = path.hasPrefix("/") ? path : "/" + path + guard elasticsearchEndpoint(of: rooted) == "_doc" else { return false } + let route = rooted.prefix { $0 != "?" && $0 != "#" } + let segments = Array(route.split(separator: "/", omittingEmptySubsequences: false).dropFirst()) + guard segments.count == 3, segments[1] == "_doc", let idStart = segments[2].first, idStart != "_", + let indexStart = segments[0].first, !"_-+.".contains(indexStart) + else { return false } + return !segments[0].contains { elasticsearchIndexExpressionCharacters.contains($0) } + } + + private static let elasticsearchIndexExpressionCharacters: Set = [ + "*", "?", ",", "<", ">", ":", "|", "\"" + ] + /// The first path segment naming an API, `_search` in `/orders/_search/template`, since an index /// name cannot start with `_` and the body or an index may spell `_search` anywhere. Nil when the /// URL the driver builds could reach another path: a `.` or `..` segment resolves away, and a @@ -326,7 +381,14 @@ private extension QueryClassifier { guard !route.contains(where: { $0.isWhitespace || $0 == "%" || $0 == "\\" }) else { return nil } let segments = route.split(separator: "/") guard !segments.contains(where: { $0 == "." || $0 == ".." }) else { return nil } - return segments.first { $0.hasPrefix("_") }.map(String.init) + // `_all` is a target, not an API: `GET /_all/_search`. + return segments.first { $0.hasPrefix("_") && $0 != "_all" }.map(String.init) + } + + // An async ES|QL query keeps its results on the cluster, as `_async_search` does. + static func elasticsearchKeepsResults(_ path: String) -> Bool { + let segments = path.prefix { $0 != "?" && $0 != "#" }.split(separator: "/") + return zip(segments, segments.dropFirst()).contains { $0 == "_query" && $1 == "async" } } // A snapshot writes to a server path and `/keys` mints API keys. @@ -384,11 +446,18 @@ private extension QueryClassifier { return QueryClassification(tier: .safe, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) } if verb == "DELETE" { - return QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) + let tier: QueryTier = route.map(typesenseNamesOneDocument) == true ? .write : .destructive + return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) } return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) } + // `/collections//documents/`. Without the id, a DELETE removes every document its `filter_by` matches. + static func typesenseNamesOneDocument(_ route: [String]) -> Bool { + route.count == 4 && route[0] == "COLLECTIONS" && route[2] == "DOCUMENTS" + && !["IMPORT", "EXPORT", "SEARCH"].contains(route[3]) + } + /// A bare operation, a `{"query": ...}` envelope and a console body are the same request, and /// the driver forwards the envelope verbatim, so the read-only gate has to read all three. static func weaviateDeclaresMutation(_ body: String) -> Bool { @@ -403,15 +472,18 @@ private extension QueryClassifier { return query.trimmingCharacters(in: .whitespacesAndNewlines).lowercased().hasPrefix("mutation") } - /// Read in the order the driver reads it, with the driver's own console parser, so the request tiered is the one - /// the driver sends: `DELETE` and its path may be split by any whitespace. + // Read in the order the driver reads it, with the driver's own console parser, so the request tiered is the one the + // driver sends: `DELETE` and its path may be split by any whitespace. static func weaviateClassification(_ statement: String) -> QueryClassification { let trimmed = statement.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.lowercased() == "select 1" || WeaviateBrowseQuery.isTagged(trimmed) { return .safe } if WeaviateWriteCodec.isTagged(trimmed) { - return WeaviateWriteCodec.decode(trimmed)?.method.uppercased() == "DELETE" ? .destructive : .write + guard let tagged = WeaviateWriteCodec.decode(trimmed), tagged.method.uppercased() == "DELETE" else { + return .write + } + return weaviateDeletesOneObject(path: tagged.path) ? .write : .destructive } if let request = WeaviateConsoleParser.parse(trimmed) { return weaviateClassification(of: request) @@ -423,12 +495,12 @@ private extension QueryClassifier { return .write } - private static func weaviateClassification(of request: WeaviateConsoleRequest) -> QueryClassification { + static func weaviateClassification(of request: WeaviateConsoleRequest) -> QueryClassification { switch request.method { case "GET", "HEAD": return .safe case "DELETE": - return .destructive + return weaviateDeletesOneObject(path: request.path) ? .write : .destructive case "POST" where request.path.hasPrefix("/v1/graphql"): guard let body = request.body else { return .write } return weaviateDeclaresMutation(body) ? .write : .safe @@ -437,6 +509,21 @@ private extension QueryClassifier { } } + // `/v1/objects//`, or one reference on one object, + // `/v1/objects/[/]/references/`. `/v1/objects/` without a class deletes that UUID in every + // class that holds one (Weaviate's `deleteObjectFromRepo`), so it stays destructive. + static func weaviateDeletesOneObject(path: String) -> Bool { + guard let segments = weaviateSegments(of: path), segments.count >= 4, + segments[0] == "v1", segments[1] == "objects" + else { return false } + switch segments.count { + case 4: return true + case 5: return segments[3] == "references" + case 6: return segments[4] == "references" + default: return false + } + } + // The driver runs what `QdrantConsoleParser` parses, so the gate reads the same request. static func qdrantClassification(_ trimmed: String) -> QueryClassification { if QdrantOperations.isTagged(trimmed) { @@ -447,34 +534,88 @@ private extension QueryClassifier { let tier: QueryTier = switch kind.tier { case .read: .safe case .write: .write - case .remove: .destructive + case .remove: qdrantDeletesOnePoint(request) ? .write : .destructive } return QueryClassification(tier: tier, reachesFilesystemOrExecutesCode: kind.touchesServerFiles) } - /// A grid page is a tagged spec, which only reads. Anything else is read with the driver's own parser, so the path - /// tiered is the path the driver writes. A write that leaves a node holding nothing deletes it, so it ranks with DELETE. + // Qdrant's `PointsSelector` is a list of ids or a filter, either with an optional `shard_key`. Any other key, a + // filter or a second id can remove more than one point. + static func qdrantDeletesOnePoint(_ request: QdrantConsoleRequest) -> Bool { + guard request.method == "POST", + let route = QdrantPathEncoding.segments(of: request.path), route.count == 4, + route[0] == "collections", route[2] == "points", route[3] == "delete", + let data = request.body?.data(using: .utf8), + let selector = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + Set(selector.keys).isSubset(of: ["points", "shard_key"]), + let points = selector["points"] as? [Any], points.count == 1 + else { return false } + switch points[0] { + case let number as NSNumber: + return CFGetTypeID(number) != CFBooleanGetTypeID() + case is String: + return true + default: + return false + } + } + + // A grid page is a tagged spec, which only reads. Anything else is read with the driver's own parser, so the path + // tiered is the path the driver writes. A write that leaves a node holding nothing deletes it, so it ranks with + // DELETE: a write when it removes one document or one record, destructive otherwise. static func firebaseClassification(_ statement: String, databaseType: DatabaseType) -> QueryClassification { let trimmed = statement.trimmingCharacters(in: .whitespacesAndNewlines) if trimmed.unicodeScalars.starts(with: FirebaseStatement.browseTag.unicodeScalars) { return .safe } - let destructive = QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) - let write = QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) // The driver refuses any other text, and an unknown request is a write all the same. - guard case .request(let request)? = try? FirebaseStatement.parse(trimmed) else { return write } + guard case .request(let request)? = try? FirebaseStatement.parse(trimmed) else { return .write } let body = request.body ?? "" + let path = request.path.unicodeScalars.split(separator: "/").map(String.init) + let realtimeDatabase = databaseType == .firebaseRealtimeDatabase switch request.method { case .get, .query: return .safe + case .delete where databaseType == .firestore: + return firestoreNamesDocument(path) ? .write : .destructive case .delete: - return destructive - case .put where request.path.isEmpty || firebaseHoldsNothing(body): - return destructive - case .patch where databaseType == .firebaseRealtimeDatabase && firebaseRemovesChild(body): - return destructive + return realtimeDatabase && realtimeDatabaseRemovesOnlyRecords([path]) ? .write : .destructive + case .put where request.path.isEmpty: + return .destructive + // A PUT replaces the node at its path, every record under a table's node with it. + case .put where realtimeDatabase: + if firebaseHoldsNothing(body) { return realtimeDatabaseRemovesOnlyRecords([path]) ? .write : .destructive } + return path.count >= 2 ? .write : .destructive + case .put where firebaseHoldsNothing(body): + return .destructive + case .patch where realtimeDatabase: + return realtimeDatabasePatchChangesOnlyRecords(path, body: body) ? .write : .destructive case .put, .patch, .post: - return write + return .write + } + } + + // An even number of segments names one document. Firestore refuses `.` and `..` as IDs, and the driver sends them as + // written, where the front end may resolve them to another path. + static func firestoreNamesDocument(_ path: [String]) -> Bool { + !path.isEmpty && path.count.isMultiple(of: 2) && !path.contains { $0 == "." || $0 == ".." } + } + + // The plugin lists `/` and each top-level key as tables (`FirebasePluginDriver.fetchTables`), so only a node two + // levels down is one record. A key cannot hold `.`, so a path through one names the server's own node. + static func realtimeDatabaseRemovesOnlyRecords(_ nodes: [[String]]) -> Bool { + nodes.allSatisfy { node in node.count >= 2 && !node.contains { $0.unicodeScalars.contains(".") } } + } + + // Each key of a PATCH body is a path below the request's, and the PATCH replaces each child it names: one set to + // nothing is removed, any other replaces every record under it. A metadata key sets its own node. + static func realtimeDatabasePatchChangesOnlyRecords(_ path: [String], body: String) -> Bool { + guard let members = firebaseJSON(body) as? [String: Any] else { return true } + return members.allSatisfy { key, value in + var node = path + key.unicodeScalars.split(separator: "/").map(String.init) + if firebaseHoldsNothing(json: value) { return realtimeDatabaseRemovesOnlyRecords([node]) } + if let last = node.last, realtimeDatabaseMetadataKeys.contains(last) { node.removeLast() } + return node.count >= 2 } } @@ -483,11 +624,6 @@ private extension QueryClassifier { return firebaseHoldsNothing(json: json) } - static func firebaseRemovesChild(_ body: String) -> Bool { - guard let members = firebaseJSON(body) as? [String: Any] else { return false } - return members.values.contains { firebaseHoldsNothing(json: $0) } - } - private static func firebaseJSON(_ body: String) -> Any? { guard let data = body.trimmingCharacters(in: .whitespacesAndNewlines).data(using: .utf8) else { return nil } return try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) @@ -516,34 +652,17 @@ private extension QueryClassifier { } } - /// A PUT that sets a cell to null deletes that cell, so it ranks with DELETE. INSERT refuses a - /// key that already has a row, so it never overwrites one. + // Every change names one row by its key (`BigtableRequest.parse`) and runs as one `mutateRow`, so a DELETE or a PUT + // that nulls a cell removes from that row only. static func bigtableClassification(_ trimmed: String) -> QueryClassification { if trimmed.hasPrefix("BIGTABLE_BROWSE:") { return .safe } - let write = QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: false) - let destructive = QueryClassification(tier: .destructive, reachesFilesystemOrExecutesCode: false) switch trimmed.prefix(while: { !$0.isWhitespace }).uppercased() { case "GET", "SCAN", "READ": return .safe - case "DELETE": - return destructive - case "PUT": - return bigtableBodyDeletesCell(trimmed) ? destructive : write default: - return write - } - } - - /// The body is the JSON object that runs to the end of the text. A quoted row key may itself - /// hold a brace, so each brace is tried until one starts an object that parses. - private static func bigtableBodyDeletesCell(_ trimmed: String) -> Bool { - for brace in trimmed.indices where trimmed[brace] == "{" { - guard let data = String(trimmed[brace...]).data(using: .utf8), - let cells = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { continue } - return cells.values.contains { $0 is NSNull } + return .write } - return false } } diff --git a/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift b/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift index c2f9ead605..1bb897caff 100644 --- a/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift +++ b/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift @@ -36,6 +36,7 @@ internal enum ResultJsonSerializer { displayIDs: [RowID]?, selectedDisplayIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columns projection: VisibleColumnProjection ) -> Output { let read = DisplayedResultReader.read( @@ -43,6 +44,7 @@ internal enum ResultJsonSerializer { displayIDs: displayIDs, selectedDisplayIndices: selectedDisplayIndices, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, columns: projection ) diff --git a/apple/TablePro/Core/Utilities/SQL/RowSortComparator.swift b/apple/TablePro/Core/Utilities/SQL/RowSortComparator.swift index f6282825df..4a49e1ae78 100644 --- a/apple/TablePro/Core/Utilities/SQL/RowSortComparator.swift +++ b/apple/TablePro/Core/Utilities/SQL/RowSortComparator.swift @@ -2,24 +2,25 @@ // RowSortComparator.swift // TablePro // -// Type-aware row value comparator for grid sorting. -// import Foundation -/// Type-aware row value comparator for grid sorting. -/// Uses String.compare with .numeric option and type-specific fast paths for integer/decimal columns. internal enum RowSortComparator { internal static func compare(_ lhs: String, _ rhs: String, columnType: ColumnType?) -> ComparisonResult { if let columnType { switch columnType { case .integer: if let l = Int64(lhs), let r = Int64(rhs) { - return l < r ? .orderedAscending : (l > r ? .orderedDescending : .orderedSame) + return order(l, r) } case .decimal: if let l = Double(lhs), let r = Double(rhs) { - return l < r ? .orderedAscending : (l > r ? .orderedDescending : .orderedSame) + return order(l, r) + } + case .boolean: + // MySQL's TINYINT(1) is a boolean that stores any integer from -128 to 127. + if let l = booleanNumber(lhs), let r = booleanNumber(rhs) { + return order(l, r) } default: break @@ -27,4 +28,14 @@ internal enum RowSortComparator { } return lhs.compare(rhs, options: [.numeric]) } + + // One number per value, so `1`, `2` and `true` order the same way whichever pair is compared. + private static func booleanNumber(_ text: String) -> Int64? { + if let number = Int64(text) { return number } + return StoredBoolean.value(of: text).map { $0 ? 1 : 0 } + } + + private static func order(_ lhs: Value, _ rhs: Value) -> ComparisonResult { + lhs < rhs ? .orderedAscending : (lhs > rhs ? .orderedDescending : .orderedSame) + } } diff --git a/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift b/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift index d337c97ebf..dc9896b0f4 100644 --- a/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift +++ b/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift @@ -13,11 +13,11 @@ internal enum RowValueCopyFormatter { return nil case .text(let value): if columnType?.isBlobType ?? false { - return value.formattedAsCompactHex() ?? value + return BlobFormattingService.shared.format(value, for: .copy) ?? value } return value case .bytes(let data): - return String(data: data, encoding: .isoLatin1)?.formattedAsCompactHex() ?? "" + return BlobFormattingService.shared.format(data, for: .copy) ?? "" } } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift b/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift index 9672323deb..a501f6cbb9 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift @@ -36,7 +36,7 @@ internal enum SQLClientEncodingRewriter { replacements = mysqlClientCharacterSets(&cursor, in: text) case .postgres, .redshift, .cockroach: replacements = postgresClientEncoding(&cursor, in: text).map { [Replacement(range: $0, text: postgresEncoding)] } ?? [] - case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .other: + case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .teradata, .other: return nil } guard !replacements.isEmpty else { return nil } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift new file mode 100644 index 0000000000..87ef982eaa --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift @@ -0,0 +1,153 @@ +// +// SQLFileParser+CopyRows.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +extension SQLFileParser { + static let defaultChunkSize = 65_536 + + // A UTF-16 unit takes at most three UTF-8 bytes, so a piece of COPY rows stays within 1 MiB. + static let defaultCopyPieceLength = 1_048_576 / 3 + + struct ReadLimits { + let batchCutLength: Int + let chunkSize: Int + let copyPieceLength: Int + } + + // Reads the rows as psql 17 and 18 do (SQLPsqlScript.copyRows), from a buffer that holds part of the file. + struct CopyRowsScanner { + enum Step { + // Read more, then scan again from the same start. + case needsMoreText + // `length` units of data, then `skipped` units that are never sent: the rest of the `;`'s line, or the + // `\.` line with its line break. + case advance(length: Int, skipped: Int, lineFeeds: Int, endsRows: Bool) + } + + private let pieceLength: Int + private var passesCommandLine = true + // Where the scan stands relative to `start`, which stays put until a step hands units out. + private var lineStart = 0 + private var searchedTo = 0 + private var lineFeeds = 0 + private var checksLine = true + + init(pieceLength: Int) { + // A cut inside a line steps back off a lead surrogate and must still take a unit. + self.pieceLength = max(2, pieceLength) + } + + mutating func scan(_ text: NSString, from start: Int, atEndOfInput: Bool) -> Step { + let available = text.length - start + if passesCommandLine { + return passCommandLine(text, from: start, available: available, atEndOfInput: atEndOfInput) + } + while true { + if checksLine { + if lineStart >= pieceLength { + return cut(at: lineStart, insideLine: false) + } + if available - lineStart < SQLPsqlScript.endOfDataMarkerLookahead, !atEndOfInput { + return .needsMoreText + } + if let markerEnd = SQLPsqlScript.endOfDataMarker(in: text, at: start + lineStart, length: text.length) { + return endAtMarker(text, from: start, markerEnd: markerEnd) + } + if lineStart == available { + return .advance(length: lineStart, skipped: 0, lineFeeds: lineFeeds, endsRows: true) + } + checksLine = false + searchedTo = lineStart + } + let found = text.range( + of: "\n", + options: .literal, + range: NSRange(location: start + searchedTo, length: available - searchedTo) + ) + guard found.location != NSNotFound else { + if available > pieceLength { + return cutBeforeOverflow(text, from: start) + } + if atEndOfInput { + return .advance(length: available, skipped: 0, lineFeeds: lineFeeds, endsRows: true) + } + searchedTo = available + return .needsMoreText + } + let lineEnd = found.location - start + 1 + if lineEnd > pieceLength { + return cutBeforeOverflow(text, from: start) + } + lineFeeds += 1 + lineStart = lineEnd + checksLine = true + } + } + + private mutating func passCommandLine( + _ text: NSString, + from start: Int, + available: Int, + atEndOfInput: Bool + ) -> Step { + let found = text.range(of: "\n", options: .literal, range: NSRange(location: start, length: available)) + if found.location != NSNotFound { + passesCommandLine = false + return .advance(length: 0, skipped: found.location - start + 1, lineFeeds: 1, endsRows: false) + } + guard available > 0 || atEndOfInput else { return .needsMoreText } + return .advance(length: 0, skipped: available, lineFeeds: 0, endsRows: atEndOfInput) + } + + // The line being read runs past the piece: end the piece at the line before it, or inside it when it is the + // piece's first. + private mutating func cutBeforeOverflow(_ text: NSString, from start: Int) -> Step { + if lineStart > 0 { + return cut(at: lineStart, insideLine: false) + } + let splitsPair = UTF16.isLeadSurrogate(text.character(at: start + pieceLength - 1)) + return cut(at: splitsPair ? pieceLength - 1 : pieceLength, insideLine: true) + } + + private mutating func cut(at length: Int, insideLine: Bool) -> Step { + let step = Step.advance(length: length, skipped: 0, lineFeeds: lineFeeds, endsRows: false) + lineStart = 0 + searchedTo = 0 + lineFeeds = 0 + checksLine = !insideLine + return step + } + + private func endAtMarker(_ text: NSString, from start: Int, markerEnd: Int) -> Step { + var end = markerEnd + var feeds = lineFeeds + if end < text.length, text.character(at: end) == SqlLexer.carriageReturn { + end += 1 + } + if end < text.length, text.character(at: end) == SqlLexer.newline { + end += 1 + feeds += 1 + } + return .advance(length: lineStart, skipped: end - start - lineStart, lineFeeds: feeds, endsRows: true) + } + } +} + +extension SQLFileParser.ParserContext { + // SQL resumes on the line after a COPY's rows with nothing of a statement read. + mutating func resumeAfterCopyRows() { + SQLFileParser.resetStatement(&self) + state = .normal + lineHasCode = false + lineHoldsOnlyBlanks = true + previousUnit = SQLFileParser.kNewline + previousUnitInWord = false + pendingSlashLine = false + pendingSlashTrailing.removeAll() + separatorLineSearched = 0 + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift new file mode 100644 index 0000000000..bdf0e0b269 --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift @@ -0,0 +1,235 @@ +import Foundation +import TableProSQLGrammar + +// A literal or quoted name runs to its closer here, and stops at the end of a chunk when the next one decides it. +extension SQLFileParser { + static func processAlternativeQuote( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + if ch == ctx.alternativeQuoteCloser { + let quoteAt = pos + (ctx.alternativeQuoteCloserTrail == nil ? 1 : 2) + if quoteAt >= bufLen { + if ctx.atEndOfInput { + pos += 1 + continue + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + let trailMatches = ctx.alternativeQuoteCloserTrail.map { nsBuffer.character(at: pos + 1) == $0 } ?? true + if trailMatches, nsBuffer.character(at: quoteAt) == kSingleQuote { + pos = quoteAt + 1 + ctx.state = .normal + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + } + pos += 1 + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + /// Runs a triple-quoted literal to the three quotes that close it, holding back a quote or a backslash the buffer + /// ends on until the next chunk shows what follows. + static func processTripleQuotedString( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + if ctx.backslashEscapesActive && ch == kBackslash { + guard pos + 1 < bufLen || ctx.atEndOfInput else { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + pos += 2 + continue + } + if ch == ctx.quoteChar { + guard pos + 2 < bufLen || ctx.atEndOfInput else { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + if SqlLexer.startsTripleQuote(nsBuffer, at: pos, length: bufLen) { + pos += 3 + ctx.state = .normal + ctx.backslashEscapesActive = false + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + } + pos += 1 + } + appendRange(&ctx, from: start, to: min(pos, bufLen), in: nsBuffer) + i = min(pos, bufLen) + return StepResult(advanced: true, deferred: false) + } + + /// Runs a `[...]` identifier to its `]`, which `]]` does not close where the grammar escapes it. + static func processBracketedIdentifier( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + let doubledEscapes = ctx.grammar.contains(.doubledClosingBracketEscapes) + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + guard ch == kCloseBracket else { + pos += 1 + continue + } + if doubledEscapes { + guard pos + 1 < bufLen || ctx.atEndOfInput else { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + if pos + 1 < bufLen, nsBuffer.character(at: pos + 1) == kCloseBracket { + pos += 2 + continue + } + } + pos += 1 + ctx.state = .normal + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + static func processQuotedString( + _ ctx: inout ParserContext, + quoteChar: unichar, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + let escapesActive = ctx.backslashEscapesActive + + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + + if escapesActive && ch == kBackslash { + if pos + 1 >= bufLen { + if ctx.atEndOfInput { + pos += 1 + continue + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + let next = nsBuffer.character(at: pos + 1) + if next == kNewline { ctx.currentLine += 1 } + pos += 2 + continue + } + + if ch == quoteChar { + if pos + 1 >= bufLen && !ctx.atEndOfInput { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + let next: unichar? = pos + 1 < bufLen ? nsBuffer.character(at: pos + 1) : nil + if next == quoteChar { + pos += 2 + continue + } + pos += 1 + ctx.state = .normal + ctx.backslashEscapesActive = false + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + pos += 1 + } + + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + static func processDollarQuote( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + let closeLen = (ctx.dollarTag as NSString).length + 2 + + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + + if ch == kDollar { + if pos + closeLen > bufLen { + if ctx.atEndOfInput { + pos += 1 + continue + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + if SqlDollarQuote.matchesClose(at: pos, tag: ctx.dollarTag, in: nsBuffer, bufLen: bufLen) { + pos += closeLen + ctx.state = .normal + ctx.dollarTag = "" + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + } + pos += 1 + } + + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 4a4445a695..047b9e3885 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -9,6 +9,23 @@ import TableProPluginKit import TableProSQLGrammar import TableProUtilities +/// A file the import refuses before it sends the statement that needs what it cannot give. +enum SQLFileParseError: LocalizedError, Equatable { + case copyFromStandardInput(line: Int) + + var errorDescription: String? { + switch self { + case .copyFromStandardInput(let line): + return String( + format: String( + localized: "Line %lld runs COPY FROM STDIN, and this connection cannot load its rows. Run the file with psql, or dump it again with --inserts." + ), + line + ) + } + } +} + final class SQLFileParser: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFileParser") @@ -19,23 +36,30 @@ final class SQLFileParser: Sendable { /// and held one piece at a time rather than whole. static let defaultBatchCutLength = 67_108_864 - private let batchCutLength: Int + private let limits: ReadLimits - init(batchCutLength: Int = SQLFileParser.defaultBatchCutLength) { - self.batchCutLength = batchCutLength + init( + batchCutLength: Int = SQLFileParser.defaultBatchCutLength, + chunkSize: Int = SQLFileParser.defaultChunkSize, + copyPieceLength: Int = SQLFileParser.defaultCopyPieceLength + ) { + limits = ReadLimits(batchCutLength: batchCutLength, chunkSize: chunkSize, copyPieceLength: copyPieceLength) } /// One statement the file holds, or on a grammar that cuts scripts into batches, one batch, with how many times /// the script runs it. - private struct ParsedStatement { + struct ParsedStatement { let statement: String let lineNumber: Int let repeatCount: Int + var readsCopyRows = false } - private enum ParserState { + enum ParserState { case normal case inSingleLineComment + /// A psql `\restrict` line, which runs to its line feed whatever ends a comment. + case inClientLine case inMultiLineComment case inSingleQuotedString case inDoubleQuotedString @@ -47,24 +71,24 @@ final class SQLFileParser: Sendable { } private static let kSemicolon: unichar = 0x3B - private static let kSingleQuote: unichar = 0x27 + static let kSingleQuote: unichar = 0x27 private static let kDoubleQuote: unichar = 0x22 private static let kBacktick: unichar = 0x60 - private static let kBackslash: unichar = 0x5C + static let kBackslash: unichar = 0x5C private static let kDash: unichar = 0x2D private static let kSlash: unichar = 0x2F private static let kStar: unichar = 0x2A private static let kHash: unichar = 0x23 - private static let kNewline: unichar = 0x0A + static let kNewline: unichar = 0x0A private static let kSpace: unichar = 0x20 private static let kTab: unichar = 0x09 private static let kCarriageReturn: unichar = 0x0D - private static let kDollar: unichar = 0x24 + static let kDollar: unichar = 0x24 private static let kCapitalE: unichar = 0x45 private static let kSmallE: unichar = 0x65 private static let kSmallB: unichar = 0x62 private static let kOpenBracket: unichar = 0x5B - private static let kCloseBracket: unichar = 0x5D + static let kCloseBracket: unichar = 0x5D private static let kCapitalG: unichar = 0x47 private static let kSmallG: unichar = 0x67 @@ -101,7 +125,7 @@ final class SQLFileParser: Sendable { return char == kCloseBracket case .inMultiLineComment: return char == kStar || char == kSlash - case .inSingleLineComment: + case .inSingleLineComment, .inClientLine: return false case .inDollarQuote: return char == kDollar @@ -137,34 +161,37 @@ final class SQLFileParser: Sendable { return true } - private static let delimiterPrefix = "DELIMITER " - private static let delimiterPrefixLength = 10 - - // mysqldump writes `DELIMITER ;;`, whose new delimiter is the rest of the line, so a `;` in it ends nothing. + // mysqldump writes `DELIMITER ;;`, so a `;` on the directive's line ends nothing. private static func readsDelimiterDirective(_ ctx: ParserContext) -> Bool { - guard ctx.grammar.contains(.delimiterDirective), let statement = ctx.currentStatement else { return false } + guard ctx.grammar.contains(.delimiterDirective), ctx.statementOpensLine, + let statement = ctx.currentStatement + else { return false } let length = statement.length var index = 0 while index < length, isWhitespace(statement.character(at: index)) { index += 1 } - let prefix = delimiterPrefix as NSString - guard length - index >= prefix.length, - statement.substring(with: NSRange(location: index, length: prefix.length)).uppercased() == delimiterPrefix - else { return false } + guard SQLDelimiterDirective.startsLine(in: statement, at: index, length: length) else { return false } let rest = NSRange(location: index, length: length - index) return statement.range(of: "\n", options: [], range: rest).location == NSNotFound } - private static func extractDelimiterChange(_ text: String) -> String? { - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard trimmed.uppercased().hasPrefix(delimiterPrefix) else { return nil } - let newDelim = String(trimmed.dropFirst(delimiterPrefixLength)) - .trimmingCharacters(in: .whitespacesAndNewlines) - return newDelim.isEmpty ? nil : newDelim + /// The `DELIMITER` line the statement read so far is, where it opened its line as the mysql client requires. + private static func delimiterDirective(_ ctx: ParserContext) -> SQLDelimiterDirective? { + guard ctx.grammar.contains(.delimiterDirective), ctx.statementOpensLine, + let statement = ctx.currentStatement + else { return nil } + let length = statement.length + var start = 0 + while start < length, isWhitespace(statement.character(at: start)) { + start += 1 + } + let rest = NSRange(location: start, length: length - start) + guard statement.range(of: "\n", options: [], range: rest).location == NSNotFound else { return nil } + return SQLDelimiterDirective.line(startingAt: start, in: statement, length: length) } - private struct ParserContext { + struct ParserContext { var grammar: SQLLexicalGrammar var state: ParserState = .normal let currentStatement: NSMutableString? @@ -215,11 +242,19 @@ final class SQLFileParser: Sendable { /// other dialect has always split an import at each `;` and relies on `DELIMITER` or dollar quoting for a /// routine body, and keeps doing so. var boundaries: (any SQLStatementBoundaryTracking)? + + /// A `;` that ended the statement while the tracker could still take it back at the next token. + var heldEnd: HeldEnd? var word: [unichar] = [] + /// Where the word being read starts in the statement text, and on which line. + var wordStart = 0 + var wordStartLine = 1 var alternativeQuoteCloser: unichar = 0 // A delimiter outside the Basic Multilingual Plane is two units, and the literal closes at both. var alternativeQuoteCloserTrail: unichar? var lineHasCode = false + /// Whether the statement's first unit was the first code on its line, where a `DELIMITER` can stand. + var statementOpensLine = false var pendingSlashLine = false var pendingSlashTrailing: [unichar] = [] @@ -240,7 +275,7 @@ final class SQLFileParser: Sendable { self.currentStatement = currentStatement self.readsBatches = readsBatches self.batchCutLength = batchCutLength - self.boundaries = !readsBatches && SQLStatementBoundaries.statementsCanOwnSemicolons(in: grammar) + self.boundaries = !readsBatches && Self.readsStatementsWithTracker(grammar) ? SQLStatementBoundaries.makeTracker(for: grammar) : nil } @@ -249,27 +284,62 @@ final class SQLFileParser: Sendable { mutating func adopt(_ next: SQLLexicalGrammar) { guard state == .normal, !hasStatementContent, next != grammar else { return } grammar = next - boundaries = !readsBatches && SQLStatementBoundaries.statementsCanOwnSemicolons(in: next) + boundaries = !readsBatches && Self.readsStatementsWithTracker(next) ? SQLStatementBoundaries.makeTracker(for: next) : nil } + // A psql script needs the tracker to find a `COPY ... FROM STDIN`, and it then holds a `BEGIN ATOMIC` body + // and a rule's action list together as the editor does. + private static func readsStatementsWithTracker(_ grammar: SQLLexicalGrammar) -> Bool { + SQLStatementBoundaries.statementsCanOwnSemicolons(in: grammar) || grammar.contains(.psqlScripts) + } + // A SQL Server statement is a batch of its own and keeps the comments inside it, not the ones before it. var keepsComments: Bool { readsBatches || (grammar.contains(.batchSeparatorLines) && hasStatementContent) || insideConditionalComment } } + /// The statement's text before the held `;`, and whether the `;` belongs to it. + struct HeldEnd { + let textLength: Int + let keepsSemicolon: Bool + } + + /// Settles a held end at the first token after it, `word` being that token when it is a word. When the `;` ended + /// the statement, the text up to it is yielded and the word, if any, starts the next statement. + private static func settleHeldEnd(_ ctx: inout ParserContext, word: String?) { + guard let held = ctx.heldEnd else { return } + ctx.heldEnd = nil + guard ctx.boundaries?.continuesStatement(with: word) != true else { return } + var next = "" + if let statement = ctx.currentStatement { + if word != nil, ctx.wordStart < statement.length { + next = statement.substring(from: ctx.wordStart) + } + let kept = held.textLength + (held.keepsSemicolon ? 1 : 0) + statement.deleteCharacters(in: NSRange(location: kept, length: statement.length - kept)) + } + yieldAndReset(&ctx) + guard word != nil else { return } + ctx.currentStatement?.setString(next) + ctx.hasStatementContent = true + ctx.statementStartLine = ctx.wordStartLine + } + private static func trimmedStatement(_ ctx: ParserContext) -> String { (ctx.currentStatement as NSString?)? .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" } - private static func resetStatement(_ ctx: inout ParserContext) { + static func resetStatement(_ ctx: inout ParserContext) { ctx.currentStatement?.setString("") ctx.hasStatementContent = false ctx.insideConditionalComment = false ctx.boundaries?.reset() + ctx.heldEnd = nil + ctx.statementOpensLine = false ctx.word.removeAll(keepingCapacity: true) } @@ -283,6 +353,7 @@ final class SQLFileParser: Sendable { guard !ctx.word.isEmpty else { return } let word = String(utf16CodeUnits: ctx.word, count: ctx.word.count).uppercased() ctx.word.removeAll(keepingCapacity: true) + settleHeldEnd(&ctx, word: word) ctx.boundaries?.observeWord(word) } @@ -308,9 +379,11 @@ final class SQLFileParser: Sendable { flushWord(&ctx) } guard SqlBlockStructure.startsWord(nsBuffer, at: i, length: bufLen, grammar: ctx.grammar) else { return .none } - // `E'...'` is a literal, and the escape-string branch that reads it never runs once `E` starts a word. - if ctx.grammar.contains(.escapeStringPrefix), char == kCapitalE || char == kSmallE, - !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { + // `E'...'` and CockroachDB's `b'...'` are literals, and the branch that reads them never runs once the prefix + // starts a word. + let startsEscapeLiteral = (ctx.grammar.contains(.escapeStringPrefix) && (char == kCapitalE || char == kSmallE)) + || (ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB) + if startsEscapeLiteral, !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar) { if i + 1 < bufLen { if nsBuffer.character(at: i + 1) == kSingleQuote { return .none } } else if !ctx.atEndOfInput { @@ -318,7 +391,7 @@ final class SQLFileParser: Sendable { return .needsMoreData } } - if readsAlternativeQuotes, !SQLNonCodeSpan.isWordUnit(ctx.previousUnit), + if readsAlternativeQuotes, !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar), let quoteLength = alternativeQuotePrefixLength(nsBuffer, at: i, bufLen: bufLen) { if quoteLength > 0 { return .opensAlternativeQuote(prefixLength: quoteLength) @@ -330,7 +403,9 @@ final class SQLFileParser: Sendable { } guard !followsWord else { return .none } ctx.previousUnitInWord = true - if ctx.boundaries?.needsWords == true { + if ctx.heldEnd != nil || ctx.boundaries?.needsWords == true { + ctx.wordStart = ctx.currentStatement?.length ?? 0 + ctx.wordStartLine = ctx.currentLine ctx.word.append(char) } return .continues @@ -400,19 +475,36 @@ final class SQLFileParser: Sendable { return false } + // A refused argument leaves the delimiter as it was, and the line still runs nothing. private static func processDelimiterChange(_ ctx: inout ParserContext, char: unichar) { - guard ctx.grammar.contains(.delimiterDirective) else { return } - guard char == kNewline && ctx.hasStatementContent else { return } - let text = trimmedStatement(ctx) - if let newDelim = extractDelimiterChange(text) { + guard char == kNewline, ctx.hasStatementContent, let directive = delimiterDirective(ctx) else { return } + if let newDelim = directive.delimiter { ctx.currentDelimiter = newDelim as NSString ctx.isSingleCharDelimiter = ctx.currentDelimiter.length == 1 && ctx.currentDelimiter.character(at: 0) == kSemicolon - resetStatement(&ctx) } + resetStatement(&ctx) } - private struct StepResult { + // pg_dump's `\restrict` and `\unrestrict` limit psql's meta-commands, and none runs here, so the line is read as a + // comment. + private static func skipRestrictLine( + _ ctx: inout ParserContext, + at i: Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult? { + guard ctx.grammar.contains(.psqlScripts), !ctx.hasStatementContent, !ctx.lineHasCode else { return nil } + if i + SQLPsqlScript.restrictLineLookahead > bufLen, !ctx.atEndOfInput { + return StepResult(advanced: false, deferred: true) + } + guard SQLPsqlScript.startsRestrictLine(in: nsBuffer, at: i, length: bufLen) else { return nil } + ctx.state = .inClientLine + ctx.boundaries?.observeGap() + return StepResult(advanced: false, deferred: false) + } + + struct StepResult { var advanced: Bool var deferred: Bool } @@ -434,8 +526,15 @@ final class SQLFileParser: Sendable { return step } + if char == kBackslash, let step = skipRestrictLine(&ctx, at: i, nsBuffer: nsBuffer, bufLen: bufLen) { + return step + } + switch observeWordCharacter(&ctx, char: char, i: i, nsBuffer: nsBuffer, bufLen: bufLen) { case .continues: + if !ctx.hasStatementContent { + ctx.statementOpensLine = !ctx.lineHasCode + } (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) @@ -443,6 +542,7 @@ final class SQLFileParser: Sendable { case .needsMoreData: return StepResult(advanced: false, deferred: true) case let .opensAlternativeQuote(prefixLength): + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendRange(&ctx, from: i, to: i + prefixLength, in: nsBuffer) @@ -502,7 +602,8 @@ final class SQLFileParser: Sendable { let startsEscapeBytes = ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB if (startsEscapeString || startsEscapeBytes) && nextChar == kSingleQuote - && !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { + && !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar) { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) @@ -516,6 +617,7 @@ final class SQLFileParser: Sendable { } if char == kOpenBracket, ctx.grammar.contains(.bracketQuotedIdentifiers) { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) @@ -525,8 +627,16 @@ final class SQLFileParser: Sendable { } if let style = ctx.grammar.dollarQuoteStyle, char == kDollar { - switch SqlDollarQuote.scanOpener(at: i, in: nsBuffer, bufLen: bufLen, style: style) { + let grammar = ctx.grammar + switch SqlDollarQuote.scanOpener( + at: i, + in: nsBuffer, + bufLen: bufLen, + style: style, + continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) } + ) { case .opener(let length, let tag): + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) if let target = ctx.currentStatement { @@ -579,17 +689,24 @@ final class SQLFileParser: Sendable { return StepResult(advanced: true, deferred: false) } - if !ctx.hasStatementContent && !isWhitespace(char) { - ctx.statementStartLine = ctx.currentLine - ctx.hasStatementContent = true - } + appendPlainUnit(&ctx, char: char) + return StepResult(advanced: false, deferred: false) + } + + // Whitespace, or a symbol no rule above claimed. + private static func appendPlainUnit(_ ctx: inout ParserContext, char: unichar) { if isWhitespace(char) { ctx.boundaries?.observeGap() } else { + settleHeldEnd(&ctx, word: nil) + if !ctx.hasStatementContent { + ctx.statementStartLine = ctx.currentLine + ctx.hasStatementContent = true + ctx.statementOpensLine = !ctx.lineHasCode + } ctx.boundaries?.observeSymbol(char) } appendChar(char, to: ctx.currentStatement) - return StepResult(advanced: false, deferred: false) } /// A `/*` opens a comment, or the code of a conditional comment the server runs, which a `*/` read as code closes. @@ -614,6 +731,7 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: true) } if SqlLexer.executableCommentOpenerLength(nsBuffer, at: i, length: bufLen) != nil { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) ctx.insideConditionalComment = true @@ -654,57 +772,38 @@ final class SQLFileParser: Sendable { return } flushWord(&ctx) + // T-SQL also reads further `;`s after an `IF`'s body as part of it, up to its `ELSE`. + if ctx.heldEnd != nil { + _ = ctx.boundaries?.observeSemicolon() + appendChar(kSemicolon, to: ctx.currentStatement) + return + } let endsStatement = ctx.boundaries?.observeSemicolon() ?? true guard endsStatement else { appendChar(kSemicolon, to: ctx.currentStatement) return } + if ctx.boundaries?.copiesFromStandardInput == true { + ctx.collected.append(ParsedStatement( + statement: trimmedStatement(ctx), lineNumber: ctx.statementStartLine, repeatCount: 1, readsCopyRows: true + )) + resetStatement(&ctx) + return + } + if ctx.boundaries?.awaitsContinuation == true, ctx.hasStatementContent { + ctx.heldEnd = HeldEnd( + textLength: ctx.currentStatement?.length ?? 0, + keepsSemicolon: ctx.boundaries?.terminator == .partOfStatement + ) + appendChar(kSemicolon, to: ctx.currentStatement) + return + } if ctx.boundaries?.terminator == .partOfStatement, ctx.hasStatementContent { appendChar(kSemicolon, to: ctx.currentStatement) } yieldAndReset(&ctx) } - private static func processAlternativeQuote( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - if ch == ctx.alternativeQuoteCloser { - let quoteAt = pos + (ctx.alternativeQuoteCloserTrail == nil ? 1 : 2) - if quoteAt >= bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let trailMatches = ctx.alternativeQuoteCloserTrail.map { nsBuffer.character(at: pos + 1) == $0 } ?? true - if trailMatches, nsBuffer.character(at: quoteAt) == kSingleQuote { - pos = quoteAt + 1 - ctx.state = .normal - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - private static func isQuote(_ char: unichar, grammar: SQLLexicalGrammar) -> Bool { grammar.isQuote(char) } @@ -715,6 +814,7 @@ final class SQLFileParser: Sendable { nextChar: unichar? ) -> Bool? { guard isQuote(char, grammar: ctx.grammar) else { return nil } + settleHeldEnd(&ctx, word: nil) if let next = nextChar, next == char { (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) @@ -736,6 +836,7 @@ final class SQLFileParser: Sendable { } private static func openTripleQuote(_ ctx: inout ParserContext, quote: unichar, at i: Int, in buffer: NSString) { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendRange(&ctx, from: i, to: i + 3, in: buffer) @@ -745,93 +846,6 @@ final class SQLFileParser: Sendable { ctx.boundaries?.observeQuotedToken(opener: quote) } - /// Runs a triple-quoted literal to the three quotes that close it, holding back a quote or a backslash the buffer - /// ends on until the next chunk shows what follows. - private static func processTripleQuotedString( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - if ctx.backslashEscapesActive && ch == kBackslash { - guard pos + 1 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - pos += 2 - continue - } - if ch == ctx.quoteChar { - guard pos + 2 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if SqlLexer.startsTripleQuote(nsBuffer, at: pos, length: bufLen) { - pos += 3 - ctx.state = .normal - ctx.backslashEscapesActive = false - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - appendRange(&ctx, from: start, to: min(pos, bufLen), in: nsBuffer) - i = min(pos, bufLen) - return StepResult(advanced: true, deferred: false) - } - - /// Runs a `[...]` identifier to its `]`, which `]]` does not close where the grammar escapes it. - private static func processBracketedIdentifier( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let doubledEscapes = ctx.grammar.contains(.doubledClosingBracketEscapes) - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - guard ch == kCloseBracket else { - pos += 1 - continue - } - if doubledEscapes { - guard pos + 1 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if pos + 1 < bufLen, nsBuffer.character(at: pos + 1) == kCloseBracket { - pos += 2 - continue - } - } - pos += 1 - ctx.state = .normal - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - private static func yieldAndReset(_ ctx: inout ParserContext) { if ctx.hasStatementContent { let text = trimmedStatement(ctx) @@ -935,7 +949,7 @@ final class SQLFileParser: Sendable { return false } - private static func appendRange( + static func appendRange( _ ctx: inout ParserContext, from start: Int, to end: Int, @@ -945,109 +959,6 @@ final class SQLFileParser: Sendable { target.append(buffer.substring(with: NSRange(location: start, length: end - start))) } - private static func processQuotedString( - _ ctx: inout ParserContext, - quoteChar: unichar, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let escapesActive = ctx.backslashEscapesActive - - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - - if escapesActive && ch == kBackslash { - if pos + 1 >= bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let next = nsBuffer.character(at: pos + 1) - if next == kNewline { ctx.currentLine += 1 } - pos += 2 - continue - } - - if ch == quoteChar { - if pos + 1 >= bufLen && !ctx.atEndOfInput { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let next: unichar? = pos + 1 < bufLen ? nsBuffer.character(at: pos + 1) : nil - if next == quoteChar { - pos += 2 - continue - } - pos += 1 - ctx.state = .normal - ctx.backslashEscapesActive = false - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - pos += 1 - } - - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - private static func processDollarQuote( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let closeLen = (ctx.dollarTag as NSString).length + 2 - - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - - if ch == kDollar { - if pos + closeLen > bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if SqlDollarQuote.matchesClose(at: pos, tag: ctx.dollarTag, in: nsBuffer, bufLen: bufLen) { - pos += closeLen - ctx.state = .normal - ctx.dollarTag = "" - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - func parseFile(url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { parseFile(url: url, encoding: encoding, grammar: { grammar }) } @@ -1056,25 +967,43 @@ final class SQLFileParser: Sendable { func parseFile( url: URL, encoding: String.Encoding, - grammar: @escaping @Sendable () -> SQLLexicalGrammar + grammar: @escaping @Sendable () -> SQLLexicalGrammar, + onRead: (@Sendable (_ bytesRead: Int64, _ fileSize: Int64) -> Void)? = nil ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - let session = ParseSession( - url: url, encoding: encoding, grammar: grammar, countOnly: false, batchCutLength: batchCutLength - ) + let session = ParseSession(url: url, encoding: encoding, grammar: grammar, countOnly: false, limits: limits, onRead: onRead) return AsyncThrowingStream(unfolding: { try await session.nextRun() }) } - // Runs are added up rather than walked: a `GO n` count can reach `Int32.max`. + func parseElements( + url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar + ) -> AsyncThrowingStream { + parseElements(url: url, encoding: encoding, grammar: { grammar }) + } + + // The statements as parseFile yields them, and each `COPY ... FROM STDIN` with a reader over its rows. + func parseElements( + url: URL, + encoding: String.Encoding, + grammar: @escaping @Sendable () -> SQLLexicalGrammar, + onRead: (@Sendable (_ bytesRead: Int64, _ fileSize: Int64) -> Void)? = nil + ) -> AsyncThrowingStream { + let session = ParseSession(url: url, encoding: encoding, grammar: grammar, countOnly: false, limits: limits, onRead: onRead) + return AsyncThrowingStream(unfolding: { + try await session.nextElement() + }) + } + + // Runs are added up rather than walked: a `GO n` count can reach `Int32.max`. A COPY counts as one. func countStatements( url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar ) async throws -> Int { - let session = ParseSession( - url: url, encoding: encoding, grammar: { grammar }, countOnly: true, batchCutLength: batchCutLength - ) + let session = ParseSession(url: url, encoding: encoding, grammar: { grammar }, countOnly: true, limits: limits) var count = 0 while let statement = try await session.nextStatement() { @@ -1100,10 +1029,12 @@ extension SQLFileParser { private let url: URL private let encoding: String.Encoding private let grammar: @Sendable () -> SQLLexicalGrammar - private let batchCutLength: Int - private let chunkSize = 65_536 + private let limits: ReadLimits + private let onRead: (@Sendable (Int64, Int64) -> Void)? private var fileHandle: FileHandle? + private var fileSize: Int64 = 0 + private var bytesRead: Int64 = 0 private var ctx: ParserContext private let nsBuffer = NSMutableString() // Text before it is lexed; it is deleted when the next chunk is read, not once per statement. @@ -1115,24 +1046,35 @@ extension SQLFileParser { private var repeating: ParsedStatement? private var remainingRuns = 0 + // The rows of the COPY last yielded while any are unread, and which COPY that is, so a reader handed out + // for an earlier one answers nil. + private var copyRows: CopyRowsScanner? + private var copyGeneration = 0 + private var rowsFailure: Error? + init( url: URL, encoding: String.Encoding, grammar: @escaping @Sendable () -> SQLLexicalGrammar, countOnly: Bool, - batchCutLength: Int, - reading: Reading = .fromFile + limits: ReadLimits, + reading: Reading = .fromFile, + onRead: (@Sendable (Int64, Int64) -> Void)? = nil ) { self.url = url self.encoding = encoding self.grammar = grammar - self.batchCutLength = batchCutLength + self.limits = limits + self.onRead = onRead self.decoder = SQLChunkDecoder(encoding: encoding) self.readingSettled = reading == .batches + let initialGrammar = grammar() + // A `DELIMITER` line is read from the statement's text, so a count needs the text too. + let keepsText = !countOnly || initialGrammar.contains(.delimiterDirective) self.ctx = ParserContext( - grammar: grammar(), - currentStatement: countOnly ? nil : NSMutableString(), - batchCutLength: batchCutLength, + grammar: initialGrammar, + currentStatement: keepsText ? NSMutableString() : nil, + batchCutLength: limits.batchCutLength, readsBatches: reading == .batches ) } @@ -1142,21 +1084,48 @@ extension SQLFileParser { } func nextRun() async throws -> (statement: String, lineNumber: Int)? { + let element = try await nextElement() + // The rows after the `COPY` would otherwise reach the server as statements. + if case let .copyFromStandardInput(_, line, _)? = element { + finished = true + closeFile() + throw SQLFileParseError.copyFromStandardInput(line: line) + } + guard case let .statement(statement, lineNumber)? = element else { return nil } + return (statement, lineNumber) + } + + func nextElement() async throws -> PluginImportElement? { if let repeating, remainingRuns > 0 { remainingRuns -= 1 - return (repeating.statement, repeating.lineNumber) + return .statement(repeating.statement, lineNumber: repeating.lineNumber) } guard let next = try await nextStatement() else { return nil } - repeating = next - remainingRuns = next.repeatCount - 1 - return (next.statement, next.lineNumber) + guard next.readsCopyRows else { + repeating = next + remainingRuns = next.repeatCount - 1 + return .statement(next.statement, lineNumber: next.lineNumber) + } + let generation = copyGeneration + // The driver reads pieces on its own queue while the import awaits the copy, and the import asks for the + // next element only once the copy returns, so the session is never used from two places at once. + let rows = PluginCopyData { [self] in try nextRowsPiece(generation: generation) } + return .copyFromStandardInput(statement: next.statement, lineNumber: next.lineNumber, rows: rows) } func nextStatement() async throws -> ParsedStatement? { + if let failure = rowsFailure { + rowsFailure = nil + throw failure + } while true { if emitIndex < ctx.collected.count { let item = ctx.collected[emitIndex] emitIndex += 1 + if item.readsCopyRows { + copyGeneration += 1 + copyRows = CopyRowsScanner(pieceLength: limits.copyPieceLength) + } return item } ctx.collected.removeAll(keepingCapacity: true) @@ -1167,6 +1136,7 @@ extension SQLFileParser { } do { + try passUnreadRows() try settleReading() guard !Task.isCancelled else { finished = true @@ -1195,14 +1165,14 @@ extension SQLFileParser { encoding: encoding, grammar: { grammar }, countOnly: true, - batchCutLength: batchCutLength, + limits: limits, reading: .batches ) guard try scan.findsBatchSeparator() else { return } ctx = ParserContext( grammar: grammar, currentStatement: ctx.currentStatement, - batchCutLength: batchCutLength, + batchCutLength: limits.batchCutLength, readsBatches: true ) } @@ -1216,17 +1186,29 @@ extension SQLFileParser { return ctx.sawBatchSeparator } + // A MySQL count keeps each statement's text, so the cut drains per statement as well as per chunk. private func readNextStatement() throws { - processBuffer() - guard ctx.collected.isEmpty else { return } - guard !ctx.atEndOfInput else { - emitTrailingStatement() - finished = true - closeFile() - return + try autoreleasepool { + processBuffer() + guard ctx.collected.isEmpty else { return } + guard !ctx.atEndOfInput else { + emitTrailingStatement() + finished = true + closeFile() + return + } + try readChunk() } + } + + private func readChunk() throws { let handle = try openFileIfNeeded() - let rawData = handle.readData(ofLength: chunkSize) + // A count, a GO scan and a driver's COPY read chunk after chunk with nothing draining the autorelease pool. + let rawData = try autoreleasepool { try handle.read(upToCount: limits.chunkSize) } ?? Data() + if !rawData.isEmpty, let onRead { + bytesRead += Int64(rawData.count) + onRead(bytesRead, fileSize) + } if rawData.isEmpty && !decoder.hasPendingBytes { ctx.atEndOfInput = true @@ -1292,6 +1274,16 @@ extension SQLFileParser { || (char == SqlLexer.formFeed && ctx.grammar.contains(.formFeedEndsLineComments)) { ctx.state = .normal } + // `DELIMITER // -- note` ends at this line feed, which the comment took. + if char == SQLFileParser.kNewline { + SQLFileParser.processDelimiterChange(&ctx, char: char) + } + step = StepResult(advanced: false, deferred: false) + + case .inClientLine: + if char == SQLFileParser.kNewline { + ctx.state = .normal + } step = StepResult(advanced: false, deferred: false) case .inMultiLineComment: @@ -1347,6 +1339,48 @@ extension SQLFileParser { readOffset = min(i, bufLen) } + private func nextRowsPiece(generation: Int) throws -> Data? { + do { + return try autoreleasepool { + guard let piece = try nextRowsRange(generation: generation) else { return nil } + return Data(nsBuffer.substring(with: piece).utf8) + } + } catch { + rowsFailure = error + copyRows = nil + finished = true + closeFile() + SQLFileParser.logger.error("SQL file parsing failed: \(error.localizedDescription)") + throw error + } + } + + private func passUnreadRows() throws { + while copyRows != nil, !finished, !Task.isCancelled { + _ = try nextRowsRange(generation: copyGeneration) + } + } + + // The next piece's range in the buffer, valid until the next chunk is read. + private func nextRowsRange(generation: Int) throws -> NSRange? { + while !finished, generation == copyGeneration, + let step = copyRows?.scan(nsBuffer, from: readOffset, atEndOfInput: ctx.atEndOfInput) { + guard case let .advance(length, skipped, lineFeeds, endsRows) = step else { + try readChunk() + continue + } + let piece = NSRange(location: readOffset, length: length) + readOffset += length + skipped + ctx.currentLine += lineFeeds + if endsRows { + copyRows = nil + ctx.resumeAfterCopyRows() + } + if length > 0 { return piece } + } + return nil + } + private func emitTrailingStatement() { ctx.pendingSlashLine = false ctx.pendingSlashTrailing.removeAll() @@ -1354,9 +1388,11 @@ extension SQLFileParser { SQLFileParser.yieldBatch(&ctx, repeatCount: 1, nextBatchStart: ctx.unitsBeforeBuffer + readOffset) return } + SQLFileParser.flushWord(&ctx) + SQLFileParser.settleHeldEnd(&ctx, word: nil) guard ctx.hasStatementContent else { return } let text = SQLFileParser.trimmedStatement(ctx) - if SQLFileParser.extractDelimiterChange(text) == nil { + if SQLFileParser.delimiterDirective(ctx) == nil { ctx.collected.append( ParsedStatement(statement: text, lineNumber: ctx.statementStartLine, repeatCount: 1) ) @@ -1369,6 +1405,11 @@ extension SQLFileParser { } let handle = try FileHandle(forReadingFrom: url) fileHandle = handle + // Taken from the open handle; a size of 0 leaves the import's bar on statements. + if onRead != nil { + var info = stat() + fileSize = fstat(handle.fileDescriptor, &info) == 0 ? Int64(info.st_size) : 0 + } return handle } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift index dab2c5e5b4..05251416e1 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift @@ -55,7 +55,9 @@ enum SQLParameterExtractor { } if let param = paramLookup[name] { - values.append(param.isNull ? nil : param.value) + values.append( + param.isNull ? nil : TypedParameterValue(text: param.value, kind: param.type.pluginKind(for: param.value)) + ) } else { values.append(nil) } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift index 4eac9054f4..acadf5177a 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift @@ -14,8 +14,8 @@ import TableProPluginKit internal struct SQLRowToStatementConverter { /// One row of the grid, as it is shown and as the server has it. /// - /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid spells that as the - /// text `__DEFAULT__`, which a stored value can also be, so only a cell named here is read as the marker. + /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid stages that as the + /// keyword DEFAULT marked as SQL, and a stored value can read the same, so only a cell named here is the default. /// `expressionColumns` names the cells staged as SQL, which go out unquoted; text anywhere else is a literal. internal struct SourceRow { internal let values: [PluginCellValue] @@ -228,18 +228,6 @@ internal struct SQLRowToStatementConverter { } private func formatBinaryLiteral(_ data: Data) -> String { - var hex = "" - hex.reserveCapacity(data.count * 2) - for byte in data { - hex += String(format: "%02X", byte) - } - switch databaseType { - case .postgresql, .redshift, .cockroachdb, .pglite, .kingbasees: - return "'\\x\(hex)'::bytea" - case .mssql: - return "0x\(hex)" - default: - return "X'\(hex)'" - } + CompareSQLLiteral.binaryLiteral(for: data, databaseType: databaseType) ?? "X'\(data.uppercaseHex())'" } } diff --git a/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift b/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift index 214686aad0..1edd37db8c 100644 --- a/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift +++ b/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift @@ -78,7 +78,7 @@ struct TableEditDialect: Sendable, Equatable { return .sqlServer case .oracle: return .oracle - case .cockroach, .redis, .other: + case .cockroach, .redis, .teradata, .other: return nil } } diff --git a/apple/TablePro/Core/Utilities/UI/AlertHelper.swift b/apple/TablePro/Core/Utilities/UI/AlertHelper.swift index 1c6f1c04de..de6cc1c69e 100644 --- a/apple/TablePro/Core/Utilities/UI/AlertHelper.swift +++ b/apple/TablePro/Core/Utilities/UI/AlertHelper.swift @@ -381,6 +381,54 @@ final class AlertHelper { } } + // MARK: - Ending a Transaction + + enum TransactionEndAnswer: Equatable, Sendable { + case commit, rollBack, cancel + } + + // The Save alert's order: Commit keeps Return, Cancel takes Escape, Roll Back takes no key. Without Commit, the roll-back alert. + static func makeTransactionEndAlert(_ prompt: TransactionEndPrompt) -> NSAlert { + let alert = NSAlert() + alert.messageText = prompt.title + alert.informativeText = prompt.message + alert.alertStyle = .warning + guard prompt.offersCommit else { + addConfirmAndCancel(to: alert, confirmButton: prompt.confirmButton, cancelButton: String(localized: "Cancel")) + return alert + } + let commit = alert.addButton(withTitle: String(localized: "Commit")) + commit.keyEquivalent = "\r" + addCancelButton(to: alert, title: String(localized: "Cancel")) + let rollBack = alert.addButton(withTitle: String(localized: "Roll Back")) + rollBack.hasDestructiveAction = true + rollBack.keyEquivalent = "" + return alert + } + + static func transactionEndAnswer(to response: NSApplication.ModalResponse, offersCommit: Bool) -> TransactionEndAnswer { + switch response { + case .alertFirstButtonReturn: + return offersCommit ? .commit : .rollBack + case .alertThirdButtonReturn where offersCommit: + return .rollBack + default: + return .cancel + } + } + + // On the window's frontmost sheet: work still running can hold a progress sheet there. + static func answerTransactionEnd(_ prompt: TransactionEndPrompt, window: NSWindow?) async -> TransactionEndAnswer { + let parent = resolveWindow(window).map(AlertOperationConfirming.frontmostSheet(of:)) + let clicked = await response(to: makeTransactionEndAlert(prompt), in: parent) + return transactionEndAnswer(to: clicked, offersCommit: prompt.offersCommit) + } + + // Quit and relaunch need the answer before they return. + static func answerTransactionEndModally(_ prompt: TransactionEndPrompt) -> TransactionEndAnswer { + transactionEndAnswer(to: makeTransactionEndAlert(prompt).runModal(), offersCommit: prompt.offersCommit) + } + // MARK: - Three-Way Confirmations // `detail` is the text the answer is given against, so it goes whole into a scrolling box that reveals invisible diff --git a/apple/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift b/apple/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift index 9010bba72d..ba84225491 100644 --- a/apple/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift +++ b/apple/TablePro/Core/VersionHistory/LinkedFileVersionHistoryProvider.swift @@ -70,7 +70,7 @@ internal struct LinkedFileVersionHistoryProvider: VersionHistoryProvider { throw VersionHistoryError.undecodableContent } return content - case .savedQueryVersion: + case .savedQueryVersion, .tabVersion: throw VersionHistoryError.versionNotFound } } diff --git a/apple/TablePro/Core/VersionHistory/QueryTabVersionHistoryProvider.swift b/apple/TablePro/Core/VersionHistory/QueryTabVersionHistoryProvider.swift new file mode 100644 index 0000000000..132e22be01 --- /dev/null +++ b/apple/TablePro/Core/VersionHistory/QueryTabVersionHistoryProvider.swift @@ -0,0 +1,85 @@ +// +// QueryTabVersionHistoryProvider.swift +// TablePro +// + +import Foundation + +internal struct LiveQueryTab: Sendable, Equatable { + let connectionId: UUID + let text: String +} + +internal struct QueryTabVersionHistoryProvider: VersionHistoryProvider { + let tabId: UUID + let storage: QueryTabVersionStorage + // Nil once the tab is closed. + let liveTab: @MainActor @Sendable () -> LiveQueryTab? + // False when the tab is gone by the time the text arrives. + let write: @MainActor @Sendable (String) -> Bool + var isCapturePaused: @MainActor @Sendable () -> Bool = { QueryHistoryCaptureStore.isPaused } + + func loadHistory() async throws -> VersionHistoryPage { + guard await liveTab() != nil else { throw VersionHistoryError.tabClosed } + let past = storage.summaries(tabId: tabId).map { Self.entry(for: $0) } + return VersionHistoryPage( + entries: [VersionHistoryEntry(reference: .current)] + past, + notice: .keepsRecentVersions(count: QueryTabVersionStorage.retainedVersionCount, days: storage.retentionDays) + ) + } + + func content(of reference: VersionHistoryReference) async throws -> String { + switch reference { + case .current: + guard let live = await liveTab() else { throw VersionHistoryError.tabClosed } + return live.text + case .tabVersion(let id): + guard let version = storage.version(id: id, tabId: tabId) else { + throw VersionHistoryError.versionNotFound + } + return version.query + case .savedQueryVersion, .gitRevision: + throw VersionHistoryError.versionNotFound + } + } + + // Restore is recoverable through the version it records first, since the target editor is rarely mounted. + func prepareRestore(_ reference: VersionHistoryReference) async throws -> VersionRestorePlan { + guard case .tabVersion(let id) = reference, let version = storage.version(id: id, tabId: tabId) else { + throw VersionHistoryError.versionNotFound + } + guard await liveTab() != nil else { throw VersionHistoryError.tabClosed } + return VersionRestorePlan(replacesUncommittedChanges: false) { + guard let live = await liveTab() else { throw VersionHistoryError.tabClosed } + if await !isCapturePaused() { + await storage.record( + tabId: tabId, + connectionId: live.connectionId, + text: live.text, + reason: .beforeRestore + ) + } + guard await write(version.query) else { throw VersionHistoryError.tabClosed } + } + } + + static func entry(for summary: QueryTabVersionSummary) -> VersionHistoryEntry { + VersionHistoryEntry( + reference: .tabVersion(id: summary.id), + summary: summary.reason.summary, + date: summary.savedAt + ) + } +} + +extension QueryTabVersionReason { + // An idle snapshot has no label, so its title is its date. + var summary: String? { + switch self { + case .run: String(localized: "Run") + case .idle: nil + case .beforeReplace: String(localized: "Before Replace") + case .beforeRestore: String(localized: "Before Restore") + } + } +} diff --git a/apple/TablePro/Core/VersionHistory/QueryTabVersionRecorder.swift b/apple/TablePro/Core/VersionHistory/QueryTabVersionRecorder.swift new file mode 100644 index 0000000000..31a456e4b5 --- /dev/null +++ b/apple/TablePro/Core/VersionHistory/QueryTabVersionRecorder.swift @@ -0,0 +1,28 @@ +// +// QueryTabVersionRecorder.swift +// TablePro +// + +import Foundation + +@MainActor +internal struct QueryTabVersionRecorder { + let storage: QueryTabVersionStorage + // The Query History pause means "record nothing I run on this Mac", and a tab's drafts are the same text. + var isCapturePaused: @MainActor @Sendable () -> Bool + + static let live = QueryTabVersionRecorder(storage: .shared, isCapturePaused: { QueryHistoryCaptureStore.isPaused }) + + // The text is read now, so a later edit cannot change what this version holds. + @discardableResult + func record(_ tab: QueryTab?, connectionId: UUID, reason: QueryTabVersionReason) -> Task? { + guard let tab, tab.tabType == .query, !isCapturePaused() else { return nil } + let text = tab.content.query + guard QueryTabVersionStorage.isRecordable(text) else { return nil } + let storage = storage + let tabId = tab.id + return Task { + await storage.record(tabId: tabId, connectionId: connectionId, text: text, reason: reason) + } + } +} diff --git a/apple/TablePro/Core/VersionHistory/SavedQueryVersionHistoryProvider.swift b/apple/TablePro/Core/VersionHistory/SavedQueryVersionHistoryProvider.swift index 7e4cfc4571..6b6abf5d2a 100644 --- a/apple/TablePro/Core/VersionHistory/SavedQueryVersionHistoryProvider.swift +++ b/apple/TablePro/Core/VersionHistory/SavedQueryVersionHistoryProvider.swift @@ -38,7 +38,7 @@ internal struct SavedQueryVersionHistoryProvider: VersionHistoryProvider { return favorite.query case .savedQueryVersion(let id): return try await version(id: id).query - case .gitRevision: + case .gitRevision, .tabVersion: throw VersionHistoryError.versionNotFound } } diff --git a/apple/TablePro/Core/VersionHistory/VersionHistoryModels.swift b/apple/TablePro/Core/VersionHistory/VersionHistoryModels.swift index a394028ad3..63fe7e18b2 100644 --- a/apple/TablePro/Core/VersionHistory/VersionHistoryModels.swift +++ b/apple/TablePro/Core/VersionHistory/VersionHistoryModels.swift @@ -8,12 +8,14 @@ import Foundation internal enum VersionHistorySubject: Codable, Hashable, Sendable { case savedQuery(id: UUID) case linkedFile(url: URL) + case queryTab(id: UUID) } internal enum VersionHistoryReference: Hashable, Sendable { case current case savedQueryVersion(id: Int64) case gitRevision(commit: String, path: String) + case tabVersion(id: Int64) var shortRevision: String? { guard case .gitRevision(let commit, _) = self else { return nil } @@ -35,6 +37,7 @@ internal struct VersionHistoryEntry: Identifiable, Hashable, Sendable { internal enum VersionHistoryNotice: Hashable, Sendable { case keepsLatestVersions(Int) case showsLatestCommits(Int) + case keepsRecentVersions(count: Int, days: Int) } internal struct VersionHistoryPage: Equatable, Sendable { @@ -64,6 +67,7 @@ internal enum VersionHistoryError: LocalizedError, Equatable { case storedInLargeFileStorage case fileChangedBeforeWriting case restoreFailed(String) + case tabClosed var errorDescription: String? { switch self { @@ -87,6 +91,8 @@ internal enum VersionHistoryError: LocalizedError, Equatable { return String(localized: "The file or its staged version changed before it could be written. Nothing was replaced.") case .restoreFailed(let message): return String(format: String(localized: "The version could not be restored: %@"), message) + case .tabClosed: + return String(localized: "This tab is closed. Reopen it to see its history.") } } } diff --git a/apple/TablePro/Core/Vim/VimEngine+Characters.swift b/apple/TablePro/Core/Vim/VimEngine+Characters.swift new file mode 100644 index 0000000000..186864c896 --- /dev/null +++ b/apple/TablePro/Core/Vim/VimEngine+Characters.swift @@ -0,0 +1,32 @@ +import Foundation + +extension VimEngine { + func characterStart(at offset: Int, in buffer: VimTextBuffer) -> Int { + buffer.characterRange(at: offset).location + } + + // At or past the end of the text this is the end of the text. + func characterEnd(at offset: Int, in buffer: VimTextBuffer) -> Int { + NSMaxRange(buffer.characterRange(at: offset)) + } + + // Stops at the last step that stays within `limit`; an offset already past it comes back to `limit`. + func characterOffset(after offset: Int, count: Int, limit: Int, in buffer: VimTextBuffer) -> Int { + var position = offset + for _ in 0.. position, next <= limit else { break } + position = next + } + return min(position, limit) + } + + func characterOffset(before offset: Int, count: Int, limit: Int, in buffer: VimTextBuffer) -> Int { + var position = offset + for _ in 0.. limit else { break } + position = characterStart(at: position - 1, in: buffer) + } + return max(position, limit) + } +} diff --git a/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift b/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift index ea721bd704..2a53ae73f8 100644 --- a/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift +++ b/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift @@ -24,7 +24,7 @@ extension VimEngine { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) if pos > lineRange.location { - buffer.setSelectedRange(NSRange(location: pos - 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterStart(at: pos - 1, in: buffer), length: 0)) } } return true @@ -41,7 +41,7 @@ extension VimEngine { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) if pos > lineRange.location { - buffer.setSelectedRange(NSRange(location: pos - 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterStart(at: pos - 1, in: buffer), length: 0)) } return true } @@ -64,27 +64,27 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - if pos < contentEnd { - let overwritten = NSRange(location: pos, length: 1) - replaceModeEdits.append(VimReplaceModeEdit(offset: pos, original: buffer.string(in: overwritten))) - buffer.replaceCharacters(in: overwritten, with: String(char)) - } else { - replaceModeEdits.append(VimReplaceModeEdit(offset: pos, original: nil)) - buffer.replaceCharacters(in: NSRange(location: pos, length: 0), with: String(char)) - } + let typed = String(char) + let insertedLength = (typed as NSString).length + let overwritten = pos < contentEnd ? buffer.characterRange(at: pos) : NSRange(location: pos, length: 0) + let original = overwritten.length > 0 ? buffer.string(in: overwritten) : nil + replaceModeEdits.append(VimReplaceModeEdit(offset: pos, original: original, insertedLength: insertedLength)) + buffer.replaceCharacters(in: overwritten, with: typed) return true } func backspaceInReplace(in buffer: VimTextBuffer) { let pos = buffer.selectedRange().location guard pos > 0 else { return } - if let last = replaceModeEdits.last, last.offset == pos - 1 { + if let last = replaceModeEdits.last, last.offset + last.insertedLength == pos { replaceModeEdits.removeLast() - buffer.replaceCharacters(in: NSRange(location: pos - 1, length: 1), with: last.original ?? "") + let inserted = NSRange(location: last.offset, length: last.insertedLength) + buffer.replaceCharacters(in: inserted, with: last.original ?? "") + buffer.setSelectedRange(NSRange(location: last.offset, length: 0)) } else { replaceModeEdits.removeAll() + buffer.setSelectedRange(NSRange(location: characterStart(at: pos - 1, in: buffer), length: 0)) } - buffer.setSelectedRange(NSRange(location: pos - 1, length: 0)) } nonisolated static func isUnwritableControl(_ char: Character) -> Bool { @@ -140,7 +140,8 @@ extension VimEngine { let pos = buffer.selectedRange().location let lineStart = lineStartBeforeCaret(in: buffer) guard pos > lineStart else { return pos } - return max(lineStart, buffer.wordBoundary(forward: false, from: pos)) + // The word scan goes by code unit, so its boundary can fall inside a character. + return max(lineStart, characterStart(at: buffer.wordBoundary(forward: false, from: pos), in: buffer)) } func lineStartBeforeCaret(in buffer: VimTextBuffer) -> Int { diff --git a/apple/TablePro/Core/Vim/VimEngine+Motions.swift b/apple/TablePro/Core/Vim/VimEngine+Motions.swift index 8de33a21ff..10e46b619c 100644 --- a/apple/TablePro/Core/Vim/VimEngine+Motions.swift +++ b/apple/TablePro/Core/Vim/VimEngine+Motions.swift @@ -9,7 +9,7 @@ extension VimEngine { func moveLeft(_ count: Int, in buffer: VimTextBuffer) { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) - let newPos = max(lineRange.location, pos - count) + let newPos = characterOffset(before: pos, count: count, limit: lineRange.location, in: buffer) buffer.setSelectedRange(NSRange(location: newPos, length: 0)) goalColumn = nil } @@ -24,8 +24,11 @@ extension VimEngine { } else { contentEnd = lineEnd } - let maxPos = max(lineRange.location, contentEnd - 1) - let newPos = min(maxPos, pos + count) + // With an operator pending, `l` reaches past the last character, so `dl` there deletes it. + let maxPos = pendingOperator != nil || contentEnd == lineRange.location + ? contentEnd + : characterStart(at: contentEnd - 1, in: buffer) + let newPos = characterOffset(after: pos, count: count, limit: maxPos, in: buffer) buffer.setSelectedRange(NSRange(location: newPos, length: 0)) goalColumn = nil } @@ -90,7 +93,7 @@ extension VimEngine { } else { contentEnd = lineEnd } - let finalPos = contentEnd > lineRange.location ? contentEnd - 1 : lineRange.location + let finalPos = contentEnd > lineRange.location ? characterStart(at: contentEnd - 1, in: buffer) : lineRange.location buffer.setSelectedRange(NSRange(location: finalPos, length: 0)) } @@ -151,7 +154,7 @@ extension VimEngine { if pos >= contentEnd && contentEnd > lineRange.location { pos = contentEnd - 1 } - return pos + return characterStart(at: pos, in: buffer) } func wordBackward(_ count: Int, in buffer: VimTextBuffer) { @@ -235,8 +238,7 @@ extension VimEngine { func executeFindChar(_ char: Character, request: VimFindCharRequest, in buffer: VimTextBuffer) -> Bool { lastFindChar = VimLastFindChar(char: char, forward: request.forward, till: request.till) - guard let scalar = char.unicodeScalars.first else { return true } - let target = unichar(scalar.value) + let target = String(char) let count = consumeCount() let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) @@ -244,37 +246,44 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd + let matches: (NSRange) -> Bool = { buffer.string(in: $0) == target } + // An offset below the line start ends the backward scan. + let previous: (Int) -> Int = { offset in + offset > lineRange.location ? self.characterStart(at: offset - 1, in: buffer) : offset - 1 + } var resolved: Int? if request.forward { - let initial = request.till ? pos + 2 : pos + 1 - var scanStart = min(initial, contentEnd) + var scanStart = characterEnd(at: pos, in: buffer) + if request.till { scanStart = characterEnd(at: scanStart, in: buffer) } + scanStart = min(scanStart, contentEnd) for _ in 0..= lineRange.location { - if buffer.character(at: idx) == target { + if matches(buffer.characterRange(at: idx)) { resolved = idx - scanStart = idx - 1 + scanStart = previous(idx) break } - idx -= 1 + idx = previous(idx) } if resolved == nil { break } } @@ -282,7 +291,7 @@ extension VimEngine { guard var finalPos = resolved else { return true } if request.till { - finalPos += request.forward ? -1 : 1 + finalPos = request.forward ? previous(finalPos) : characterEnd(at: finalPos, in: buffer) } executeMotion(in: buffer, inclusive: true) { buffer.setSelectedRange(NSRange(location: finalPos, length: 0)) diff --git a/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift b/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift index 52fd42aac5..1d689ecab8 100644 --- a/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift +++ b/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift @@ -85,7 +85,8 @@ extension VimEngine { switch char { case "h": - moveLeft(consumeCount(), in: buffer) + let count = consumeCount() + executeMotion(in: buffer) { self.moveLeft(count, in: buffer) } return true case "j": moveDown(consumeCount(), in: buffer) @@ -94,7 +95,8 @@ extension VimEngine { moveUp(consumeCount(), in: buffer) return true case "l": - moveRight(consumeCount(), in: buffer) + let count = consumeCount() + executeMotion(in: buffer) { self.moveRight(count, in: buffer) } return true case "w": let count = consumeCount() @@ -189,7 +191,7 @@ extension VimEngine { countPrefix = 0 let pos = buffer.selectedRange().location if pos < buffer.length { - buffer.setSelectedRange(NSRange(location: pos + 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterEnd(at: pos, in: buffer), length: 0)) } setMode(.insert) return true @@ -236,8 +238,7 @@ extension VimEngine { let pos = buffer.selectedRange().location visualAnchor = pos cursorOffset = pos - let initialLen = pos < buffer.length ? 1 : 0 - buffer.setSelectedRange(NSRange(location: pos, length: initialLen)) + buffer.setSelectedRange(buffer.characterRange(at: pos)) setMode(.visual(linewise: false)) return true case "V": diff --git a/apple/TablePro/Core/Vim/VimEngine+Operators.swift b/apple/TablePro/Core/Vim/VimEngine+Operators.swift index 9c03d4dbbe..582aa26f7a 100644 --- a/apple/TablePro/Core/Vim/VimEngine+Operators.swift +++ b/apple/TablePro/Core/Vim/VimEngine+Operators.swift @@ -20,7 +20,7 @@ extension VimEngine { let rangeStart = min(startPos, endPos) var rangeEnd = max(startPos, endPos) if inclusive && rangeEnd < buffer.length && buffer.character(at: rangeEnd) != 0x0A { - rangeEnd += 1 + rangeEnd = characterEnd(at: rangeEnd, in: buffer) } let range = NSRange(location: rangeStart, length: rangeEnd - rangeStart) @@ -28,8 +28,11 @@ extension VimEngine { pendingOperator = nil } - func executeOperatorOnRange(_ op: VimOperator, range: NSRange, linewise: Bool, in buffer: VimTextBuffer) { - guard range.length > 0 else { return } + func executeOperatorOnRange(_ op: VimOperator, range proposed: NSRange, linewise: Bool, in buffer: VimTextBuffer) { + guard proposed.length > 0 else { return } + // Text objects scan by code unit, so their ranges can end inside a character. + let start = characterStart(at: proposed.location, in: buffer) + let range = NSRange(location: start, length: characterEnd(at: NSMaxRange(proposed) - 1, in: buffer) - start) switch op { case .delete: @@ -37,8 +40,7 @@ extension VimEngine { writeToActiveRegister(text: text, linewise: linewise, asDelete: true) buffer.replaceCharacters(in: range, with: "") adjustMarksForEdit(in: range, replacementLength: 0) - let newPos = min(range.location, max(0, buffer.length - 1)) - buffer.setSelectedRange(NSRange(location: max(0, newPos), length: 0)) + buffer.setSelectedRange(NSRange(location: clampToContentPosition(range.location, in: buffer), length: 0)) case .yank: let text = buffer.string(in: range) writeToActiveRegister(text: text, linewise: linewise, asDelete: false) @@ -193,7 +195,7 @@ extension VimEngine { } } else { if after { - let insertPos = min(pos + 1, buffer.length) + let insertPos = characterEnd(at: pos, in: buffer) buffer.replaceCharacters(in: NSRange(location: insertPos, length: 0), with: source.text) let newPos = insertPos + (source.text as NSString).length - 1 buffer.setSelectedRange(NSRange(location: max(insertPos, newPos), length: 0)) @@ -212,16 +214,16 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - let deleteCount = min(count, max(0, contentEnd - pos)) - guard deleteCount > 0 else { return } - let range = NSRange(location: pos, length: deleteCount) + let end = characterOffset(after: pos, count: count, limit: contentEnd, in: buffer) + guard end > pos else { return } + let range = NSRange(location: pos, length: end - pos) writeToActiveRegister(text: buffer.string(in: range), linewise: false, asDelete: true) adjustMarksForEdit(in: range, replacementLength: 0) noteEdit(at: pos, in: buffer) buffer.replaceCharacters(in: range, with: "") - let newContentEnd = contentEnd - deleteCount + let newContentEnd = contentEnd - range.length if pos >= newContentEnd && newContentEnd > lineRange.location { - buffer.setSelectedRange(NSRange(location: newContentEnd - 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterStart(at: newContentEnd - 1, in: buffer), length: 0)) } else { buffer.setSelectedRange(NSRange(location: pos, length: 0)) } @@ -230,10 +232,9 @@ extension VimEngine { func deleteCharBeforeCursor(_ count: Int, in buffer: VimTextBuffer) { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) - let deleteCount = min(count, pos - lineRange.location) - guard deleteCount > 0 else { return } - let start = pos - deleteCount - let range = NSRange(location: start, length: deleteCount) + let start = characterOffset(before: pos, count: count, limit: lineRange.location, in: buffer) + guard start < pos else { return } + let range = NSRange(location: start, length: pos - start) writeToActiveRegister(text: buffer.string(in: range), linewise: false, asDelete: true) buffer.replaceCharacters(in: range, with: "") buffer.setSelectedRange(NSRange(location: start, length: 0)) @@ -246,9 +247,9 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - let deleteCount = min(count, max(0, contentEnd - pos)) - guard deleteCount > 0 else { setMode(.insert); return } - let range = NSRange(location: pos, length: deleteCount) + let end = characterOffset(after: pos, count: count, limit: contentEnd, in: buffer) + guard end > pos else { setMode(.insert); return } + let range = NSRange(location: pos, length: end - pos) writeToActiveRegister(text: buffer.string(in: range), linewise: false, asDelete: true) buffer.replaceCharacters(in: range, with: "") buffer.setSelectedRange(NSRange(location: pos, length: 0)) @@ -262,13 +263,16 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - let toggleCount = min(count, max(0, contentEnd - pos)) - guard toggleCount > 0 else { return } - let range = NSRange(location: pos, length: toggleCount) + let end = characterOffset(after: pos, count: count, limit: contentEnd, in: buffer) + guard end > pos else { return } + let range = NSRange(location: pos, length: end - pos) let transformed = toggleCaseTransform(buffer.string(in: range)) buffer.replaceCharacters(in: range, with: transformed) - let newPos = min(pos + toggleCount, contentEnd > lineRange.location ? contentEnd - 1 : lineRange.location) - buffer.setSelectedRange(NSRange(location: newPos, length: 0)) + // A case change can change the length, as ß becomes SS. + let newEnd = pos + (transformed as NSString).length + let newContentEnd = contentEnd + newEnd - end + let lastCharacter = characterStart(at: newContentEnd - 1, in: buffer) + buffer.setSelectedRange(NSRange(location: min(newEnd, lastCharacter), length: 0)) } func applyCaseToLine(_ op: VimOperator, count: Int, in buffer: VimTextBuffer) { @@ -384,16 +388,16 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - guard pos + count <= contentEnd else { return true } - let replacement: String - if char == "\r" || char == "\n" { - replacement = String(repeating: "\n", count: count) - } else { - replacement = String(repeating: char, count: count) + var end = pos + for _ in 0.. 0 else { return } lastVisualStart = sel.location - lastVisualEnd = sel.location + sel.length - 1 + lastVisualEnd = characterStart(at: NSMaxRange(sel) - 1, in: buffer) lastVisualLinewise = linewise } @@ -104,8 +138,8 @@ extension VimEngine { buffer.setSelectedRange(NSRange(location: lineStart, length: lineEnd - lineStart)) setMode(.visual(linewise: true)) } else { - let length = clampedEnd - clampedStart + (clampedEnd < buffer.length ? 1 : 0) - buffer.setSelectedRange(NSRange(location: clampedStart, length: length)) + let selectionEnd = clampedEnd < buffer.length ? characterEnd(at: clampedEnd, in: buffer) : clampedEnd + buffer.setSelectedRange(NSRange(location: clampedStart, length: selectionEnd - clampedStart)) setMode(.visual(linewise: false)) } } diff --git a/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift b/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift index 55c7a9aaae..1f3a12c37b 100644 --- a/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift +++ b/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift @@ -53,8 +53,8 @@ extension VimEngine { let cursorPos = visualCursorEnd(buffer: buffer) let newPos: Int switch char { - case "h": newPos = max(0, cursorPos - 1) - case "l": newPos = min(buffer.length, cursorPos + 1) + case "h": newPos = characterOffset(before: cursorPos, count: 1, limit: 0, in: buffer) + case "l": newPos = characterOffset(after: cursorPos, count: 1, limit: buffer.length, in: buffer) case "j": let (line, col) = buffer.lineAndColumn(forOffset: cursorPos) let targetLine = min(buffer.lineCount - 1, line + 1) @@ -205,12 +205,17 @@ extension VimEngine { func visualCursorEnd(buffer: VimTextBuffer) -> Int { let sel = buffer.selectedRange() if sel.location == visualAnchor { - return sel.location + max(sel.length, 1) - 1 + return lastSelectedCharacter(sel, in: buffer) } return sel.location } - func updateVisualSelection(cursorPos: Int, linewise: Bool, in buffer: VimTextBuffer) { + private func lastSelectedCharacter(_ sel: NSRange, in buffer: VimTextBuffer) -> Int { + sel.length > 0 ? characterStart(at: NSMaxRange(sel) - 1, in: buffer) : sel.location + } + + func updateVisualSelection(cursorPos proposedCursor: Int, linewise: Bool, in buffer: VimTextBuffer) { + let cursorPos = characterStart(at: proposedCursor, in: buffer) cursorOffset = cursorPos let start = min(visualAnchor, cursorPos) let end = max(visualAnchor, cursorPos) @@ -222,17 +227,15 @@ extension VimEngine { let lineEnd = endLineRange.location + endLineRange.length buffer.setSelectedRange(NSRange(location: lineStart, length: lineEnd - lineStart)) } else { - let length = end - start + (end < buffer.length ? 1 : 0) - buffer.setSelectedRange(NSRange(location: start, length: length)) + let selectionEnd = end < buffer.length ? characterEnd(at: end, in: buffer) : end + buffer.setSelectedRange(NSRange(location: start, length: selectionEnd - start)) } } func swapVisualAnchorAndCursor(in buffer: VimTextBuffer, linewise: Bool) { let sel = buffer.selectedRange() let cursor = visualCursorEnd(buffer: buffer) - let otherEnd = cursor == sel.location - ? sel.location + max(0, sel.length - 1) - : sel.location + let otherEnd = cursor == sel.location ? lastSelectedCharacter(sel, in: buffer) : sel.location visualAnchor = cursor cursorOffset = otherEnd updateVisualSelection(cursorPos: otherEnd, linewise: linewise, in: buffer) @@ -259,13 +262,14 @@ extension VimEngine { guard sel.length > 0 else { setMode(.normal); return } var replacement = "" replacement.reserveCapacity(sel.length) - for i in 0.. NSRange - /// Returns (0-based line index, 0-based column) for the given offset + // 0-based line and column. func lineAndColumn(forOffset offset: Int) -> (line: Int, column: Int) - /// Returns the offset for a given 0-based line and column func offset(forLine line: Int, column: Int) -> Int - /// Returns the UTF-16 code unit at the given offset — must be O(1) + // One UTF-16 code unit, in O(1). func character(at offset: Int) -> unichar - /// Returns the offset of the next/previous word boundary from the given offset + // An empty range at the clamped offset when `offset` is outside the text. + func characterRange(at offset: Int) -> NSRange + func wordBoundary(forward: Bool, from offset: Int) -> Int - /// Returns the offset of the end of the current word from the given offset func wordEnd(from offset: Int) -> Int - /// Returns the offset of the previous word-end (backward analog of wordEnd) — for ge + // The backward analog of `wordEnd`, for ge. func wordEndBackward(from offset: Int) -> Int - /// Returns the next/previous WORD boundary (whitespace-delimited, punctuation included) + // WORD motions split on whitespace only. func bigWordBoundary(forward: Bool, from offset: Int) -> Int - /// Returns the end of the current WORD (whitespace-delimited) func bigWordEnd(from offset: Int) -> Int - /// Returns the previous WORD-end (whitespace-delimited) func bigWordEndBackward(from offset: Int) -> Int - /// Returns the offset of the matching bracket pair at the given offset, or nil if none. - /// Supported pairs: () [] {}. Handles nested pairs correctly. + // Pairs () [] {}, counting nested pairs. func matchingBracket(at offset: Int) -> Int? - /// Returns the inclusive 0-based line range currently visible in the editor. - /// Test mocks return the whole buffer range. + // Inclusive 0-based lines. Test mocks return the whole buffer. func visibleLineRange() -> (firstLine: Int, lastLine: Int) - /// Returns the indent string (" " for 4-space, "\t" for tab) used by >> and <<. + // The unit >> and << add or strip. func indentString() -> String - /// Returns the indent width in columns. func indentWidth() -> Int - /// Returns the currently selected range func selectedRange() -> NSRange - /// Returns the string in the given range func string(in range: NSRange) -> String - /// Sets the selected range and scrolls to make it visible + // Widens the range to whole characters and scrolls it into view. func setSelectedRange(_ range: NSRange) - /// Replaces characters in the given range with new text func replaceCharacters(in range: NSRange, with string: String) - /// Undo the last change func undo() - /// Redo the last undone change func redo() } diff --git a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift index 4d012d8074..53ee9d9c44 100644 --- a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift +++ b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift @@ -1,210 +1,123 @@ -// -// VimTextBufferAdapter.swift -// TablePro -// -// Adapts TableProTextEngine's TextView to the VimTextBuffer protocol -// - import AppKit import Foundation import TableProTextEngine -/// Bridges TableProTextEngine's TextView to VimTextBuffer for the Vim engine @MainActor final class VimTextBufferAdapter: VimTextBuffer { private weak var textView: TextView? + private struct LineStorageTrust { + weak var storage: NSTextStorage? + let length: Int + let trusted: Bool + } + + private var lineStorageTrust: LineStorageTrust? + init(textView: TextView) { self.textView = textView } - private var cachedLineCount: Int? + private var text: NSString? { + textView?.textStorage?.mutableString + } // MARK: - VimTextBuffer var length: Int { - guard let textView else { return 0 } - return (textView.string as NSString).length + textView?.textStorage?.length ?? 0 } var lineCount: Int { - if let cached = cachedLineCount { return cached } - guard let textView else { return 1 } - let nsString = textView.string as NSString - if nsString.length == 0 { - cachedLineCount = 1 - return 1 + guard let text, text.length > 0 else { return 1 } + let endsWithLineBreak = isLineBreak(text.character(at: text.length - 1)) + // The line storage keeps an empty line after a final line ending, which Vim does not count. + if let lines = lineStorage(matching: text), let last = lines.last, (last.range.length == 0) == endsWithLineBreak { + return max(1, endsWithLineBreak ? lines.count - 1 : lines.count) } var count = 0 - var index = 0 - while index < nsString.length { - let lineRange = nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) + var location = 0 + while location < text.length { + location = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0))) count += 1 - index = lineRange.location + lineRange.length } - let result = max(1, count) - cachedLineCount = result - return result + return max(1, count) } func invalidateLineCache() { - cachedLineCount = nil - } - - /// Incrementally update the cached line count based on text change delta. - /// Avoids a full O(n) recount on every keystroke. - func textDidChange(in range: NSRange, replacementLength: Int) { - guard let textView else { - cachedLineCount = nil - return - } - - guard let cached = cachedLineCount else { return } - - let nsString = textView.string as NSString - - // Pure insertion: count newlines in the new text and apply delta - if range.length == 0 { - var addedNewlines = 0 - let end = range.location + replacementLength - if replacementLength > 0 && end <= nsString.length { - for i in range.location.. 0 && range.location + range.length <= oldNs.length { - let end = range.location + range.length - for i in range.location.. 0 && replacementEnd <= nsString.length { - for i in range.location.. NSRange { - guard let textView else { return NSRange(location: 0, length: 0) } - let nsString = textView.string as NSString - let clampedOffset = min(max(0, offset), nsString.length) - return nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: clampedOffset, length: 0)) + guard let text else { return NSRange(location: 0, length: 0) } + let clampedOffset = min(max(0, offset), text.length) + return text.lineRangeBreakingAtLineEndings(for: NSRange(location: clampedOffset, length: 0)) } func lineAndColumn(forOffset offset: Int) -> (line: Int, column: Int) { - guard let textView else { return (0, 0) } - let nsString = textView.string as NSString - let clampedOffset = min(max(0, offset), nsString.length) - - if nsString.length == 0 { return (0, 0) } - - let safeOffset = min(clampedOffset, max(0, nsString.length - 1)) - let lineRange = nsString.lineRangeBreakingAtLineEndings( - for: NSRange(location: safeOffset, length: 0) - ) - let column = clampedOffset - lineRange.location - - // Count newlines before lineRange.location — uses fast NSString search - var line = 0 - var searchStart = 0 - while searchStart < lineRange.location { - let found = nsString.range(of: "\n", range: NSRange(location: searchStart, length: lineRange.location - searchStart)) - if found.location == NSNotFound { break } - line += 1 - searchStart = found.location + found.length - } - - return (line, column) + guard let text, text.length > 0 else { return (0, 0) } + let clampedOffset = min(max(0, offset), text.length) + // The end of the document is a column past the end of the last line, never the empty line after a final + // line ending. + let anchor = min(clampedOffset, text.length - 1) + let range = text.lineRangeBreakingAtLineEndings(for: NSRange(location: anchor, length: 0)) + return (lineIndex(of: range, in: text), clampedOffset - range.location) } func offset(forLine line: Int, column: Int) -> Int { - guard let textView else { return 0 } - let nsString = textView.string as NSString - var currentLine = 0 - var index = 0 - while index < nsString.length && currentLine < line { - let lineRange = nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) - currentLine += 1 - index = lineRange.location + lineRange.length - } - // Now index is at the start of the target line - let lineRange = nsString.lineRangeBreakingAtLineEndings(for: NSRange(location: min(index, nsString.length), length: 0)) - // Content length excludes trailing newline + guard let text else { return 0 } + let range = lineRange(atIndex: max(0, line), in: text) + let lineEnd = NSMaxRange(range) let contentLength: Int - let lineEnd = lineRange.location + lineRange.length - if lineEnd > lineRange.location && lineEnd <= nsString.length - && nsString.character(at: lineEnd - 1) == 0x0A { - contentLength = lineRange.length - 1 + if lineEnd > range.location && lineEnd <= text.length && text.character(at: lineEnd - 1) == 0x0A { + contentLength = range.length - 1 } else { - contentLength = lineRange.length + contentLength = range.length } - let clampedCol = min(column, max(0, contentLength - 1)) - return lineRange.location + max(0, clampedCol) + let clampedColumn = min(column, max(0, contentLength - 1)) + return range.location + max(0, clampedColumn) } func character(at offset: Int) -> unichar { - guard let textView else { return 0 } - let nsString = textView.string as NSString - guard offset >= 0 && offset < nsString.length else { return 0 } - return nsString.character(at: offset) + guard let text, offset >= 0 && offset < text.length else { return 0 } + return text.character(at: offset) + } + + func characterRange(at offset: Int) -> NSRange { + guard let text, offset >= 0, offset < text.length else { + return NSRange(location: min(max(0, offset), length), length: 0) + } + return text.rangeOfComposedCharacterSequence(at: offset) } func wordBoundary(forward: Bool, from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } + guard let text else { return offset } + guard text.length > 0 else { return 0 } if forward { - var pos = min(offset, nsString.length - 1) - let startClass = charClass(nsString.character(at: pos)) + var pos = min(offset, text.length - 1) + let startClass = charClass(text.character(at: pos)) if startClass == .whitespace { - // Skip whitespace, then stop at start of next word/punctuation - while pos < nsString.length && charClass(nsString.character(at: pos)) == .whitespace { + while pos < text.length && charClass(text.character(at: pos)) == .whitespace { pos += 1 } } else { - while pos < nsString.length && charClass(nsString.character(at: pos)) == startClass { + while pos < text.length && charClass(text.character(at: pos)) == startClass { pos += 1 } - while pos < nsString.length && charClass(nsString.character(at: pos)) == .whitespace { + while pos < text.length && charClass(text.character(at: pos)) == .whitespace { pos += 1 } } - return min(pos, nsString.length) + return min(pos, text.length) } else { - var pos = min(offset, nsString.length) + var pos = min(offset, text.length) if pos > 0 { pos -= 1 } - while pos > 0 && charClass(nsString.character(at: pos)) == .whitespace { + while pos > 0 && charClass(text.character(at: pos)) == .whitespace { pos -= 1 } - let cls = charClass(nsString.character(at: pos)) - while pos > 0 && charClass(nsString.character(at: pos - 1)) == cls { + let cls = charClass(text.character(at: pos)) + while pos > 0 && charClass(text.character(at: pos - 1)) == cls { pos -= 1 } return max(0, pos) @@ -212,20 +125,19 @@ final class VimTextBufferAdapter: VimTextBuffer { } func wordEnd(from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } + guard let text else { return offset } + guard text.length > 0 else { return 0 } - var pos = min(offset + 1, nsString.length - 1) - while pos < nsString.length && charClass(nsString.character(at: pos)) == .whitespace { + var pos = min(offset + 1, text.length - 1) + while pos < text.length && charClass(text.character(at: pos)) == .whitespace { pos += 1 } - guard pos < nsString.length else { return nsString.length - 1 } - let cls = charClass(nsString.character(at: pos)) - while pos < nsString.length - 1 && charClass(nsString.character(at: pos + 1)) == cls { + guard pos < text.length else { return text.length - 1 } + let cls = charClass(text.character(at: pos)) + while pos < text.length - 1 && charClass(text.character(at: pos + 1)) == cls { pos += 1 } - return min(pos, nsString.length - 1) + return min(pos, text.length - 1) } func selectedRange() -> NSRange { @@ -234,28 +146,35 @@ final class VimTextBufferAdapter: VimTextBuffer { } func string(in range: NSRange) -> String { - guard let textView else { return "" } - let nsString = textView.string as NSString + guard let text else { return "" } let clampedRange = NSRange( location: max(0, range.location), - length: min(range.length, nsString.length - max(0, range.location)) + length: min(range.length, text.length - max(0, range.location)) ) guard clampedRange.length > 0 else { return "" } - return nsString.substring(with: clampedRange) + return text.substring(with: clampedRange) } func setSelectedRange(_ range: NSRange) { guard let textView else { return } - let clampedLocation = max(0, min(range.location, (textView.string as NSString).length)) - let maxLength = (textView.string as NSString).length - clampedLocation - let clampedLength = max(0, min(range.length, maxLength)) - let clampedRange = NSRange(location: clampedLocation, length: clampedLength) + let documentLength = length + let clampedLocation = max(0, min(range.location, documentLength)) + let clampedLength = max(0, min(range.length, documentLength - clampedLocation)) + let wholeRange = wholeCharacters(NSRange(location: clampedLocation, length: clampedLength)) let currentRange = textView.selectedRange() - guard clampedRange != currentRange else { return } + guard wholeRange != currentRange else { return } - textView.selectionManager.setSelectedRange(clampedRange) - textView.scrollToRange(clampedRange) + textView.selectionManager.setSelectedRange(wholeRange) + textView.scrollToRange(wholeRange) + } + + // A caret or selection never splits a composed character sequence. + private func wholeCharacters(_ range: NSRange) -> NSRange { + guard range.length > 0, let text else { + return NSRange(location: characterRange(at: range.location).location, length: 0) + } + return text.rangeOfComposedCharacterSequences(for: range) } func replaceCharacters(in range: NSRange, with string: String) { @@ -274,104 +193,99 @@ final class VimTextBufferAdapter: VimTextBuffer { } func wordEndBackward(from offset: Int) -> Int { - guard let textView else { return 0 } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } - var pos = min(max(0, offset), nsString.length - 1) + guard let text else { return 0 } + guard text.length > 0 else { return 0 } + var pos = min(max(0, offset), text.length - 1) if pos > 0 { pos -= 1 } - if charClass(nsString.character(at: pos)) == .whitespace { - while pos > 0 && charClass(nsString.character(at: pos)) == .whitespace { + if charClass(text.character(at: pos)) == .whitespace { + while pos > 0 && charClass(text.character(at: pos)) == .whitespace { pos -= 1 } return pos } - let cls = charClass(nsString.character(at: pos)) - while pos > 0 && charClass(nsString.character(at: pos - 1)) == cls { + let cls = charClass(text.character(at: pos)) + while pos > 0 && charClass(text.character(at: pos - 1)) == cls { pos -= 1 } guard pos > 0 else { return 0 } pos -= 1 - while pos > 0 && charClass(nsString.character(at: pos)) == .whitespace { + while pos > 0 && charClass(text.character(at: pos)) == .whitespace { pos -= 1 } return pos } func bigWordBoundary(forward: Bool, from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } + guard let text else { return offset } + guard text.length > 0 else { return 0 } if forward { - var pos = min(offset, nsString.length - 1) - let startWS = isWhitespace(nsString.character(at: pos)) + var pos = min(offset, text.length - 1) + let startWS = isWhitespace(text.character(at: pos)) if startWS { - while pos < nsString.length && isWhitespace(nsString.character(at: pos)) { + while pos < text.length && isWhitespace(text.character(at: pos)) { pos += 1 } } else { - while pos < nsString.length && !isWhitespace(nsString.character(at: pos)) { + while pos < text.length && !isWhitespace(text.character(at: pos)) { pos += 1 } - while pos < nsString.length && isWhitespace(nsString.character(at: pos)) { + while pos < text.length && isWhitespace(text.character(at: pos)) { pos += 1 } } - return min(pos, nsString.length) + return min(pos, text.length) } - var pos = min(offset, nsString.length) + var pos = min(offset, text.length) if pos > 0 { pos -= 1 } - while pos > 0 && isWhitespace(nsString.character(at: pos)) { + while pos > 0 && isWhitespace(text.character(at: pos)) { pos -= 1 } - while pos > 0 && !isWhitespace(nsString.character(at: pos - 1)) { + while pos > 0 && !isWhitespace(text.character(at: pos - 1)) { pos -= 1 } return max(0, pos) } func bigWordEnd(from offset: Int) -> Int { - guard let textView else { return offset } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } - var pos = min(offset + 1, nsString.length - 1) - while pos < nsString.length && isWhitespace(nsString.character(at: pos)) { + guard let text else { return offset } + guard text.length > 0 else { return 0 } + var pos = min(offset + 1, text.length - 1) + while pos < text.length && isWhitespace(text.character(at: pos)) { pos += 1 } - guard pos < nsString.length else { return nsString.length - 1 } - while pos < nsString.length - 1 && !isWhitespace(nsString.character(at: pos + 1)) { + guard pos < text.length else { return text.length - 1 } + while pos < text.length - 1 && !isWhitespace(text.character(at: pos + 1)) { pos += 1 } - return min(pos, nsString.length - 1) + return min(pos, text.length - 1) } func bigWordEndBackward(from offset: Int) -> Int { - guard let textView else { return 0 } - let nsString = textView.string as NSString - guard nsString.length > 0 else { return 0 } - var pos = min(max(0, offset), nsString.length - 1) + guard let text else { return 0 } + guard text.length > 0 else { return 0 } + var pos = min(max(0, offset), text.length - 1) if pos > 0 { pos -= 1 } - if isWhitespace(nsString.character(at: pos)) { - while pos > 0 && isWhitespace(nsString.character(at: pos)) { + if isWhitespace(text.character(at: pos)) { + while pos > 0 && isWhitespace(text.character(at: pos)) { pos -= 1 } return pos } - while pos > 0 && !isWhitespace(nsString.character(at: pos - 1)) { + while pos > 0 && !isWhitespace(text.character(at: pos - 1)) { pos -= 1 } guard pos > 0 else { return 0 } pos -= 1 - while pos > 0 && isWhitespace(nsString.character(at: pos)) { + while pos > 0 && isWhitespace(text.character(at: pos)) { pos -= 1 } return pos } func matchingBracket(at offset: Int) -> Int? { - guard let textView else { return nil } - let nsString = textView.string as NSString - guard offset >= 0 && offset < nsString.length else { return nil } - let ch = nsString.character(at: offset) + guard let text else { return nil } + guard offset >= 0 && offset < text.length else { return nil } + let ch = text.character(at: offset) let pairs: [unichar: (close: unichar, forward: Bool)] = [ 0x28: (0x29, true), 0x5B: (0x5D, true), 0x7B: (0x7D, true), 0x29: (0x28, false), 0x5D: (0x5B, false), 0x7D: (0x7B, false) @@ -380,8 +294,8 @@ final class VimTextBufferAdapter: VimTextBuffer { let step = pair.forward ? 1 : -1 var depth = 1 var pos = offset + step - while pos >= 0 && pos < nsString.length { - let cur = nsString.character(at: pos) + while pos >= 0 && pos < text.length { + let cur = text.character(at: pos) if cur == ch { depth += 1 } else if cur == pair.close { @@ -398,12 +312,12 @@ final class VimTextBufferAdapter: VimTextBuffer { return (0, max(0, lineCount - 1)) } let visible = scrollView.documentVisibleRect - let nsString = textView.string as NSString - guard nsString.length > 0 else { return (0, 0) } + let documentLength = length + guard documentLength > 0 else { return (0, 0) } let topGlyph = textView.layoutManager.textOffsetAtPoint(visible.origin) ?? 0 let bottomGlyph = textView.layoutManager.textOffsetAtPoint( CGPoint(x: visible.origin.x, y: visible.maxY - 1) - ) ?? max(0, nsString.length - 1) + ) ?? max(0, documentLength - 1) let topLine = lineAndColumn(forOffset: topGlyph).line let bottomLine = lineAndColumn(forOffset: bottomGlyph).line return (min(topLine, bottomLine), max(topLine, bottomLine)) @@ -417,6 +331,53 @@ final class VimTextBufferAdapter: VimTextBuffer { AppSettingsManager.shared.editor.clampedTabWidth } + // MARK: - Line Storage + + // The layout manager's line storage finds a line by index or offset in O(log n) and breaks lines where + // `lineRangeBreakingAtLineEndings` does. While an edit is still being applied it describes the old text, and an + // edit across a CR LF pair can leave it off the text's lines, so either case sends the query back to the text. + private func lineStorage(matching text: NSString) -> TextLineStorage? { + guard let storage = textView?.textStorage, let lines = textView?.layoutManager?.lineStorage, !lines.isEmpty, + lines.length == text.length, !storage.editedMask.contains(.editedCharacters) else { + return nil + } + if let trust = lineStorageTrust, trust.storage === storage, trust.length == text.length { + return trust.trusted ? lines : nil + } + let trusted = text.range(of: "\r", options: .literal).location == NSNotFound || lines.breaksLines(like: text) + lineStorageTrust = LineStorageTrust(storage: storage, length: text.length, trusted: trusted) + return trusted ? lines : nil + } + + private func lineIndex(of range: NSRange, in text: NSString) -> Int { + if let line = lineStorage(matching: text)?.getLine(atOffset: range.location), line.range == range { + return line.index + } + var index = 0 + var location = 0 + while location < range.location { + location = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0))) + index += 1 + } + return index + } + + // An index past the last line resolves to the last line, which is the empty one after a final line ending. + private func lineRange(atIndex index: Int, in text: NSString) -> NSRange { + if let lines = lineStorage(matching: text), + let line = lines.getLine(atIndex: min(index, lines.count - 1)), + line.range == text.lineRangeBreakingAtLineEndings(for: NSRange(location: line.range.location, length: 0)) { + return line.range + } + var current = 0 + var location = 0 + while location < text.length && current < index { + location = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0))) + current += 1 + } + return text.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0)) + } + // MARK: - Helpers private enum CharClass { @@ -435,4 +396,8 @@ final class VimTextBufferAdapter: VimTextBuffer { private func isWhitespace(_ char: unichar) -> Bool { char == 0x20 || char == 0x09 || char == 0x0A || char == 0x0D } + + private func isLineBreak(_ char: unichar) -> Bool { + char == 0x0A || char == 0x0D + } } diff --git a/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift b/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift index 74b11c7a5b..63f9e24d11 100644 --- a/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift +++ b/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift @@ -7,16 +7,15 @@ import Foundation import TableProPluginKit internal extension PluginCellValue { - /// The sentinel a pending inserted row carries for a column the server fills in, written by - /// `RowOperationsManager` and stripped again by `SQLStatementGenerator`. - /// - /// It is an internal marker, never a value, so every view that renders a cell has to recognise - /// it. The grid shows it as a placeholder; anything that renders the same rows some other way - /// has to agree, or the marker leaks out as literal text. - static let defaultMarkerText = "__DEFAULT__" + // A cell left to the column's default is the SQL keyword, marked as SQL beside the value like any expression. The + // mark decides, never the text: a stored or typed value can read the same. + static let columnDefaultKeyword = "DEFAULT" + static let columnDefault = PluginCellValue.text(columnDefaultKeyword) - var isDefaultMarker: Bool { - guard case .text(let value) = self else { return false } - return value == Self.defaultMarkerText + // The spelling driver plugins read as the default in the rows the host hands them. + static let pluginDefaultMarker = PluginCellValue.text("__DEFAULT__") + + func isColumnDefault(isExpression: Bool) -> Bool { + isExpression && self == Self.columnDefault } } diff --git a/apple/TablePro/Extensions/String+HexDump.swift b/apple/TablePro/Extensions/String+HexDump.swift index 60f9a6fba7..1c0d352d80 100644 --- a/apple/TablePro/Extensions/String+HexDump.swift +++ b/apple/TablePro/Extensions/String+HexDump.swift @@ -37,109 +37,148 @@ extension String { data(using: .isoLatin1) ?? Data(utf8) } - /// Returns a classic hex dump representation of this string's bytes, or nil if empty. - /// - /// Format per line: `OFFSET HH HH HH HH HH HH HH HH HH HH HH HH HH HH HH HH |ASCII...........|` - /// - Parameter maxBytes: Maximum bytes to display before truncating (default 10KB). + // The first `maxBytes` of `storedBytes` and how many there are in all, without encoding the rest: + // a cell shows 64 bytes of a value that can run to megabytes. + private func storedBytesPrefix(_ maxBytes: Int) -> (bytes: Data, count: Int) { + // A lead byte of 0xC4 or more starts a scalar past U+00FF. Counting in UInt8 runs of 255 lets the + // loop vectorize: under 1 ms for 6 MB against 13 ms with an Int counter, measured. + let scan = utf8.withContiguousStorageIfAvailable { buffer -> (fitsLatin1: Bool, scalars: Int) in + var highest: UInt8 = 0 + var continuations = 0 + var index = 0 + while index < buffer.count { + let end = Swift.min(index + 255, buffer.count) + var run: UInt8 = 0 + while index < end { + let byte = buffer[index] + highest = Swift.max(highest, byte) + run &+= byte >> 6 == 2 ? 1 : 0 + index += 1 + } + continuations += Int(run) + if highest >= 0xC4 { return (false, 0) } + } + return (true, buffer.count - continuations) + } + guard let scan else { + let bytes = storedBytes + return (bytes.prefix(maxBytes), bytes.count) + } + guard scan.fitsLatin1 else { + return (Data(utf8.prefix(maxBytes)), utf8.count) + } + var bytes = Data(capacity: Swift.min(maxBytes, scan.scalars)) + for scalar in unicodeScalars.prefix(maxBytes) { + bytes.append(UInt8(truncatingIfNeeded: scalar.value)) + } + return (bytes, scan.scalars) + } + func formattedAsHexDump(maxBytes: Int = 10_240) -> String? { - let bytes = storedBytes + let stored = storedBytesPrefix(maxBytes) + return HexText.dump(stored.bytes, totalCount: stored.count, maxBytes: maxBytes) + } - let totalCount = bytes.count + func formattedAsEditableHex(maxBytes: Int = 10_240) -> String? { + let stored = storedBytesPrefix(maxBytes) + return HexText.editable(stored.bytes, totalCount: stored.count, maxBytes: maxBytes) + } + + func formattedAsCompactHex(maxBytes: Int = 64) -> String? { + let stored = storedBytesPrefix(maxBytes) + return HexText.compact(stored.bytes, totalCount: stored.count, maxBytes: maxBytes) + } +} + +extension Data { + func formattedAsHexDump(maxBytes: Int = 10_240) -> String? { + HexText.dump(prefix(maxBytes), totalCount: count, maxBytes: maxBytes) + } + + func formattedAsEditableHex(maxBytes: Int = 10_240) -> String? { + HexText.editable(prefix(maxBytes), totalCount: count, maxBytes: maxBytes) + } + + func formattedAsCompactHex(maxBytes: Int = 64) -> String? { + HexText.compact(prefix(maxBytes), totalCount: count, maxBytes: maxBytes) + } +} + +// Each takes the first bytes of a value and the value's whole byte count. +private enum HexText { + private static let digits = Array("0123456789ABCDEF".utf8) + + private static func append(_ byte: UInt8, to text: inout [UInt8]) { + text.append(digits[Int(byte >> 4)]) + text.append(digits[Int(byte & 0x0F)]) + } + + static func dump(_ bytes: Data, totalCount: Int, maxBytes: Int) -> String? { guard totalCount > 0 else { return nil } let displayCount = min(totalCount, maxBytes) let bytesArray = [UInt8](bytes.prefix(displayCount)) - - var lines: [String] = [] let bytesPerLine = HexDumpLayout.bytesPerLine - lines.reserveCapacity(displayCount / bytesPerLine + 2) + var text: [UInt8] = [] + text.reserveCapacity((displayCount / bytesPerLine + 1) * (HexDumpLayout.lineWidthInCharacters + 1)) var offset = 0 - while offset < displayCount { let lineEnd = min(offset + bytesPerLine, displayCount) - let lineBytes = bytesArray[offset.. 0 { text.append(UInt8(ascii: "\n")) } - // Hex columns: two groups of 8 bytes - for i in 0..> shift) & 0x0F]) } + text.append(contentsOf: " ".utf8) - // ASCII column - line += " |" - for byte in lineBytes { - if byte >= 0x20, byte <= 0x7E { - line += String(UnicodeScalar(byte)) + for column in 0..= 0x20 && byte <= 0x7E ? byte : UInt8(ascii: ".")) + } + text.append(UInt8(ascii: "|")) offset += bytesPerLine } + var lines = text.isEmpty ? [] : [String(bytes: text, encoding: .ascii) ?? ""] if totalCount > maxBytes { let formattedTotal = totalCount.formatted(.number) lines.append("… (truncated, \(formattedTotal) bytes total)") } - return lines.joined(separator: "\n") } - /// Returns a space-separated hex representation suitable for editing. - /// - /// Format: `48 65 6C 6C 6F` — one hex byte pair separated by spaces, no offset or ASCII columns. - /// - Parameter maxBytes: Maximum bytes to display before truncating (default 10KB). - func formattedAsEditableHex(maxBytes: Int = 10_240) -> String? { - let bytes = storedBytes - - let totalCount = bytes.count + static func editable(_ bytes: Data, totalCount: Int, maxBytes: Int) -> String? { guard totalCount > 0 else { return nil } let displayCount = min(totalCount, maxBytes) - let bytesArray = [UInt8](bytes.prefix(displayCount)) - - var hex = bytesArray.map { String(format: "%02X", $0) }.joined(separator: " ") + var text: [UInt8] = [] + text.reserveCapacity(displayCount * 3 + 4) + for (index, byte) in bytes.prefix(displayCount).enumerated() { + if index > 0 { text.append(UInt8(ascii: " ")) } + append(byte, to: &text) + } + var hex = String(bytes: text, encoding: .ascii) ?? "" if totalCount > maxBytes { hex += " …" } - return hex } - /// Returns a compact single-line hex representation for data grid cells. - /// - /// Format: `0x48656C6C6F` for short values, truncated with `…` for longer ones. - /// - Parameter maxBytes: Maximum bytes to show before truncating (default 64). - func formattedAsCompactHex(maxBytes: Int = 64) -> String? { - let bytes = storedBytes - - let totalCount = bytes.count + static func compact(_ bytes: Data, totalCount: Int, maxBytes: Int) -> String? { guard totalCount > 0 else { return nil } - - let displayCount = min(totalCount, maxBytes) - let bytesArray = [UInt8](bytes.prefix(displayCount)) - - var hex = "0x" - for byte in bytesArray { - hex += String(format: "%02X", byte) - } - - if totalCount > maxBytes { - hex += "…" - } - - return hex + let hex = "0x" + bytes.uppercaseHex(maxBytes: maxBytes) + return totalCount > maxBytes ? hex + "…" : hex } } diff --git a/apple/TablePro/Extensions/TextLineStorage+LineBreaks.swift b/apple/TablePro/Extensions/TextLineStorage+LineBreaks.swift new file mode 100644 index 0000000000..e1571c672d --- /dev/null +++ b/apple/TablePro/Extensions/TextLineStorage+LineBreaks.swift @@ -0,0 +1,28 @@ +import Foundation +import TableProTextEngine + +extension TextLineStorage where Data == TextLine { + // The layout manager updates its lines edit by edit and can split or join a CR LF pair an edit crosses, after + // which its lines no longer end where the text's line breaks do. Without a CR in the text that cannot happen. + func breaksLines(like text: NSString) -> Bool { + guard length == text.length else { return false } + let lineBreaks = CharacterSet(charactersIn: "\r\n") + var start = 0 + var index = 0 + while let line = getLine(atIndex: index), line.range.location == start { + let rest = NSRange(location: start, length: text.length - start) + let found = text.rangeOfCharacter(from: lineBreaks, options: .literal, range: rest) + guard found.location != NSNotFound else { + return NSMaxRange(line.range) == text.length && index == count - 1 + } + var end = NSMaxRange(found) + if text.character(at: found.location) == 0x0D, end < text.length, text.character(at: end) == 0x0A { + end += 1 + } + guard NSMaxRange(line.range) == end else { return false } + start = end + index += 1 + } + return false + } +} diff --git a/apple/TablePro/Models/AI/AIModels.swift b/apple/TablePro/Models/AI/AIModels.swift index 778b3a5af8..d806eb1044 100644 --- a/apple/TablePro/Models/AI/AIModels.swift +++ b/apple/TablePro/Models/AI/AIModels.swift @@ -104,21 +104,21 @@ enum AIProviderType: String, Codable, CaseIterable, Identifiable, Sendable { /// The types `OpenAICompatibleProvider` serves, which share one registration. static let openAICompatibleFamily: [AIProviderType] = [.openRouter, .openCode, .ollama, .llamaCpp, .mlx, .custom] - var symbolName: String { + var icon: BrandIcon { switch self { - case .copilot: return "chevron.left.forwardslash.chevron.right" - case .chatgptCodex: return "bubble.left.and.bubble.right" - case .claude: return "brain" - case .claudeAgent: return "terminal" - case .openAI: return "cpu" - case .openRouter: return "globe" - case .gemini: return "wand.and.stars" - case .xai: return "x.circle" - case .ollama: return "desktopcomputer" - case .llamaCpp: return "memorychip" - case .mlx: return "m.square" - case .openCode: return "sparkles" - case .custom: return "server.rack" + case .copilot: return BrandIcon(slug: "githubcopilot", fallbackSystemImage: "chevron.left.forwardslash.chevron.right") + case .chatgptCodex: return BrandIcon(slug: "openai", fallbackSystemImage: "bubble.left.and.bubble.right") + case .claude: return BrandIcon(slug: "claude", fallbackSystemImage: "brain") + case .claudeAgent: return BrandIcon(slug: "claudecode", fallbackSystemImage: "terminal") + case .openAI: return BrandIcon(slug: "openai", fallbackSystemImage: "cpu") + case .openRouter: return BrandIcon(slug: "openrouter", fallbackSystemImage: "globe") + case .gemini: return BrandIcon(slug: "googlegemini", fallbackSystemImage: "wand.and.stars") + case .xai: return BrandIcon(slug: "xai", fallbackSystemImage: "x.circle") + case .ollama: return BrandIcon(slug: "ollama", fallbackSystemImage: "desktopcomputer") + case .llamaCpp: return BrandIcon(slug: "llamacpp", fallbackSystemImage: "memorychip") + case .mlx: return BrandIcon(slug: "mlx", fallbackSystemImage: "m.square") + case .openCode: return BrandIcon(slug: "opencode", fallbackSystemImage: "sparkles") + case .custom: return BrandIcon(slug: nil, fallbackSystemImage: "server.rack") } } } @@ -195,7 +195,7 @@ struct AIProviderConfig: Codable, Equatable, Identifiable, Sendable { name.isEmpty ? kindName : name } - var symbolName: String { preset?.symbolName ?? type.symbolName } + var icon: BrandIcon { preset?.icon ?? type.icon } var authStyle: AIProviderType.AuthStyle { preset?.authStyle ?? type.authStyle } @@ -257,7 +257,7 @@ enum AIChatMode: String, Codable, CaseIterable, Identifiable, Sendable { case .ask: return String(localized: "Ask: read-only schema lookups. AI can browse but not run queries.") case .edit: - return String(localized: "Edit: read-only tools plus running queries. Destructive DDL stays blocked.") + return String(localized: "Edit: read-only tools plus queries and data changes. Schema, privilege and session commands need Agent mode.") case .agent: return String(localized: "Agent: full tool access including destructive DDL. Safe mode still gates execution.") } @@ -269,7 +269,10 @@ enum AIChatMode: String, Codable, CaseIterable, Identifiable, Sendable { return "You are in Ask mode. Tools are read-only: schema lookups only. You cannot run queries or modify data." case .edit: return """ - You are in Edit mode. You can read schema and run SELECT/INSERT/UPDATE/DELETE via execute_query. \ + You are in Edit mode. You can read schema and run queries and data changes, such as \ + SELECT, INSERT, UPDATE, DELETE and MERGE, via execute_query. execute_query refuses schema, privilege, \ + session and transaction commands, such as CREATE, ALTER, GRANT, SET or BEGIN, and their counterparts in \ + a database's own language, such as Redis ACL or MongoDB createIndex: they need Agent mode. \ Destructive statements, such as DROP, TRUNCATE or a DELETE with no WHERE, are blocked. """ case .agent: @@ -326,6 +329,16 @@ struct AISettings: Codable, Equatable, Sendable { composerHighlightEnabled: true ) + // A reset keeps the providers, whose keys can be iCloud Keychain items other Macs still use, and the switch that + // keeps them from being used. + var resetKeepingProviders: AISettings { + var reset = AISettings.default + reset.enabled = enabled + reset.providers = providers + reset.activeProviderID = activeProviderID + return reset + } + init( enabled: Bool = true, providers: [AIProviderConfig] = [], diff --git a/apple/TablePro/Models/AI/AIProviderPreset.swift b/apple/TablePro/Models/AI/AIProviderPreset.swift index a25b599e33..9021669d47 100644 --- a/apple/TablePro/Models/AI/AIProviderPreset.swift +++ b/apple/TablePro/Models/AI/AIProviderPreset.swift @@ -5,34 +5,113 @@ import Foundation -/// A named vendor that speaks the OpenAI-compatible wire format. -/// -/// A preset is stored as a `.custom` provider carrying the preset's id, so a build that has never -/// heard of the vendor still decodes it as a working custom provider. A new `AIProviderType` case -/// cannot do that: its raw value fails to decode on every older build that syncs the same settings. +// Stored as `.custom` plus an id: a new `AIProviderType` case fails to decode on older synced builds. struct AIProviderPreset: Identifiable, Equatable, Sendable { let id: String let displayName: String let endpoint: String - let symbolName: String + let icon: BrandIcon let authStyle: AIProviderType.AuthStyle - /// Most servers answer a wrong key with 401. One that answers 403 would otherwise be reported - /// as a server error, which the chat offers to retry. + // A 403 for a wrong key would otherwise read as a server error, which the chat offers to retry. let rejectsBadKeyWithForbidden: Bool + // Ranked: a new provider starts on the first. + let curatedModels: [CuratedModel] + let modelIDs: AIModelIDFilter + // What every chat model here accepts when the server's model list says nothing about reasoning. + let reasoning: AIReasoningSupport? + + init( + id: String, + displayName: String, + endpoint: String, + icon: BrandIcon, + authStyle: AIProviderType.AuthStyle, + rejectsBadKeyWithForbidden: Bool, + curatedModels: [CuratedModel] = [], + modelIDs: AIModelIDFilter = .all, + reasoning: AIReasoningSupport? = nil + ) { + self.id = id + self.displayName = displayName + self.endpoint = endpoint + self.icon = icon + self.authStyle = authStyle + self.rejectsBadKeyWithForbidden = rejectsBadKeyWithForbidden + self.curatedModels = curatedModels + self.modelIDs = modelIDs + self.reasoning = reasoning + } static let requesty = AIProviderPreset( id: "requesty", displayName: "Requesty", endpoint: "https://router.requesty.ai", - symbolName: "arrow.triangle.branch", + icon: BrandIcon(slug: "requesty", fallbackSystemImage: "arrow.triangle.branch"), authStyle: .apiKey, rejectsBadKeyWithForbidden: true ) - static let all: [AIProviderPreset] = [.requesty] + // No Off: without an effort Meta runs its default, max on Muse Wonder and Muse Ember. + private static let museReasoning = AIReasoningSupport( + mode: .effortOnly, + effortLevels: [.minimal, .low, .medium, .high, .xhigh], + defaultEffort: .high, + isMandatory: true + ) + + static let meta = AIProviderPreset( + id: "meta", + displayName: "Meta", + endpoint: "https://api.meta.ai", + icon: BrandIcon(slug: "meta", fallbackSystemImage: "infinity.circle"), + authStyle: .apiKey, + rejectsBadKeyWithForbidden: false, + curatedModels: [ + CuratedModel(id: "muse-spark-1.3", displayName: "Muse Spark 1.3"), + CuratedModel(id: "muse-wonder-1.0", displayName: "Muse Wonder"), + CuratedModel(id: "muse-ember-1.0", displayName: "Muse Ember") + ], + // Meta may train on what a contributor model is sent, so one is used only when typed under Other…. + modelIDs: AIModelIDFilter( + families: ["muse-spark-", "muse-wonder-", "muse-ember-"], + unlistedParts: ["-contributor"] + ), + reasoning: museReasoning + ) + + static let all: [AIProviderPreset] = [.requesty, .meta] static func preset(withID id: String?) -> AIProviderPreset? { guard let id else { return nil } return all.first { $0.id == id } } + + // A gateway in the vendor's place names its models its own way, so its list is kept whole. + func modelIDFilter(forEndpoint endpoint: String) -> AIModelIDFilter { + guard let host = URL(string: endpoint.trimmingCharacters(in: .whitespacesAndNewlines))?.host?.lowercased(), + host == URL(string: self.endpoint)?.host?.lowercased() + else { return .all } + return modelIDs + } + + func reasoning(forModelID modelID: String) -> AIReasoningSupport? { + guard let reasoning, modelIDs.isChatFamily(modelID) else { return nil } + return reasoning + } +} + +// For a model list with no capability field, where chat models are told apart by their id. +struct AIModelIDFilter: Equatable, Sendable { + let families: [String] + let unlistedParts: [String] + + static let all = AIModelIDFilter(families: [], unlistedParts: []) + + func isChatFamily(_ modelID: String) -> Bool { + families.isEmpty || families.contains { modelID.hasPrefix($0) } + } + + func lists(_ modelID: String) -> Bool { + isChatFamily(modelID) && !unlistedParts.contains { modelID.contains($0) } + } } diff --git a/apple/TablePro/Models/AI/AgentSession.swift b/apple/TablePro/Models/AI/AgentSession.swift index a2b68570b2..a0d48c1433 100644 --- a/apple/TablePro/Models/AI/AgentSession.swift +++ b/apple/TablePro/Models/AI/AgentSession.swift @@ -6,22 +6,12 @@ import Combine import Foundation -/// One conversation with the assistant about one connection. -/// -/// A session outlives the window that opened it. Closing a window stops its sessions and keeps -/// their transcripts; opening one again continues where it stopped, and nothing is replayed. -/// -/// It owns exactly one `AIChatViewModel`, which is what makes "one session, two presentations" -/// true: the trailing-pane chat in Browse mode and the conversation column in Agent mode render -/// this same view model, so there is one transcript, one composer draft and one scroll position -/// however the user is looking at it. +// Both presentations render this one `AIChatViewModel`, so there is one transcript, draft and scroll position. @MainActor internal final class AgentSession: ObservableObject, Identifiable { internal let id: UUID internal let connectionId: UUID - /// The engine. Built with this session's id so a restored session is the same session rather - /// than a new one wearing its transcript. internal let viewModel: AIChatViewModel @Published internal private(set) var status: AgentSessionStatus @@ -29,32 +19,15 @@ internal final class AgentSession: ObservableObject, Identifiable { internal let startedAt: Date - /// When the session last went to work: the start of the reply it last began, or its own start. - /// The rail lists the latest first. - /// - /// Opening a session is not work, so it does not move one. It used to, and the rail was ordered - /// by start time while it did, so the stamp changed nothing anyone could see; ordered by this, a - /// session double-clicked in the middle of the list would have jumped to the top from under the - /// pointer that opened it. + // The rail's order. Opening a session is not work, or a row would jump from under the pointer. @Published internal private(set) var lastActiveAt: Date - /// Which of the result column's views this session is showing. - /// - /// On the session rather than in the column: the column is one hosting controller per window, - /// drawing whichever session is open, so view state there carried one session's choice into the - /// next. Not stored, so a relaunch opens on the statements. + // The result column is one hosting controller per window, so state kept there leaks between sessions. @Published internal var resultSegment: AgentResultSegment = .sql - /// Text typed before the session could send it, which a connect long enough to notice a typo in - /// needs. Cleared before it is dispatched so three flush sites still send once. internal var pendingPrompt: String? - /// Hands the pending prompt over once the connection is up, and clears it as it goes. - /// - /// The conversation asks from a `task` keyed on the session, the connect and the prompt, and a - /// reparent re-runs every such task on the same view: a mode toggle and a connection switch are - /// both one. Taking rather than reading is what keeps each of those re-runs from sending the - /// prompt a second time. + // A reparent re-runs the task that asks for this, so taking rather than reading sends it once. internal func takePendingPrompt(isConnecting: Bool) -> String? { guard !isConnecting, let prompt = pendingPrompt else { return nil } pendingPrompt = nil @@ -82,27 +55,16 @@ internal final class AgentSession: ObservableObject, Identifiable { observeEngine() } - /// The conversation this session's transcript is stored under, or nil before the first reply. + // A session restored from disk or released holds its conversation's id before its transcript. internal var conversationId: UUID? { - viewModel.activeConversationID + viewModel.activeConversationID ?? viewModel.conversationToRestore } - /// What the rail, the conversation and the delete confirmation call the session. A session names - /// itself from its first question, so one that has not been asked anything goes by the command - /// that made it. internal var displayTitle: String { title.isEmpty ? String(localized: "New Session") : title } - /// Republishes the engine's changes as the session's own, so a rail row bound to the session - /// redraws when the transcript moves. `AIChatViewModel` is an `ObservableObject` of its own and - /// nothing else forwards it. - /// Republishes the engine's changes, and its turns'. - /// - /// A `ChatTurn` is an `ObservableObject` of its own, and appending a tool-use block mutates the - /// turn rather than the view model's `messages` array, so the view model never announced it. - /// The result pane could not show a proposed statement until some later top-level change - /// happened to fire, which for a card waiting on an answer is never. + // A tool-use block mutates its turn rather than `messages`, so each turn is observed as well. private func observeEngine() { viewModel.objectWillChange .sink { [weak self] _ in @@ -132,14 +94,7 @@ internal final class AgentSession: ObservableObject, Identifiable { } } - /// A stopped session keeps the state it ended on; a failed one does not. - /// - /// Retry is offered on a failure and moves the engine back through idle, loading and streaming, - /// so freezing on `.failed` left the rail reporting Failed for the whole of a successful retry - /// and session resolution still treating it as ended. - /// - /// Going to work is what makes a session the latest, so the rail moves it up as a reply starts - /// rather than as someone looks at it. + // A failed session follows the engine, since Retry moves it back through streaming. private func refreshDerivedState() { let engineStatus = derivedStatus() if status != .stopped, status != engineStatus { @@ -148,15 +103,15 @@ internal final class AgentSession: ObservableObject, Identifiable { } status = engineStatus } + // Empty lists say nothing about the title of a transcript that is not read yet. + guard !viewModel.isAwaitingTranscript else { return } let derivedTitle = derivedTitle() if title != derivedTitle { title = derivedTitle } } - /// Waiting is asked before working, because an ordinary tool card leaves `streamingState` on - /// `.streaming`: the stream is parked on the answer, not producing. `.awaitingApproval` is the - /// connection's own AI-access confirmation and is a different question. + // A tool card leaves `streamingState` on `.streaming` while the stream waits on the answer. private func derivedStatus() -> AgentSessionStatus { if ToolApprovalCenter.shared.hasPending(sessionId: id) { return .waitingOnYou } if case .awaitingApproval = viewModel.streamingState { return .waitingOnYou } @@ -190,45 +145,24 @@ internal final class AgentSession: ObservableObject, Identifiable { status = newStatus } - /// Ends the session without touching what it said. - /// - /// Window close, disconnect and a lost session all reach here, and none of them is the user - /// throwing a conversation away, so the transcript is written first and kept. + // None of the callers is the user throwing a conversation away, so the transcript is written first. internal func stop() { viewModel.cancelStream() - settlePendingApprovals() viewModel.persistCurrentConversation() mark(.stopped) } - /// Puts a stopped session back to work. Nothing is replayed: a statement that was waiting for an - /// answer when the window closed was cancelled by the stop, and the model is asked again rather - /// than the call being re-issued behind the user's back. - /// - /// Resuming is opening, not working, so the session keeps its place in the rail until it is - /// asked something. + // Only once no window shows the session; its next showing reads back from disk what this drops. + internal func release() { + viewModel.releaseLoadedState() + } + + // Nothing is replayed: a statement waiting for an answer was cancelled by the stop. internal func resume() { guard status.isEnded else { return } mark(.ready) } - /// Settles a card that is still waiting before the transcript is written. - /// - /// `cancelStream()` resumes the continuation, but the suspended task only marks the block - /// `.cancelled` once it is back on the main actor, and the snapshot was taken before that. The - /// restored transcript then held a `.pending` card with no continuation behind it, whose Run - /// button could never work. - private func settlePendingApprovals() { - for turn in viewModel.messages { - for block in turn.blocks { - guard case .toolUse(var use) = block.kind, - case .pending = use.approvalState else { continue } - use.approvalState = .cancelled - block.setKind(.toolUse(use)) - } - } - } - internal var record: AgentSessionRecord { AgentSessionRecord( id: id, @@ -242,8 +176,7 @@ internal final class AgentSession: ObservableObject, Identifiable { } } -/// What a session is on disk. It points at a conversation rather than copying its turns, so the -/// transcript has one home and a restored session cannot disagree with it. +// Points at a conversation rather than copying its turns, so the transcript has one home. internal struct AgentSessionRecord: Codable, Equatable, Identifiable, Sendable { internal static let currentSchemaVersion = 1 diff --git a/apple/TablePro/Models/AuditEntry.swift b/apple/TablePro/Models/AuditEntry.swift index 8b09a5f2f1..b19520f65c 100644 --- a/apple/TablePro/Models/AuditEntry.swift +++ b/apple/TablePro/Models/AuditEntry.swift @@ -1,8 +1,3 @@ -// -// AuditEntry.swift -// TablePro -// - import Foundation enum AuditCategory: String, Codable, CaseIterable, Sendable, Identifiable { @@ -63,6 +58,9 @@ struct AuditEntry: Codable, Identifiable, Sendable, Equatable, Hashable { let action: String let outcome: String let details: String? + // The name the client reported for itself, never its token. Nil for rows no client caused and for + // rows written before the log recorded it. + let clientName: String? init( id: UUID = UUID(), @@ -73,7 +71,8 @@ struct AuditEntry: Codable, Identifiable, Sendable, Equatable, Hashable { connectionId: UUID? = nil, action: String, outcome: String, - details: String? = nil + details: String? = nil, + clientName: String? = nil ) { self.id = id self.timestamp = timestamp @@ -84,6 +83,7 @@ struct AuditEntry: Codable, Identifiable, Sendable, Equatable, Hashable { self.action = action self.outcome = outcome self.details = details + self.clientName = clientName } init( @@ -95,7 +95,8 @@ struct AuditEntry: Codable, Identifiable, Sendable, Equatable, Hashable { connectionId: UUID? = nil, action: String, outcome: AuditOutcome, - details: String? = nil + details: String? = nil, + clientName: String? = nil ) { self.init( id: id, @@ -106,7 +107,8 @@ struct AuditEntry: Codable, Identifiable, Sendable, Equatable, Hashable { connectionId: connectionId, action: action, outcome: outcome.rawValue, - details: details + details: details, + clientName: clientName ) } } diff --git a/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift b/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift index baa347a335..afe2544e04 100644 --- a/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift +++ b/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift @@ -2,15 +2,13 @@ // ClickHousePartInfo.swift // TablePro // -// Model for ClickHouse partition/part information from system.parts. -// import Foundation -/// Represents a single part from the ClickHouse system.parts table struct ClickHousePartInfo: Identifiable { let id = UUID() let partition: String + let partitionId: String let name: String let rows: UInt64 let bytesOnDisk: UInt64 diff --git a/apple/TablePro/Models/Connection/ConnectionCommitMode.swift b/apple/TablePro/Models/Connection/ConnectionCommitMode.swift new file mode 100644 index 0000000000..8245a4edb7 --- /dev/null +++ b/apple/TablePro/Models/Connection/ConnectionCommitMode.swift @@ -0,0 +1,11 @@ +// +// ConnectionCommitMode.swift +// TablePro +// + +import Foundation + +enum ConnectionCommitMode: String, Sendable, Equatable, CaseIterable { + case automatic + case manual +} diff --git a/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift b/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift index bf327fcfe4..83b77f45bb 100644 --- a/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift +++ b/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift @@ -77,6 +77,37 @@ internal enum ConnectionFailureClassifier { return nsError.domain == NSCocoaErrorDomain && nsError.code == NSUserCancelledError } + private static let invalidAuthorizationSQLState = "28000" + private static let mysqlAccessDeniedErrorCode = 1_045 + + internal static func isAuthenticationFailure(_ error: Error) -> Bool { + if let pluginError = error as? any PluginDriverError { + if pluginError.pluginSqlState == invalidAuthorizationSQLState { + return true + } + if pluginError.pluginErrorCode == mysqlAccessDeniedErrorCode { + return true + } + return messageIndicatesAuthenticationFailure(pluginError.pluginErrorMessage) + } + return messageIndicatesAuthenticationFailure(error.localizedDescription) + } + + // SQL Server, Oracle, Redis, Cassandra and Snowflake word a refused login their own way. + private static let refusedLoginPhrases = [ + "access denied", "authentication failed", "invalid credentials", "login failed for user", "ora-01017", + "wrongpass", "password are incorrect", "incorrect username or password", + ] + + // SQL Server says "Login failed for user" while it is upgrading or in single-user mode too, and connects again after. + private static let passingLoginRefusals = ["script upgrade mode", "single user mode"] + + private static func messageIndicatesAuthenticationFailure(_ message: String) -> Bool { + let lowered = message.lowercased() + guard !passingLoginRefusals.contains(where: { lowered.contains($0) }) else { return false } + return refusedLoginPhrases.contains { lowered.contains($0) } + } + internal static func info(for error: Error) -> ConnectionFailureInfo { if error is SSLHandshakeError { return ConnectionFailureInfo(message: SSLHandshakeError.formatted(error)) diff --git a/apple/TablePro/Models/Connection/ConnectionSession.swift b/apple/TablePro/Models/Connection/ConnectionSession.swift index bfac87017e..d576075016 100644 --- a/apple/TablePro/Models/Connection/ConnectionSession.swift +++ b/apple/TablePro/Models/Connection/ConnectionSession.swift @@ -24,6 +24,7 @@ struct ConnectionSession: Identifiable { // The level in force; the toolbar changes it, so `connection.safeModeLevel` goes stale. var safeModeLevel: SafeModeLevel + var commitMode: ConnectionCommitMode // Per-connection state var selectedTables: Set = [] @@ -64,6 +65,7 @@ struct ConnectionSession: Identifiable { globalSeconds: 0 ) self.safeModeLevel = connection.safeModeLevel + self.commitMode = connection.commitMode self.connectedAt = Date() self.lastActiveAt = Date() } diff --git a/apple/TablePro/Models/Connection/DatabaseConnection+Destination.swift b/apple/TablePro/Models/Connection/DatabaseConnection+Destination.swift index e4bcfd6b39..c03966c6e6 100644 --- a/apple/TablePro/Models/Connection/DatabaseConnection+Destination.swift +++ b/apple/TablePro/Models/Connection/DatabaseConnection+Destination.swift @@ -21,6 +21,21 @@ extension DatabaseConnection { ) } + // Read through the linked SSH profile as it stands now, which is where the next connect dials. + @MainActor var dialedDestination: ConnectionDestination { + SSHProfileStorage.shared.refreshingLinkedProfile(self).destination + } + + // The SSH server and jump hosts this copy names, in order. A profile counts by its servers, never its id. + var destinationTunnel: ConnectionDestination.Tunnel { + let config = resolvedSSHConfig + guard config.enabled else { return .none } + return .ssh( + server: ConnectionDestination.Endpoint(host: config.host, port: config.port), + jumps: config.jumpHosts.map { ConnectionDestination.Endpoint(host: $0.host, port: $0.port) } + ) + } + // Host and port first, then every other server or project the connection may sign in to, so a prompt never // hides the real host behind a value the engine ignores, or a server behind "(+N more)". Where plugin fields name // the server, as DuckDB's Remote fields do, they stand in for Host, which nothing dials. @@ -87,19 +102,4 @@ extension DatabaseConnection { case .verifyIdentity: return .verifyIdentity } } - - private var destinationTunnel: ConnectionDestination.Tunnel { - switch sshTunnelMode { - case .disabled: - return .none - case .profile(let profileId, _): - return .profile(profileId) - case .inline(let config): - guard config.enabled else { return .none } - return .ssh( - server: ConnectionDestination.Endpoint(host: config.host, port: config.port), - jumps: config.jumpHosts.map { ConnectionDestination.Endpoint(host: $0.host, port: $0.port) } - ) - } - } } diff --git a/apple/TablePro/Models/Connection/DatabaseConnection+IAMSignIn.swift b/apple/TablePro/Models/Connection/DatabaseConnection+IAMSignIn.swift index 8b3c0fa05e..ee33750411 100644 --- a/apple/TablePro/Models/Connection/DatabaseConnection+IAMSignIn.swift +++ b/apple/TablePro/Models/Connection/DatabaseConnection+IAMSignIn.swift @@ -36,8 +36,24 @@ enum GoogleCloudSQLCredentialKind: Equatable, Sendable { extension DatabaseConnection { static let iamSignInFieldId = "awsAuth" + // Only a value the type's own Authentication picker offers. A definition from someone else can carry the field on + // an engine that takes no IAM token, which would sign one and send it to whatever servers that engine dials. The + // Keyspaces engines are the exception for AWS: their plugin signs in itself and this host mints nothing for them. var iamSignIn: IAMSignInMethod? { - IAMSignInMethod(pickerValue: additionalFields[Self.iamSignInFieldId]) + guard let value = additionalFields[Self.iamSignInFieldId], + let method = IAMSignInMethod(pickerValue: value) + else { return nil } + if resolvesAWSIAMInDriver, case .aws = method { + return method + } + return Self.offersSignIn(value, for: type) ? method : nil + } + + private static func offersSignIn(_ value: String, for type: DatabaseType) -> Bool { + let field = PluginMetadataRegistry.shared.snapshot(for: type)?.connection.additionalConnectionFields + .first { $0.id == iamSignInFieldId } + guard case .dropdown(let options) = field?.fieldType else { return false } + return options.contains { $0.value == value } } var usesAWSIAM: Bool { diff --git a/apple/TablePro/Models/Connection/DatabaseConnection.swift b/apple/TablePro/Models/Connection/DatabaseConnection.swift index f981693eb3..3b90b24275 100644 --- a/apple/TablePro/Models/Connection/DatabaseConnection.swift +++ b/apple/TablePro/Models/Connection/DatabaseConnection.swift @@ -156,6 +156,7 @@ struct DatabaseConnection: Identifiable, Hashable { static let postDisconnectScriptKey = "postDisconnectScript" static let scriptTimeoutSecondsKey = "scriptTimeoutSeconds" static let completionIdentifierCaseKey = "completionIdentifierCase" + static let commitModeKey = "commitMode" static let binaryInlineLimitKilobytesKey = "binaryInlineLimitKB" static let binaryInlineLimitKilobytesRange = 1 ... 1_048_576 @@ -354,6 +355,19 @@ struct DatabaseConnection: Identifiable, Hashable { } } + var commitMode: ConnectionCommitMode { + get { + additionalFields[Self.commitModeKey].flatMap(ConnectionCommitMode.init(rawValue:)) ?? .automatic + } + set { + if newValue == .automatic { + additionalFields.removeValue(forKey: Self.commitModeKey) + } else { + additionalFields[Self.commitModeKey] = newValue.rawValue + } + } + } + /// A socket path stands in for the built-in Host and Port, so only a `.network` driver can be /// reached through one. A server named in plugin fields, like DuckDB's Quack, listens on TCP /// alone, and a path stored for it would send the tunnel somewhere nothing answers. diff --git a/apple/TablePro/Models/Connection/SafeModeFloor.swift b/apple/TablePro/Models/Connection/SafeModeFloor.swift index a68045e83e..f6e618ea5d 100644 --- a/apple/TablePro/Models/Connection/SafeModeFloor.swift +++ b/apple/TablePro/Models/Connection/SafeModeFloor.swift @@ -99,4 +99,11 @@ internal extension SafeModeFloor { static func levels(allowedBy floor: SafeModeFloor?) -> [SafeModeLevel] { SafeModeLevel.allCases.filter { floor?.allows($0) ?? true } } + + // The level a refusal at Read-Only tells the user to switch to: the weakest the menu offers above `floor`, never + // below Alert. Nil when the floor itself is Read-Only. + static func levelThatWrites(above floor: SafeModeLevel?) -> SafeModeLevel? { + guard let floor, floor.strictness > SafeModeLevel.alert.strictness else { return .alert } + return floor.blocksAllWrites ? nil : floor + } } diff --git a/apple/TablePro/Models/Database/TableSchema.swift b/apple/TablePro/Models/Database/TableSchema.swift index 578a8df7d4..b0229f39f7 100644 --- a/apple/TablePro/Models/Database/TableSchema.swift +++ b/apple/TablePro/Models/Database/TableSchema.swift @@ -29,9 +29,6 @@ struct TableSchema { primaryKeyColumns.compactMap { columns.firstIndex(of: $0) } } - /// Get index of first primary key column - var primaryKeyIndex: Int? { primaryKeyIndices.first } - /// Check if a column name exists func hasColumn(_ name: String) -> Bool { columns.contains(name) diff --git a/apple/TablePro/Models/ERDiagram/ERDiagramLayout.swift b/apple/TablePro/Models/ERDiagram/ERDiagramLayout.swift index 732808f04b..f0aa4263b1 100644 --- a/apple/TablePro/Models/ERDiagram/ERDiagramLayout.swift +++ b/apple/TablePro/Models/ERDiagram/ERDiagramLayout.swift @@ -26,7 +26,13 @@ enum ERDiagramLayout { let size: CGSize } - static func compute(graph: ERDiagramGraph) -> [UUID: CGPoint] { + // Off the caller's actor, so a superseded layout's task cancels the work itself and not only its result. + @concurrent + static func layOut(_ graph: ERDiagramGraph) async throws -> [UUID: CGPoint] { + try compute(graph: graph) + } + + static func compute(graph: ERDiagramGraph) throws -> [UUID: CGPoint] { guard !graph.nodes.isEmpty else { return [:] } let sizes = nodeSizes(graph: graph) @@ -45,7 +51,7 @@ enum ERDiagramLayout { var blocks: [Block] = [] for clusterId in componentGroups.keys.sorted() { let members = componentGroups[clusterId] ?? [] - let local = forceDirected(members: members, adjacency: adjacency, sizes: sizes) + let local = try forceDirected(members: members, adjacency: adjacency, sizes: sizes) blocks.append(makeBlock(centers: local, members: members, sizes: sizes)) } if !singletons.isEmpty { @@ -83,67 +89,70 @@ enum ERDiagramLayout { // MARK: - Force-Directed Component Layout + // A component is indexed by its members' order, so each step reads and writes arrays rather than + // hashing a UUID per pair: a 300-table component took 1.1 s that way and takes 11 ms this way, measured. private static func forceDirected( members: [UUID], adjacency: [UUID: [UUID]], sizes: [UUID: CGSize] - ) -> [UUID: CGPoint] { + ) throws -> [UUID: CGPoint] { guard let first = members.first else { return [:] } guard members.count > 1 else { return [first: .zero] } let count = members.count let spacing = idealDistance(members: members, sizes: sizes) let edges = uniqueEdges(members: members, adjacency: adjacency) - var degree: [UUID: Int] = [:] + var degree = ContiguousArray(repeating: 0, count: count) for (source, target) in edges { - degree[source, default: 0] += 1 - degree[target, default: 0] += 1 + degree[source] += 1 + degree[target] += 1 } - var positions = circularInit(members: members, idealDistance: spacing) + var positions = circularInit(count: count, idealDistance: spacing) let iterations = max(60, min(300, 2_000 / count)) var temperature = spacing * 2 for _ in 0.. [UUID: CGPoint] { - guard members.count > 2 else { return positions } + private static func rotateToLandscape(_ positions: inout ContiguousArray) { + guard positions.count > 2 else { return } var centerX: CGFloat = 0 var centerY: CGFloat = 0 - for id in members { - centerX += positions[id]?.x ?? 0 - centerY += positions[id]?.y ?? 0 + for position in positions { + centerX += position.x + centerY += position.y } - centerX /= CGFloat(members.count) - centerY /= CGFloat(members.count) + centerX /= CGFloat(positions.count) + centerY /= CGFloat(positions.count) var sxx: CGFloat = 0 var syy: CGFloat = 0 var sxy: CGFloat = 0 - for id in members { - guard let position = positions[id] else { continue } + for position in positions { let dx = position.x - centerX let dy = position.y - centerY sxx += dx * dx @@ -154,27 +163,27 @@ enum ERDiagramLayout { let theta = 0.5 * atan2(2 * sxy, sxx - syy) let cosT = cos(-theta) let sinT = sin(-theta) - var rotated: [UUID: CGPoint] = [:] - for id in members { - guard let position = positions[id] else { continue } - let dx = position.x - centerX - let dy = position.y - centerY - rotated[id] = CGPoint(x: centerX + dx * cosT - dy * sinT, y: centerY + dx * sinT + dy * cosT) + for index in positions.indices { + let dx = positions[index].x - centerX + let dy = positions[index].y - centerY + positions[index] = CGPoint(x: centerX + dx * cosT - dy * sinT, y: centerY + dx * sinT + dy * cosT) } - return rotated } - private static func uniqueEdges(members: [UUID], adjacency: [UUID: [UUID]]) -> [(UUID, UUID)] { - let memberSet = Set(members) - var seen: Set = [] - var edges: [(UUID, UUID)] = [] - for source in members { - for target in adjacency[source] ?? [] where memberSet.contains(target) { - let key = source.uuidString < target.uuidString - ? source.uuidString + target.uuidString - : target.uuidString + source.uuidString - guard !seen.contains(key) else { continue } - seen.insert(key) + // Each pair once, in the order the members first reach it. + private static func uniqueEdges(members: [UUID], adjacency: [UUID: [UUID]]) -> [(Int, Int)] { + var indexOf: [UUID: Int] = [:] + indexOf.reserveCapacity(members.count) + for (index, id) in members.enumerated() { + indexOf[id] = index + } + var seen: Set = [] + var edges: [(Int, Int)] = [] + for (source, id) in members.enumerated() { + for neighbor in adjacency[id] ?? [] { + guard let target = indexOf[neighbor] else { continue } + let key = min(source, target) * members.count + max(source, target) + guard seen.insert(key).inserted else { continue } edges.append((source, target)) } } @@ -182,31 +191,27 @@ enum ERDiagramLayout { } private static func forceStep( - members: [UUID], - edges: [(UUID, UUID)], - degree: [UUID: Int], - positions: [UUID: CGPoint], + edges: [(Int, Int)], + degree: ContiguousArray, + positions: ContiguousArray, idealDistance: CGFloat - ) -> [UUID: CGVector] { - var displacement: [UUID: CGVector] = [:] - for id in members { displacement[id] = .zero } + ) -> ContiguousArray { + let count = positions.count + var displacement = ContiguousArray(repeating: CGVector.zero, count: count) var centerX: CGFloat = 0 var centerY: CGFloat = 0 - for id in members { - centerX += positions[id]?.x ?? 0 - centerY += positions[id]?.y ?? 0 + for position in positions { + centerX += position.x + centerY += position.y } - centerX /= CGFloat(members.count) - centerY /= CGFloat(members.count) + centerX /= CGFloat(count) + centerY /= CGFloat(count) - let count = members.count for i in 0.. [UUID: CGPoint] { - let count = members.count + private static func circularInit(count: Int, idealDistance: CGFloat) -> ContiguousArray { let radius = idealDistance * CGFloat(count) / (2 * .pi) + idealDistance - var positions: [UUID: CGPoint] = [:] - for (index, id) in members.enumerated() { + var positions = ContiguousArray() + positions.reserveCapacity(count) + for index in 0.., + sizes: ContiguousArray + ) throws { + let count = positions.count let padding = horizontalGap * 0.5 let passes = max(20, min(count, 60)) for _ in 0.. 0, overlapY > 0 else { continue } moved = true if overlapX < overlapY { let shift = overlapX / 2 * (posLhs.x >= posRhs.x ? 1 : -1) - positions[lhs]?.x += shift - positions[rhs]?.x -= shift + positions[i].x += shift + positions[j].x -= shift } else { let shift = overlapY / 2 * (posLhs.y >= posRhs.y ? 1 : -1) - positions[lhs]?.y += shift - positions[rhs]?.y -= shift + positions[i].y += shift + positions[j].y -= shift } } } diff --git a/apple/TablePro/Models/ERDiagram/ERDiagramModels.swift b/apple/TablePro/Models/ERDiagram/ERDiagramModels.swift index 2a9c246d06..143b244cfc 100644 --- a/apple/TablePro/Models/ERDiagram/ERDiagramModels.swift +++ b/apple/TablePro/Models/ERDiagram/ERDiagramModels.swift @@ -3,7 +3,7 @@ import Foundation // MARK: - Table Node -struct ERTableNode: Identifiable, Sendable { +struct ERTableNode: Identifiable, Equatable, Sendable { let id: UUID let tableName: String let columns: [ERColumnDisplay] @@ -12,7 +12,7 @@ struct ERTableNode: Identifiable, Sendable { var isJunctionTable: Bool = false } -struct ERColumnDisplay: Identifiable, Sendable { +struct ERColumnDisplay: Identifiable, Equatable, Sendable { let id: String let name: String let dataType: String @@ -38,7 +38,7 @@ enum ERCardinality: Sendable { case manyToMany } -struct EREdge: Identifiable, Sendable { +struct EREdge: Identifiable, Equatable, Sendable { let id: UUID let fkName: String let fromTable: String diff --git a/apple/TablePro/Models/Query/DatabaseObjectRef.swift b/apple/TablePro/Models/Query/DatabaseObjectRef.swift index 0c0814ec70..dd35c068ea 100644 --- a/apple/TablePro/Models/Query/DatabaseObjectRef.swift +++ b/apple/TablePro/Models/Query/DatabaseObjectRef.swift @@ -14,8 +14,10 @@ enum DatabaseObjectKind: String, Codable, Sendable, Hashable { case userType case view case materializedView + // A kind the driver declares; the ref's `serverKind` says which one. + case serverObject - var sidebarObjectKind: SidebarObjectKind { + var sidebarObjectKind: SidebarObjectKind? { switch self { case .procedure: return .procedure case .function: return .function @@ -23,6 +25,7 @@ enum DatabaseObjectKind: String, Codable, Sendable, Hashable { case .userType: return .type case .view: return .view case .materializedView: return .materializedView + case .serverObject: return nil } } @@ -37,11 +40,30 @@ enum DatabaseObjectKind: String, Codable, Sendable, Hashable { } var displayName: String { - sidebarObjectKind.displayName + sidebarObjectKind?.displayName ?? String(localized: "Object") } var iconName: String { - sidebarObjectKind.iconName + sidebarObjectKind?.iconName ?? "shippingbox" + } +} + +struct ServerObjectKindRef: Hashable, Codable, Sendable { + let id: String + let displayName: String + let iconName: String + // Optional so a ref written without it decodes, and reads as a container kind. + let scope: SidebarServerObjectKind.Scope? + + init(id: String, displayName: String, iconName: String, scope: SidebarServerObjectKind.Scope? = nil) { + self.id = id + self.displayName = displayName + self.iconName = iconName + self.scope = scope + } + + init(_ kind: SidebarServerObjectKind) { + self.init(id: kind.id, displayName: kind.displayName, iconName: kind.iconName, scope: kind.scope) } } @@ -69,6 +91,9 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { /// re-list an entire schema to recover it, and short enough to persist with the tab. let attributes: [ObjectAttribute] + // Server objects only, and optional so a tab persisted before them still decodes. + let serverKind: ServerObjectKindRef? + init( kind: DatabaseObjectKind, name: String, @@ -78,7 +103,8 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { identity: String? = nil, argumentSignature: String? = nil, typeKind: UserDefinedTypeInfo.Kind? = nil, - attributes: [ObjectAttribute] = [] + attributes: [ObjectAttribute] = [], + serverKind: ServerObjectKindRef? = nil ) { self.kind = kind self.name = name @@ -89,6 +115,7 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { self.argumentSignature = argumentSignature self.typeKind = typeKind self.attributes = attributes + self.serverKind = serverKind } init(routine: RoutineInfo, database: String) { @@ -132,8 +159,19 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { ) } - /// What the tab is titled and what the viewer's header shows: enough to tell two overloads - /// apart, and enough to tell two same-named triggers on different tables apart. + init(serverObject: ServerObjectInfo, kind: SidebarServerObjectKind, database: String) { + self.init( + kind: .serverObject, + name: serverObject.name, + database: database, + schema: serverObject.schema, + identity: serverObject.identity, + attributes: serverObject.attributes, + serverKind: ServerObjectKindRef(kind) + ) + } + + // Enough to tell two overloads, or two same-named triggers on different tables, apart. var displayIdentity: String { switch kind { case .procedure, .function: @@ -144,19 +182,31 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { return String(format: String(localized: "%1$@ on %2$@"), qualifiedName, table) case .userType, .view, .materializedView: return qualifiedName + case .serverObject: + return isDatabaseScoped ? name : qualifiedName } } - /// The kind's name for the header capsule. A type says which kind of type it is, because an - /// enum and a domain are edited differently and the reader should know which one opened. + // A database-wide object's schema is where it keeps its members, never part of its name. + var isDatabaseScoped: Bool { + kind == .serverObject && serverKind?.scope == .database + } + + // A type names its own kind, since an enum and a domain are edited differently. var kindDisplayName: String { - guard kind == .userType else { return kind.displayName } - return (typeKind ?? .other).displayName + switch kind { + case .userType: return (typeKind ?? .other).displayName + case .serverObject: return serverKind?.displayName ?? kind.displayName + case .procedure, .function, .trigger, .view, .materializedView: return kind.displayName + } } var kindIconName: String { - guard kind == .userType else { return kind.iconName } - return (typeKind ?? .other).iconName + switch kind { + case .userType: return (typeKind ?? .other).iconName + case .serverObject: return serverKind?.iconName ?? kind.iconName + case .procedure, .function, .trigger, .view, .materializedView: return kind.iconName + } } var qualifiedName: String { @@ -164,8 +214,7 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { return "\(schema).\(name)" } - /// A ref built where no database was selected carries an empty one, and an empty database is - /// server-scoped. Resolving it once keeps the tab-dedup key and the loader's scope agreeing. + // An empty database is server-scoped; resolving it once keeps the tab-dedup key and the loader's scope agreeing. func resolvingDatabase(_ fallback: String) -> DatabaseObjectRef { guard database.isEmpty, !fallback.isEmpty else { return self } return DatabaseObjectRef( @@ -177,7 +226,8 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { identity: identity, argumentSignature: argumentSignature, typeKind: typeKind, - attributes: attributes + attributes: attributes, + serverKind: serverKind ) } @@ -191,7 +241,7 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { argumentSignature: argumentSignature, identity: identity ) - case .trigger, .userType, .view, .materializedView: + case .trigger, .userType, .view, .materializedView, .serverObject: return nil } } @@ -212,16 +262,26 @@ struct DatabaseObjectRef: Hashable, Codable, Sendable { ) } - /// Whether two refs name one object. The attributes are what a listing learned and change with - /// a comment edit, so they are left out. An identity is only compared, never trusted alone: some - /// drivers fill it with the object's kind or its signature, which many objects share. + var serverObject: ServerObjectInfo? { + guard kind == .serverObject, let serverKind else { return nil } + return ServerObjectInfo( + kindId: serverKind.id, + name: name, + schema: schema, + identity: identity, + attributes: attributes + ) + } + + // Attributes change with a comment edit, and some drivers fill identity with a value many objects share. func namesSameObject(as other: DatabaseObjectRef) -> Bool { guard kind == other.kind, database == other.database, name == other.name, schema == other.schema, table == other.table, - argumentSignature == other.argumentSignature + argumentSignature == other.argumentSignature, + serverKind?.id == other.serverKind?.id else { return false } guard let identity, let otherIdentity = other.identity else { return true } return identity == otherIdentity diff --git a/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift b/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift index e87b1d2e84..25d3fe0501 100644 --- a/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift +++ b/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift @@ -6,15 +6,12 @@ import Foundation import TableProPluginKit -/// Turns a per-column value filter into the order the grid displays. -/// -/// Pure, so both the grid and the coordinator that has to answer while the grid is unmounted can -/// call it and cannot disagree. Matching is done on the displayed string rather than the raw value, -/// because that is what the filter popover offered the user to pick from. +// Turns a per-column value filter into the order the grid displays. Pure, so the grid and the +// coordinator that answers while the grid is unmounted cannot disagree. Matching is on the displayed +// string rather than the raw value, because that is what the filter popover offered. @MainActor internal enum GridDisplayOrderResolver { - /// - Returns: the displayed row ids, or nil when no filter is active and the display order is - /// the storage order. + // Nil when no filter is active and the display order is the storage order. internal static func resolve( tableRows: TableRows, valueFilter: GridValueFilterState, @@ -38,27 +35,42 @@ internal enum GridDisplayOrderResolver { return result } - /// The displayed form of a cell, which is what a value filter stores and matches against. + // The displayed form of a cell, which is what a value filter stores and matches against. internal static func displayValue( _ rawValue: PluginCellValue, columnType: ColumnType?, displayFormat: ValueDisplayFormat?, databaseType: DatabaseType? + ) -> String { + displayValue( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer + ) + } + + nonisolated internal static func displayValue( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer ) -> String { CellDisplayFormatter.format( rawValue, columnType: columnType, displayFormat: displayFormat, - databaseType: databaseType + databaseType: databaseType, + dates: dates ) ?? rawValue.asText ?? "" } - /// Drops filters whose column has moved or disappeared since they were set. - /// - /// A filter is keyed by column index and remembers the name it was set on, so a result set of a - /// different shape would otherwise apply one column's chosen values to another column's cells. - /// Reading it out rather than mutating keeps this callable from a getter; the mutating - /// `GridValueFilterState.prune` still tidies the stored state at the points that replace rows. + // Drops filters whose column has moved or disappeared since they were set. A filter is keyed by + // column index and remembers the name it was set on, so a result of a different shape would apply + // one column's chosen values to another's cells. Reading it out rather than mutating keeps this + // callable from a getter; `GridValueFilterState.prune` tidies the stored state where rows are replaced. private static func activeFilters( _ valueFilter: GridValueFilterState, columns: [String] diff --git a/apple/TablePro/Models/Query/QueryBatchResult.swift b/apple/TablePro/Models/Query/QueryBatchResult.swift index df45727465..8f2b2f79e1 100644 --- a/apple/TablePro/Models/Query/QueryBatchResult.swift +++ b/apple/TablePro/Models/Query/QueryBatchResult.swift @@ -14,6 +14,8 @@ struct QueryBatchResult { let errors: [PluginBatchError] let discardedResultSetCount: Int let executionTime: TimeInterval + // Earlier result sets `followedKeepingLatest(by:)` let go to keep later ones. + var supersededResultSetCount = 0 static let empty = QueryBatchResult( resultSets: [], @@ -49,4 +51,28 @@ struct QueryBatchResult { executionTime: executionTime + next.executionTime ) } + + // The editor shows the last results of a run, so past the same ceiling it keeps the newest result sets. + func followedKeepingLatest(by next: QueryBatchResult) -> QueryBatchResult { + let combined = resultSets + next.resultSets + let superseded = max(combined.count - Self.keptResultSetLimit, 0) + let shifted = { (error: PluginBatchError, preceding: Int) in + PluginBatchError( + message: error.message, + code: error.code, + line: error.line, + procedure: error.procedure, + precedingResultSetCount: max(preceding - superseded, 0) + ) + } + return QueryBatchResult( + resultSets: Array(combined.suffix(Self.keptResultSetLimit)), + rowsAffected: rowsAffected + next.rowsAffected, + errors: errors.map { shifted($0, $0.precedingResultSetCount) } + + next.errors.map { shifted($0, resultSets.count + $0.precedingResultSetCount) }, + discardedResultSetCount: discardedResultSetCount + next.discardedResultSetCount, + executionTime: executionTime + next.executionTime, + supersededResultSetCount: supersededResultSetCount + next.supersededResultSetCount + superseded + ) + } } diff --git a/apple/TablePro/Models/Query/QueryHistoryEntry.swift b/apple/TablePro/Models/Query/QueryHistoryEntry.swift index 139776f907..5355cb10f0 100644 --- a/apple/TablePro/Models/Query/QueryHistoryEntry.swift +++ b/apple/TablePro/Models/Query/QueryHistoryEntry.swift @@ -97,26 +97,95 @@ struct QueryHistoryEntry: Identifiable, Codable, Hashable, Sendable { ) } + // Reads only the ends of the query: the Quick Switcher builds this for every history item it lists. var queryPreview: String { - var trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + let trimmable = CharacterSet.whitespacesAndNewlines + let scalars = query.unicodeScalars + guard let first = scalars.firstIndex(where: { !trimmable.contains($0) }), + let last = scalars.lastIndex(where: { !trimmable.contains($0) }) else { + return ";" + } + var head = PreviewHead(limit: 100) + for scalar in scalars[first...last] { + head.append(scalar) + if head.isComplete { return head.preview } + } + var trimmed = head.text if !trimmed.hasSuffix(";") { trimmed += ";" } - if (trimmed as NSString).length > 100 { - return String(trimmed.prefix(100)) + "…" - } - return trimmed + return PreviewHead.shortened(trimmed, to: head.limit) } + // Stops reading once the cut is known: a history row builds this every time it renders. var singleLinePreview: String { - let collapsed = query - .components(separatedBy: .newlines) - .map { $0.trimmingCharacters(in: .whitespaces) } - .filter { !$0.isEmpty } - .joined(separator: " ") - if (collapsed as NSString).length > 160 { - return String(collapsed.prefix(160)) + "…" + let newlines = CharacterSet.newlines + let whitespaces = CharacterSet.whitespaces + let scalars = query.unicodeScalars + var head = PreviewHead(limit: 160) + var lineHasText = false + var gapStart: String.UnicodeScalarView.Index? + var index = scalars.startIndex + while index < scalars.endIndex { + let scalar = scalars[index] + if newlines.contains(scalar) { + lineHasText = false + gapStart = nil + } else if whitespaces.contains(scalar) { + if lineHasText && gapStart == nil { + gapStart = index + } + } else { + if !lineHasText { + lineHasText = true + if !head.text.isEmpty { + head.append(" ") + } + } else if let start = gapStart { + gapStart = nil + for gapScalar in scalars[start..= nextCount else { return } + nextCount = scalarCount * 2 + isComplete = text.count > limit + } + + // Tests the UTF-16 length but cuts at a Character count, as the previews always have. + static func shortened(_ text: String, to limit: Int) -> String { + guard (text as NSString).length > limit else { return text } + return String(text.prefix(limit)) + "…" } } diff --git a/apple/TablePro/Models/Query/QueryParameter.swift b/apple/TablePro/Models/Query/QueryParameter.swift index d7a7cc4db8..56550eab6a 100644 --- a/apple/TablePro/Models/Query/QueryParameter.swift +++ b/apple/TablePro/Models/Query/QueryParameter.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProPluginKit enum QueryParameterType: String, Codable, CaseIterable, Sendable { case string @@ -28,3 +29,56 @@ struct QueryParameter: Identifiable, Codable, Equatable, Hashable, Sendable { self.isNull = isNull } } + +extension QueryParameterType { + // String is the panel's default, so a plain number in it still goes as a number (`LIMIT :n`, `id = :id`). Anything + // else is text, a code like `1E5` included; Decimal sends that as a number. + func pluginKind(for value: String) -> PluginColumnKind? { + switch self { + case .string: return Self.isPlainNumber(value) ? nil : .text + case .integer: return .integer + case .decimal: return .decimal + case .boolean: return .boolean + case .date: return .other + } + } + + // Digits with an optional sign and an optional fraction: `42`, `-3.5`, `.5`. + private static func isPlainNumber(_ value: String) -> Bool { + var scalars = value.unicodeScalars[...] + if scalars.first == "-" || scalars.first == "+" { scalars = scalars.dropFirst() } + var sawDot = false + var digitsAfterDot = 0 + var digits = 0 + for scalar in scalars { + if ("0"..."9").contains(scalar) { + digits += 1 + if sawDot { digitsAfterDot += 1 } + } else if scalar == ".", !sawDot { + sawDot = true + } else { + return false + } + } + return digits > 0 && (!sawDot || digitsAfterDot > 0) + } +} + +// A panel value with the type the user gave it, so a driver that writes values into the statement writes it as that. +struct TypedParameterValue: Equatable, Sendable { + let text: String + let kind: PluginColumnKind? + + // A run's value as the rerun state keeps it; plain text from a caller that gave no type stays text with no kind. + static func kept(_ value: Any?) -> TypedParameterValue? { + if let typed = value as? TypedParameterValue { return typed } + return (value as? String).map { TypedParameterValue(text: $0, kind: nil) } + } + + // Nil when no value carries a type, so a driver reads the values as it did before types were passed. + static func kinds(of parameters: [Any?]?) -> [PluginColumnKind?]? { + guard let parameters else { return nil } + let kinds = parameters.map { ($0 as? TypedParameterValue)?.kind ?? nil } + return kinds.contains { $0 != nil } ? kinds : nil + } +} diff --git a/apple/TablePro/Models/Query/QueryTab.swift b/apple/TablePro/Models/Query/QueryTab.swift index f5f32b7f12..314d4336f9 100644 --- a/apple/TablePro/Models/Query/QueryTab.swift +++ b/apple/TablePro/Models/Query/QueryTab.swift @@ -66,6 +66,9 @@ struct QueryTab: Identifiable, Equatable { var sessionHighlightRules: [HighlightRule] = [] // Set only by Reveal, after the user authenticated; never saved, so a reopened tab masks again. var revealsMaskedValues = false + // Never saved: a relaunch must not start polling a server with nobody watching. + var autoRefreshInterval: TableAutoRefreshInterval = .off + var autoRefreshPause: TableAutoRefreshPause? var pagination: PaginationState var chartConfiguration: ResultChartConfiguration var mapConfiguration: ResultMapConfiguration @@ -375,6 +378,8 @@ struct QueryTab: Identifiable, Equatable { && lhs.sortState == rhs.sortState && lhs.valueFilter == rhs.valueFilter && lhs.sessionHighlightRules == rhs.sessionHighlightRules + && lhs.autoRefreshInterval == rhs.autoRefreshInterval + && lhs.autoRefreshPause == rhs.autoRefreshPause && lhs.chartConfiguration == rhs.chartConfiguration && lhs.mapConfiguration == rhs.mapConfiguration && lhs.display == rhs.display diff --git a/apple/TablePro/Models/Query/QueryTabManager.swift b/apple/TablePro/Models/Query/QueryTabManager.swift index 42d045985b..02dd5453ad 100644 --- a/apple/TablePro/Models/Query/QueryTabManager.swift +++ b/apple/TablePro/Models/Query/QueryTabManager.swift @@ -526,7 +526,8 @@ final class QueryTabManager: ObservableObject { var newTab = QueryTab(title: Self.objectSourceTitle(for: objectRef), tabType: .objectSource) newTab.tableContext.isEditable = false newTab.tableContext.databaseName = objectRef.database - newTab.tableContext.schemaName = objectRef.schema + // A database-wide object's schema is where it installs its objects, not a namespace to browse. + newTab.tableContext.schemaName = objectRef.isDatabaseScoped ? nil : objectRef.schema newTab.display.objectRef = objectRef newTab.hasUserInteraction = true tabs.append(newTab) @@ -639,6 +640,8 @@ final class QueryTabManager: ObservableObject { case .userType: format = String(localized: "Type: %@") case .view: format = String(localized: "View: %@") case .materializedView: format = String(localized: "Materialized View: %@") + case .serverObject: + return String(format: String(localized: "%1$@: %2$@"), objectRef.kindDisplayName, objectRef.displayIdentity) } return String(format: format, objectRef.displayIdentity) } @@ -697,6 +700,8 @@ final class QueryTabManager: ObservableObject { tab.restoredSortSource = .unset tab.selectionSnapshot = .empty tab.pendingChanges = TabChangeSnapshot() + tab.autoRefreshInterval = .off + tab.autoRefreshPause = nil tab.hasUserInteraction = false tab.filterState = TabFilterState() tab.valueFilter = GridValueFilterState() diff --git a/apple/TablePro/Models/Query/QueryTabState.swift b/apple/TablePro/Models/Query/QueryTabState.swift index 796c43ee88..48b832dd2e 100644 --- a/apple/TablePro/Models/Query/QueryTabState.swift +++ b/apple/TablePro/Models/Query/QueryTabState.swift @@ -288,10 +288,10 @@ struct PaginationState: Equatable { var hasMoreRows: Bool = false var isLoadingMore: Bool = false var baseQueryForMore: String? - var baseQueryParameterValues: [String?]? + var baseQueryParameterValues: [TypedParameterValue?]? // Set together: stale bindings sent with a placeholder-free query made Fetch All fail on the server. - mutating func setBaseQueryForMore(_ sql: String?, parameterValues: [String?]?) { + mutating func setBaseQueryForMore(_ sql: String?, parameterValues: [TypedParameterValue?]?) { baseQueryForMore = sql baseQueryParameterValues = parameterValues } diff --git a/apple/TablePro/Models/Query/ResultOrigin.swift b/apple/TablePro/Models/Query/ResultOrigin.swift index ef3dfd03cb..1f8c805752 100644 --- a/apple/TablePro/Models/Query/ResultOrigin.swift +++ b/apple/TablePro/Models/Query/ResultOrigin.swift @@ -62,6 +62,7 @@ enum ResultEditRefusal: Equatable { case unknownTable case view(PluginViewReadOnlyReason?) case databaseMoved + case schemaMoved case keysUnresolved case masked @@ -73,6 +74,8 @@ enum ResultEditRefusal: Equatable { return Self.viewMessage(reason) case .databaseMoved: return String(localized: "These rows came from another database than the one this tab is now using.") + case .schemaMoved: + return String(localized: "These rows came from another schema than the one this tab is now using.") case .keysUnresolved: return String(localized: "TablePro has not looked up this table's key columns, so it cannot tell which row an edit would change.") case .masked: @@ -122,7 +125,11 @@ extension PluginViewRowEditing { // Fail-closed: a result with no captured origin refuses rather than borrowing the tab's current table. enum ResultEditability { - static func refusal(for origin: ResultOrigin?, currentDatabaseName: String) -> ResultEditRefusal? { + static func refusal( + for origin: ResultOrigin?, + currentDatabaseName: String, + currentSchemaName: String? + ) -> ResultEditRefusal? { guard let origin else { return .unknownTable } // A query tab learns its relation is a view only from the driver's answer. if origin.isView || origin.viewRowEditing != nil { @@ -134,11 +141,20 @@ enum ResultEditability { return .unknownTable } guard origin.keysResolved else { return .keysUnresolved } - guard origin.databaseName.isEmpty || currentDatabaseName.isEmpty - || origin.databaseName == currentDatabaseName - else { - return .databaseMoved - } + guard !origin.wasRead(elsewhereThanDatabase: currentDatabaseName) else { return .databaseMoved } + // The schema picker moves only the schema, so the database check above cannot see it. + guard origin.schemaName == currentSchemaName else { return .schemaMoved } return nil } } + +extension ResultOrigin { + // An unknown database on either side is not a move. + func wasRead(elsewhereThanDatabase database: String) -> Bool { + !databaseName.isEmpty && !database.isEmpty && databaseName != database + } + + func wasRead(elsewhereThanDatabase database: String, schema: String?) -> Bool { + wasRead(elsewhereThanDatabase: database) || schemaName != schema + } +} diff --git a/apple/TablePro/Models/Query/ResultSet.swift b/apple/TablePro/Models/Query/ResultSet.swift index 96319f8afa..44e62e8543 100644 --- a/apple/TablePro/Models/Query/ResultSet.swift +++ b/apple/TablePro/Models/Query/ResultSet.swift @@ -39,7 +39,7 @@ final class ResultSet: ObservableObject, Identifiable { @Published var isPinned: Bool = false @Published var isTruncated: Bool = false @Published var baseQuery: String? - @Published var baseQueryParameterValues: [String?]? + @Published var baseQueryParameterValues: [TypedParameterValue?]? /// The statement a parameterized run produced these rows from, as written, with the values it bound. `baseQuery` /// and `baseQueryParameterValues` are the same run in the driver's positional form, which Fetch All sends as it diff --git a/apple/TablePro/Models/Query/ResultSetMenuModel.swift b/apple/TablePro/Models/Query/ResultSetMenuModel.swift index 65399f008e..fbe9fe7442 100644 --- a/apple/TablePro/Models/Query/ResultSetMenuModel.swift +++ b/apple/TablePro/Models/Query/ResultSetMenuModel.swift @@ -21,6 +21,21 @@ struct ResultSetMenuModel: Equatable { let entries: [ResultSetMenuEntry] let activeOrdinal: Int let total: Int + // Counted once here: the pull-down names every entry, and counting per entry made a 10,000-statement run quadratic. + private let positions: [UUID: Int] + private let occurrences: [Int?] + + init(entries: [ResultSetMenuEntry], activeOrdinal: Int, total: Int) { + self.entries = entries + self.activeOrdinal = activeOrdinal + self.total = total + positions = Dictionary(entries.enumerated().map { ($1.id, $0) }) { first, _ in first } + occurrences = Self.occurrenceNumbers(in: entries.map(\.label)) + } + + static func == (lhs: ResultSetMenuModel, rhs: ResultSetMenuModel) -> Bool { + lhs.entries == rhs.entries && lhs.activeOrdinal == rhs.activeOrdinal && lhs.total == rhs.total + } // HIG, Tab views: "Avoid providing more than six tabs". Past that the pull-down alone lists them. static let maximumTabCount = 6 @@ -48,28 +63,39 @@ struct ResultSetMenuModel: Equatable { } func displayLabel(of entry: ResultSetMenuEntry) -> String { - guard let index = entries.firstIndex(where: { $0.id == entry.id }) else { return entry.label } - return Self.displayLabel(at: index, in: entries.map(\.label)) + guard let index = positions[entry.id] else { return entry.label } + return Self.counted(entries[index].label, occurrence: occurrences[index]) } // The name is cut before the occurrence is added, so "(2)" always survives. func tabTitle(of entry: ResultSetMenuEntry) -> String { - guard let index = entries.firstIndex(where: { $0.id == entry.id }) else { return entry.label } - let labels = entries.map(\.label) - let name = Self.truncated(entry.label, limit: Self.tabTitleCharacterLimit) - guard let occurrence = Self.occurrence(at: index, in: labels) else { return name } - return String(format: String(localized: "%1$@ (%2$d)"), name, occurrence) + guard let index = positions[entry.id] else { return entry.label } + return Self.counted(Self.truncated(entry.label, limit: Self.tabTitleCharacterLimit), occurrence: occurrences[index]) } // Two results read from one table share a name; the later ones carry their occurrence so two never look alike. static func displayLabel(at index: Int, in labels: [String]) -> String { - guard let occurrence = occurrence(at: index, in: labels) else { return labels[index] } - return String(format: String(localized: "%1$@ (%2$d)"), labels[index], occurrence) + let earlier = labels[.. 0 ? earlier + 1 : nil) } - private static func occurrence(at index: Int, in labels: [String]) -> Int? { - let earlier = labels[.. 0 ? earlier + 1 : nil + // Every name at once, for a caller naming each result of a tab. + static func displayLabels(for labels: [String]) -> [String] { + zip(labels, occurrenceNumbers(in: labels)).map { counted($0, occurrence: $1) } + } + + private static func occurrenceNumbers(in labels: [String]) -> [Int?] { + var seen: [String: Int] = [:] + return labels.map { label in + let earlier = seen[label, default: 0] + seen[label] = earlier + 1 + return earlier > 0 ? earlier + 1 : nil + } + } + + private static func counted(_ name: String, occurrence: Int?) -> String { + guard let occurrence else { return name } + return String(format: String(localized: "%1$@ (%2$d)"), name, occurrence) } // A pinned result is pinned precisely so that nothing takes it away. diff --git a/apple/TablePro/Models/Query/ResultStatusModel.swift b/apple/TablePro/Models/Query/ResultStatusModel.swift index 665075df18..b7efcf26d7 100644 --- a/apple/TablePro/Models/Query/ResultStatusModel.swift +++ b/apple/TablePro/Models/Query/ResultStatusModel.swift @@ -44,6 +44,7 @@ struct ResultStatusControls: Equatable { var showsColumns = false var showsHighlightRules = false var showsFilters = false + var showsAutoRefresh = false var showsPagination = false /// First, Previous, Next, Last and the page number, which an engine that cannot skip rows has /// no use for. The rows-per-page menu stays, because it still sets how many leading rows load. @@ -130,6 +131,8 @@ struct ResultStatusModel: Equatable { controls.showsHighlightRules = viewMode == .data && describesAResult controls.showsSelectionSummary = viewMode == .data && describesAResult controls.showsFilters = viewMode.showsRowFilters && isTable && snapshot.hasTableName + // Data only, the one mode a tick runs in; a query tab's SQL may write, so it never re-runs on a timer. + controls.showsAutoRefresh = viewMode == .data && isTable && snapshot.hasTableName controls.showsPagination = viewMode.showsResultScope && isTable && snapshot.hasTableName controls.showsPageNavigation = controls.showsPagination && snapshot.paginationCapability.allowsSeeking diff --git a/apple/TablePro/Models/Query/ResultStatusPresentation.swift b/apple/TablePro/Models/Query/ResultStatusPresentation.swift index 54c3832824..1b34bb2bf7 100644 --- a/apple/TablePro/Models/Query/ResultStatusPresentation.swift +++ b/apple/TablePro/Models/Query/ResultStatusPresentation.swift @@ -52,6 +52,8 @@ struct ResultStatusPresentation: Equatable { /// ("2/4"). It never leaves the bar: Pin and Close live in its menu and have no other one-click /// route, which is the same reason nothing that opens a popover may leave. let resultSetMenuIsSpelledOut: Bool + // Opens a menu, not a popover, and Database > Auto Refresh carries the same choice. + let showsAutoRefresh: Bool init(tier: StatusBarTier) { switch tier { @@ -61,18 +63,21 @@ struct ResultStatusPresentation: Equatable { showsEdgePageButtons = true pageSizeIsInline = true resultSetMenuIsSpelledOut = true + showsAutoRefresh = true case .compact: showsControlTitles = false modeSwitcherIsSegmented = true showsEdgePageButtons = false pageSizeIsInline = true resultSetMenuIsSpelledOut = false + showsAutoRefresh = true case .narrow: showsControlTitles = false modeSwitcherIsSegmented = false showsEdgePageButtons = false pageSizeIsInline = false resultSetMenuIsSpelledOut = false + showsAutoRefresh = false } } } diff --git a/apple/TablePro/Models/Query/SelectionSummary.swift b/apple/TablePro/Models/Query/SelectionSummary.swift index af5da02cb1..923812aa44 100644 --- a/apple/TablePro/Models/Query/SelectionSummary.swift +++ b/apple/TablePro/Models/Query/SelectionSummary.swift @@ -32,8 +32,8 @@ struct SelectionSummaryInput: Sendable { let dataColumnsByDisplayPosition: [Int] let policy: SelectionSummaryColumnPolicy let deletedRowIDs: Set - let insertedRowIDs: Set - let modifiedCells: [RowID: Set] + // The cells a pending edit set to the column's default, which hold the keyword as text. + let defaultedCells: [RowID: Set] init( selection: GridSelection, @@ -42,8 +42,7 @@ struct SelectionSummaryInput: Sendable { dataColumnsByDisplayPosition: [Int], policy: SelectionSummaryColumnPolicy, deletedRowIDs: Set = [], - insertedRowIDs: Set = [], - modifiedCells: [RowID: Set] = [:] + defaultedCells: [RowID: Set] = [:] ) { self.selection = selection self.tableRows = tableRows @@ -51,8 +50,7 @@ struct SelectionSummaryInput: Sendable { self.dataColumnsByDisplayPosition = dataColumnsByDisplayPosition self.policy = policy self.deletedRowIDs = deletedRowIDs - self.insertedRowIDs = insertedRowIDs - self.modifiedCells = modifiedCells + self.defaultedCells = defaultedCells } } diff --git a/apple/TablePro/Models/Query/ServerObjectInfo.swift b/apple/TablePro/Models/Query/ServerObjectInfo.swift new file mode 100644 index 0000000000..1aa231984e --- /dev/null +++ b/apple/TablePro/Models/Query/ServerObjectInfo.swift @@ -0,0 +1,90 @@ +// +// ServerObjectInfo.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// One object of a kind the driver declares. Equality follows `id`, so a definition read later never +// makes a reload stop matching the row it replaces. +struct ServerObjectInfo: Identifiable, Hashable, Sendable { + let kindId: String + let name: String + let schema: String? + let identity: String? + let detail: String? + let isEnabled: Bool? + let definition: String? + let attributes: [ObjectAttribute] + + init( + kindId: String, + name: String, + schema: String? = nil, + identity: String? = nil, + detail: String? = nil, + isEnabled: Bool? = nil, + definition: String? = nil, + attributes: [ObjectAttribute] = [] + ) { + self.kindId = kindId + self.name = name + self.schema = schema + self.identity = identity + self.detail = detail + self.isEnabled = isEnabled + self.definition = definition + self.attributes = attributes + } + + var id: String { + let path = IdentityPath.qualified(name: name, schema: schema) + guard let identity, !identity.isEmpty else { return "\(kindId)_\(path)" } + return "\(kindId)_\(path)_\(identity)" + } + + static func == (lhs: ServerObjectInfo, rhs: ServerObjectInfo) -> Bool { + lhs.id == rhs.id + } + + func hash(into hasher: inout Hasher) { + hasher.combine(id) + } + + func adoptingSchema(_ fallback: String?) -> ServerObjectInfo { + guard schema?.isEmpty != false, let fallback, !fallback.isEmpty else { return self } + return ServerObjectInfo( + kindId: kindId, name: name, schema: fallback, identity: identity, detail: detail, + isEnabled: isEnabled, definition: definition, attributes: attributes + ) + } +} + +extension ServerObjectInfo { + init(_ object: PluginServerObjectInfo) { + self.init( + kindId: object.kindId, + name: object.name, + schema: object.schema, + identity: object.identity, + detail: object.detail, + isEnabled: object.isEnabled, + definition: object.definition, + attributes: object.attributes.map(ObjectAttribute.init) + ) + } + + var pluginObject: PluginServerObjectInfo { + PluginServerObjectInfo( + kindId: kindId, + name: name, + schema: schema, + identity: identity, + detail: detail, + isEnabled: isEnabled, + definition: definition, + attributes: attributes.map(\.pluginAttribute) + ) + } +} diff --git a/apple/TablePro/Models/Query/TabSessionRegistry.swift b/apple/TablePro/Models/Query/TabSessionRegistry.swift index 6b2429bd89..377a09e64b 100644 --- a/apple/TablePro/Models/Query/TabSessionRegistry.swift +++ b/apple/TablePro/Models/Query/TabSessionRegistry.swift @@ -45,10 +45,13 @@ final class TabSessionRegistry { session.rowSetRevision &+= 1 } + // `mutate` must not read this tab's rows back from the registry: they are out of the session while it runs. @discardableResult func updateTableRows(for tabId: UUID, _ mutate: (inout TableRows) -> Delta) -> Delta { let session = ensureSession(for: tabId) var rows = session.tableRows + // A buffer the session still held would be copied whole by the first write, once per edit. + session.tableRows = TableRows() let delta = mutate(&rows) session.tableRows = rows session.dataRevision &+= 1 diff --git a/apple/TablePro/Models/Query/TableAutoRefreshGate.swift b/apple/TablePro/Models/Query/TableAutoRefreshGate.swift new file mode 100644 index 0000000000..11a638ee1b --- /dev/null +++ b/apple/TablePro/Models/Query/TableAutoRefreshGate.swift @@ -0,0 +1,96 @@ +// +// TableAutoRefreshGate.swift +// TablePro +// + +import Foundation + +enum TableAutoRefreshPause: Equatable, Sendable { + case notVisible + case disconnected + case otherViewMode + case openTransaction + case unsavedChanges + case editing + case interacting + case valueWindowOpen + case loading + case safeModePrompt + case unkeyedSelection + + // Nil where nobody can read the bar's help anyway, or the pause lasts only as long as a gesture. + var help: String? { + switch self { + case .disconnected: + String(localized: "Auto Refresh waits until the connection is back.") + case .openTransaction: + String(localized: "Auto Refresh waits while a transaction is open.") + case .unsavedChanges: + String(localized: "Auto Refresh waits while the table has unsaved changes.") + case .editing: + String(localized: "Auto Refresh waits while a cell is being edited.") + case .valueWindowOpen: + String(localized: "Auto Refresh waits while a value is open in its own window.") + case .loading: + String(localized: "Auto Refresh waits while rows load.") + case .unkeyedSelection: + String(localized: "Auto Refresh waits while rows are selected in a table without a primary key.") + case .notVisible, .otherViewMode, .interacting, .safeModePrompt: + nil + } + } +} + +enum TableAutoRefreshDecision: Equatable, Sendable { + case refresh + case wait(TableAutoRefreshPause) + case stop +} + +struct TableAutoRefreshGate: Equatable, Sendable { + enum Load: Equatable, Sendable { + case idle + case busy + } + + var isTableTab = true + var previousTickFailed = false + var hasLoaded = true + var windowIsVisible = true + var isSelectedInWindow = true + var isConnected = true + var viewMode: ResultsViewMode = .data + var holdsTransaction = false + var holdsUnsavedWork = false + var hasSidebarDataWork = false + var isEditingCell = false + var gridHoldsTransientUI = false + var gridIsTrackingInteraction = false + var inspectorHoldsValueWindow = false + var load: Load = .idle + var isCountPending = false + var isLoadingMore = false + var showsSafeModePrompt = false + var hasSelection = false + var hasPrimaryKey = true + + // Never the app being active: watching a jobs table while working in another app is the point. + var decision: TableAutoRefreshDecision { + guard isTableTab, !previousTickFailed else { return .stop } + guard windowIsVisible, isSelectedInWindow else { return .wait(.notVisible) } + guard isConnected else { return .wait(.disconnected) } + guard viewMode == .data else { return .wait(.otherViewMode) } + guard !holdsTransaction else { return .wait(.openTransaction) } + // Ahead of unsaved work, which counts an open editor too. + guard !isEditingCell else { return .wait(.editing) } + guard !holdsUnsavedWork, !hasSidebarDataWork else { return .wait(.unsavedChanges) } + // A reload closes the grid's viewers and the value filter, and moves rows under a drag or a scroll. + guard !gridHoldsTransientUI, !gridIsTrackingInteraction else { return .wait(.interacting) } + guard !inspectorHoldsValueWindow else { return .wait(.valueWindowOpen) } + guard hasLoaded, load == .idle, !isCountPending, !isLoadingMore else { return .wait(.loading) } + guard !showsSafeModePrompt else { return .wait(.safeModePrompt) } + // Rows are kept across a reload by primary key, so without one the selection would be dropped. + guard !hasSelection || hasPrimaryKey else { return .wait(.unkeyedSelection) } + return .refresh + } +} diff --git a/apple/TablePro/Models/Query/TableAutoRefreshInterval.swift b/apple/TablePro/Models/Query/TableAutoRefreshInterval.swift new file mode 100644 index 0000000000..b41b60ff92 --- /dev/null +++ b/apple/TablePro/Models/Query/TableAutoRefreshInterval.swift @@ -0,0 +1,38 @@ +// +// TableAutoRefreshInterval.swift +// TablePro +// + +import Foundation + +enum TableAutoRefreshInterval: Int, CaseIterable, Identifiable, Sendable { + case off = 0 + case fiveSeconds = 5 + case tenSeconds = 10 + case thirtySeconds = 30 + case oneMinute = 60 + case fiveMinutes = 300 + + var id: Int { rawValue } + + var isOn: Bool { self != .off } + + var duration: Duration { .seconds(rawValue) } + + var title: String { + switch self { + case .off: String(localized: "Off") + case .fiveSeconds: String(localized: "Every 5 Seconds") + case .tenSeconds: String(localized: "Every 10 Seconds") + case .thirtySeconds: String(localized: "Every 30 Seconds") + case .oneMinute: String(localized: "Every Minute") + case .fiveMinutes: String(localized: "Every 5 Minutes") + } + } + + // The status bar's regular tier names the interval beside the glyph, so it has to stay short in every language. + var shortTitle: String { + guard isOn else { return String(localized: "Off") } + return duration.formatted(.units(allowed: [.minutes, .seconds], width: .narrow)) + } +} diff --git a/apple/TablePro/Models/Query/TableLoadTrigger.swift b/apple/TablePro/Models/Query/TableLoadTrigger.swift index 744f343fd2..13067926e3 100644 --- a/apple/TablePro/Models/Query/TableLoadTrigger.swift +++ b/apple/TablePro/Models/Query/TableLoadTrigger.swift @@ -8,6 +8,31 @@ import Foundation internal enum TableLoadTrigger { case userInitiated case restore + case autoRefresh + + // A timer's read at 5 s would write 720 rows an hour per tab. + var recordsHistory: Bool { + switch self { + case .userInitiated, .restore: true + case .autoRefresh: false + } + } + + // At 5 s a tick's trace would push every real load out of the diagnostics store within hours. + var tracesLoad: Bool { + switch self { + case .userInitiated, .restore: true + case .autoRefresh: false + } + } + + // A count slower than the interval would otherwise run on every tick and never land; only a manual Refresh recounts. + var countsRows: Bool { + switch self { + case .userInitiated, .restore: true + case .autoRefresh: false + } + } } // Keyed by tab: a window-wide trigger was consumed against whatever tab was selected when the connection came back. diff --git a/apple/TablePro/Models/Query/TableRows.swift b/apple/TablePro/Models/Query/TableRows.swift index c175fc3c73..09592ca312 100644 --- a/apple/TablePro/Models/Query/TableRows.swift +++ b/apple/TablePro/Models/Query/TableRows.swift @@ -395,29 +395,15 @@ struct TableRows: Sendable { absentCells: [Int: Set] = [:], unloadedCells: [Int: [Int: Int]] = [:] ) -> Delta { - var rebuilt = ContiguousArray() - rebuilt.reserveCapacity(replacementRows.count) - var rebuiltIndex = [RowID: Int]() - rebuiltIndex.reserveCapacity(replacementRows.count) - for (idx, values) in replacementRows.enumerated() { - let normalized = Self.normalize(values: masks.row(values), toCount: columns.count) - let row = Row( - id: .existing(offset + idx), - values: normalized, - absentColumns: Self.clamp(absentCells[idx] ?? [], toCount: columns.count), - unloadedColumns: Self.clamp(unloadedCells[idx] ?? [:], toCount: columns.count), - exportSnapshot: Self.exportSnapshot( - documents: exportDocuments, index: idx, count: replacementRows.count, - columns: columns, values: normalized, rawValues: values, absent: absentCells[idx] ?? [], masks: masks, - policy: exportPolicy, table: exportTable - ) - ) - rebuilt.append(row) - rebuiltIndex[row.id] = idx - } - rows = rebuilt - indexByID = rebuiltIndex - rowLocators = Self.locators(replacementLocators, rowCount: replacementRows.count, offset: offset) + adopt(Self.prepare( + replacementRows, + offset: offset, + rowLocators: replacementLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: rowShape + )) return .fullReplace } @@ -510,25 +496,50 @@ struct TableRows: Sendable { exportTable: String? = nil, columnOrigins: [PluginColumnOrigin?]? = nil ) -> TableRows { - var rows = ContiguousArray() - rows.reserveCapacity(queryRows.count) - for (index, values) in queryRows.enumerated() { - let normalized = normalize(values: masks.row(values), toCount: columns.count) - rows.append(Row( - id: .existing(index), - values: normalized, - absentColumns: clamp(absentCells[index] ?? [], toCount: columns.count), - unloadedColumns: clamp(unloadedCells[index] ?? [:], toCount: columns.count), - exportSnapshot: exportSnapshot( - documents: exportDocuments, index: index, count: queryRows.count, - columns: columns, values: normalized, rawValues: values, absent: absentCells[index] ?? [], masks: masks, - policy: exportPolicy, table: exportTable - ) - )) - } + from( + prepared: prepare( + queryRows, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: RowShape(columns: columns, masks: masks, exportPolicy: exportPolicy, exportTable: exportTable) + ), + columnTypes: columnTypes, + columnDefaults: columnDefaults, + columnForeignKeys: columnForeignKeys, + columnEnumValues: columnEnumValues, + columnNullable: columnNullable, + columnComments: columnComments, + columnIdentity: columnIdentity, + columnDeclaredTypes: columnDeclaredTypes, + generatedColumns: generatedColumns, + rowMatchPolicy: rowMatchPolicy, + hasAuthoritativeSchema: hasAuthoritativeSchema, + foreignKeysFetched: foreignKeysFetched, + columnOrigins: columnOrigins + ) + } + + static func from( + prepared: PreparedRows, + columnTypes: [ColumnType], + columnDefaults: [String: String?] = [:], + columnForeignKeys: [String: ForeignKeyInfo] = [:], + columnEnumValues: [String: [String]] = [:], + columnNullable: [String: Bool] = [:], + columnComments: [String: String] = [:], + columnIdentity: [String: IdentityKind] = [:], + columnDeclaredTypes: [String: String] = [:], + generatedColumns: Set = [], + rowMatchPolicy: RowMatchPolicy = .none, + hasAuthoritativeSchema: Bool = false, + foreignKeysFetched: Bool = false, + columnOrigins: [PluginColumnOrigin?]? = nil + ) -> TableRows { + let shape = prepared.shape var tableRows = TableRows( - rows: rows, - columns: columns, + columns: shape.columns, columnTypes: columnTypes, columnDefaults: columnDefaults, columnForeignKeys: columnForeignKeys, @@ -542,14 +553,125 @@ struct TableRows: Sendable { hasAuthoritativeSchema: hasAuthoritativeSchema, foreignKeysFetched: foreignKeysFetched ) - tableRows.rowLocators = locators(rowLocators, rowCount: queryRows.count, offset: 0) - tableRows.masks = masks - tableRows.exportPolicy = exportPolicy - tableRows.exportTable = exportTable - tableRows.reportedOrigins = columnOrigins.map { (columns: columns, origins: $0) } + tableRows.adopt(prepared) + tableRows.masks = shape.masks + tableRows.exportPolicy = shape.exportPolicy + tableRows.exportTable = shape.exportTable + tableRows.reportedOrigins = columnOrigins.map { (columns: shape.columns, origins: $0) } return tableRows } + var rowShape: RowShape { + RowShape(columns: columns, masks: masks, exportPolicy: exportPolicy, exportTable: exportTable) + } + + // Nil when the rows were built for other columns or masks, as after a masking rule saved while they were built. + mutating func install(_ prepared: PreparedRows) -> Delta? { + guard prepared.fits(rowShape) else { return nil } + adopt(prepared) + return .fullReplace + } + + private mutating func adopt(_ prepared: PreparedRows) { + rows = prepared.rows + indexByID = prepared.indexByID + rowLocators = prepared.rowLocators + } + + static let cancellationCheckInterval = 4_096 + + // A Fetch All or a first page past the row cap is millions of rows, too many to build on the main actor. + // A Stop ends the build within one interval of rows; the result is then incomplete and never installs. + @concurrent + static func preparing( + _ queryRows: Source, + offset: Int = 0, + rowLocators: [String?]? = nil, + exportDocuments: [PluginExportDocument]? = nil, + absentCells: [Int: Set] = [:], + unloadedCells: [Int: [Int: Int]] = [:], + shape: RowShape + ) async -> PreparedRows where Source.Element == [PluginCellValue] { + build( + queryRows, + offset: offset, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: shape, + stopsOnCancel: true + ) + } + + // Never stops early: it is also the fallback that has to land every row, inside a task a Stop may have cancelled. + static func prepare( + _ queryRows: Source, + offset: Int = 0, + rowLocators: [String?]? = nil, + exportDocuments: [PluginExportDocument]? = nil, + absentCells: [Int: Set] = [:], + unloadedCells: [Int: [Int: Int]] = [:], + shape: RowShape + ) -> PreparedRows where Source.Element == [PluginCellValue] { + build( + queryRows, + offset: offset, + rowLocators: rowLocators, + exportDocuments: exportDocuments, + absentCells: absentCells, + unloadedCells: unloadedCells, + shape: shape, + stopsOnCancel: false + ) + } + + private static func build( + _ queryRows: Source, + offset: Int, + rowLocators: [String?]?, + exportDocuments: [PluginExportDocument]?, + absentCells: [Int: Set], + unloadedCells: [Int: [Int: Int]], + shape: RowShape, + stopsOnCancel: Bool + ) -> PreparedRows where Source.Element == [PluginCellValue] { + let count = queryRows.count + let columnCount = shape.columns.count + var rows = ContiguousArray() + rows.reserveCapacity(count) + var index = [RowID: Int]() + index.reserveCapacity(count) + for (position, rowIndex) in zip(0..., queryRows.indices) { + if stopsOnCancel, position.isMultiple(of: cancellationCheckInterval), Task.isCancelled { + return PreparedRows(shape: shape, rows: [], indexByID: [:], rowLocators: [:], isComplete: false) + } + let values = queryRows[rowIndex] + let normalized = normalize(values: shape.masks.row(values), toCount: columnCount) + let absent = absentCells[position] ?? [] + let row = Row( + id: .existing(offset + position), + values: normalized, + absentColumns: clamp(absent, toCount: columnCount), + unloadedColumns: clamp(unloadedCells[position] ?? [:], toCount: columnCount), + exportSnapshot: exportSnapshot( + documents: exportDocuments, index: position, count: count, + columns: shape.columns, values: normalized, rawValues: values, absent: absent, masks: shape.masks, + policy: shape.exportPolicy, table: shape.exportTable + ) + ) + rows.append(row) + index[row.id] = position + } + return PreparedRows( + shape: shape, + rows: rows, + indexByID: index, + rowLocators: locators(rowLocators, rowCount: count, offset: offset), + isComplete: true + ) + } + private static func exportSnapshot( documents: [PluginExportDocument]?, index: Int, count: Int, columns: [String], values: ContiguousArray, rawValues: [PluginCellValue], @@ -646,3 +768,27 @@ extension TableRows { return columnEnumValues.filter { !setColumns.contains($0.key) } } } + +extension TableRows { + // What building a row reads from the rows it joins: the masks, and the column names export documents match. + struct RowShape: Equatable, Sendable { + let columns: [String] + let masks: ColumnMasks + let exportPolicy: MaskingPolicy + let exportTable: String? + } + + // Rows built from one result with their index, which install by assignment while the shape still holds. + struct PreparedRows: Sendable { + let shape: RowShape + let rows: ContiguousArray + fileprivate let indexByID: [RowID: Int] + fileprivate let rowLocators: [RowID: String] + // False when a Stop ended the build early: the rows are not the result. + let isComplete: Bool + + func fits(_ target: RowShape) -> Bool { + isComplete && shape == target + } + } +} diff --git a/apple/TablePro/Models/Schema/ClickHousePartStatements.swift b/apple/TablePro/Models/Schema/ClickHousePartStatements.swift index 5436cfced7..c11617c1c6 100644 --- a/apple/TablePro/Models/Schema/ClickHousePartStatements.swift +++ b/apple/TablePro/Models/Schema/ClickHousePartStatements.swift @@ -2,15 +2,10 @@ // ClickHousePartStatements.swift // TablePro // -// The statements the Parts tab issues, built from the tab's own container. -// import Foundation -/// ClickHouse carries the database as a request parameter rather than in the session, and the app -/// moves that parameter whenever any tab runs somewhere else. An unqualified name therefore names -/// whichever database the connection was last pinned to, which for `DROP PARTITION` is data loss in -/// a database the user was not looking at. +// ClickHouse takes the database from a request parameter any tab can move, so every statement names its own. internal enum ClickHousePartStatements { internal static func qualifiedName( database: String, @@ -32,42 +27,49 @@ internal enum ClickHousePartStatements { internal static func dropPartition( database: String, table: String, - partition: String, - quote: (String) -> String, - escape: (String) -> String - ) -> String { + partitionId: String, + quote: (String) -> String + ) -> String? { + guard isValidPartitionId(partitionId) else { return nil } let name = qualifiedName(database: database, table: table, quote: quote) - return "ALTER TABLE \(name) DROP PARTITION '\(escape(partition))'" + return "ALTER TABLE \(name) DROP PARTITION ID '\(partitionId)'" } internal static func detachPartition( database: String, table: String, - partition: String, - quote: (String) -> String, - escape: (String) -> String - ) -> String { + partitionId: String, + quote: (String) -> String + ) -> String? { + guard isValidPartitionId(partitionId) else { return nil } let name = qualifiedName(database: database, table: table, quote: quote) - return "ALTER TABLE \(name) DETACH PARTITION '\(escape(partition))'" + return "ALTER TABLE \(name) DETACH PARTITION ID '\(partitionId)'" + } + + // The server's own rule (MergeTreePartInfo::validatePartitionID); it also keeps quotes out of the literal. + internal static func isValidPartitionId(_ id: String) -> Bool { + !id.isEmpty && id.unicodeScalars.allSatisfy { scalar in + switch scalar { + case "0"..."9", "a"..."z", "A"..."Z", "-": true + default: false + } + } } - /// A connection saved with no database of its own has none to name, and ClickHouse then resolves - /// an unqualified request against its own configured default, which `currentDatabase()` reports. - /// Comparing against an empty string instead matches nothing and empties the tab. + // With no database of its own, ClickHouse resolves a request against its configured default. private static func databasePredicate(_ database: String, escape: (String) -> String) -> String { guard !database.isEmpty else { return "currentDatabase()" } return "'\(escape(database))'" } - /// Filtered on the named database rather than `currentDatabase()`, which answers with the - /// request parameter and so listed another database's parts under this table's name. + // `partition` renders a tuple key as `(202401,'eu')` and no key as `tuple()`, so statements name `partition_id`. internal static func parts( database: String, table: String, escape: (String) -> String ) -> String { """ - SELECT partition, name, rows, bytes_on_disk, + SELECT partition, partition_id, name, rows, bytes_on_disk, toString(modification_time) AS mod_time, active FROM system.parts WHERE database = \(databasePredicate(database, escape: escape)) AND table = '\(escape(table))' diff --git a/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift b/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift index d21d7ca284..58ab041028 100644 --- a/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift +++ b/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift @@ -28,14 +28,14 @@ enum DashboardRefreshInterval: Double, CaseIterable, Identifiable { var displayLabel: String { switch self { case .off: return String(localized: "Off") - default: return "\(Int(rawValue))s" + default: return Duration.seconds(rawValue).formatted(.units(allowed: [.seconds], width: .narrow)) } } } // MARK: - Dashboard Session -struct DashboardSession: Identifiable { +struct DashboardSession: Identifiable, Equatable { let id: String let user: String let database: String @@ -49,7 +49,7 @@ struct DashboardSession: Identifiable { // MARK: - Dashboard Metric -struct DashboardMetric: Identifiable { +struct DashboardMetric: Identifiable, Equatable { let id: String let label: String let value: String @@ -65,4 +65,8 @@ struct DashboardSlowQuery: Identifiable { let query: String let user: String let database: String + + func hasSameContent(as other: DashboardSlowQuery) -> Bool { + duration == other.duration && query == other.query && user == other.user && database == other.database + } } diff --git a/apple/TablePro/Models/Settings/AppSettings.swift b/apple/TablePro/Models/Settings/AppSettings.swift index a18e4aeb1b..9d4d1cf751 100644 --- a/apple/TablePro/Models/Settings/AppSettings.swift +++ b/apple/TablePro/Models/Settings/AppSettings.swift @@ -51,7 +51,6 @@ struct AppearanceSettings: Codable, Equatable { // MARK: - Data Grid Settings -/// Row height options for data grid enum DataGridRowHeight: Int, Codable, CaseIterable, Identifiable { case compact = 20 case normal = 24 @@ -109,7 +108,6 @@ enum RowCopyFormat: String, Codable, CaseIterable, Identifiable { } } -/// Data grid settings struct DataGridSettings: Codable, Equatable { var rowHeight: DataGridRowHeight var dateFormat: DateFormatOption @@ -126,6 +124,8 @@ struct DataGridSettings: Codable, Equatable { var defaultSortDirection: SortDirection var rowCopyFormat: RowCopyFormat var showInvisibleCharacters: Bool + // 0 keeps every result a run returns. + var resultSetsKeptPerRun: Int static let `default` = DataGridSettings( rowHeight: .normal, @@ -142,7 +142,8 @@ struct DataGridSettings: Codable, Equatable { defaultSortBehavior: .none, defaultSortDirection: .ascending, rowCopyFormat: .rows, - showInvisibleCharacters: true + showInvisibleCharacters: true, + resultSetsKeptPerRun: 0 ) init( @@ -160,7 +161,8 @@ struct DataGridSettings: Codable, Equatable { defaultSortBehavior: DefaultSortBehavior = .none, defaultSortDirection: SortDirection = .ascending, rowCopyFormat: RowCopyFormat = .rows, - showInvisibleCharacters: Bool = true + showInvisibleCharacters: Bool = true, + resultSetsKeptPerRun: Int = 0 ) { self.rowHeight = rowHeight self.dateFormat = dateFormat @@ -177,6 +179,7 @@ struct DataGridSettings: Codable, Equatable { self.defaultSortDirection = defaultSortDirection self.rowCopyFormat = rowCopyFormat self.showInvisibleCharacters = showInvisibleCharacters + self.resultSetsKeptPerRun = resultSetsKeptPerRun } init(from decoder: Decoder) throws { @@ -204,29 +207,27 @@ struct DataGridSettings: Codable, Equatable { showInvisibleCharacters = container.decodeSetting( .showInvisibleCharacters, default: fallback.showInvisibleCharacters ) + resultSetsKeptPerRun = container.decodeSetting(.resultSetsKeptPerRun, default: fallback.resultSetsKeptPerRun) } // MARK: - Validated Properties - /// Validated and sanitized nullDisplay (max 20 chars, no newlines) var validatedNullDisplay: String { let sanitized = nullDisplay.sanitized let maxLength = SettingsValidationRules.nullDisplayMaxLength if sanitized.isEmpty { - return "NULL" // Fallback to default + return "NULL" } else if sanitized.count > maxLength { return String(sanitized.prefix(maxLength)) } return sanitized } - /// Validated defaultPageSize (10 to 100,000) var validatedDefaultPageSize: Int { defaultPageSize.clamped(to: SettingsValidationRules.defaultPageSizeRange) } - /// Validation error for nullDisplay (for UI feedback) var nullDisplayValidationError: String? { let sanitized = nullDisplay.sanitized let maxLength = SettingsValidationRules.nullDisplayMaxLength @@ -241,7 +242,6 @@ struct DataGridSettings: Codable, Equatable { return nil } - /// Validation error for defaultPageSize (for UI feedback) var defaultPageSizeValidationError: String? { let range = SettingsValidationRules.defaultPageSizeRange if defaultPageSize < range.lowerBound || defaultPageSize > range.upperBound { @@ -250,13 +250,16 @@ struct DataGridSettings: Codable, Equatable { return nil } - /// Validated queryResultRowCap (100 to 500,000; 0 means unlimited) + // Nil keeps every result. + var resultSetLimitPerRun: Int? { + resultSetsKeptPerRun > 0 ? resultSetsKeptPerRun : nil + } + var validatedQueryResultRowCap: Int { if queryResultRowCap == 0 { return 0 } return queryResultRowCap.clamped(to: SettingsValidationRules.queryResultRowCapRange) } - /// Validation error for queryResultRowCap (for UI feedback) var queryResultRowCapValidationError: String? { let range = SettingsValidationRules.queryResultRowCapRange if queryResultRowCap != 0 && (queryResultRowCap < range.lowerBound || queryResultRowCap > range.upperBound) { @@ -270,18 +273,31 @@ struct DataGridSettings: Codable, Equatable { } } +// MARK: - Value Color Settings + +struct ValueColorSettings: Codable, Equatable { + var colorsValuesByType: Bool + + static let `default` = ValueColorSettings() + + init(colorsValuesByType: Bool = false) { + self.colorsValuesByType = colorsValuesByType + } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + colorsValuesByType = container.decodeSetting(.colorsValuesByType, default: Self.default.colorsValuesByType) + } +} + // MARK: - History Settings -/// History settings struct HistorySettings: Codable, Equatable { var maxEntries: Int // 0 = unlimited var maxDays: Int // 0 = unlimited var autoCleanup: Bool - /// Keep what rows looked like before each save, so a committed save can be restored. - /// - /// This is the only setting in the app that governs storing real row values on disk, which is - /// why it is a switch rather than an assumption. + // The only setting that governs storing real row values on disk, so it is a switch rather than an assumption. var keepRewindHistory: Bool static let `default` = HistorySettings( @@ -308,17 +324,14 @@ struct HistorySettings: Codable, Equatable { // MARK: - Validated Properties - /// Validated maxEntries (>= 0) var validatedMaxEntries: Int { max(0, maxEntries) } - /// Validated maxDays (>= 0) var validatedMaxDays: Int { max(0, maxDays) } - /// Validation error for maxEntries var maxEntriesValidationError: String? { if maxEntries < 0 { return String(localized: "Maximum entries cannot be negative") @@ -326,7 +339,6 @@ struct HistorySettings: Codable, Equatable { return nil } - /// Validation error for maxDays var maxDaysValidationError: String? { if maxDays < 0 { return String(localized: "Maximum days cannot be negative") @@ -337,11 +349,9 @@ struct HistorySettings: Codable, Equatable { // MARK: - Tab Settings -/// Tab behavior settings struct TabSettings: Codable, Equatable { var enablePreviewTabs: Bool = true - /// What the strip does once the tabs stop fitting. `scroll` is the system's answer and the - /// default: every tab bar Apple ships keeps one row and scrolls it. + // `scroll` is the system's answer: every tab bar Apple ships keeps one row and scrolls it. var overflow: EditorTabStripOverflow = .scroll static let `default` = TabSettings() diff --git a/apple/TablePro/Models/Settings/AppSettingsCategory.swift b/apple/TablePro/Models/Settings/AppSettingsCategory.swift index eaab17b259..89aa8b3a76 100644 --- a/apple/TablePro/Models/Settings/AppSettingsCategory.swift +++ b/apple/TablePro/Models/Settings/AppSettingsCategory.swift @@ -20,6 +20,8 @@ internal enum AppSettingsCategory { internal static let notifications = "notifications" // Not a field of `editor`: an older build re-uploads that blob without it, which would reset it here. internal static let formatter = "formatter" + // Not a field of `dataGrid`, for the same reason. + internal static let valueColors = "valueColors" internal static let appearanceMode = "appearanceMode" // `sync` holds the switch that turns syncing on, `mcp` a port and server setup for one machine, @@ -27,6 +29,6 @@ internal enum AppSettingsCategory { internal static let deviceLocal: Set = ["sync", "mcp", appearanceMode] internal static let synced: [String] = [ - general, appearance, editor, dataGrid, history, tabs, keyboard, ai, notifications, formatter + general, appearance, editor, dataGrid, history, tabs, keyboard, ai, notifications, formatter, valueColors ] } diff --git a/apple/TablePro/Models/Settings/MCPSettings.swift b/apple/TablePro/Models/Settings/MCPSettings.swift index a6d7eca727..2738d6364e 100644 --- a/apple/TablePro/Models/Settings/MCPSettings.swift +++ b/apple/TablePro/Models/Settings/MCPSettings.swift @@ -10,6 +10,9 @@ struct MCPSettings: Codable, Equatable { var connectionApproval: MCPConnectionApproval var allowsHiddenConnectionListing: Bool + // Linux takes the same range, so one client setup works against either app. + static let portRange: ClosedRange = 1_024...65_535 + static let `default` = MCPSettings( enabled: false, port: 23_508, @@ -46,7 +49,7 @@ struct MCPSettings: Codable, Equatable { let fallback = Self.default enabled = container.decodeSetting(.enabled, default: fallback.enabled) let rawPort = container.decodeSetting(.port, default: fallback.port) - port = (1...65_535).contains(rawPort) ? rawPort : fallback.port + port = Self.portRange.contains(rawPort) ? rawPort : fallback.port defaultRowLimit = container.decodeSetting(.defaultRowLimit, default: fallback.defaultRowLimit) maxRowLimit = container.decodeSetting(.maxRowLimit, default: fallback.maxRowLimit) queryTimeoutSeconds = container.decodeSetting(.queryTimeoutSeconds, default: fallback.queryTimeoutSeconds) @@ -61,6 +64,10 @@ struct MCPSettings: Codable, Equatable { queryTimeoutSeconds = validatedQueryTimeoutSeconds } + var validatedPort: Int { + port.clamped(to: Self.portRange) + } + var validatedMaxRowLimit: Int { maxRowLimit.clamped(to: SettingsValidationRules.mcpRowLimitRange) } diff --git a/apple/TablePro/Models/Settings/SyncSettings.swift b/apple/TablePro/Models/Settings/SyncSettings.swift index bf8f97395b..55954bf595 100644 --- a/apple/TablePro/Models/Settings/SyncSettings.swift +++ b/apple/TablePro/Models/Settings/SyncSettings.swift @@ -77,6 +77,11 @@ struct SyncSettings: Codable, Equatable { syncSQLFavorites: true, syncTableFolders: true ) + + // Whether a secret saved now goes to iCloud Keychain. + var syncsPasswords: Bool { + enabled && syncConnections && syncPasswords + } } internal extension SyncSettings { diff --git a/apple/TablePro/Models/Sidebar/SidebarObjectKind.swift b/apple/TablePro/Models/Sidebar/SidebarObjectKind.swift index 3ce26cf891..56b5962f37 100644 --- a/apple/TablePro/Models/Sidebar/SidebarObjectKind.swift +++ b/apple/TablePro/Models/Sidebar/SidebarObjectKind.swift @@ -6,16 +6,16 @@ struct DatabaseTreeObjectGroup: Hashable, Sendable { let kind: SidebarObjectKind } -/// Which model a kind's rows are drawn from. Every helper that used to ask `isRoutine` and pick -/// one of two buckets asks this instead, so a kind added later cannot silently read the wrong one. +// Which model a kind's rows are drawn from, so a kind added later cannot read the wrong bucket. enum SidebarObjectCategory: Sendable, Hashable { case table case routine case trigger case type + case server } -enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { +enum SidebarObjectKind: Sendable, Hashable { case table case view case materializedView @@ -25,6 +25,50 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { case function case trigger case type + case server(SidebarServerObjectKind) + + static let builtIn: [SidebarObjectKind] = [ + .table, .view, .materializedView, .foreignTable, .sequence, .procedure, .function, .trigger, .type, + ] + + static func ordered(declared: [SidebarServerObjectKind]) -> [SidebarObjectKind] { + builtIn + declared.map { SidebarObjectKind.server($0) } + } + + private static let serverPersistencePrefix = "server." + + // Built-in spellings are shipped in UserDefaults keys, the tree expansion JSON and node ids. + var persistenceName: String { + switch self { + case .table: return "table" + case .view: return "view" + case .materializedView: return "materializedView" + case .foreignTable: return "foreignTable" + case .sequence: return "sequence" + case .procedure: return "procedure" + case .function: return "function" + case .trigger: return "trigger" + case .type: return "type" + case .server(let kind): return Self.serverPersistencePrefix + kind.id + } + } + + // A declared kind decodes before any driver exists, so it carries only its id. + init?(persistenceName: String) { + if let builtIn = Self.builtIn.first(where: { $0.persistenceName == persistenceName }) { + self = builtIn + return + } + guard persistenceName.hasPrefix(Self.serverPersistencePrefix) else { return nil } + let id = String(persistenceName.dropFirst(Self.serverPersistencePrefix.count)) + guard !id.isEmpty else { return nil } + self = .server(SidebarServerObjectKind(id: id)) + } + + var serverKind: SidebarServerObjectKind? { + if case .server(let kind) = self { return kind } + return nil + } var displayName: String { switch self { @@ -37,6 +81,7 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { case .trigger: return String(localized: "Trigger") case .type: return String(localized: "Type") case .sequence: return String(localized: "Sequence") + case .server(let kind): return kind.displayName } } @@ -51,6 +96,7 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { case .trigger: return String(localized: "Triggers") case .type: return String(localized: "Types") case .sequence: return String(localized: "Sequences") + case .server(let kind): return kind.pluralDisplayName } } @@ -65,6 +111,7 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { case .trigger: return String(localized: "No triggers") case .type: return String(localized: "No types") case .sequence: return String(localized: "No sequences") + case .server: return String(localized: "No items") } } @@ -87,6 +134,7 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { case .trigger: return "bolt" case .type: return "cube" case .sequence: return "number" + case .server(let kind): return kind.iconName } } @@ -96,6 +144,7 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { case .procedure, .function: return .routine case .trigger: return .trigger case .type: return .type + case .server: return .server } } @@ -113,22 +162,13 @@ enum SidebarObjectKind: String, CaseIterable, Sendable, Hashable { self == .table } - /// Which kinds a container lists, in declaration order: the kinds holding objects, and the kinds in - /// `keeping`. Both sidebar layouts ask the same question, so toggling the layout never changes which - /// kinds are on screen. - /// - /// A kind is listed for what its driver returned, never for what the plugin declares. Gating on a - /// capability flag hid objects that came back, and listing every declared kind filled the sidebar - /// with Procedures, Triggers and Types sections that held nothing. - /// - /// `keeping` is for a section with something to say and no rows to say it with: a table kind whose - /// objects all sit in folders, which is where a drag takes one back out, and a kind whose fetch - /// failed, which shows the error where its rows would be. + // Listed for its rows or for `keeping` (filed into folders, or a failed fetch), never for being declared. static func visible( itemCounts: [SidebarObjectKind: Int], - keeping: Set = [] + keeping: Set = [], + declared: [SidebarServerObjectKind] = [] ) -> [SidebarObjectKind] { - allCases.filter { kind in + ordered(declared: declared).filter { kind in itemCounts[kind, default: 0] > 0 || keeping.contains(kind) } } diff --git a/apple/TablePro/Models/Sidebar/SidebarServerObjectKind.swift b/apple/TablePro/Models/Sidebar/SidebarServerObjectKind.swift new file mode 100644 index 0000000000..ebf242fbff --- /dev/null +++ b/apple/TablePro/Models/Sidebar/SidebarServerObjectKind.swift @@ -0,0 +1,83 @@ +// +// SidebarServerObjectKind.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// An object kind a driver declares, such as MySQL events. The app reads it as an opaque descriptor and +// compares it by id alone, so a kind decoded from storage with no driver equals the declared one. +struct SidebarServerObjectKind: Sendable, Hashable { + enum Scope: String, Sendable, Hashable, Codable { + case container + case database + } + + let id: String + let displayName: String + let pluralDisplayName: String + let iconName: String + let scope: Scope + + init(id: String, displayName: String, pluralDisplayName: String, iconName: String, scope: Scope) { + self.id = id + self.displayName = displayName + self.pluralDisplayName = pluralDisplayName + self.iconName = iconName + self.scope = scope + } + + init(id: String) { + self.init(id: id, displayName: id, pluralDisplayName: id, iconName: "shippingbox", scope: .container) + } + + init(_ kind: PluginServerObjectKind) { + let scope: Scope + switch kind.scope { + case .container: scope = .container + case .database: scope = .database + @unknown default: scope = .container + } + self.init( + id: kind.id, + displayName: kind.displayName, + pluralDisplayName: kind.pluralDisplayName, + iconName: kind.iconName, + scope: scope + ) + } + + static func == (lhs: SidebarServerObjectKind, rhs: SidebarServerObjectKind) -> Bool { + lhs.id == rhs.id + } + + func hash(into hasher: inout Hasher) { + hasher.combine(id) + } + + // Menu validation and every redraw ask, and a driver answers the same for its whole life. + @MainActor private static let probed = NSMapTable.weakToStrongObjects() + + @MainActor + static func declared(by driver: DatabaseDriver?) -> [SidebarServerObjectKind] { + guard let driver else { return [] } + if let known = probed.object(forKey: driver) { + return known.kinds + } + var seen = Set() + let kinds = driver.serverObjectKinds + .map(SidebarServerObjectKind.init) + .filter { !$0.id.isEmpty && seen.insert($0.id).inserted } + probed.setObject(DeclaredKinds(kinds), forKey: driver) + return kinds + } + + private final class DeclaredKinds { + let kinds: [SidebarServerObjectKind] + + init(_ kinds: [SidebarServerObjectKind]) { + self.kinds = kinds + } + } +} diff --git a/apple/TablePro/Models/UI/DataGridConfiguration.swift b/apple/TablePro/Models/UI/DataGridConfiguration.swift index 3f05e7c2b8..7e2cfd4dc4 100644 --- a/apple/TablePro/Models/UI/DataGridConfiguration.swift +++ b/apple/TablePro/Models/UI/DataGridConfiguration.swift @@ -24,6 +24,11 @@ struct DataGridConfiguration: Equatable { // A result's rows, not a list of columns: the Data Grid sort preferences apply. var appliesRowSortPreferences: Bool = false + var colorsValuesByType: Bool = false + + // The theme editor shows a theme's value colors even while Color values by type is off. + var overridesValueColorSetting: Bool = false + // Off on the Structure grids, whose Copy writes definitions a paste reads back. var appliesRowCopyFormat: Bool = false diff --git a/apple/TablePro/Models/UI/JsonFieldEditingModel.swift b/apple/TablePro/Models/UI/JsonFieldEditingModel.swift index 8b86f24408..bbf7ffe86f 100644 --- a/apple/TablePro/Models/UI/JsonFieldEditingModel.swift +++ b/apple/TablePro/Models/UI/JsonFieldEditingModel.swift @@ -13,9 +13,9 @@ import Foundation /// /// The rule is ``lastSynced``: the editor's text and the stored value are known to agree at that /// string, so a value arriving from the store that still means the same thing is this editor's own -/// write coming back and is ignored. What the field stages is a compacted form of what the editor -/// shows, so the two agree in meaning rather than in bytes, and a comparison against the editor's -/// displayed text cannot tell an echo from an external change on its own. +/// write coming back and is ignored. A value stored on one line is staged compacted, so the two agree +/// in meaning rather than in bytes, and a comparison against the editor's displayed text cannot tell +/// an echo from an external change on its own. /// /// Adopting an echo is what let a keystroke be overwritten by the value it had just replaced, and /// then pushed back, forever: one keystroke produced 6,944 body evaluations in 30 seconds at 100% diff --git a/apple/TablePro/Models/UI/KeyboardShortcutModels.swift b/apple/TablePro/Models/UI/KeyboardShortcutModels.swift index 81f7d56b94..ee2a1a2244 100644 --- a/apple/TablePro/Models/UI/KeyboardShortcutModels.swift +++ b/apple/TablePro/Models/UI/KeyboardShortcutModels.swift @@ -68,6 +68,8 @@ enum ShortcutAction: String, Codable, CaseIterable, Identifiable { case moveTabToConnection case showServerDashboard case showQueryInsights + case commitTransaction + case rollbackTransaction // Editor & Query case openFile @@ -197,7 +199,7 @@ enum ShortcutAction: String, Codable, CaseIterable, Identifiable { var category: ShortcutCategory { switch self { case .manageConnections, .newConnection, .openDatabase, .switchConnection, .moveTabToConnection, - .showServerDashboard, .showQueryInsights: + .showServerDashboard, .showQueryInsights, .commitTransaction, .rollbackTransaction: return .connections case .openFile, .saveChanges, .saveAs, .copyPath, .copyRelativePath, .executeQuery, .executeAllStatements, .executeQueryWithoutLimit, .cancelQuery, .explainQuery, .formatQuery, @@ -307,6 +309,8 @@ enum ShortcutAction: String, Codable, CaseIterable, Identifiable { case .moveTabToConnection: return String(localized: "Move Tab to Connection") case .showServerDashboard: return String(localized: "Server Dashboard") case .showQueryInsights: return String(localized: "Query Insights") + case .commitTransaction: return String(localized: "Commit Transaction") + case .rollbackTransaction: return String(localized: "Roll Back Transaction") case .saveChanges: return String(localized: "Save Changes") case .saveAs: return String(localized: "Save As") case .copyPath: return String(localized: "Copy Path") @@ -660,7 +664,8 @@ struct KeyboardSettings: Codable, Equatable { // MARK: - Default Shortcuts - // An action absent here shows as unassigned until the user binds it. + // An action absent here shows as unassigned until the user binds it. Commit and Roll Back Transaction ship + // unbound: they are infrequent, and a stray Roll Back discards work. static let defaultShortcuts: [ShortcutAction: BoundKey] = [ // Connections .newConnection: .character("n", command: true), diff --git a/apple/TablePro/Models/UI/MultiRowEditState.swift b/apple/TablePro/Models/UI/MultiRowEditState.swift index 26c98ce2f2..2842465463 100644 --- a/apple/TablePro/Models/UI/MultiRowEditState.swift +++ b/apple/TablePro/Models/UI/MultiRowEditState.swift @@ -7,8 +7,19 @@ import Combine import Foundation import TableProPluginKit +// A column, whatever row is selected. +struct FieldEditID: Hashable { + let columnIndex: Int + let columnName: String +} + struct FieldEditState: Identifiable { - var id = UUID() + // Per column, so a new selection updates each field of the inspector's list in place. + var id: FieldEditID { FieldEditID(columnIndex: columnIndex, columnName: columnName) } + + // Reissued with each selection and each value read, so an editor seeded from the old value starts over. + var editorID = UUID() + let columnIndex: Int let columnName: String let columnTypeEnum: ColumnType @@ -70,7 +81,7 @@ struct FieldEditState: Identifiable { var effectiveValue: String? { if isPendingDefault { - return "__DEFAULT__" + return PluginCellValue.columnDefaultKeyword } else if isPendingNull { return nil } else { @@ -127,7 +138,7 @@ final class MultiRowEditState: ObservableObject { @Published private(set) var expressionCells: [Set] = [] @Published private(set) var columns: [String] = [] @Published private(set) var columnTypes: [ColumnType] = [] - // By column, not field id: every selection change reissues the ids. Cleared when the columns change. + // By column, not editor id: every selection change reissues those. Cleared when the columns change. @Published private(set) var expandedColumn: Int? private var nonNullableColumns: Set = [] @@ -214,7 +225,7 @@ final class MultiRowEditState: ObservableObject { originalValue = values.first.flatMap { $0 } } - var preservedId: UUID? + var preservedEditorID: UUID? var pendingValue: String? var isPendingNull = false var isPendingDefault = false @@ -226,7 +237,7 @@ final class MultiRowEditState: ObservableObject { let oldField = fields[colIndex] if oldField.originalValue == originalValue && oldField.hasMultipleValues == hasMultipleValues && oldField.isAbsent == isAbsent && oldField.isUnloaded == isUnloaded { - preservedId = oldField.id + preservedEditorID = oldField.editorID pendingValue = oldField.pendingValue isPendingNull = oldField.isPendingNull isPendingDefault = oldField.isPendingDefault @@ -244,6 +255,8 @@ final class MultiRowEditState: ObservableObject { hasCommittedEdit = true } else if originalValue == nil { isPendingNull = true + } else if expression.first == true, originalValue == PluginCellValue.columnDefaultKeyword { + isPendingDefault = true } else { pendingValue = originalValue isPendingExpression = expression.first == true @@ -279,8 +292,8 @@ final class MultiRowEditState: ObservableObject { isPendingRemoval: isPendingRemoval ) newField.isUnloaded = isUnloaded - if let preservedId { - newField.id = preservedId + if let preservedEditorID { + newField.editorID = preservedEditorID } // Same display format as the grid, or a column set to JSON or PHP-serialized opened a plain text field here. newField.resolvedEditor = FieldEditorResolver.resolve( @@ -462,11 +475,10 @@ final class MultiRowEditState: ObservableObject { private static func resolvePendingValue(_ value: String?, original: String?, isJson: Bool) -> String? { if isJson, let value, !value.isEmpty { - let normalized = JsonReindenter.normalize(value) - if let original, JsonReindenter.normalize(original) == normalized { + if let original, JsonReindenter.normalize(original) == JsonReindenter.normalize(value) { return nil } - return normalized + return JsonReindenter.staged(value, keepingLayoutOf: original) } if value == original || (original == nil && value == "") { return nil @@ -508,7 +520,7 @@ final class MultiRowEditState: ObservableObject { guard index < fields.count else { return } resetPending(at: index) fields[index].isPendingDefault = true - onFieldChanged?(index, .text("__DEFAULT__"), .discrete) + onFieldExpressionChanged?(index, PluginCellValue.columnDefaultKeyword, .discrete) } func setFieldToExpression(at index: Int, expression: CellExpression) { @@ -619,7 +631,7 @@ final class MultiRowEditState: ObservableObject { columnName: field.columnName, newValue: field.effectiveValue, removesField: field.isPendingRemoval, - isExpression: field.isPendingExpression + isExpression: field.isPendingExpression || field.isPendingDefault ) } } diff --git a/apple/TablePro/Models/UI/WindowSidebarState.swift b/apple/TablePro/Models/UI/WindowSidebarState.swift index 5dc1aed215..cd77ef5e06 100644 --- a/apple/TablePro/Models/UI/WindowSidebarState.swift +++ b/apple/TablePro/Models/UI/WindowSidebarState.swift @@ -18,6 +18,12 @@ struct DatabaseTableKey: Hashable, Sendable, Codable { let table: String } +struct PendingTableOperation: Identifiable, Equatable, Sendable { + let id = UUID() + let operation: TableOperationType + let tables: [DatabaseTreeTableRef] +} + @MainActor internal final class WindowSidebarState: ObservableObject { private let connectionId: UUID? @@ -55,6 +61,19 @@ internal final class WindowSidebarState: ObservableObject { var acceptsObjectMarkRefresh: Bool { selectedTables.isEmpty && selectedRowCount == 0 } + + // The connection's queues are shared by every window hosting it, but a Truncate or Drop prompt belongs to the window that asked. + @Published private(set) var pendingTableOperation: PendingTableOperation? + + func requestTableOperation(_ operation: TableOperationType, on tables: [DatabaseTreeTableRef]) { + pendingTableOperation = PendingTableOperation(operation: operation, tables: tables) + } + + func resolveTableOperation(_ request: PendingTableOperation) { + guard pendingTableOperation?.id == request.id else { return } + pendingTableOperation = nil + } + @Published var expandedTreeSchemas: Set = [] { didSet { persistExpansion() } } @Published var expandedTreeDatabases: Set = [] { didSet { persistExpansion() } } @Published var expandedTreeDatabaseSchemas: Set = [] { didSet { persistExpansion() } } @@ -111,8 +130,7 @@ internal final class WindowSidebarState: ObservableObject { isLoaded = true } - /// The kind rides as its raw string. Decoding it as `SidebarObjectKind` throws on a value this - /// build does not know, and one throw inside this blob discards every other expansion set. + // The kind rides as its persistence name, so a value this build does not know drops only that entry. private struct PersistedObjectGroup: Codable { let database: String let schema: String? @@ -176,7 +194,7 @@ internal final class WindowSidebarState: ObservableObject { from persisted: [PersistedObjectGroup] ) -> [DatabaseTreeObjectGroup: Bool] { persisted.reduce(into: [:]) { result, entry in - guard let kind = SidebarObjectKind(rawValue: entry.kind) else { return } + guard let kind = SidebarObjectKind(persistenceName: entry.kind) else { return } let group = DatabaseTreeObjectGroup(database: entry.database, schema: entry.schema, kind: kind) result[group] = entry.expanded } @@ -189,7 +207,7 @@ internal final class WindowSidebarState: ObservableObject { PersistedObjectGroup( database: group.database, schema: group.schema, - kind: group.kind.rawValue, + kind: group.kind.persistenceName, expanded: expanded ) } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 549c36a1a9..aa4f93e18d 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -1072,6 +1072,9 @@ } } } + }, + "“%@” is used more than once." : { + }, "“%@” isn't a port or a range of ports, such as 4000-4999." : { "extractionState" : "manual", @@ -1289,6 +1292,9 @@ } } } + }, + "“%@” will be disconnected and its token revoked. Any in-flight requests will be cancelled." : { + }, "\"%@\" will be moved to Trash. You can recover it from there." : { "localizations" : { @@ -1489,6 +1495,12 @@ } } } + }, + "“%@”'s in-flight requests will be cancelled. It reconnects the next time it starts; to stop it, remove TablePro from it in Connect a Client." : { + + }, + "“%@”'s running requests will be cancelled, but it can send new ones right away. To stop it, remove TablePro from it in Connect a Client, or turn off the MCP server." : { + }, "“%1$@” could not be written. %2$@" : { "localizations" : { @@ -3173,6 +3185,9 @@ } } } + }, + "%@ connected" : { + }, "%@ connections do not support adding rows from Shortcuts." : { @@ -3216,6 +3231,9 @@ } } } + }, + "%@ connects with a token from Settings > MCP > Authentication, which can limit what it reaches. TablePro leaves it as is." : { + }, "%@ contains characters Spanner does not allow." : { "extractionState" : "manual", @@ -3274,6 +3292,9 @@ } } } + }, + "%@ could not be read as valid settings, so TablePro left it unchanged." : { + }, "%@ could not be updated" : { @@ -3292,6 +3313,9 @@ }, "%@ DDL" : { + }, + "%@ defines TablePro in a form TablePro cannot edit safely, so it left the file unchanged." : { + }, "%@ does not accept this column change." : { "localizations" : { @@ -3449,6 +3473,9 @@ "%@ during the Bolt handshake. If the server requires TLS, turn on SSL." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "%@ earlier results were dropped." : { + }, "%@ ends on %@ and does not renew." : { "localizations" : { @@ -3563,6 +3590,9 @@ } } } + }, + "%@ has an unexpected shape where TablePro would add itself, so TablePro left it unchanged." : { + }, "%@ has no column named %@" : { "localizations" : { @@ -3613,6 +3643,9 @@ }, "%@ has no recorded reads." : { + }, + "%@ has TablePro in one project only. Install adds it for every project." : { + }, "%@ holds a character macOS would turn into command line punctuation." : { "extractionState" : "manual", @@ -3709,6 +3742,9 @@ } } } + }, + "%@ is a binary column, and this value is not 0x followed by hex digits." : { + }, "%@ is a boolean field, so it matches only true or false, not %@." : { "extractionState" : "manual", @@ -4234,6 +4270,18 @@ } } } + }, + "%@ is set up for a copy of TablePro that is no longer there." : { + + }, + "%@ is set up for a different copy of TablePro. Update points it to this one." : { + + }, + "%@ is set up with an address or token that no longer works." : { + + }, + "%@ is set up, but one project has its own older TablePro setup that wins there. Remove TablePro from that project's settings." : { + }, "%@ is showing another sheet. Close it, then try again." : { "localizations" : { @@ -4286,6 +4334,9 @@ "%@ isn't compiled for debugging." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "%@ kept changing while TablePro wrote it. Try again." : { + }, "%@ lines have unreadable characters" : { "localizations" : { @@ -4339,6 +4390,9 @@ }, "%@ May Still Be Running" : { + }, + "%@ more messages." : { + }, "%@ more, in the script beside this" : { "localizations" : { @@ -4560,6 +4614,9 @@ } } } + }, + "%@ names the same key twice, so TablePro left it unchanged." : { + }, "%@ needs %@." : { "extractionState" : "manual", @@ -4829,6 +4886,9 @@ } } } + }, + "%@ picks up the change right away." : { + }, "%@ Plan" : { @@ -4924,6 +4984,9 @@ } } } + }, + "%@ reaches MCP servers only from its maker's cloud, so it cannot reach TablePro on this Mac." : { + }, "%@ read" : { "extractionState" : "manual", @@ -5413,12 +5476,18 @@ } } } + }, + "%@ starts MCP servers only as local commands, so it cannot reach TablePro's HTTP server." : { + }, "%@ statement" : { }, "%@ statements" : { + }, + "%@ Statements" : { + }, "%@ stops being a computed column" : { "localizations" : { @@ -5439,6 +5508,9 @@ } } } + }, + "%@ stops reaching your databases through TablePro." : { + }, "%@ succeeded" : { "extractionState" : "manual", @@ -7016,6 +7088,9 @@ }, "%1$@ column of %2$@" : { + }, + "%1$@ contrast is %2$@:1. Aim for at least 4.5:1." : { + }, "%1$@ could not be read again before generating the script. %2$@" : { "localizations" : { @@ -7296,6 +7371,9 @@ "%1$@ is not a level %2$@ accepts. Use one of %3$@." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "%1$@ is set up, but the project %2$@ has its own older TablePro setup that wins there. Remove TablePro from that project's settings." : { + }, "%1$@ is the local port, %2$@ and %3$@ are this connection's host and port. The command runs directly rather than through a shell, so start it with /usr/bin/env if you need environment variables." : { "localizations" : { @@ -7562,6 +7640,9 @@ }, "%1$@ rows modified since it was last analyzed, more than the %2$@ that start an autoanalyze." : { + }, + "%1$@ stops reaching your databases through TablePro. The current %2$@ is kept next to it as %3$@." : { + }, "%1$@ tables hold “%2$@”." : { @@ -13544,6 +13625,9 @@ } } } + }, + "1 earlier result was dropped." : { + }, "1 field" : { @@ -13583,6 +13667,9 @@ } } } + }, + "1 more message." : { + }, "1 new column" : { @@ -16237,6 +16324,10 @@ }, "A label cannot be removed or moved once it exists." : { + }, + "A large object value is longer than the 64 MiB TablePro reads from SAP HANA, so it cannot be copied or exported whole." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "A larger binary value stays on the server until you open, copy or export it. Applies to tables with a primary key." : { @@ -16903,6 +16994,9 @@ } } } + }, + "A run that returns more results with rows keeps only the newest. Pinned results always stay." : { + }, "a running query" : { @@ -16963,6 +17057,10 @@ } } }, + "A script cannot hold binary COPY data, so no rows were sent. Dump the rows as text or CSV." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A script wants to run a query on \"%@\"" : { "localizations" : { "fr" : { @@ -17211,6 +17309,12 @@ } } } + }, + "A statement failed in the transaction, so it can only be rolled back." : { + + }, + "A statement is still running in the transaction, so it cannot be committed now." : { + }, "A statement on '%1$@' matched %2$d rows instead of %3$d, so nothing was saved." : { "localizations" : { @@ -17570,6 +17674,9 @@ } } } + }, + "A transaction is already open on this connection, or it cannot say whether one is. Commit or roll it back first." : { + }, "A transaction is already open. Commit or roll it back first." : { "extractionState" : "manual", @@ -17582,18 +17689,30 @@ } } } + }, + "A Transaction Is Open" : { + }, "A transaction is open on %@. Switching databases reconnects, and the server would roll it back. Commit or roll back the transaction, then switch again." : { + }, + "A transaction is open on this connection, and a failed statement would abort it. Commit or roll back in TablePro first." : { + }, "A transaction is open on this connection, and its uncommitted rows aren't in these counts." : { }, "A transaction is open on this connection, and this change cannot run inside it. Nothing was run. Commit or roll back the transaction, then try again." : { + }, + "A transaction is open on this connection, and this change would have to end it. Commit or roll back the transaction, then try again." : { + }, "A transaction is open on this connection. Commit or roll it back in TablePro, then run the shortcut again." : { + }, + "A transaction is still open on %@." : { + }, "A transaction is still open on this connection. Commit or roll it back." : { @@ -19720,6 +19839,9 @@ }, "Add column “%@”" : { + }, + "Add Column Pair" : { + }, "Add Column…" : { "localizations" : { @@ -20352,6 +20474,9 @@ } } } + }, + "Add it in %@'s settings" : { + }, "Add Jump Host" : { "localizations" : { @@ -20805,6 +20930,9 @@ } } } + }, + "Add this to %@:" : { + }, "Add to Favorites" : { "comment" : "A label that describes an action to add an item to a user's favorites.", @@ -21236,6 +21364,9 @@ } } } + }, + "Adds through the %@ command" : { + }, "Adjust Limit" : { "localizations" : { @@ -22686,6 +22817,9 @@ } } } + }, + "All clients" : { + }, "All collections" : { "extractionState" : "manual", @@ -24329,6 +24463,12 @@ } } } + }, + "Always Allow on an assistant tool card adds the tool here. Forget makes its next call ask again." : { + + }, + "Always allowed" : { + }, "Always count" : { "localizations" : { @@ -29699,6 +29839,33 @@ } } } + }, + "Auto Refresh" : { + + }, + "Auto Refresh waits until the connection is back." : { + + }, + "Auto Refresh waits while a cell is being edited." : { + + }, + "Auto Refresh waits while a transaction is open." : { + + }, + "Auto Refresh waits while a value is open in its own window." : { + + }, + "Auto Refresh waits while rows are selected in a table without a primary key." : { + + }, + "Auto Refresh waits while rows load." : { + + }, + "Auto Refresh waits while the table has unsaved changes." : { + + }, + "Auto Refresh: %@" : { + }, "AUTO_INCREMENT columns whose next value has reached 75% of the largest value their type holds." : { @@ -31535,6 +31702,12 @@ }, "Before" : { + }, + "Before Replace" : { + + }, + "Before Restore" : { + }, "BEFORE, AFTER, or INSTEAD OF" : { "localizations" : { @@ -31575,6 +31748,9 @@ } } } + }, + "Begin, commit, or roll back a transaction on the connection's shared session. The transaction stays open across calls until committed or rolled back, and it is the same session the user's own tabs run on, so leave nothing open. Begin is refused while a transaction is already open, and every action is refused on a connection with Manual Commit on, where the user commits in TablePro. Needs tools:write." : { + }, "Begin, commit, or roll back a transaction on the connection's shared session. The transaction stays open across calls until committed or rolled back, and it is the same session the user's own tabs run on, so leave nothing open. Needs tools:write." : { "localizations" : { @@ -31805,6 +31981,10 @@ }, "Billing…" : { + }, + "Binary" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Binary content" : { "localizations" : { @@ -32405,6 +32585,9 @@ } } } + }, + "Bring an open tab to the front, using a tab id from list_recent_tabs. TablePro asks the user about the tab's connection first, as it does for any call that names a connection." : { + }, "Broadcast" : { "extractionState" : "manual", @@ -32503,6 +32686,9 @@ } } } + }, + "Browse All Versions…" : { + }, "Browse needs a TableName" : { "extractionState" : "manual", @@ -33718,6 +33904,9 @@ } } } + }, + "Can't read settings" : { + }, "Can't read this folder" : { @@ -34692,6 +34881,12 @@ } } } + }, + "Cannot run this query at Read-Only: it calls a function that can change data. Switch Safe Mode to %@ to run it." : { + + }, + "Cannot run this query: it calls a function that can change data, and Safe Mode is held at Read-Only for this connection." : { + }, "Cannot save changes to '%@'." : { "localizations" : { @@ -36531,6 +36726,9 @@ }, "Changes Unavailable" : { + }, + "Changes you don’t commit are rolled back." : { + }, "Changing %@ from %@ to %@ can truncate existing values." : { "localizations" : { @@ -37160,6 +37358,15 @@ } } } + }, + "ChatGPT sign-in failed: %@" : { + + }, + "ChatGPT sign-in was denied: %@" : { + + }, + "ChatGPT sign-in was denied." : { + }, "Check" : { @@ -37973,6 +38180,9 @@ }, "Choose a character set first" : { + }, + "Choose a column and a referenced column in every row." : { + }, "Choose a connection" : { "localizations" : { @@ -39464,6 +39674,15 @@ } } } + }, + "Claude Agent can't use this connection: TablePro couldn't create its access token. Try again." : { + + }, + "Claude Agent can't use this connection: TablePro couldn't save its access token to the Keychain. Try again." : { + + }, + "Claude Agent can't use this connection: TablePro couldn't write its MCP settings file. Try again." : { + }, "Claude Code" : { "localizations" : { @@ -41248,6 +41467,9 @@ } } } + }, + "Client List Unavailable" : { + }, "Client Secret" : { "localizations" : { @@ -41329,6 +41551,18 @@ } } } + }, + "Client: %@" : { + + }, + "Clients can connect once the MCP server is running." : { + + }, + "Clients can connect while the MCP server is on. Turn it on in Settings > MCP." : { + + }, + "Clients cannot connect until it runs. Open Settings > MCP to see why and try again." : { + }, "Clients will appear here while they have an active MCP session." : { "localizations" : { @@ -43208,6 +43442,9 @@ } } } + }, + "Code expires in ^[%lld second](inflect: true)" : { + }, "Code folding" : { "localizations" : { @@ -43883,6 +44120,9 @@ } } } + }, + "Color values by type" : { + }, "Color, omitted when none" : { "localizations" : { @@ -44045,6 +44285,18 @@ } } }, + "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to DECIMAL or VARCHAR in the query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to VARCHAR in the query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "column %@ first appears in a later chunk" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Column %@ has a type, default, generation expression or check that is not one SQL expression." : { "extractionState" : "manual", "generatesSymbol" : false @@ -44070,6 +44322,9 @@ } } } + }, + "Column → Referenced Column" : { + }, "Column comment" : { "extractionState" : "manual", @@ -45537,9 +45792,16 @@ }, "Commit Message" : { + }, + "Commit or roll back the open transaction before switching database." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Commit the transaction" : { + }, + "Commit Transaction" : { + }, "Commit…" : { @@ -46820,6 +47082,9 @@ } } } + }, + "Configuration" : { + }, "Configure an active provider or turn on Apple Intelligence to enable inline suggestions." : { @@ -47910,6 +48175,9 @@ } } } + }, + "Connected just now" : { + }, "Connected Threads" : { "localizations" : { @@ -49748,6 +50016,9 @@ } } } + }, + "Connections on this profile share one database user. TablePro renews its lease while in use, revokes it once the last connection disconnects or after 10 minutes unused, and never saves the generated password." : { + }, "Connections the client may use, ordered by name" : { "localizations" : { @@ -51083,6 +51354,10 @@ }, "Copy as UPDATE Statements" : { + }, + "COPY at line %@: %@ lines sent" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Copy Average" : { @@ -51179,6 +51454,12 @@ } } } + }, + "Copy Command" : { + + }, + "Copy Configuration" : { + }, "Copy Connection ID" : { "localizations" : { @@ -51551,6 +51832,13 @@ }, "Copy Fix SQL" : { + }, + "COPY FROM STDIN takes its rows from the lines after it, which only the query editor and SQL import send. Run it there." : { + + }, + "COPY FROM STDIN takes its rows from the lines after the line its ; ends, so no rows were sent." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Copy ID" : { "localizations" : { @@ -52235,6 +52523,9 @@ } } } + }, + "Copy Setup for %@" : { + }, "Copy Special" : { "localizations" : { @@ -52368,6 +52659,9 @@ } } } + }, + "Copy the command and run it in Terminal" : { + }, "Copy the full error to the clipboard" : { "localizations" : { @@ -52408,6 +52702,9 @@ } } } + }, + "Copy the setup into %@" : { + }, "Copy to" : { "localizations" : { @@ -53018,6 +53315,9 @@ } } } + }, + "Could Not Commit the Transaction" : { + }, "Could Not Compare" : { "localizations" : { @@ -54534,6 +54834,10 @@ }, "Could not read the table." : { + }, + "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Could not read which rows are already in the table" : { @@ -55039,6 +55343,12 @@ } } } + }, + "Couldn't Create the Token" : { + + }, + "Couldn't Detach Partition" : { + }, "Couldn't Discard Changes" : { "localizations" : { @@ -55052,6 +55362,9 @@ }, "Couldn't download cloud-sql-proxy. The server answered with HTTP status %lld." : { + }, + "Couldn't Drop Partition" : { + }, "Couldn't Duplicate the Theme" : { @@ -55070,6 +55383,9 @@ }, "Couldn't Import Formatter Rules" : { + }, + "Couldn't install" : { + }, "Couldn't Load" : { @@ -55119,6 +55435,9 @@ }, "Couldn't Move the Theme to the Trash" : { + }, + "Couldn't Optimize Table" : { + }, "Couldn't publish to the team catalog" : { "localizations" : { @@ -55254,6 +55573,9 @@ }, "Couldn't Read the DDL" : { + }, + "Couldn't Read the Parts" : { + }, "Couldn't read the referenced table" : { "localizations" : { @@ -55289,6 +55611,9 @@ }, "Couldn't refresh the team library" : { + }, + "Couldn't remove" : { + }, "Couldn't Rename the Theme" : { @@ -55388,12 +55713,18 @@ } } } + }, + "Couldn't Save the Deletion" : { + }, "Couldn't Save the Revocation" : { }, "Couldn't start" : { + }, + "Couldn't update" : { + }, "Count" : { "localizations" : { @@ -57626,6 +57957,9 @@ } } } + }, + "cursors" : { + }, "CURTIME()" : { "extractionState" : "stale", @@ -58460,6 +58794,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid Binary" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data Grid Bool False" : { "extractionState" : "manual", "generatesSymbol" : false @@ -58468,6 +58806,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid Date & Time" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data Grid Deleted" : { "extractionState" : "manual", "generatesSymbol" : false @@ -58476,6 +58818,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid Enum" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data Grid Focus Border" : { "extractionState" : "manual", "generatesSymbol" : false @@ -58537,6 +58883,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid JSON" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data Grid Line" : { "extractionState" : "manual", "generatesSymbol" : false @@ -58549,6 +58899,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid Number" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data Grid Row Number" : { "extractionState" : "manual", "generatesSymbol" : false @@ -58561,10 +58915,18 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid String" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data Grid Text" : { "extractionState" : "manual", "generatesSymbol" : false }, + "Data Grid UUID" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Data loss" : { "localizations" : { "fr" : { @@ -60749,6 +61111,10 @@ } } }, + "Date & Time" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Date and Time" : { }, @@ -61994,6 +62360,10 @@ } } }, + "Default Version" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Default view:" : { "localizations" : { "fr" : { @@ -62085,6 +62455,10 @@ } } }, + "Definer" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Definition" : { "localizations" : { "fr" : { @@ -62244,6 +62618,9 @@ } } } + }, + "Delete “/%@”?" : { + }, "Delete \"%@\"" : { "localizations" : { @@ -62358,6 +62735,9 @@ } } } + }, + "Delete %d tokens?" : { + }, "Delete %lld columns?" : { "localizations" : { @@ -65455,6 +65835,10 @@ } } }, + "Disabled on Replica" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Discard" : { "localizations" : { "fr" : { @@ -66502,6 +66886,33 @@ } } } + }, + "Do you want to commit the open transaction before closing “%@”?" : { + + }, + "Do you want to commit the open transaction before closing the database “%@”?" : { + + }, + "Do you want to commit the open transaction before disconnecting from “%@”?" : { + + }, + "Do you want to commit the open transaction before quitting?" : { + + }, + "Do you want to commit the open transaction before relaunching?" : { + + }, + "Do you want to commit the open transactions before closing “%@”?" : { + + }, + "Do you want to commit the open transactions before disconnecting from “%@”?" : { + + }, + "Do you want to commit the open transactions before quitting?" : { + + }, + "Do you want to commit the open transactions before relaunching?" : { + }, "Do you want to save changes?" : { "localizations" : { @@ -67271,6 +67682,9 @@ } } } + }, + "Draw strings, numbers, dates and other typed values in the colors of the theme's Values section" : { + }, "Drawing %1$d shapes in SRID %2$d." : { "localizations" : { @@ -69660,6 +70074,13 @@ } } }, + "e.g., Cursor on my Mac" : { + + }, + "Each array element must be a JSON object" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Each Browse filter needs an Attribute and an Operator" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -70516,6 +70937,9 @@ } } } + }, + "Edit mode runs only queries and data changes, such as SELECT, INSERT, UPDATE, DELETE and MERGE. A schema, privilege, session or transaction statement, or a change to the database's own tables, needs Agent mode, or run it yourself in a query tab." : { + }, "Edit Profile…" : { "localizations" : { @@ -70753,6 +71177,12 @@ } } } + }, + "Edit: read-only tools plus queries and data changes. On SQL databases, schema, privilege and session statements need Agent mode." : { + + }, + "Edit: read-only tools plus queries and data changes. Schema, privilege and session commands need Agent mode." : { + }, "Edit: read-only tools plus running queries. Destructive DDL stays blocked." : { "localizations" : { @@ -71132,6 +71562,12 @@ "Editor Text" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Edits %@" : { + + }, + "Edits saved with Manual Commit on are not committed yet." : { + }, "Effective" : { "localizations" : { @@ -72036,6 +72472,8 @@ } }, "Enabled" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -72342,6 +72780,10 @@ } } }, + "Ends" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Ends on %@" : { }, @@ -73515,6 +73957,10 @@ } } }, + "Enum" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Enum %@ has no symbols." : { }, @@ -74486,6 +74932,10 @@ } } }, + "etcd compacted the revision this export started from, so it stopped part way. Run it again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "etcd has no %@ column." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -74595,6 +75045,8 @@ "generatesSymbol" : false }, "Event" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -74634,6 +75086,10 @@ } } }, + "Event Scheduler" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Event Streaming" : { "localizations" : { "fr" : { @@ -74715,6 +75171,8 @@ } }, "Events" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -74736,6 +75194,12 @@ }, "Every 5 Minutes" : { + }, + "Every 5 Seconds" : { + + }, + "Every 10 Seconds" : { + }, "Every 15 minutes" : { "localizations" : { @@ -74759,6 +75223,9 @@ } } } + }, + "Every 30 Seconds" : { + }, "Every call that reaches a connection asks first." : { "localizations" : { @@ -74802,6 +75269,9 @@ } } } + }, + "Every custom shortcut goes back to its default." : { + }, "Every database you picked is written into this folder." : { "localizations" : { @@ -75477,6 +75947,9 @@ } } } + }, + "Except NULL and booleans, these colors show in the grid only while Color values by type is on in Settings > Data." : { + }, "exclamation mark" : { "extractionState" : "manual", @@ -75689,6 +76162,9 @@ } } } + }, + "Exclude from iCloud Sync?" : { + }, "Exclude the AUTO_INCREMENT counter" : { "extractionState" : "stale", @@ -79210,6 +79686,10 @@ } } }, + "Extension" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Extension File" : { "localizations" : { "fr" : { @@ -84060,6 +84540,9 @@ }, "Findings the check reported" : { + }, + "Finish in %@" : { + }, "Finish Signing In" : { "localizations" : { @@ -85478,6 +85961,9 @@ } } } + }, + "Forget %@" : { + }, "Forget All" : { "localizations" : { @@ -87396,6 +87882,9 @@ } } } + }, + "Give your client this address and header:" : { + }, "global" : { "localizations" : { @@ -95772,6 +96261,9 @@ } } } + }, + "Insert rows into a table using bound parameters, so values are never spliced into SQL text. Needs tools:write and goes through the connection's Safe Mode approval. With Manual Commit on, the rows stay pending until the user commits in TablePro." : { + }, "Insert rows the target is missing" : { "localizations" : { @@ -96158,6 +96650,12 @@ }, "Install %@…" : { + }, + "Install adds TablePro to a client on this Mac. Select a client to see its settings and set it up by hand." : { + + }, + "Install in %@" : { + }, "Install it with “brew install libpq” and link it." : { "localizations" : { @@ -96334,6 +96832,9 @@ }, "Install Update…" : { + }, + "Install…" : { + }, "Installed" : { "localizations" : { @@ -99306,6 +99807,9 @@ } } } + }, + "Keeps up to %1$d versions from the last %2$d days on this Mac." : { + }, "Keeps what rows looked like before each save, on this Mac only, for 7 days. It is encrypted, and never synced." : { "localizations" : { @@ -101033,6 +101537,10 @@ } } }, + "Last Executed" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Last Hour" : { "localizations" : { "fr" : { @@ -103358,6 +103866,12 @@ } } } + }, + "Line %lld runs COPY FROM STDIN, and this connection cannot load its rows. Run the file with psql, or dump it again with --inserts." : { + + }, + "Line %lld runs COPY FROM STDIN, whose rows TablePro cannot send. Run the file with psql, or dump it again with --inserts." : { + }, "Line break" : { "extractionState" : "manual", @@ -105156,6 +105670,9 @@ } } } + }, + "List the tabs open in TablePro. With connection_id, TablePro asks the user about that connection first. Without it, the list covers only connections this client may already reach without asking. Each entry carries the tab id that focus_query_tab takes and the window id that open_table_tab returns." : { + }, "List the triggers attached to one table, with their timing, event, and body." : { "localizations" : { @@ -105342,6 +105859,9 @@ }, "Listen for Notifications" : { + }, + "LISTEN registrations" : { + }, "Listens on all interfaces (0.0.0.0), reachable from your local network." : { "localizations" : { @@ -106467,6 +106987,10 @@ } } }, + "Local socket" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Locale Provider" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -107116,6 +107640,9 @@ } } } + }, + "macOS could not generate a secure token. Try again." : { + }, "macOS will ask for your login password" : { "localizations" : { @@ -107648,6 +108175,15 @@ } } } + }, + "Manual Commit" : { + + }, + "Manual Commit is on for this connection, so this change is pending until the user commits it in TablePro." : { + + }, + "Manual Commit is on for this connection. Commit or roll back in TablePro." : { + }, "Many Databases, One App" : { "localizations" : { @@ -109573,6 +110109,18 @@ } } } + }, + "MCP Server Couldn't Start" : { + + }, + "MCP Server Not Running" : { + + }, + "MCP Server Off" : { + + }, + "MCP Server Starting" : { + }, "MCP Server: Running (%d clients)" : { "extractionState" : "stale", @@ -111909,6 +112457,9 @@ }, "Move “%@” to Connection" : { + }, + "Move “%@” to the Trash?" : { + }, "Move Column Down" : { "localizations" : { @@ -112911,6 +113462,9 @@ } } } + }, + "Moving this tab to another database or schema will discard all unsaved changes." : { + }, "Moving through this tab's history replaces what it is showing and will discard all unsaved changes." : { "localizations" : { @@ -113002,6 +113556,9 @@ } } } + }, + "Multiple Results" : { + }, "Multiple statements are not permitted for this client" : { "localizations" : { @@ -116099,7 +116656,8 @@ } }, "No" : { - + "extractionState" : "manual", + "generatesSymbol" : false }, "No %@" : { "localizations" : { @@ -120441,6 +120999,10 @@ } } }, + "No row has this primary key, so nothing was updated. Another client may have deleted it." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "No row holds the values it was loaded with. Another client may have changed or deleted it." : { }, @@ -120453,6 +121015,9 @@ } } } + }, + "No row took the change. Another client may have changed or deleted it, or a security policy or trigger skipped it." : { + }, "No rows" : { "localizations" : { @@ -123274,6 +123839,9 @@ } } } + }, + "Not found" : { + }, "Not found (404) at %@. Check the Base URL and the model." : { "localizations" : { @@ -126316,6 +126884,9 @@ } } } + }, + "Old setup in a project" : { + }, "Ollama is running but has no models. Run \"ollama pull \" to download one." : { "localizations" : { @@ -126423,6 +126994,10 @@ }, "On a partitioned table the drop cannot run concurrently, and it removes the index from every partition." : { + }, + "On Completion" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "On Delete" : { "localizations" : { @@ -126692,6 +127267,9 @@ } } } + }, + "On This Mac" : { + }, "On Update" : { "extractionState" : "manual", @@ -126892,6 +127470,9 @@ } } } + }, + "One project only" : { + }, "One report a day: hashed Mac ID, versions, architecture, language, database types, connection count, license status, update settings, first-use dates. Never hostnames, credentials, queries, or rows." : { "localizations" : { @@ -127042,6 +127623,10 @@ } } }, + "One Time" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "One username and password, shared by any number of connections. Change it here and every connection using it signs in with the new one." : { "localizations" : { "fr" : { @@ -129694,6 +130279,9 @@ } } } + }, + "Opens %@ to confirm" : { + }, "Opens a database connection in TablePro" : { @@ -130760,6 +131348,12 @@ } } } + }, + "Other Client" : { + + }, + "Other Clients" : { + }, "Other rows hold exactly the same values, so the change cannot reach this row alone." : { @@ -130820,6 +131414,9 @@ } } } + }, + "Out of date" : { + }, "out-of-band check" : { "extractionState" : "manual", @@ -131377,6 +131974,8 @@ } }, "Owner" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -136160,6 +136759,19 @@ } } } + }, + "Port %d cannot be used. Choose a port from %d to %d." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Port %1$d cannot be used. Choose a port from %2$d to %3$d." + } + } + } + }, + "Port %d was in use, so this address changes when TablePro restarts." : { + }, "Port %d was in use." : { @@ -136967,6 +137579,9 @@ } } } + }, + "prepared statements" : { + }, "Preparing Import…" : { "localizations" : { @@ -137067,6 +137682,9 @@ } } } + }, + "Present and true when the answer's size limit cut plan_text or left the parsed plan or rows out" : { + }, "Preserve all values as strings" : { "extractionState" : "stale", @@ -141425,6 +142043,9 @@ } } } + }, + "Query History Was Damaged" : { + }, "Query Insights" : { "localizations" : { @@ -142352,6 +142973,9 @@ } } } + }, + "Quit and reopen %@ to load it." : { + }, "Quit Anyway" : { "localizations" : { @@ -142459,6 +143083,9 @@ }, "Quit while work is still running?" : { + }, + "Quit with an Open Transaction?" : { + }, "Quit Without Saving" : { @@ -142926,6 +143553,9 @@ } } } + }, + "Ran %@ statements that returned no rows." : { + }, "ran %@×" : { "comment" : "Run count in a detail line, %@ is a count", @@ -143623,6 +144253,9 @@ } } } + }, + "Reaches only servers on the internet" : { + }, "Read" : { "localizations" : { @@ -144247,6 +144880,9 @@ } } } + }, + "Read-Only Safe Mode does not allow a commit." : { + }, "Read-only SQL over Iceberg tables in R2" : { "localizations" : { @@ -145177,6 +145813,12 @@ "Reconnecting (%lld of %lld)" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Reconnecting could not reopen the database %1$@: %2$@" : { + + }, + "Reconnecting could not reopen the schema %1$@: %2$@" : { + }, "Reconnecting failed after %d attempts." : { "localizations" : { @@ -145333,6 +145975,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Recurring" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Red" : { "localizations" : { "fr" : { @@ -145797,6 +146443,9 @@ } } } + }, + "Referenced Column" : { + }, "Referenced row not found" : { "localizations" : { @@ -146079,6 +146728,10 @@ } } }, + "Refresh before changing this item, so the save can check that it has not changed since it was loaded." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Refresh concurrently" : { "localizations" : { "fr" : { @@ -146534,6 +147187,9 @@ }, "Relaunch while work is still running?" : { + }, + "Relaunch with an Open Transaction?" : { + }, "Release" : { "localizations" : { @@ -146795,6 +147451,9 @@ } } } + }, + "Reload %@ to load it." : { + }, "Reload from Disk" : { "localizations" : { @@ -146936,6 +147595,10 @@ } } }, + "Relocatable" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Remote (Quack, experimental)" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -147481,6 +148144,9 @@ } } } + }, + "Remove Column Pair" : { + }, "Remove Comment" : { "localizations" : { @@ -148405,6 +149071,9 @@ "Remove SET transaction_timeout from the routine to debug it." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Remove TablePro from %@?" : { + }, "Remove tag %@" : { "comment" : "A label for a button that removes a tag from a list of selected tags. The argument is the name of the tag to be removed.", @@ -148599,6 +149268,9 @@ }, "Removing" : { + }, + "Removing…" : { + }, "Rename" : { "localizations" : { @@ -150099,6 +150771,9 @@ } } } + }, + "Reset All Shortcuts?" : { + }, "Reset column widths, order, and visibility to defaults" : { "localizations" : { @@ -151374,6 +152049,12 @@ } } } + }, + "Restore Row" : { + + }, + "Restore Rows" : { + }, "Restore the previous values of a save" : { "localizations" : { @@ -151748,6 +152429,9 @@ } } } + }, + "Restrict to one connection. Omit for every connection already approved." : { + }, "Restrict to one connection. Omit for every readable connection." : { "localizations" : { @@ -152089,6 +152773,9 @@ } } } + }, + "Results kept per run:" : { + }, "Resume" : { "localizations" : { @@ -152638,6 +153325,9 @@ }, "Revert Changes" : { + }, + "Revert To" : { + }, "review" : { "extractionState" : "stale", @@ -153300,6 +153990,9 @@ }, "Roll Back the Transaction?" : { + }, + "Roll Back Transaction" : { + }, "Roll it back to continue." : { @@ -155715,6 +156408,9 @@ }, "Run one statement, or on SQL Server a whole script. Reads need tools:read; anything that writes needs tools:write and is subject to the connection's Safe Mode, which asks the user to approve it. Destructive statements, such as DROP, TRUNCATE, ALTER...DROP or a DELETE with no WHERE, go through confirm_destructive_operation instead. Statements that read or write files, or run server-side code, are refused. Send one statement per call, except on SQL Server, where a script runs batch by batch at its GO lines and result_sets lists every result set when there is more than one." : { + }, + "Run one statement, or on SQL Server a whole script. Reads need tools:read; anything that writes needs tools:write and is subject to the connection's Safe Mode, which asks the user to approve it. Destructive statements, such as DROP, TRUNCATE, ALTER...DROP or a DELETE with no WHERE, go through confirm_destructive_operation instead. Statements that read or write files, or run server-side code, are refused. Send one statement per call, except on SQL Server, where a script runs batch by batch at its GO lines and result_sets lists every result set when there is more than one. A statement that starts a transaction while one is open is refused. With Manual Commit on, a write stays pending until the user commits it in TablePro, and statements that begin, commit or roll back a transaction are refused. While a transaction is open on PostgreSQL or CockroachDB a statement runs inside a savepoint, so its failure leaves the transaction usable; on Redshift and DuckDB it is refused." : { + }, "Run one statement, or on SQL Server a whole script. Reads need tools:read; anything that writes needs tools:write and is subject to the connection's Safe Mode, which asks the user to approve it. DROP and TRUNCATE go through confirm_destructive_operation instead. Statements that read or write files, or run server-side code, are refused. Send one statement per call, except on SQL Server, where a script runs batch by batch at its GO lines and result_sets lists every result set when there is more than one." : { "localizations" : { @@ -156136,6 +156832,9 @@ } } } + }, + "Run this in Terminal:" : { + }, "Run WATCH again before starting the block." : { "extractionState" : "manual", @@ -156151,6 +156850,9 @@ }, "Run what execute_query refuses as destructive, such as DROP, TRUNCATE, ALTER...DROP or a DELETE with no WHERE: one statement, or on SQL Server a whole script. Anything else is refused. The user approves it before it runs: through your elicitation prompt when your client supports elicitation, otherwise through TablePro's own confirmation dialog on the user's Mac. Needs tools:write and a connection the user left writable for external clients." : { + }, + "Run what execute_query refuses as destructive, such as DROP, TRUNCATE, ALTER...DROP or a DELETE with no WHERE: one statement, or on SQL Server a whole script. Anything else is refused. The user approves it before it runs: through your elicitation prompt when your client supports elicitation, otherwise through TablePro's own confirmation dialog on the user's Mac. Needs tools:write and a connection the user left writable for external clients. The transaction rules of execute_query apply, Manual Commit's among them." : { + }, "Run Without Limit" : { "localizations" : { @@ -156578,6 +157280,14 @@ }, "Safe Mode is read-only for this connection." : { + }, + "Safe Mode is set to Read-Only for this connection, and it has a transaction open. Roll it back, then try again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Safe Mode is set to Read-Only for this connection, and the database refused the statement." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Safe mode level" : { "localizations" : { @@ -157966,6 +158676,9 @@ } } } + }, + "Saved connection “%@”" : { + }, "Saved Connections" : { "localizations" : { @@ -158157,6 +158870,9 @@ } } } + }, + "Saved passwords are removed from iCloud Keychain, and your other devices will need them entered again." : { + }, "Saved passwords can only be exported to an encrypted file." : { "extractionState" : "manual", @@ -158250,6 +158966,9 @@ }, "Saved queries import for every connection, in a folder named “%@”." : { + }, + "Saved Queries Were Damaged" : { + }, "Saved Query" : { "localizations" : { @@ -158350,6 +159069,9 @@ } } } + }, + "Saving removes its passwords from iCloud Keychain. Your other devices will need them entered again." : { + }, "Saving your choice" : { @@ -158508,6 +159230,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Schedule" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Schema" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -160849,6 +161575,9 @@ } } } + }, + "Search the query history TablePro keeps on this Mac. With connection_id, TablePro asks the user about that connection first. Without it, results cover only connections this client may already reach without asking." : { + }, "Search values" : { "localizations" : { @@ -161650,6 +162379,9 @@ } } } + }, + "Select a client to see how TablePro connects to it." : { + }, "Select a column" : { @@ -164346,6 +165078,12 @@ } } } + }, + "session locks" : { + + }, + "session settings" : { + }, "Session SID %1$d, SERIAL# %2$d. %3$@ A user with ALTER SYSTEM can end it now with %4$@." : { "extractionState" : "manual", @@ -164994,6 +165732,15 @@ } } } + }, + "Set up" : { + + }, + "Set up an app to use TablePro's MCP server" : { + + }, + "Set up with a token" : { + }, "Set Value" : { "localizations" : { @@ -165155,6 +165902,9 @@ } } } + }, + "Settings file" : { + }, "Settings:" : { "extractionState" : "stale", @@ -167374,6 +168124,9 @@ } } } + }, + "Show the connection this conversation works on, with its status." : { + }, "Show the messages that arrived while paused" : { @@ -167997,6 +168750,9 @@ } } } + }, + "Showing the last %1$@ of %2$@ results that returned rows." : { + }, "Shown beside each key" : { "localizations" : { @@ -170729,6 +171485,9 @@ } } } + }, + "Some tabs have unsaved edits. Relaunching will discard these changes." : { + }, "Some values are set outside this file" : { "localizations" : { @@ -170809,6 +171568,9 @@ } } } + }, + "Some values were longer than 64 KB and were cut there; each one cut ends in a mark with its full length. Read such a value in parts with a narrower query." : { + }, "Someone in %@ changed this item after TablePro last downloaded the team library. Choose the version the team library keeps." : { @@ -173972,6 +174734,9 @@ }, "Start" : { + }, + "Start a new %@ session to load it." : { + }, "Start one to ask about this connection." : { "localizations" : { @@ -174114,6 +174879,13 @@ } } } + }, + "Starts" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Starts only local command servers" : { + }, "starts with" : { "localizations" : { @@ -174209,6 +174981,9 @@ } } } + }, + "Startup commands failed, so this session runs without them:\n%@" : { + }, "Startup SQL" : { "comment" : "A section header for the startup SQL commands of a connection.", @@ -174402,6 +175177,21 @@ } } } + }, + "Statement %1$d/%2$d is a DDL statement, and the session takes nothing after it but COMMIT or ROLLBACK. The run stopped before statement %3$d. Click Commit on the Transaction Open bar, then begin the rest with “%4$@”." : { + + }, + "Statement %1$d/%2$d is a DDL statement, and the session takes nothing after it but COMMIT or ROLLBACK. The run stopped before statement %3$d. Click Commit on the Transaction Open bar, then run the rest." : { + + }, + "Statement %1$d/%2$d is a DDL statement, and the session takes nothing after it but COMMIT or ROLLBACK. The run stopped before statement %3$d. Click Commit on the Transaction Open bar. A new run starts in the tab's database, so begin the rest with “%4$@”." : { + + }, + "Statement %1$d/%2$d moves the session to another database, which this session takes only outside a transaction. The run stopped before it. Click Commit or Roll Back on the Transaction Open bar, then run the script again from statement %1$d." : { + + }, + "Statement %1$d/%2$d moves the session to another database, which this session takes only outside a transaction. The run stopped before it. Click Roll Back on the Transaction Open bar, move the statement above the script's BT, then run the script again from there." : { + }, "Statement %lld" : { "extractionState" : "stale", @@ -174974,6 +175764,8 @@ } }, "Status" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -181205,6 +181997,9 @@ }, "TablePro cannot tell which Redshift cluster or workgroup \"%@\" is. Set Host to the endpoint AWS issued, such as my-cluster.abc123.us-east-1.redshift.amazonaws.com, or enter that endpoint as the Redshift Endpoint when you connect through a port forward." : { + }, + "TablePro cannot tell which Redshift cluster or workgroup \"%@\" is. Set Host to the endpoint AWS issued, such as my-cluster.abc123.us-east-1.redshift.amazonaws.com, or to a custom domain name that points to it, or enter that endpoint as the Redshift Endpoint." : { + }, "TablePro cannot upload changes until there is space in iCloud. Changes from this Mac are kept here and upload once you free up space or add storage. To manage storage, open System Settings, click your name, then click iCloud." : { @@ -181305,6 +182100,9 @@ }, "TablePro could not check whether %@ sends the password in plain text without TLS. Try again, or set SSL Mode to Required." : { + }, + "TablePro could not confirm its change to %@. Check the file before you try again." : { + }, "TablePro could not do this for you. Run this in Terminal instead:" : { "extractionState" : "stale", @@ -181347,6 +182145,10 @@ } } }, + "TablePro could not look up \"%@\" in DNS to find the AWS endpoint it points to. Check the network and try again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "TablePro could not open its query history database, so nothing is being recorded and nothing can be shown. Your existing history is still on disk." : { "comment" : "A description of the error that occurs when the query history database cannot be opened.", "isCommentAutoGenerated" : true, @@ -181418,6 +182220,18 @@ }, "TablePro could not reach the Schema Registry: %@" : { + }, + "TablePro could not read %@'s settings file, so it leaves the file alone. Copy the setup and add it by hand." : { + + }, + "TablePro could not read its list of clients. Reinstall TablePro to restore it." : { + + }, + "TablePro could not read its query history, so it moved the file aside as “%@” and started a new one. The copy stays until you delete it or clear all of your history." : { + + }, + "TablePro could not read its saved queries, so it moved the file aside as “%@” and started an empty one. Nothing in that file was deleted." : { + }, "TablePro could not read its Table Health history for this connection." : { @@ -181438,6 +182252,9 @@ }, "TablePro could not read the Table Health history: %@" : { + }, + "TablePro could not read the tokens saved in the Keychain, so it will not save a new one over them. Restart TablePro and try again." : { + }, "TablePro could not read what this connection is holding, so it kept the file." : { "extractionState" : "manual", @@ -181462,6 +182279,9 @@ }, "TablePro could not save the Table Health history: %@" : { + }, + "TablePro could not save the token to the Keychain, so it was not created. Try again." : { + }, "TablePro could not tell how this Dameng server treats backslashes, so it will not send a value that contains one. Reconnect, then try again." : { "extractionState" : "manual", @@ -181487,6 +182307,9 @@ }, "TablePro could not verify your license with the server. Connect to the internet, then check again." : { + }, + "TablePro could not write %@." : { + }, "TablePro could not write the CA bundle DuckDB reads: %@" : { "extractionState" : "manual", @@ -181498,6 +182321,9 @@ "TablePro couldn't confirm that the routine ended. It may still be running on the server." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "TablePro couldn't create a token for this client, so the client wasn't paired. Try pairing again." : { + }, "TablePro couldn't reach the plugin registry to update this plugin. Check your connection and reopen TablePro." : { "localizations" : { @@ -181544,6 +182370,9 @@ }, "TablePro couldn't reach the team library." : { + }, + "TablePro couldn't save a token for this client to the Keychain, so the client wasn't paired. Try pairing again." : { + }, "TablePro Data Was Removed from iCloud" : { "extractionState" : "manual", @@ -182091,6 +182920,12 @@ } } } + }, + "TablePro is turned off in %1$@. Turn it on in %1$@'s MCP settings to use it." : { + + }, + "TablePro moved" : { + }, "TablePro on iPhone and iPad can't connect to %1$@ with the SSL mode %2$@." : { "extractionState" : "manual", @@ -182396,6 +183231,9 @@ } } } + }, + "TablePro's client list has a setup for this client that it cannot fill in." : { + }, "TablePro's data is no longer in iCloud. It may have been deleted in iCloud settings. Upload what this Mac has again, or turn off sync to keep it on this Mac only." : { @@ -182481,6 +183319,9 @@ } } } + }, + "TablePro's Safe Mode is held at Read-Only for this connection, so this tool cannot run." : { + }, "TablePro's Safe Mode is set to read-only for this connection. Destructive operations are not permitted." : { "localizations" : { @@ -182561,6 +183402,9 @@ } } } + }, + "TablePro's Safe Mode is set to Read-Only for this connection. Switch it to %@ to allow this tool." : { + }, "tables" : { "localizations" : { @@ -183774,6 +184618,9 @@ }, "Temporary Table" : { + }, + "temporary tables" : { + }, "Teradata uses TLS on its HTTPS port, which the tunnel does not forward. Set SSL Mode to Disabled or Preferred to connect through the tunnel, or turn the tunnel off." : { "extractionState" : "manual", @@ -185038,6 +185885,19 @@ } } } + }, + "The %@ command did not finish within %d seconds." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The %1$@ command did not finish within %2$d seconds." + } + } + } + }, + "The %@ command is no longer installed." : { + }, "The %@ command line tool is not installed." : { "localizations" : { @@ -185079,6 +185939,16 @@ } } }, + "The %@ command stopped with exit code %d." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The %1$@ command stopped with exit code %2$d." + } + } + } + }, "The %@ filter is not available for Neo4j." : { "extractionState" : "manual", "generatesSymbol" : false @@ -186243,6 +187113,9 @@ } } } + }, + "The change did not run." : { + }, "The change to this table's structure will be lost." : { @@ -186959,6 +187832,9 @@ } } } + }, + "The command is removed from this Mac and from every Mac that syncs settings. This cannot be undone." : { + }, "The command must contain %@, where the local port goes" : { "localizations" : { @@ -186990,9 +187866,19 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The condition of check constraint %@ cannot be written as SQL." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The condition of index %@ cannot be written as SQL." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The connection “%@” has an open transaction. Changes you don’t commit are rolled back." : { + + }, + "The connection “%@” has open transactions. Changes you don’t commit are rolled back." : { + }, "The connection closed." : { @@ -187259,6 +188145,12 @@ } } } + }, + "The connection to %1$@ was lost while it waited and has been reconnected. The new session does not have what the old one set up: %2$@. Set it up again, then run the statements." : { + + }, + "The connection to %1$@ was lost while it waited, and the server rolled back its open transaction. The new session does not have what the old one set up either: %2$@. Set it up again, then run the statements." : { + }, "The connection to Redis dropped after EXEC was sent, so whether the block ran is unknown." : { "extractionState" : "manual", @@ -187785,6 +188677,9 @@ }, "The data has no values to insert into %@." : { + }, + "The database “%@” has an open transaction. Changes you don’t commit are rolled back." : { + }, "The database “%@” has an open transaction. Closing rolls it back and discards its uncommitted changes." : { "localizations" : { @@ -187932,6 +188827,9 @@ } } } + }, + "The database refused the new user Vault created for it for %lld seconds. Some databases load new users on a timer, so try again in a moment." : { + }, "The database server enforced this, not TablePro's Safe Mode. You are most likely connected to a read replica, or the server sets new transactions to read-only. Connect to the primary server to write.\n\nServer response: %@" : { "localizations" : { @@ -189240,6 +190138,10 @@ }, "The global_instrumentation consumer is disabled, so the server records no index reads." : { + }, + "The grid shows only the start of this value, so it cannot be edited here. Change it with a command in the query editor." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "the group coordinator is not available" : { "extractionState" : "manual", @@ -189443,8 +190345,28 @@ } } }, + "The item changed since its CAS value was read, so nothing was saved. Refresh to see its current value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The item does not exist, so nothing was saved. Refresh to see the current keys." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The JSON array must contain only objects." : { + }, + "The JSON file ends before its last value is complete." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The JSON file has text that is not valid in its encoding." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The JSON file is not valid at byte %@." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The JSON is nested more deeply than DynamoDB allows" : { "extractionState" : "manual", @@ -190182,6 +191104,15 @@ }, "The MCP server could not prepare its private socket folder." : { + }, + "The MCP server is not running, so clients you add cannot connect yet. Turn it on in Settings > MCP." : { + + }, + "The MCP server is not running, so TablePro cannot change its tokens now. Start the server, then try again." : { + + }, + "The MCP server is not running, so TablePro cannot save a token now. Start the server, then try again." : { + }, "The MCP server is turned off. Turn it on in Settings > MCP." : { @@ -190429,6 +191360,9 @@ }, "The open transaction failed. Roll it back yourself." : { + }, + "The open transaction on this connection ended while this statement ran." : { + }, "The operation failed." : { "localizations" : { @@ -190772,6 +191706,9 @@ } } } + }, + "The parsed plan did not fit in TablePro's %@ limit for one answer, so it is left out. Read plan_text instead." : { + }, "The partial backup file is removed." : { @@ -191005,6 +191942,9 @@ } } } + }, + "The plugin can't be loaded. You can put it back from the Trash." : { + }, "The plugin depends on a component that's missing or incompatible with this Mac." : { "localizations" : { @@ -191819,6 +192759,10 @@ } } }, + "The RDS Endpoint \"%1$@\" is not the database at \"%2$@\", where the connection goes. Clear the RDS Endpoint unless you connect through a port forward." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The Realtime Database cannot filter with %@. It supports =, <, <=, >, >=, BETWEEN, STARTS WITH and IS NULL." : { "extractionState" : "manual", "generatesSymbol" : false @@ -191877,6 +192821,9 @@ }, "The Redshift Endpoint applies only when Host is a local port forward or that same endpoint, and \"%@\" is neither." : { + }, + "The Redshift Endpoint applies only when Host is a local port forward, a custom domain name or that same endpoint, and \"%@\" is none of them." : { + }, "the reference “%@” is inside itself" : { "extractionState" : "manual", @@ -191890,6 +192837,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The referential action of foreign key %@ is not one SQL Server has: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "the replica is not available on this broker" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -191998,6 +192949,12 @@ } } } + }, + "The response stopped at the model's output token limit." : { + + }, + "The response stopped because the conversation filled the model's context window." : { + }, "The REST endpoint is not a valid URL." : { "extractionState" : "manual", @@ -192126,6 +193083,12 @@ }, "The rows go in as one transaction. Cancel rolls it back." : { + }, + "The rows reached TablePro's %@ limit for one answer, so a result marked truncated holds only the rows that fit. Ask for fewer columns or rows." : { + + }, + "The rows reached TablePro's %@ limit for one answer, so an export marked truncated holds only the rows that fit, and the exports after it may hold none. Export fewer tables, columns or rows per call." : { + }, "The rules file is not a regular file." : { @@ -193150,6 +194113,9 @@ } } } + }, + "The server reported a partition ID that cannot be used in a statement, so nothing was run." : { + }, "The server reported an error." : { "localizations" : { @@ -193984,6 +194950,9 @@ "The SSH port %lld is not between 1 and 65535." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The SSH profile this connection uses now points to another server. Click Reconnect in its window to use it." : { + }, "The SSH server closed the connection during the handshake." : { @@ -194352,6 +195321,12 @@ } } } + }, + "The startup command %@ failed, so this session runs without it." : { + + }, + "The startup command %1$@ failed (%2$@), so this session runs without it." : { + }, "The statement before it stays applied." : { "localizations" : { @@ -194505,6 +195480,13 @@ } } }, + "The statement was still running in the database when it was stopped, and it runs to the end. It may have been applied; check before running it again." : { + + }, + "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The statement's placeholders do not match the %d values given." : { "extractionState" : "manual", "generatesSymbol" : false @@ -195964,6 +196946,10 @@ }, "The value ends early, at byte %lld." : { + }, + "The value in %@ expired after the grid read it. Refresh, then edit the row again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query." : { "extractionState" : "manual", @@ -196077,6 +197063,10 @@ }, "The wait/io/table/sql/handler instrument is disabled, so the server records no index reads." : { + }, + "The workbook has more sheets than an XLSX file can hold." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The workbook has no worksheet to read." : { "extractionState" : "manual", @@ -196225,10 +197215,18 @@ } } }, + "Theme format %1$d is no longer supported. This version of TablePro reads format %2$d and later." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Theme format %1$d is no longer supported. This version of TablePro reads format %2$d." : { "extractionState" : "manual", "generatesSymbol" : false }, + "Theme format %1$d was made for a newer TablePro. This version reads format %2$d and earlier." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Theme format %1$d was made for a newer TablePro. This version reads format %2$d." : { "extractionState" : "manual", "generatesSymbol" : false @@ -196527,6 +197525,9 @@ }, "These columns would leave too little room to scroll the rest." : { + }, + "These connections have open transactions: %@. Changes you don’t commit are rolled back." : { + }, "These connections open their tunnel by running a command on this Mac, every time they connect:\n\n%@" : { "extractionState" : "stale", @@ -196544,6 +197545,9 @@ }, "These connections run commands every time they connect: a tunnel command on this Mac, SQL on the server, or both.\n\n%@" : { + }, + "These databases have open transactions: %@. Changes you don’t commit are rolled back." : { + }, "These databases have open transactions: %@. Closing rolls them back and discards their uncommitted changes." : { "localizations" : { @@ -196619,6 +197623,9 @@ } } } + }, + "These rows came from another schema than the one this tab is now using." : { + }, "These rows cannot be edited because TablePro cannot tell which table they came from." : { "localizations" : { @@ -196729,6 +197736,13 @@ }, "These statements ran inside the transaction already open on this connection, so rolling it back undoes them:" : { + }, + "These theme keys need theme format %1$d: %2$@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "These tokens will be permanently deleted. External clients using them will lose access immediately." : { + }, "These two engines cannot share a script." : { "localizations" : { @@ -197000,6 +198014,9 @@ }, "This change does not apply to this kind of type." : { + }, + "This change is pending in a transaction the statement opened. Other statements are refused until it is committed or rolled back." : { + }, "This change needs %@ to be recreated, and it cannot be recreated from the definition the database stored for it. Virtual tables, such as full-text search tables, are the usual reason." : { "localizations" : { @@ -197010,6 +198027,12 @@ } } } + }, + "This change runs on its own connection and can wait until you commit or roll back the open transaction." : { + + }, + "This change sets session options around its statements, so it can't wait for Commit. Turn off Manual Commit in the Database menu to run it." : { + }, "This change was not authorized" : { "localizations" : { @@ -197207,6 +198230,9 @@ } } } + }, + "This connection can't hold this change until you commit, so it did not run. Turn off Manual Commit in the Database menu to run it." : { + }, "This connection can't open the separate session live messages need." : { @@ -197218,6 +198244,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This connection cannot load the rows of COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This connection comes from a file in a linked folder, so its author chose the server and how it signs in." : { }, @@ -197276,6 +198306,9 @@ "This connection has a transaction open. Commit or roll it back in TablePro, then try again." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "This connection has an open transaction. Closing rolls it back and discards its uncommitted changes." : { + }, "This connection has an open transaction. Commit or roll it back first." : { "extractionState" : "manual", @@ -197288,6 +198321,9 @@ } } } + }, + "This connection has an open transaction. Disconnecting rolls it back and discards its uncommitted changes." : { + }, "This connection has another database attached, which closing the file would detach." : { "extractionState" : "manual", @@ -197971,6 +199007,9 @@ } } } + }, + "This database can't report an open transaction, so it commits each change." : { + }, "This database cannot be renamed" : { "extractionState" : "manual", @@ -198172,6 +199211,9 @@ }, "This database cannot store a table comment." : { + }, + "This database commits schema and maintenance statements on its own, so changes can't wait for Commit." : { + }, "This database did not return a query plan for the statement." : { "comment" : "Text displayed in a description when the database did not return a query plan for the statement.", @@ -198312,6 +199354,10 @@ } } }, + "This database has no column defaults, so a cell cannot be left to one. Enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This database has no Create Table form" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -198432,6 +199478,9 @@ }, "This database's driver cannot write SQL into %@. Update the driver." : { + }, + "This database's driver may read the text __DEFAULT__ in %@ as the column default, so it cannot be saved as text." : { + }, "This deletes only the queries listed here, across every connection. Queries the source, date, outcome and search filters are hiding stay. You cannot undo this." : { "comment" : "Message for the \"Delete this connection's query history\" button.", @@ -198833,6 +199882,10 @@ } } }, + "This driver cannot send rows to COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This driver has no TLS fallback. Preferred forces TLS, same as Required." : { "localizations" : { "fr" : { @@ -199775,6 +200828,12 @@ } } } + }, + "This import runs each statement on its own, so nothing can wait for Commit. Turn on Wrap in transaction or turn off Manual Commit." : { + + }, + "This import turns off foreign key checks, which can't wait for Commit. Turn off Disable foreign key checks or Manual Commit to run it." : { + }, "This index needs a name." : { "localizations" : { @@ -201462,6 +202521,9 @@ } } } + }, + "This query calls a function that can change data" : { + }, "This query language has no formatter." : { "localizations" : { @@ -201473,6 +202535,14 @@ } } }, + "This record cannot be found again from what was read. Refresh and try again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "This record's id reads like a UUID, array or object id, so its text does not say which record it is. Open the table to edit it, or use a query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This replaces %@ on this Mac." : { }, @@ -201529,6 +202599,10 @@ }, "This routine takes no arguments." : { + }, + "This row cannot be found again from what was read. Reload the table." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "This row does not name a stored document." : { "extractionState" : "manual", @@ -201567,6 +202641,13 @@ }, "This row has no value for this field." : { + }, + "This row has no values, and %@ has no way to insert a row of only defaults." : { + + }, + "This row holds only the part of a value the grid shows, so adding it would store that part. Add the key with a command in the query editor." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names." : { "extractionState" : "manual", @@ -201623,6 +202704,9 @@ } } } + }, + "This run mixes changes with a statement that can't run in a transaction. Run that statement on its own, or turn off Manual Commit." : { + }, "This save also edits key columns, and the counts don't include those edits." : { @@ -202221,6 +203305,9 @@ }, "This Tab" : { + }, + "This tab is closed. Reopen it to see its history." : { + }, "This tab is on %@. Switch the connection to that database to run it." : { "localizations" : { @@ -202610,6 +203697,10 @@ } } }, + "This version of SQL Import cannot run line %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This version of TablePro can no longer sync with iCloud. Update TablePro to resume." : { }, @@ -202963,6 +204054,9 @@ } } } + }, + "This will permanently delete all query history entries, the saved changes kept for restoring, and the earlier versions of your query tabs. This action cannot be undone." : { + }, "This will reset all settings across every section to their default values." : { "localizations" : { @@ -203003,6 +204097,9 @@ } } } + }, + "This will reset all settings across every section to their default values. AI providers, their API keys and Enable AI Features are kept." : { + }, "This workbook has no worksheets." : { "localizations" : { @@ -204676,6 +205773,12 @@ } } } + }, + "Tokens are not available" : { + + }, + "Tokens load while the MCP server is running." : { + }, "Too Large to Compare" : { @@ -206938,6 +208041,9 @@ } } } + }, + "Turned off" : { + }, "Two columns are both named “%@”. Rename one, then save again." : { "localizations" : { @@ -207473,6 +208579,18 @@ } } } + }, + "Uncommitted changes on “%@” are rolled back when TablePro quits." : { + + }, + "Uncommitted changes on “%@” are rolled back when TablePro relaunches." : { + + }, + "Uncommitted changes on %@ are rolled back when TablePro quits." : { + + }, + "Uncommitted changes on %@ are rolled back when TablePro relaunches." : { + }, "Under MCP Servers click \"+ Add Custom Server\", select the Local tab, paste the JSON below, then click Add Server" : { "localizations" : { @@ -209970,6 +211088,9 @@ } } } + }, + "Update in %@" : { + }, "Update Now" : { "localizations" : { @@ -210240,6 +211361,9 @@ } } } + }, + "Update…" : { + }, "Updated" : { "localizations" : { @@ -211814,6 +212938,9 @@ } } } + }, + "user variables" : { + }, "User-Defined" : { "localizations" : { @@ -212069,6 +213196,9 @@ } } } + }, + "Uses another copy" : { + }, "Uses the access key stored in ~/.aws/credentials." : { "localizations" : { @@ -212214,6 +213344,8 @@ } }, "UUID" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -213139,6 +214271,14 @@ } } }, + "Values" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Values a spreadsheet would read as formulas, written with a leading single quote: %lld. Turn off Sanitize formula-like values to export them unchanged." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Values from %d loaded rows" : { "extractionState" : "stale", "localizations" : { @@ -213359,6 +214499,9 @@ }, "Vault Address" : { + }, + "Vault could not complete the request: %@" : { + }, "Vault could not issue credentials from %@: %@" : { "localizations" : { @@ -213394,6 +214537,9 @@ }, "Vault returned a sign-in page that is not an https address, so TablePro did not open it." : { + }, + "Vault returned a sign-in page without a state, so TablePro could not check the browser's answer." : { + }, "Vault returned no sign-in page. The OIDC role must allow the redirect URI http://localhost:8250/oidc/callback." : { @@ -213599,6 +214745,10 @@ } } }, + "Verify CA needs the server's CA certificate. Choose it under SSL, or use Verify Identity to check the server against this Mac's trusted certificates." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Verify certificate and hostname" : { "localizations" : { "fr" : { @@ -213793,6 +214943,8 @@ } }, "Version" : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -214587,6 +215739,9 @@ }, "Waiting for a compatible build" : { + }, + "Waiting for the database to accept the new Vault user" : { + }, "Waiting for the tunnel" : { "localizations" : { @@ -215235,6 +216390,9 @@ } } } + }, + "What happened to the open transaction, when it matters" : { + }, "What is switched on here is ignored when two structures are compared. These drift between environments by design." : { "localizations" : { @@ -216517,6 +217675,16 @@ } } }, + "Whether the row limit or the answer's size limit left rows out" : { + + }, + "Whether the row limit or the answer's size limit left rows out of any export" : { + + }, + "Whether the row limit or the answer's size limit left rows out of the result" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Whether the session is open" : { "localizations" : { "fr" : { @@ -216556,6 +217724,9 @@ } } } + }, + "Whether the size limit left out matching entries" : { + }, "Whether the statement names an object, the database, or either" : { "localizations" : { @@ -216958,6 +218129,13 @@ }, "Why fix_sql is left out: running it with the other fixes would do harm" : { + }, + "Why rows were left out or values cut, when they were" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Why rows were left out, when the size limit left them out" : { + }, "Why the check was skipped or failed" : { @@ -218200,6 +219378,10 @@ } } }, + "XLSX file exceeds 4 GB ZIP limit" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "XXXXX-XXXXX-XXXXX-XXXXX-XXXXX" : { "extractionState" : "stale", "localizations" : { @@ -218325,7 +219507,8 @@ } }, "Yes" : { - + "extractionState" : "manual", + "generatesSymbol" : false }, "Yesterday" : { "localizations" : { @@ -218835,6 +220018,9 @@ } } } + }, + "Your first change opens a transaction, and nothing is saved until you commit. Structure changes, Users & Roles, Copy To, Generate Rows and Shortcuts actions still save at once." : { + }, "Your license has expired" : { "localizations" : { diff --git a/apple/TablePro/Resources/Themes/tablepro.catppuccin-latte.json b/apple/TablePro/Resources/Themes/tablepro.catppuccin-latte.json index 8cf7be7217..6ba703010b 100644 --- a/apple/TablePro/Resources/Themes/tablepro.catppuccin-latte.json +++ b/apple/TablePro/Resources/Themes/tablepro.catppuccin-latte.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#E6E9EF", "background": "#EFF1F5", - "boolFalse": "#D20F39", - "boolTrue": "#40A02B", + "binary": "#B4008A", + "boolFalse": "#C10031", + "boolTrue": "#226E0E", + "dateTime": "#0551E4", "deleted": "#D20F3926", "deletedText": "#D20F3980", + "enum": "#7E29E2", "focusBorder": "#8839EF", "gridLine": "#BCC0CC", "headerBackground": "#E6E9EF", "headerText": "#4C4F69", "inactiveSelection": "#CCD0DA", "inserted": "#40A02B26", + "json": "#875400", "modified": "#DF8E1D26", "nullValue": "#7C7F93", + "number": "#A73D00", "rowNumber": "#8C8FA1", "selection": "#CCD0DA", "selectionText": "#4C4F69", - "text": "#4C4F69" + "string": "#4653C5", + "text": "#4C4F69", + "uuid": "#006A70" }, "editor": { "background": "#EFF1F5", @@ -43,7 +50,7 @@ }, "id": "tablepro.catppuccin-latte", "name": "Catppuccin Latte", - "schema": 3, + "schema": 4, "window": { "background": "#EFF1F5" } diff --git a/apple/TablePro/Resources/Themes/tablepro.catppuccin-mocha.json b/apple/TablePro/Resources/Themes/tablepro.catppuccin-mocha.json index 2ce5a2d0e8..929bf39cf4 100644 --- a/apple/TablePro/Resources/Themes/tablepro.catppuccin-mocha.json +++ b/apple/TablePro/Resources/Themes/tablepro.catppuccin-mocha.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#181825", "background": "#1E1E2E", + "binary": "#F5C2E7", "boolFalse": "#F38BA8", "boolTrue": "#A6E3A1", + "dateTime": "#89B4FA", "deleted": "#F38BA826", "deletedText": "#F38BA880", + "enum": "#CBA6F7", "focusBorder": "#CBA6F7", "gridLine": "#45475A", "headerBackground": "#181825", "headerText": "#CDD6F4", "inactiveSelection": "#313244", "inserted": "#A6E3A126", + "json": "#F9E2AF", "modified": "#F9E2AF26", "nullValue": "#9399B2", + "number": "#FAB387", "rowNumber": "#7F849C", "selection": "#313244", "selectionText": "#CDD6F4", - "text": "#CDD6F4" + "string": "#B4BEFE", + "text": "#CDD6F4", + "uuid": "#94E2D5" }, "editor": { "background": "#1E1E2E", @@ -43,7 +50,7 @@ }, "id": "tablepro.catppuccin-mocha", "name": "Catppuccin Mocha", - "schema": 3, + "schema": 4, "window": { "background": "#1E1E2E" } diff --git a/apple/TablePro/Resources/Themes/tablepro.dracula.json b/apple/TablePro/Resources/Themes/tablepro.dracula.json index 94a1575329..06197213f5 100644 --- a/apple/TablePro/Resources/Themes/tablepro.dracula.json +++ b/apple/TablePro/Resources/Themes/tablepro.dracula.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#21222C", "background": "#282A36", - "boolFalse": "#FF5555", + "binary": "#9DAFE5", + "boolFalse": "#FF928B", "boolTrue": "#50FA7B", + "dateTime": "#8BE9FD", "deleted": "#FF555526", "deletedText": "#FF555580", + "enum": "#A4FFFF", "focusBorder": "#BD93F9", "gridLine": "#44475A", "headerBackground": "#21222C", "headerText": "#F8F8F2", "inactiveSelection": "#44475A80", "inserted": "#50FA7B26", - "modified": "#FFB86C4D", + "json": "#FFB86C", + "modified": "#FFB86C26", "nullValue": "#6272A4", + "number": "#C59EFF", "rowNumber": "#6272A4", "selection": "#44475A", "selectionText": "#F8F8F2", - "text": "#F8F8F2" + "string": "#F1FA8C", + "text": "#F8F8F2", + "uuid": "#FF89CB" }, "editor": { "background": "#282A36", @@ -43,7 +50,7 @@ }, "id": "tablepro.dracula", "name": "Dracula", - "schema": 3, + "schema": 4, "window": { "background": "#282A36" } diff --git a/apple/TablePro/Resources/Themes/tablepro.github-dark.json b/apple/TablePro/Resources/Themes/tablepro.github-dark.json index cb921d592e..3e747ef26b 100644 --- a/apple/TablePro/Resources/Themes/tablepro.github-dark.json +++ b/apple/TablePro/Resources/Themes/tablepro.github-dark.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#161B22", "background": "#0D1117", - "boolFalse": "#F85149", + "binary": "#FF9BCE", + "boolFalse": "#FF574F", "boolTrue": "#3FB950", + "dateTime": "#D2A8FF", "deleted": "#F8514926", "deletedText": "#F8514980", + "enum": "#E3B341", "focusBorder": "#1F6FEB", "gridLine": "#30363D", "headerBackground": "#161B22", "headerText": "#E6EDF3", "inactiveSelection": "#343941", "inserted": "#2EA04326", + "json": "#FFA657", "modified": "#BB800926", "nullValue": "#8B949E", + "number": "#79C0FF", "rowNumber": "#6E7681", "selection": "#142744", "selectionText": "#E6EDF3", - "text": "#E6EDF3" + "string": "#A5D6FF", + "text": "#E6EDF3", + "uuid": "#FFA28B" }, "editor": { "background": "#0D1117", @@ -43,7 +50,7 @@ }, "id": "tablepro.github-dark", "name": "GitHub Dark", - "schema": 3, + "schema": 4, "window": { "background": "#0D1117" } diff --git a/apple/TablePro/Resources/Themes/tablepro.github-light.json b/apple/TablePro/Resources/Themes/tablepro.github-light.json index b18d295e08..424566e106 100644 --- a/apple/TablePro/Resources/Themes/tablepro.github-light.json +++ b/apple/TablePro/Resources/Themes/tablepro.github-light.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#F6F8FA", "background": "#FFFFFF", - "boolFalse": "#CF222E", - "boolTrue": "#1A7F37", + "binary": "#B93384", + "boolFalse": "#CE212E", + "boolTrue": "#127A33", + "dateTime": "#7D4BD9", "deleted": "#FF818226", "deletedText": "#CF222E80", + "enum": "#916000", "focusBorder": "#0969DA", "gridLine": "#D0D7DE", "headerBackground": "#F6F8FA", "headerText": "#1F2328", "inactiveSelection": "#EFF1F3", "inserted": "#4AC26B26", + "json": "#953800", "modified": "#D4A72C26", "nullValue": "#6E7781", + "number": "#0550AE", "rowNumber": "#8C959F", "selection": "#CEE1F8", "selectionText": "#1F2328", - "text": "#1F2328" + "string": "#0A3069", + "text": "#1F2328", + "uuid": "#0066D8" }, "editor": { "background": "#FFFFFF", @@ -43,7 +50,7 @@ }, "id": "tablepro.github-light", "name": "GitHub Light", - "schema": 3, + "schema": 4, "window": { "background": "#FFFFFF" } diff --git a/apple/TablePro/Resources/Themes/tablepro.gruvbox-dark.json b/apple/TablePro/Resources/Themes/tablepro.gruvbox-dark.json index 7cb0d25f12..1ddabd1674 100644 --- a/apple/TablePro/Resources/Themes/tablepro.gruvbox-dark.json +++ b/apple/TablePro/Resources/Themes/tablepro.gruvbox-dark.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#32302F", "background": "#282828", - "boolFalse": "#FB4934", + "binary": "#FFB7CA", + "boolFalse": "#FF9381", "boolTrue": "#B8BB26", + "dateTime": "#96B8AB", "deleted": "#FB493426", "deletedText": "#FB493480", + "enum": "#FF9651", "focusBorder": "#689D6A", "gridLine": "#504945", "headerBackground": "#32302F", "headerText": "#EBDBB2", "inactiveSelection": "#3C3836", "inserted": "#B8BB2626", + "json": "#FABD2F", "modified": "#D7992126", "nullValue": "#928374", + "number": "#E99BB0", "rowNumber": "#665C54", "selection": "#384539", "selectionText": "#EBDBB2", - "text": "#EBDBB2" + "string": "#A7DA95", + "text": "#EBDBB2", + "uuid": "#8EC07C" }, "editor": { "background": "#282828", @@ -43,7 +50,7 @@ }, "id": "tablepro.gruvbox-dark", "name": "Gruvbox Dark", - "schema": 3, + "schema": 4, "window": { "background": "#282828" } diff --git a/apple/TablePro/Resources/Themes/tablepro.gruvbox-light.json b/apple/TablePro/Resources/Themes/tablepro.gruvbox-light.json index 2ef6cd9cf9..6d220a2e33 100644 --- a/apple/TablePro/Resources/Themes/tablepro.gruvbox-light.json +++ b/apple/TablePro/Resources/Themes/tablepro.gruvbox-light.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#F2E5BC", "background": "#FBF1C7", + "binary": "#76285A", "boolFalse": "#9D0006", - "boolTrue": "#79740E", + "boolTrue": "#625E00", + "dateTime": "#036476", "deleted": "#9D000626", "deletedText": "#9D000680", + "enum": "#9A4000", "focusBorder": "#427B58", "gridLine": "#D5C4A1", "headerBackground": "#F2E5BC", "headerText": "#3C3836", "inactiveSelection": "#EBDBB2", "inserted": "#79740E26", + "json": "#805100", "modified": "#D7992126", "nullValue": "#928374", + "number": "#8F3F71", "rowNumber": "#BDAE93", "selection": "#D6DCB0", "selectionText": "#3C3836", - "text": "#3C3836" + "string": "#135031", + "text": "#3C3836", + "uuid": "#2E6646" }, "editor": { "background": "#FBF1C7", @@ -43,7 +50,7 @@ }, "id": "tablepro.gruvbox-light", "name": "Gruvbox Light", - "schema": 3, + "schema": 4, "window": { "background": "#FBF1C7" } diff --git a/apple/TablePro/Resources/Themes/tablepro.nord.json b/apple/TablePro/Resources/Themes/tablepro.nord.json index a2b30e1b3b..d9d89063d9 100644 --- a/apple/TablePro/Resources/Themes/tablepro.nord.json +++ b/apple/TablePro/Resources/Themes/tablepro.nord.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#3B4252", "background": "#2E3440", - "boolFalse": "#BF616A", - "boolTrue": "#A3BE8C", + "binary": "#FFDBF8", + "boolFalse": "#FFBAC0", + "boolTrue": "#B8D4A1", + "dateTime": "#ADCFF0", "deleted": "#BF616A26", "deletedText": "#BF616A80", + "enum": "#FFBCA6", "focusBorder": "#88C0D0", "gridLine": "#434C5E", "headerBackground": "#3B4252", "headerText": "#D8DEE9", "inactiveSelection": "#434C5E80", "inserted": "#A3BE8C26", - "modified": "#EBCB8B4D", + "json": "#EBCB8B", + "modified": "#EBCB8B26", "nullValue": "#4C566A", + "number": "#E7C0E0", "rowNumber": "#4C566A", "selection": "#434C5E", "selectionText": "#ECEFF4", - "text": "#D8DEE9" + "string": "#9CD4E4", + "text": "#D8DEE9", + "uuid": "#A6D4D3" }, "editor": { "background": "#2E3440", @@ -43,7 +50,7 @@ }, "id": "tablepro.nord", "name": "Nord", - "schema": 3, + "schema": 4, "window": { "background": "#2E3440" } diff --git a/apple/TablePro/Resources/Themes/tablepro.one-dark.json b/apple/TablePro/Resources/Themes/tablepro.one-dark.json index f9098d550a..82eb6a8567 100644 --- a/apple/TablePro/Resources/Themes/tablepro.one-dark.json +++ b/apple/TablePro/Resources/Themes/tablepro.one-dark.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#21252B", "background": "#282C34", - "boolFalse": "#E06C75", + "binary": "#EEB5FF", + "boolFalse": "#FF8D94", "boolTrue": "#98C379", + "dateTime": "#67B5F5", "deleted": "#E0525226", "deletedText": "#E06C7580", + "enum": "#DF90F6", "focusBorder": "#528BFF", "gridLine": "#3E4452", "headerBackground": "#21252B", "headerText": "#ABB2BF", "inactiveSelection": "#323842", "inserted": "#43D08A26", + "json": "#E5C07B", "modified": "#E0C28526", "nullValue": "#7F848E", + "number": "#DBA36F", "rowNumber": "#636D83", "selection": "#3E4451", "selectionText": "#D7DAE0", - "text": "#ABB2BF" + "string": "#77D6E2", + "text": "#ABB2BF", + "uuid": "#5CBCC8" }, "editor": { "background": "#282C34", @@ -43,7 +50,7 @@ }, "id": "tablepro.one-dark", "name": "One Dark", - "schema": 3, + "schema": 4, "window": { "background": "#282C34" } diff --git a/apple/TablePro/Resources/Themes/tablepro.one-light.json b/apple/TablePro/Resources/Themes/tablepro.one-light.json index 0af9b52720..431c9075f8 100644 --- a/apple/TablePro/Resources/Themes/tablepro.one-light.json +++ b/apple/TablePro/Resources/Themes/tablepro.one-light.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#EAEAEB", "background": "#FAFAFA", - "boolFalse": "#E45649", - "boolTrue": "#50A14F", + "binary": "#674500", + "boolFalse": "#C40C01", + "boolTrue": "#2D6E2D", + "dateTime": "#1B56DC", "deleted": "#FF141426", "deletedText": "#E4564980", + "enum": "#A525A3", "focusBorder": "#526FFF", "gridLine": "#DBDBDC", "headerBackground": "#EAEAEB", "headerText": "#383A42", "inactiveSelection": "#DBDBDC", "inserted": "#2DB44826", + "json": "#C3003E", "modified": "#F2A60D26", "nullValue": "#A0A1A7", + "number": "#845900", "rowNumber": "#9D9D9F", "selection": "#DBDBDC", "selectionText": "#232324", - "text": "#383A42" + "string": "#005074", + "text": "#383A42", + "uuid": "#006895" }, "editor": { "background": "#FAFAFA", @@ -43,7 +50,7 @@ }, "id": "tablepro.one-light", "name": "One Light", - "schema": 3, + "schema": 4, "window": { "background": "#FAFAFA" } diff --git a/apple/TablePro/Resources/Themes/tablepro.solarized-dark.json b/apple/TablePro/Resources/Themes/tablepro.solarized-dark.json index 7f7374fcab..8d93a15b93 100644 --- a/apple/TablePro/Resources/Themes/tablepro.solarized-dark.json +++ b/apple/TablePro/Resources/Themes/tablepro.solarized-dark.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#00212B", "background": "#002B36", - "boolFalse": "#DC322F", - "boolTrue": "#859900", + "binary": "#FF96C5", + "boolFalse": "#F77471", + "boolTrue": "#91A61F", + "dateTime": "#42A3EB", "deleted": "#DC322F26", "deletedText": "#DC322F80", + "enum": "#9297DE", "focusBorder": "#2AA198", "gridLine": "#23464F", "headerBackground": "#00212B", "headerText": "#93A1A1", "inactiveSelection": "#003846", "inserted": "#85990026", + "json": "#C29620", "modified": "#B5890026", "nullValue": "#586E75", + "number": "#EF72AD", "rowNumber": "#586E75", "selection": "#005A6F", "selectionText": "#EEE8D5", - "text": "#839496" + "string": "#3AADA3", + "text": "#839496", + "uuid": "#FA7540" }, "editor": { "background": "#002B36", @@ -43,7 +50,7 @@ }, "id": "tablepro.solarized-dark", "name": "Solarized Dark", - "schema": 3, + "schema": 4, "window": { "background": "#002B36" } diff --git a/apple/TablePro/Resources/Themes/tablepro.solarized-light.json b/apple/TablePro/Resources/Themes/tablepro.solarized-light.json index 9ff51235cf..4fd142b231 100644 --- a/apple/TablePro/Resources/Themes/tablepro.solarized-light.json +++ b/apple/TablePro/Resources/Themes/tablepro.solarized-light.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#EEE8D5", "background": "#FDF6E3", - "boolFalse": "#DC322F", - "boolTrue": "#859900", + "binary": "#940053", + "boolFalse": "#C2000D", + "boolTrue": "#586500", + "dateTime": "#00639E", "deleted": "#DC322F26", "deletedText": "#DC322F80", + "enum": "#5257AC", "focusBorder": "#268BD2", "gridLine": "#DDD6C1", "headerBackground": "#EEE8D5", "headerText": "#586E75", "inactiveSelection": "#EEE0B5", "inserted": "#85990026", + "json": "#795B00", "modified": "#B5890026", "nullValue": "#93A1A1", + "number": "#B90B6A", "rowNumber": "#93A1A1", "selection": "#DFCA88", "selectionText": "#073642", - "text": "#657B83" + "string": "#006B65", + "text": "#657B83", + "uuid": "#AC3800" }, "editor": { "background": "#FDF6E3", @@ -43,7 +50,7 @@ }, "id": "tablepro.solarized-light", "name": "Solarized Light", - "schema": 3, + "schema": 4, "window": { "background": "#FDF6E3" } diff --git a/apple/TablePro/Resources/Themes/tablepro.tokyo-night-light.json b/apple/TablePro/Resources/Themes/tablepro.tokyo-night-light.json index 02342393c3..f5795fcf6d 100644 --- a/apple/TablePro/Resources/Themes/tablepro.tokyo-night-light.json +++ b/apple/TablePro/Resources/Themes/tablepro.tokyo-night-light.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#D6D8DF", "background": "#E6E7ED", - "boolFalse": "#8C4351", - "boolTrue": "#385F0D", + "binary": "#0F4B6E", + "boolFalse": "#843C4A", + "boolTrue": "#375D0A", + "dateTime": "#2150A1", "deleted": "#E8686826", "deletedText": "#8C435180", + "enum": "#65359D", "focusBorder": "#2959AA", "gridLine": "#C1C2C7", "headerBackground": "#D6D8DF", "headerText": "#343B59", "inactiveSelection": "#C9CBD6", "inserted": "#2D9C9126", + "json": "#754A00", "modified": "#8F5E1526", "nullValue": "#888B94", + "number": "#844015", "rowNumber": "#9DA0AB", "selection": "#ACB0BF", "selectionText": "#343B59", - "text": "#343B59" + "string": "#5A4A78", + "text": "#343B59", + "uuid": "#2B5B55" }, "editor": { "background": "#E6E7ED", @@ -43,7 +50,7 @@ }, "id": "tablepro.tokyo-night-light", "name": "Tokyo Night Light", - "schema": 3, + "schema": 4, "window": { "background": "#E6E7ED" } diff --git a/apple/TablePro/Resources/Themes/tablepro.tokyo-night.json b/apple/TablePro/Resources/Themes/tablepro.tokyo-night.json index 3fa07811f3..84e809759f 100644 --- a/apple/TablePro/Resources/Themes/tablepro.tokyo-night.json +++ b/apple/TablePro/Resources/Themes/tablepro.tokyo-night.json @@ -4,22 +4,29 @@ "dataGrid": { "alternateRow": "#16161E", "background": "#1A1B26", + "binary": "#7DCFFF", "boolFalse": "#F7768E", "boolTrue": "#9ECE6A", + "dateTime": "#7AA2F7", "deleted": "#DB4B4B26", "deletedText": "#F7768E80", + "enum": "#BB9AF7", "focusBorder": "#7AA2F7", "gridLine": "#232433", "headerBackground": "#16161E", "headerText": "#A9B1D6", "inactiveSelection": "#222433", "inserted": "#41A6B526", + "json": "#E0AF68", "modified": "#E0AF6826", "nullValue": "#51597D", + "number": "#FF9E64", "rowNumber": "#363B54", "selection": "#2B2F41", "selectionText": "#A9B1D6", - "text": "#A9B1D6" + "string": "#B4F9F8", + "text": "#A9B1D6", + "uuid": "#73DACA" }, "editor": { "background": "#1A1B26", @@ -43,7 +50,7 @@ }, "id": "tablepro.tokyo-night", "name": "Tokyo Night", - "schema": 3, + "schema": 4, "window": { "background": "#1A1B26" } diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/licenses.yml b/apple/TablePro/Resources/ThirdPartyLicenses/licenses.yml index ad828ffcdf..b499b5b99f 100644 --- a/apple/TablePro/Resources/ThirdPartyLicenses/licenses.yml +++ b/apple/TablePro/Resources/ThirdPartyLicenses/licenses.yml @@ -269,6 +269,21 @@ LICENSE at the tag is byte-identical to the Go distribution''s LICENSE, but this is a separate module with its own version, so it has its own entry. Reaches users only through the registry-only SAP HANA plugin.' +- id: goose + name: goose + version: vendored at bf760fd63, 2025-03-24 + spdx: Apache-2.0 + copyrights: + - Copyright 2024 Block, Inc. + homepageURL: https://github.com/aaif-goose/goose + licenseTextURL: https://github.com/aaif-goose/goose/blob/bf760fd630b9accd1eff63adba24798d6bf7413a/LICENSE + textFile: texts/goose.txt + source: manual + patched: false + platforms: [macos] + notes: The goose logo only (TablePro/Assets.xcassets/goose-icon.imageset/goose.svg), byte-identical to + documentation/static/img/goose.svg at bf760fd630b9 (block/goose, now aaif-goose/goose). No code ships. The + repository has no NOTICE file. The mark is the project's trademark; the licence covers the file only. - id: vscode-theme-gruvbox name: Gruvbox Theme version: 1.29.1 @@ -360,6 +375,20 @@ Two clause BSD license", and the inet_pton / inet_ntop implementations in src/inet.c, "copyright the Internet Systems Consortium, Inc.", ISC licence. The repo also has LICENSE-docs (documentation only, not linked).' +- id: llama-cpp + name: llama.cpp + version: vendored at ef9c13d4c, 2026-06-24 + spdx: MIT + copyrights: + - Copyright (c) 2023-2026 The ggml authors + homepageURL: https://github.com/ggml-org/llama.cpp + licenseTextURL: https://github.com/ggml-org/llama.cpp/blob/ef9c13d4c27f1ad9804341a0187c33b621609c63/LICENSE + textFile: texts/llama-cpp.txt + source: manual + patched: false + platforms: [macos] + notes: The llama.cpp logo only (TablePro/Assets.xcassets/llamacpp-icon.imageset/llamacpp.svg), byte-identical + to tools/ui/src/lib/assets/logo.svg at ef9c13d4c27f. No code ships. - id: mariadb-connector-c name: MariaDB Connector/C version: 3.4.4 @@ -387,6 +416,22 @@ LGPL: shipping the app must keep the relinking route open (object files or the shipped .a plus the licence text). The build uses Homebrew openssl@3, not the pinned OpenSSL. scripts/ios/build-mariadb-ios.sh pins the same 3.4.4.' +- id: mlx + name: MLX + version: vendored at 9ab391356, 2026-03-24 + spdx: MIT + copyrights: + - Copyright (c) 2023 Apple Inc. + homepageURL: https://github.com/ml-explore/mlx + licenseTextURL: https://github.com/ml-explore/mlx/blob/9ab3913567df16cac539dce7b4e5d569897a1c29/LICENSE + textFile: texts/mlx.txt + source: manual + patched: false + platforms: [macos] + notes: The MLX logo only (TablePro/Assets.xcassets/mlx-icon.imageset/mlx.pdf and mlx-dark.pdf), byte-identical + to docs/logo/mlx_logo.pdf and mlx_logo_dark.pdf at 9ab3913567df. The PDFs, not the SVGs beside them, because + the SVGs draw their glyphs through and render empty in AppKit. No code ships. The original LICENSE + writes the copyright sign; this YAML is ASCII-folded. - id: mongo-c-driver name: MongoDB C Driver (libmongoc / libbson) version: 1.28.1 @@ -406,6 +451,20 @@ grant), mongoc.css_t, zlib, common-md5.c and utf8proc. The full text is saved in the extra text file. Tarball is the GitHub release archive for 1.28.1, pinned by SHA-256. The script writes the flat Libs/libmongoc.a and Libs/libbson.a itself.' +- id: n8n-requesty + name: Requesty n8n node + version: vendored at 6aeafb9e9, 2026-06-02 + spdx: MIT + copyrights: + - Copyright (c) 2024 Requesty + homepageURL: https://github.com/requestyai/n8n-requesty + licenseTextURL: https://github.com/requestyai/n8n-requesty/blob/6aeafb9e98d8b2b783a1316f2a038cf1b729dab4/LICENSE + textFile: texts/n8n-requesty.txt + source: manual + patched: false + platforms: [macos] + notes: The Requesty logo only (TablePro/Assets.xcassets/requesty-icon.imageset/requesty.svg), byte-identical to + icons/requesty.svg at 6aeafb9e98d8, the commit "use official Requesty bubble logo as node icon". No code ships. - id: nord-visual-studio-code name: Nord Visual Studio Code version: vendored at 8ead09822, 2023-05-03 @@ -468,6 +527,21 @@ Atom's original One Light palette (styles/colors.less and styles/syntax-variables.less, package version 1.8.4; the repository is archived) supplies its git colors and confirms the editor values. No code or file from the project ships. +- id: openai-realtime-console + name: OpenAI Realtime Console + version: vendored at 758faf83a, 2024-12-19 + spdx: MIT + copyrights: + - Copyright (c) 2024 OpenAI + homepageURL: https://github.com/openai/openai-realtime-console + licenseTextURL: https://github.com/openai/openai-realtime-console/blob/758faf83a11d0026baafeb7bc072ee2eb9f1c2f9/LICENSE + textFile: texts/openai-realtime-console.txt + source: manual + patched: false + platforms: [macos] + notes: The OpenAI logomark only (TablePro/Assets.xcassets/openai-icon.imageset/openai.svg), byte-identical to + client/assets/openai-logomark.svg at 758faf83a11d, the same bytes as public/openai-logomark.svg in + openai/openai-realtime-agents. No code ships. The mark is an OpenAI trademark (openai.com/brand). - id: openssl name: OpenSSL version: 3.5.9 @@ -562,6 +636,21 @@ file. The COPYRIGHT text is the whole licence; SPDX calls it "PostgreSQL" (a BSD/MIT-style permissive licence), not BSD-2-Clause. The same build also emits Libs/libssl_*.a and Libs/libcrypto_*.a, which are OpenSSL, not PostgreSQL. scripts/ios/build-libpq-ios.sh pins the same 17.4. +- id: primer-octicons + name: Primer Octicons + version: copilot-24, through Simple Icons 16.37.0 + spdx: MIT + copyrights: + - Copyright (c) 2026 GitHub Inc. + homepageURL: https://github.com/primer/octicons + licenseTextURL: https://github.com/primer/octicons/blob/41c780575654fe4fef3fc4e91a46c8c0e91b233b/LICENSE + textFile: texts/primer-octicons.txt + source: manual + patched: false + platforms: [macos] + notes: The GitHub Copilot mark only (TablePro/Assets.xcassets/githubcopilot-icon.imageset/githubcopilot.svg), + byte-identical to icons/githubcopilot.svg in the simple-icons 16.37.0 npm package, whose data file records + its source as https://primer.style/foundations/icons/copilot-24 and its license as MIT. No code ships. - id: rearrange name: Rearrange version: 2.1.1 diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/texts/goose.txt b/apple/TablePro/Resources/ThirdPartyLicenses/texts/goose.txt new file mode 100644 index 0000000000..c83043d449 --- /dev/null +++ b/apple/TablePro/Resources/ThirdPartyLicenses/texts/goose.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Block, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/texts/llama-cpp.txt b/apple/TablePro/Resources/ThirdPartyLicenses/texts/llama-cpp.txt new file mode 100644 index 0000000000..e7dca554bc --- /dev/null +++ b/apple/TablePro/Resources/ThirdPartyLicenses/texts/llama-cpp.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023-2026 The ggml authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/texts/mlx.txt b/apple/TablePro/Resources/ThirdPartyLicenses/texts/mlx.txt new file mode 100644 index 0000000000..98ff47b9ef --- /dev/null +++ b/apple/TablePro/Resources/ThirdPartyLicenses/texts/mlx.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright © 2023 Apple Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/texts/n8n-requesty.txt b/apple/TablePro/Resources/ThirdPartyLicenses/texts/n8n-requesty.txt new file mode 100644 index 0000000000..f91f5414c7 --- /dev/null +++ b/apple/TablePro/Resources/ThirdPartyLicenses/texts/n8n-requesty.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Requesty + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/texts/openai-realtime-console.txt b/apple/TablePro/Resources/ThirdPartyLicenses/texts/openai-realtime-console.txt new file mode 100644 index 0000000000..91ba98f3a2 --- /dev/null +++ b/apple/TablePro/Resources/ThirdPartyLicenses/texts/openai-realtime-console.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 OpenAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apple/TablePro/Resources/ThirdPartyLicenses/texts/primer-octicons.txt b/apple/TablePro/Resources/ThirdPartyLicenses/texts/primer-octicons.txt new file mode 100644 index 0000000000..aa1fa80ed8 --- /dev/null +++ b/apple/TablePro/Resources/ThirdPartyLicenses/texts/primer-octicons.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 GitHub Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/apple/TablePro/TablePro.entitlements b/apple/TablePro/TablePro.entitlements index e094e5eb3d..c059f6ed08 100644 --- a/apple/TablePro/TablePro.entitlements +++ b/apple/TablePro/TablePro.entitlements @@ -2,6 +2,8 @@ + com.apple.developer.aps-environment + production com.apple.developer.icloud-container-identifiers iCloud.com.TablePro diff --git a/apple/TablePro/Theme/ThemeCatalog.swift b/apple/TablePro/Theme/ThemeCatalog.swift index e53c4be15e..e6bf3897f5 100644 --- a/apple/TablePro/Theme/ThemeCatalog.swift +++ b/apple/TablePro/Theme/ThemeCatalog.swift @@ -1,4 +1,3 @@ -import AppKit import Combine import Foundation import os @@ -62,14 +61,18 @@ internal struct ThemeFileTrash: Sendable { } } -/// Owns the list. A user theme is identified by its JSON `id` but lives in whatever file it was -/// loaded from, so a save, rename or move to the Trash acts on that file rather than on `.json`; -/// the two differ for any file the user named themselves. Every mutation updates the in-memory list -/// synchronously, and the list reloads whenever TablePro becomes active, so a theme fixed, dropped -/// or removed in Finder is picked up without a view asking. +// Owns the list. A user theme is identified by its JSON `id` but lives in whatever file it was +// loaded from, so a save, rename or move to the Trash acts on that file rather than on `.json`; +// the two differ for any file the user named themselves. Every mutation updates the in-memory list +// synchronously, and the shared catalog reloads when its folder changes on disk, so a theme fixed, +// dropped or removed in Finder is picked up without a view asking. @MainActor internal final class ThemeCatalog: ObservableObject { - internal static let shared = ThemeCatalog() + internal static let shared: ThemeCatalog = { + let catalog = ThemeCatalog() + catalog.startWatchingUserDirectory() + return catalog + }() @Published internal private(set) var themes: [ThemeDefinition] = BuiltInThemes.all @Published internal private(set) var rejected: [RejectedThemeRecord] = [] @@ -81,30 +84,37 @@ internal final class ThemeCatalog: ObservableObject { /// new file, which would bring back a theme deleted in Finder or on another Mac. private var vanished: [String: URL] = [:] private let userDirectory: URL - private let bundleDirectory: URL? + private let bundled: [ThemeDefinition] private let fileTrash: ThemeFileTrash private let syncTracker: SyncChangeTracker - private var activationCancellable: AnyCancellable? + private let directoryWatcher = FileSystemEventWatcher() nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "ThemeCatalog") - /// Loaded before the first activation, so a user theme in a slot is in the list the launch - /// activation resolves against. + // Loads here, so a user theme a slot names is listed before the engine first resolves it. + // Bundled themes are read only here: they cannot change while TablePro runs. internal init( userDirectory: URL = ThemeCatalog.defaultUserDirectory, bundleDirectory: URL? = Bundle.main.resourceURL, - notificationCenter: NotificationCenter = .default, fileTrash: ThemeFileTrash = .resolved(in: .shared), syncTracker: SyncChangeTracker = .shared ) { self.userDirectory = userDirectory - self.bundleDirectory = bundleDirectory + self.bundled = ThemeCatalog.loadBundled(from: bundleDirectory) self.fileTrash = fileTrash self.syncTracker = syncTracker reloadSynchronously() - activationCancellable = notificationCenter - .publisher(for: NSApplication.didBecomeActiveNotification) - .sink { [weak self] _ in self?.reloadSynchronously() } + } + + // The catalog's own writes report too; their reload finds what the list already holds and + // publishes nothing. + internal func startWatchingUserDirectory() { + directoryWatcher.onChange = { [weak self] in self?.reloadSynchronously() } + directoryWatcher.watch([userDirectory.path]) + } + + internal func stopWatchingUserDirectory() { + directoryWatcher.cancel() } internal func theme(id: String) -> ThemeDefinition? { @@ -122,7 +132,7 @@ internal final class ThemeCatalog: ObservableObject { } internal func reloadSynchronously() { - adopt(Self.loadContents(userDirectory: userDirectory, bundleDirectory: bundleDirectory)) + adopt(Self.loadContents(userDirectory: userDirectory, bundled: bundled)) } /// Writes the theme to the file it came from, or a new theme to `.json`. The list then holds @@ -361,25 +371,28 @@ internal final class ThemeCatalog: ObservableObject { // MARK: - Disk - nonisolated private static func loadContents(userDirectory: URL, bundleDirectory: URL?) -> ThemeCatalogContents { - var themes = BuiltInThemes.all + nonisolated private static func loadBundled(from directory: URL?) -> [ThemeDefinition] { + guard let directory else { return [] } + var themes: [ThemeDefinition] = [] + for file in jsonFiles(in: directory) where file.lastPathComponent.hasPrefix(ThemeDefinition.builtInPrefix) { + do { + let theme = try ThemeFileFormat.decode(try Data(contentsOf: file)) + guard !(BuiltInThemes.all + themes).contains(where: { $0.id == theme.id }) else { continue } + themes.append(theme) + } catch { + logger.error("Bundled theme \(file.lastPathComponent) rejected: \(error.publicLogShape, privacy: .public)") + } + } + return themes + } + + nonisolated private static func loadContents(userDirectory: URL, bundled: [ThemeDefinition]) -> ThemeCatalogContents { + var themes = BuiltInThemes.all + bundled var rejected: [RejectedThemeRecord] = [] var userFiles: [String: ThemeFile] = [:] ensureDirectory(userDirectory) - if let bundle = bundleDirectory { - for file in jsonFiles(in: bundle) where file.lastPathComponent.hasPrefix(ThemeDefinition.builtInPrefix) { - do { - let theme = try ThemeFileFormat.decode(try Data(contentsOf: file)) - guard !themes.contains(where: { $0.id == theme.id }) else { continue } - themes.append(theme) - } catch { - logger.error("Bundled theme \(file.lastPathComponent) rejected: \(error.publicLogShape, privacy: .public)") - } - } - } - let files = jsonFiles(in: userDirectory) .sorted { $0.lastPathComponent.localizedStandardCompare($1.lastPathComponent) == .orderedAscending } diff --git a/apple/TablePro/Theme/ThemeColor.swift b/apple/TablePro/Theme/ThemeColor.swift index 32dc09a58f..26ad51133b 100644 --- a/apple/TablePro/Theme/ThemeColor.swift +++ b/apple/TablePro/Theme/ThemeColor.swift @@ -60,7 +60,9 @@ internal extension ThemeSlot { return .textColor case .gridBackground: return .controlBackgroundColor - case .gridText, .gridBoolTrue, .gridBoolFalse: return .labelColor + case .gridText, .gridBoolTrue, .gridBoolFalse, + .gridString, .gridNumber, .gridDateTime, .gridBinary, .gridJSON, .gridUUID, .gridEnum: + return .labelColor case .gridAlternateRow: return NativeThemeColors.alternateRow case .gridHeaderBackground: return .controlBackgroundColor case .gridHeaderText: return .headerTextColor diff --git a/apple/TablePro/Theme/ThemeEngine.swift b/apple/TablePro/Theme/ThemeEngine.swift index 4c773ce87d..60126547c1 100644 --- a/apple/TablePro/Theme/ThemeEngine.swift +++ b/apple/TablePro/Theme/ThemeEngine.swift @@ -6,13 +6,6 @@ import SwiftUI import TableProEditorKit import TableProTheme -internal enum DataGridFontVariant { - static let regular = 0 - static let italic = 1 - static let medium = 2 - static let rowNumber = 3 -} - internal struct EditorFontCache: Equatable { internal let font: NSFont internal let lineNumberFont: NSFont diff --git a/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift b/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift index 414d5a6e4c..0eac80c09b 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift @@ -51,7 +51,8 @@ extension AIChatViewModel { timestamp: snapshot.timestamp, usage: snapshot.usage, modelId: snapshot.modelId, - providerId: snapshot.providerId + providerId: snapshot.providerId, + outputLimit: snapshot.outputLimit ) } diff --git a/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift b/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift index 108e8c9e2e..bf6745a14c 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift @@ -6,38 +6,52 @@ import Foundation extension AIChatViewModel { - /// Brings in this connection's conversation history, and this session's own transcript. - /// - /// Called when a session is first looked at rather than from `init`, and scoped to the - /// connection. Loading every conversation and restoring whichever happened to be newest meant a - /// second connection's window opened holding the first connection's transcript, and then saved - /// over it. + // Scoped to the connection, so a second connection's window never opens holding the first one's transcript. func restoreConversationsIfNeeded() { - guard !hasRestoredConversation else { return } + trackTableMentions() + // With no connection the history list would hold every connection's conversations. + guard connection != nil, !hasRestoredConversation else { return } markConversationRestored() let storage = chatStorage let connectionId = connection?.id - let wanted = pendingConversationToRestore - Task.detached(priority: .utility) { [weak self] in - let loaded = await storage.loadAll(connectionId: connectionId) - await MainActor.run { - guard let self else { return } - self.conversations = loaded - guard self.messages.isEmpty else { return } - guard let wanted, let stored = loaded.first(where: { $0.id == wanted }) else { return } - self.activeConversationID = stored.id - self.messages = stored.messages.map { ChatTurn(wire: $0) } + let wanted = conversationToRestore + let epoch = transcriptEpoch + let saved = lastConversationWrite + transcriptLoad = Task(priority: .utility) { [weak self] in + await saved?.value + let summaries = await storage.loadSummaries(connectionId: connectionId) + var stored: AIConversation? + if let wanted { + stored = await storage.load(id: wanted) } + guard let self, self.transcriptEpoch == epoch else { return } + self.conversations = Self.merge(summaries, keepingTurnsOf: self.conversations) + guard let wanted, self.conversationToRestore == wanted else { return } + // A message sent before the restore landed started a conversation of its own, which wins. + guard self.messages.isEmpty, self.activeConversationID == nil else { + self.conversationToRestore = nil + return + } + let readable = stored.flatMap { AIChatStorage.isShown(conversationOf: $0.connectionId, on: connectionId) ? $0 : nil } + self.showTranscript(readable) } } + // An entry still carrying turns holds what disk may have trimmed, so a re-read list keeps it. + private static func merge(_ summaries: [AIConversation], keepingTurnsOf current: [AIConversation]) -> [AIConversation] { + let kept = current.filter { !$0.messages.isEmpty } + guard !kept.isEmpty else { return summaries } + let keptByID = Dictionary(kept.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) + let listed = Set(summaries.map(\.id)) + return kept.filter { !listed.contains($0.id) } + summaries.map { keptByID[$0.id] ?? $0 } + } + func clearConversation() { cancelStream() AIProviderFactory.resetCopilotConversation(sessionId: sessionId) - let ids = conversations.map(\.id) - let storage = chatStorage - Task { for id in ids { await storage.delete(id) } } + deleteStoredConversations(conversations.map(\.id)) conversations.removeAll() + conversationToRestore = nil messages.removeAll() activeConversationID = nil clearError() @@ -47,58 +61,72 @@ extension AIChatViewModel { if activeConversationID == id { AIProviderFactory.resetCopilotConversation(sessionId: sessionId) } - Task { await chatStorage.delete(id) } + deleteStoredConversations([id]) conversations.removeAll { $0.id == id } + if conversationToRestore == id { + conversationToRestore = nil + } if activeConversationID == id { activeConversationID = nil messages.removeAll() } } - /// Writes the transcript under the conversation this session already holds. - /// - /// The id is matched against storage rather than against the in-memory list. A session holding - /// an id the list had not caught up with took the new-conversation branch, orphaning the - /// transcript the user was reading and starting a second one beside it. + // By id, not by the in-memory list, so an id the list has not caught up with keeps its own file. func persistCurrentConversation() { guard let conversation = snapshotCurrentConversation() else { return } - Task { await chatStorage.save(conversation) } + let storage = chatStorage + let previous = lastConversationWrite + lastConversationWrite = Task { + await previous?.value + await storage.save(conversation) + } } - /// The same write, finished before this call returns. Termination is the only caller. + // Behind the saves already booked, or one landing late would bring the file back. + func deleteStoredConversations(_ ids: [UUID]) { + guard !ids.isEmpty else { return } + let storage = chatStorage + let previous = lastConversationWrite + lastConversationWrite = Task { + await previous?.value + for id in ids { + await storage.delete(id) + } + } + } + + // Termination is the only caller. func persistCurrentConversationSynchronously() { guard let conversation = snapshotCurrentConversation() else { return } chatStorage.saveSynchronously(conversation) } + // Returns the whole conversation to write and keeps only its summary in `conversations`. private func snapshotCurrentConversation() -> AIConversation? { guard !messages.isEmpty else { return nil } - let wireMessages = messages.map { $0.wireSnapshot } + var conversation: AIConversation if let existingID = activeConversationID { - var conversation = conversations.first(where: { $0.id == existingID }) + conversation = conversations.first(where: { $0.id == existingID }) ?? AIConversation(id: existingID, connectionId: connection?.id, connectionName: connection?.name) - conversation.messages = wireMessages conversation.updatedAt = Date() - conversation.updateTitle() conversation.connectionName = connection?.name - - if let index = conversations.firstIndex(where: { $0.id == existingID }) { - conversations[index] = conversation - } else { - conversations.insert(conversation, at: 0) - } - return conversation + } else { + conversation = AIConversation(connectionId: connection?.id, connectionName: connection?.name) + activeConversationID = conversation.id + conversationToRestore = nil } - - var conversation = AIConversation( - messages: wireMessages, - connectionId: connection?.id, - connectionName: connection?.name - ) + conversation.messages = ChatToolCallRepair.settlingPendingApprovals(messages.map { $0.wireSnapshot }) conversation.updateTitle() - activeConversationID = conversation.id - conversations.insert(conversation, at: 0) + + var summary = conversation + summary.messages = [] + if let index = conversations.firstIndex(where: { $0.id == conversation.id }) { + conversations[index] = summary + } else { + conversations.insert(summary, at: 0) + } return conversation } } diff --git a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift index 208ee830fb..7f17df24e1 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift @@ -23,6 +23,7 @@ extension AIChatViewModel { let toolUseNames: [String: String] let toolUseInputs: [String: String] let toolUseMetadata: [String: [String: String]] + let outputLimit: ChatOutputLimit? let cancelled: Bool } @@ -221,13 +222,15 @@ extension AIChatViewModel { order: round.toolUseOrder, names: round.toolUseNames, inputs: round.toolUseInputs, - metadata: round.toolUseMetadata + metadata: round.toolUseMetadata, + outputLimit: round.outputLimit ) let context = await MainActor.run { ChatToolContext( connectionId: self.connection?.id, bridge: ChatToolBootstrap.bridge, authPolicy: ChatToolBootstrap.authPolicy, + chatMode: chatMode, sessionId: self.sessionId ) } @@ -375,9 +378,14 @@ extension AIChatViewModel { model: resolved.model, systemPrompt: systemPrompt, tools: toolSpecs, - reasoningEffort: resolved.config.reasoningEffort, + reasoningEffort: AIProviderFactory.reasoningEffort( + resolved.config.reasoningEffort, + config: resolved.config, + model: resolved.model + ), sessionId: sessionId, - connectionId: connection?.id + connectionId: connection?.id, + chatMode: chatMode ) ) @@ -466,6 +474,7 @@ extension AIChatViewModel { var toolUseNames: [String: String] = [:] var toolUseInputs: [String: String] = [:] var toolUseMetadata: [String: [String: String]] = [:] + var outputLimit: ChatOutputLimit? var hasRenderedFirstText = false for try await event in stream { @@ -515,6 +524,9 @@ extension AIChatViewModel { opaque: opaque, idMap: &reasoningIDMap.value ) + case .outputLimitReached(let limit): + outputLimit = limit + turn(withID: assistantID)?.outputLimit = limit } } @@ -523,6 +535,7 @@ extension AIChatViewModel { toolUseNames: toolUseNames, toolUseInputs: toolUseInputs, toolUseMetadata: toolUseMetadata, + outputLimit: outputLimit, cancelled: Task.isCancelled ) } @@ -663,33 +676,61 @@ extension AIChatViewModel { streamingTask = nil } + nonisolated static let unparsableToolArgumentsContent = """ + This tool call did not run: its arguments are not a complete JSON object, which usually means the \ + response hit the output token limit. Call the tool again with complete arguments. + """ + + nonisolated static let cutToolArgumentsContent = """ + This tool call did not run: the response hit the output limit before its arguments were complete. \ + Call the tool again with complete arguments, and keep them shorter if you can. + """ + + // A call whose arguments do not parse comes back already denied, so it never reaches approval or execution. + // A response cut at its limit was cut in its last call, whatever arguments a provider that parses them hands over. nonisolated static func assembleToolUseBlocks( order: [String], names: [String: String], inputs: [String: String], - metadata: [String: [String: String]] = [:] + metadata: [String: [String: String]] = [:], + outputLimit: ChatOutputLimit? = nil ) -> [ToolUseBlock] { order.compactMap { id -> ToolUseBlock? in guard let name = names[id] else { return nil } - let inputString = inputs[id] ?? "{}" - let inputValue: JsonValue - if inputString.isEmpty { - inputValue = .object([:]) - } else if let data = inputString.data(using: .utf8), - let decoded = try? JSONDecoder().decode(JsonValue.self, from: data) { - inputValue = decoded - } else { - inputValue = .object([:]) + let isCut = outputLimit != nil && id == order.last + guard !isCut, let arguments = toolArguments(from: inputs[id] ?? "") else { + return ToolUseBlock( + id: id, + name: name, + input: .object([:]), + approvalState: .denied(reason: isCut ? cutToolArgumentsContent : unparsableToolArgumentsContent), + providerMetadata: metadata[id] + ) } return ToolUseBlock( id: id, name: name, - input: inputValue, + input: arguments, providerMetadata: metadata[id] ) } } + // Every provider sends a call's arguments as one JSON object; no arguments at all reads as an empty one. + nonisolated private static func toolArguments(from raw: String) -> JsonValue? { + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return .object([:]) } + guard let decoded = try? JSONDecoder().decode(JsonValue.self, from: Data(trimmed.utf8)) else { return nil } + switch decoded { + case .object: + return decoded + case .null: + return .object([:]) + case .bool, .int, .double, .string, .array: + return nil + } + } + nonisolated static func executeToolUses( _ blocks: [ToolUseBlock], scope: ChatToolScope, diff --git a/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift b/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift index e197e82197..02d54057b0 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift @@ -29,8 +29,7 @@ struct TableMentionIndex: Sendable { TableMentionIndex(tables: tables) } - // The names are already sorted, so the first `limit` matches are the ones a sort of every match - // would keep. + // The names are already sorted, so the first `limit` matches are the ones a full sort would keep. func names(matching query: String, limit: Int) -> [String] { guard limit > 0 else { return [] } guard !query.isEmpty else { return Array(names.prefix(limit)) } @@ -49,8 +48,7 @@ struct TableMentionIndex: Sendable { return matches } - // Only ASCII folds to ASCII one character for one, which is where a byte search finds what - // `localizedCaseInsensitiveContains` finds; `ß` folds to `ss` and would match `weis`. + // Only ASCII folds one character for one; `ß` folds to `ss` and a byte search would match `weis`. private static func asciiFolded(_ text: String) -> [UInt8]? { guard text.utf8.allSatisfy({ $0 < 0x80 }) else { return nil } let folded = Array(text.folding(options: .caseInsensitive, locale: .current).utf8) @@ -82,12 +80,15 @@ extension AIChatViewModel { tableMentions.names.count > Self.mentionMenuTableLimit } - // Started by the panel, so a session whose assistant is never shown never indexes its catalog. + // A schema load on another connection leaves this session's tables as they were. func trackTableMentions() { if tableMentionObservation == nil { tableMentionObservation = services.schemaService.$generations - .map { _ in () } - .merge(with: $connection.map { _ in () }) + .combineLatest($connection.map { $0?.id }) + .map { generations, connectionId in + CatalogGeneration(connectionId: connectionId, generation: connectionId.flatMap { generations[$0] }) + } + .removeDuplicates() .receive(on: RunLoop.main) .sink { [weak self] _ in self?.refreshTableMentions() } } @@ -112,3 +113,8 @@ extension AIChatViewModel { lhs.count == rhs.count && lhs.indices.allSatisfy { lhs[$0].name == rhs[$0].name } } } + +private struct CatalogGeneration: Equatable { + let connectionId: UUID? + let generation: Int? +} diff --git a/apple/TablePro/ViewModels/AIChatViewModel+ToolApproval.swift b/apple/TablePro/ViewModels/AIChatViewModel+ToolApproval.swift index fb13b16d62..8d93cc8a57 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+ToolApproval.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+ToolApproval.swift @@ -44,7 +44,12 @@ extension AIChatViewModel { let initialBlocks = await MainActor.run { [weak self] () -> [ToolUseBlock] in guard let self else { return assembledBlocks } let initial = assembledBlocks.map { block -> ToolUseBlock in - let state = self.computeInitialApprovalState(for: block.name, registry: registry) + let state: ToolApprovalState + if case .denied = block.approvalState { + state = block.approvalState + } else { + state = self.computeInitialApprovalState(for: block.name, registry: registry) + } return ToolUseBlock( id: block.id, name: block.name, @@ -82,22 +87,24 @@ extension AIChatViewModel { sessionId: sessionId, toolUseId: block.id ) - let finalState: ToolApprovalState + let decidedState: ToolApprovalState switch decision { case .run: - finalState = .approved - explicitlyApproved.insert(block.id) + decidedState = .approved case .alwaysAllow: await MainActor.run { [weak self] in self?.persistAlwaysAllowed(toolName: block.name) } - finalState = .approved - explicitlyApproved.insert(block.id) + decidedState = .approved case .cancel: - finalState = .cancelled + decidedState = .cancelled } - await MainActor.run { [weak self] in - self?.updateApprovalState(blockID: block.id, newState: finalState, assistantID: assistantID) + let finalState = await MainActor.run { [weak self] in + self?.updateApprovalState(blockID: block.id, newState: decidedState, assistantID: assistantID) + ?? .cancelled + } + if case .approved = finalState { + explicitlyApproved.insert(block.id) } resolved.append(ToolUseBlock( id: block.id, @@ -144,19 +151,26 @@ extension AIChatViewModel { return .pending } - if let connection, connection.aiAlwaysAllowedTools.contains(toolName) { - return .approved - } - if let connection { - let safeModeLevel = liveSafeModeLevel(for: connection) - if safeModeLevel.blocksAllWrites { + // Read-Only comes first: an Always Allow grant is consent to the tool, not to a level that refuses writes. + if let connection, liveSafeModeLevel(for: connection).blocksAllWrites { + guard let level = SafeModeFloor.levelThatWrites(above: AgentModeSafeModeFloor.effectiveFloor(for: connection)?.level) + else { return .denied(reason: String( - localized: "TablePro's Safe Mode is set to read-only for this connection. Set it to Confirm Writes or higher to allow this tool." + localized: "TablePro's Safe Mode is held at Read-Only for this connection, so this tool cannot run." )) } - if !safeModeLevel.requiresConfirmation { - return .approved - } + return .denied(reason: String( + format: String( + localized: "TablePro's Safe Mode is set to Read-Only for this connection. Switch it to %@ to allow this tool." + ), + level.displayName + )) + } + if let connection, alwaysAllowedTools(for: connection).contains(toolName) { + return .approved + } + if let connection, !liveSafeModeLevel(for: connection).requiresConfirmation { + return .approved } return .pending } @@ -166,6 +180,14 @@ extension AIChatViewModel { DatabaseManager.shared.session(for: connection.id)?.safeModeLevel ?? connection.safeModeLevel } + // `connection` is read once per pane, so a grant forgotten in the form or changed by sync after + // that is only in storage. + @MainActor + private func alwaysAllowedTools(for connection: DatabaseConnection) -> Set { + services.connectionStorage.loadConnection(id: connection.id)?.aiAlwaysAllowedTools + ?? connection.aiAlwaysAllowedTools + } + @MainActor func appendPendingToolUseBlocks(_ blocks: [ToolUseBlock], assistantID: UUID) { guard let turn = turn(withID: assistantID) else { return } @@ -174,27 +196,38 @@ extension AIChatViewModel { } } + // Returns the state the card holds afterwards. Stop settles a card before a Run click already + // in flight lands, and New Conversation or a switch drops its turn: either way nothing runs. @MainActor - func updateApprovalState(blockID: String, newState: ToolApprovalState, assistantID: UUID) { - guard let turn = turn(withID: assistantID) else { return } + @discardableResult + func updateApprovalState(blockID: String, newState: ToolApprovalState, assistantID: UUID) -> ToolApprovalState { + guard let turn = turn(withID: assistantID) else { return .cancelled } for chatBlock in turn.blocks { if case .toolUse(var block) = chatBlock.kind, block.id == blockID { + if case .cancelled = block.approvalState { return .cancelled } block.approvalState = newState chatBlock.setKind(.toolUse(block)) - return + return newState } } + return .cancelled } - /// Records an Always Allow grant as a read-modify-write of the one field it changes. - /// - /// It used to save the whole `DatabaseConnection` the view model was holding. That record is - /// read once, in `AIChatPanelView`'s `onAppear` and again only when the connection's id changes, - /// so a Safe Mode level raised in the toolbar while the pane stayed mounted was still the old - /// one here, and the grant wrote it back and silently undid the change. - /// - /// A destructive operation is never granted: each DROP, TRUNCATE and ALTER...DROP is confirmed - /// on its own. + // The task marks a stopped card only after a hop back to the main actor, by which time the + // transcript has been saved or replaced. + @MainActor + func settlePendingToolApprovals() { + for turn in messages { + for block in turn.blocks { + guard case .toolUse(var use) = block.kind, case .pending = use.approvalState else { continue } + use.approvalState = .cancelled + block.setKind(.toolUse(use)) + } + } + } + + // Writes only this field: the view model's copy of the connection can be stale. A destructive + // or remote tool is never granted. @MainActor func persistAlwaysAllowed(toolName: String) { if ChatToolRegistry.shared.tool(named: toolName)?.mode == .agentOnly { @@ -204,7 +237,6 @@ extension AIChatViewModel { return } guard let connectionId = connection?.id else { return } - guard connection?.aiAlwaysAllowedTools.contains(toolName) == false else { return } guard services.connectionStorage.mutateConnections(ids: [connectionId], { stored in stored.aiAlwaysAllowedTools.insert(toolName) }) else { @@ -224,6 +256,7 @@ extension AIChatViewModel { connectionId: connection?.id, bridge: ChatToolBootstrap.bridge, authPolicy: ChatToolBootstrap.authPolicy, + chatMode: mode, sessionId: sessionId ) await handleCopilotToolInvocation( @@ -260,18 +293,20 @@ extension AIChatViewModel { sessionId: sessionId, toolUseId: block.id ) + let decidedState: ToolApprovalState switch decision { case .run: - finalState = .approved - approvalWasExplicit = true + decidedState = .approved case .alwaysAllow: persistAlwaysAllowed(toolName: block.name) - finalState = .approved - approvalWasExplicit = true + decidedState = .approved case .cancel: - finalState = .cancelled + decidedState = .cancelled + } + finalState = updateApprovalState(blockID: block.id, newState: decidedState, assistantID: assistantID) + if case .approved = finalState { + approvalWasExplicit = true } - updateApprovalState(blockID: block.id, newState: finalState, assistantID: assistantID) } else { finalState = initialState } diff --git a/apple/TablePro/ViewModels/AIChatViewModel.swift b/apple/TablePro/ViewModels/AIChatViewModel.swift index 21678b4e3f..9fd8681d99 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel.swift @@ -31,6 +31,10 @@ final class AIChatViewModel: ObservableObject { @Published var selectedProviderId: UUID? @Published var selectedModel: String? @Published var availableModels: [UUID: [String]] = [:] + @Published private(set) var providerSettingsRevision = 0 + private var modelListSources: [UUID: AIModelListSource] = [:] + private var staleModelLists: Set = [] + private var modelListGenerations: [UUID: Int] = [:] @Published var attachedContext: [ContextItem] = [] @Published var attachedImages: [ChatImageInput] = [] @Published var savedQueries: [SQLFavorite] = [] @@ -52,7 +56,7 @@ final class AIChatViewModel: ObservableObject { @Published var queryResults: String? var editorTarget: AssistantEditorTarget? - /// The composer's text. It lives on `composerDraft`, which publishes it apart from the conversation. + // Lives on `composerDraft`, which publishes it apart from the conversation. var inputText: String { get { composerDraft.text } set { composerDraft.text = newValue } @@ -113,10 +117,14 @@ final class AIChatViewModel: ObservableObject { var tableMentionSource: [TableInfo] = [] var tableMentionBuild: Task? var tableMentionObservation: AnyCancellable? + // Writes run in the order they were booked, and a transcript load waits for the ones before it. + var lastConversationWrite: Task? + var transcriptLoad: Task? + // A release bumps it, so a load still reading from disk lands in nothing. + var transcriptEpoch = 0 var bulkColumnReadTableCount: Int? var schemaCacheScope: String? - // A custom command's message carries the schema it rendered, so its tables are ranked by the - // user's own words instead. + // A custom command's message carries its rendered schema, so its tables are ranked by the user's words. var schemaQuestionOverride: (turnID: UUID, text: String)? nonisolated(unsafe) var streamingTask: Task? var prepTask: Task? @@ -125,36 +133,25 @@ final class AIChatViewModel: ObservableObject { var chatStorage: AIChatStorage { services.aiChatStorage } var cachedSavedQueries: [UUID: SQLFavorite] = [:] private var savedQueryCancellables: Set = [] + private var aiSettingsCancellables: Set = [] static let maxMessageCount = 200 - /// The session this engine belongs to. - /// - /// Injected rather than minted here, because a restored session has to be the same session: - /// identity derived inside the engine cannot round-trip, so every guarantee keyed on it - /// (reopening by id, the rail's selection, per-session provider state) silently degraded to - /// "make another one". + // Injected, so a restored session keeps the identity the rail, reopening and provider state key on. let sessionId: UUID - /// The conversation to pull in when this engine is first looked at, if it is resuming one. - /// - /// Restore is lazy on purpose: reading every stored conversation at launch is quadratic in the - /// number of sessions, and `init` used to call `loadConversations()`, so opening any connection - /// window read the whole chat history off disk even with the assistant never revealed. - private var conversationToRestore: UUID? + // Read when the session is shown: reading every stored conversation at launch is quadratic in sessions. + var conversationToRestore: UUID? private var didRestoreConversation = false - var pendingConversationToRestore: UUID? { conversationToRestore } var hasRestoredConversation: Bool { didRestoreConversation } - /// Whether the connection this session names is still being opened. - /// - /// Agent mode draws its composer over a connect on purpose, so a turn can be submitted before - /// there is a session to run its tools against. Every such turn used to open a stream anyway, - /// which reached the tools with no connection behind them and answered the user's first - /// question with a row of failures. The turn is appended to the transcript as usual and the - /// stream is held until the connect lands, which is what a live composer during a connect - /// promises. + // Restored from disk or released: the session names its conversation and holds none of it yet. + var isAwaitingTranscript: Bool { + activeConversationID == nil && conversationToRestore != nil && messages.isEmpty + } + + // A turn sent during a connect is held, so its tools never run with no connection behind them. var isAwaitingConnection = false { didSet { guard oldValue, !isAwaitingConnection else { return } @@ -162,7 +159,6 @@ final class AIChatViewModel: ObservableObject { } } - /// A turn that was submitted during a connect and has not been streamed yet. var heldTurnAwaitsConnection = false private func releaseHeldTurn() { @@ -185,6 +181,7 @@ final class AIChatViewModel: ObservableObject { self.conversationToRestore = conversationId self.queryContextMetadata = services.databaseManager observeSavedQueryUpdates() + observeAISettingsChanges() } deinit { @@ -242,8 +239,7 @@ final class AIChatViewModel: ObservableObject { return Self.acceptsImages(config: config, model: selectedModel ?? config.model, catalog: .shared) } - /// The provider type sets the envelope, and the provider's own model list narrows it: a - /// router lists text-only models beside vision ones, and an image sent to one of those fails. + // A router lists text-only models beside vision ones, so the provider's own list narrows its type. nonisolated static func acceptsImages(config: AIProviderConfig, model: String, catalog: AIModelCatalog) -> Bool { guard let descriptor = AIProviderRegistry.shared.descriptor(for: config.type.rawValue) else { return false } return descriptor.supportsImages(fetched: catalog.fetchedInfo(providerID: config.id, modelID: model)) @@ -279,6 +275,7 @@ final class AIChatViewModel: ObservableObject { streamingTask?.cancel() streamingTask = nil ToolApprovalCenter.shared.cancelAll(sessionId: sessionId) + settlePendingToolApprovals() if case .streaming(let assistantID) = streamingState, let idx = messages.firstIndex(where: { $0.id == assistantID }) { @@ -336,69 +333,105 @@ final class AIChatViewModel: ObservableObject { AIProviderFactory.resetCopilotConversation(sessionId: sessionId) cancelStream() persistCurrentConversation() + conversationToRestore = nil messages.removeAll() activeConversationID = nil clearError() } + // The transcript on screen stays until the chosen one lands, and when its file is gone or a turn started. func switchConversation(to id: UUID) { - guard let conversation = conversations.first(where: { $0.id == id }) else { return } + guard conversations.contains(where: { $0.id == id }) else { return } AIProviderFactory.resetCopilotConversation(sessionId: sessionId) cancelStream() persistCurrentConversation() - messages = conversation.messages.map { ChatTurn(wire: $0) } - activeConversationID = conversation.id clearError() + guard id != activeConversationID else { + conversationToRestore = nil + return + } + keepTurnsDiskMayTrim() + if let index = conversations.firstIndex(where: { $0.id == id }), !conversations[index].messages.isEmpty { + let kept = conversations[index] + conversations[index].messages = [] + showTranscript(kept) + return + } + conversationToRestore = id + let storage = chatStorage + let epoch = transcriptEpoch + let saved = lastConversationWrite + transcriptLoad = Task { [weak self] in + await saved?.value + let stored = await storage.load(id: id) + guard let self, self.transcriptEpoch == epoch, self.conversationToRestore == id else { return } + guard !self.isBusy else { + self.conversationToRestore = nil + return + } + self.showTranscript(stored) + } } - /// Releases everything this conversation holds, keeping what the user typed. - /// - /// Window close, disconnect and a lost session all reach here, and none of them is the user - /// throwing a conversation away. It used to empty `messages` with nothing written to disk while - /// `cancelStream()` next door persisted first, so the three ordinary ways a window goes away - /// each dropped a reply that was still arriving. - /// - /// Cancelling the task is also not enough on its own to release a turn parked on an approval - /// card: a `CheckedContinuation` is not resumed by cancellation, so the suspended turn held the - /// provider and its open stream for the life of the process. - func clearSessionData() { - ToolApprovalCenter.shared.cancelAll(sessionId: sessionId) - persistCurrentConversation() - AIProviderFactory.resetCopilotConversation(sessionId: sessionId) + // Disk may hold only the last turns of a long transcript, so the ones it may drop stay on its entry. + private func keepTurnsDiskMayTrim() { + guard messages.count > AIChatStorage.turnsKeptWhenTrimmed, + let activeConversationID, + let index = conversations.firstIndex(where: { $0.id == activeConversationID }) + else { return } + conversations[index].messages = messages.map { $0.wireSnapshot } + } + + // Nil when no transcript was found: the one on screen stays. + func showTranscript(_ conversation: AIConversation?) { + conversationToRestore = nil + guard let conversation else { return } + activeConversationID = conversation.id + // A stored card can never be answered, and older builds saved some still waiting. + messages = ChatToolCallRepair.settlingPendingApprovals(conversation.messages).map { ChatTurn(wire: $0) } + } + + // For a session no window shows; the next restore reads back what is written here. + func releaseLoadedState() { + if isBusy || streamingTask != nil { + cancelStream() + } else { + persistCurrentConversation() + } + let keepsTranscript = messages.count > AIChatStorage.turnsKeptWhenTrimmed + heldTurnAwaitsConnection = false + transcriptEpoch &+= 1 + transcriptLoad?.cancel() + transcriptLoad = nil + didRestoreConversation = false + if !keepsTranscript { + if let activeConversationID { + conversationToRestore = activeConversationID + } + activeConversationID = nil + messages = [] + } + conversations = conversations.filter { entry in + !entry.messages.isEmpty || (keepsTranscript && entry.id == activeConversationID) + } prepTask?.cancel() prepTask = nil - streamingTask?.cancel() - streamingTask = nil - AIProviderFactory.invalidateCache() - connection = nil - columnsByTable = [:] - foreignKeysByTable = [:] inFlightColumnFetches.values.forEach { $0.cancel() } inFlightColumnFetches.removeAll() inFlightSchemaLoad?.cancel() inFlightSchemaLoad = nil + columnsByTable = [:] + foreignKeysByTable = [:] relevantTableNames = [] tableMentionBuild?.cancel() tableMentionBuild = nil + tableMentionObservation = nil tableMentionSource = [] tableMentions = .empty bulkColumnReadTableCount = nil schemaCacheScope = nil schemaQuestionOverride = nil - currentQuery = nil queryResults = nil - editorTarget = nil - clearPendingWalkthrough() - messages = [] - errorMessage = nil - activeConversationID = nil - streamingState = .idle - for image in attachedImages { - if case .cacheFile(let filename, _) = image.source { - AIImageCache.shared.delete(filename: filename) - } - } - attachedImages = [] } func clearPendingWalkthrough() { @@ -406,37 +439,92 @@ final class AIChatViewModel: ObservableObject { pendingWalkthroughSource = nil } + private func observeAISettingsChanges() { + services.appEvents.aiSettingsChanged + .receive(on: RunLoop.main) + .sink { [weak self] in self?.dropSelectionOfRemovedProvider() } + .store(in: &aiSettingsCancellables) + services.appEvents.aiProviderSaved + .receive(on: RunLoop.main) + .sink { [weak self] id in self?.providerWasSaved(id) } + .store(in: &aiSettingsCancellables) + } + + private func dropSelectionOfRemovedProvider() { + guard let selectedProviderId, + !services.appSettings.ai.providers.contains(where: { $0.id == selectedProviderId }) + else { return } + self.selectedProviderId = nil + selectedModel = nil + } + + // A save can bring a new key, which nothing in the provider list shows. + private func providerWasSaved(_ id: UUID) { + staleModelLists.insert(id) + modelListGenerations[id, default: 0] += 1 + providerSettingsRevision += 1 + } + + func modelListIsStale(for config: AIProviderConfig) -> Bool { + modelListSources[config.id] != AIModelListSource(config: config) || staleModelLists.contains(config.id) + } + + private enum ModelListOutcome: Sendable { + case listed([String]) + case failed + case notReady + } + + private struct PendingModelList { + let config: AIProviderConfig + let apiKey: String? + let source: AIModelListSource + let generation: Int + let catalogRevision: Int + } + func loadAvailableModels() async { let settings = services.appSettings.ai - let pending = settings.providers.filter { availableModels[$0.id] == nil } + let providerIDs = Set(settings.providers.map(\.id)) + if availableModels.keys.contains(where: { !providerIDs.contains($0) }) { + availableModels = availableModels.filter { providerIDs.contains($0.key) } + } + modelListSources = modelListSources.filter { providerIDs.contains($0.key) } + + let pending = settings.providers.compactMap { config -> PendingModelList? in + guard modelListIsStale(for: config) else { return nil } + return PendingModelList( + config: config, + apiKey: config.authStyle.usesAPIKey ? services.aiKeyStorage.loadAPIKey(for: config.id) : nil, + source: AIModelListSource(config: config), + generation: modelListGenerations[config.id, default: 0], + catalogRevision: AIModelCatalog.shared.revision(providerID: config.id) + ) + } guard !pending.isEmpty else { return } - let results = await withTaskGroup(of: (UUID, [String]?).self) { group in - for config in pending { - let apiKey: String? - switch config.authStyle { - case .apiKey, .optionalApiKey: - apiKey = services.aiKeyStorage.loadAPIKey(for: config.id) - case .oauth, .none: - apiKey = nil - } + let results = await withTaskGroup(of: (UUID, ModelListOutcome).self) { group in + for entry in pending { + let config = entry.config + let apiKey = entry.apiKey + let catalogRevision = entry.catalogRevision group.addTask { let transport = await AIProviderFactory.createProvider(for: config, apiKey: apiKey) do { let models = try await transport.fetchAvailableModels() - AIModelCatalog.shared.store(providerID: config.id, models: models) - return (config.id, models.map(\.id)) + AIModelCatalog.shared.store(providerID: config.id, models: models, ifUnchangedSince: catalogRevision) + return (config.id, .listed(models.map(\.id))) } catch is CancellationError { - return (config.id, nil) + return (config.id, .notReady) } catch CopilotError.serverNotRunning { - return (config.id, nil) + return (config.id, .notReady) } catch { - return (config.id, []) + return (config.id, .failed) } } } - var collected: [(UUID, [String]?)] = [] + var collected: [(UUID, ModelListOutcome)] = [] for await result in group { collected.append(result) } @@ -445,23 +533,29 @@ final class AIChatViewModel: ObservableObject { guard !Task.isCancelled else { return } - for (id, models) in results { - guard let models else { continue } - if models.isEmpty { - let fallback = pending.first(where: { $0.id == id })?.model - availableModels[id] = (fallback?.isEmpty == false) ? [fallback ?? ""] : [] - } else { + for (id, outcome) in results { + // A save while this fetch was out may have brought a new key: its own fetch decides. + guard let entry = pending.first(where: { $0.config.id == id }), + modelListGenerations[id, default: 0] == entry.generation + else { continue } + switch outcome { + case .notReady: + continue + case .listed(let models) where !models.isEmpty: availableModels[id] = models + modelListSources[id] = entry.source + case .failed where modelListSources[id] == entry.source && availableModels[id]?.isEmpty == false: + break + case .listed, .failed: + // Recorded empty, so a dead server is asked again only after a change or a save. + availableModels[id] = [] + modelListSources[id] = entry.source } + staleModelLists.remove(id) } } - /// The list is read once per connection and then kept current from the app-wide favorites - /// event, the same signal the sidebar, the Quick Switcher and the editor's keyword expansion - /// already follow. Without it a query saved from the editor during a session was offered by - /// every one of those and by nothing in the assistant, for as long as the window stayed open. - /// - /// A nil payload means a global record moved, which is every connection's business. + // A nil payload means a global record moved, which is every connection's business. internal func observeSavedQueryUpdates() { AppEvents.shared.sqlFavoritesDidUpdate .receive(on: RunLoop.main) @@ -479,8 +573,7 @@ final class AIChatViewModel: ObservableObject { return } let favorites = await services.sqlFavoriteManager.fetchFavorites(connectionId: connectionId) - /// The connection can be switched while the read is in flight, and the list belongs to the - /// connection that is on screen now, not the one that asked. + // The connection can be switched while the read is in flight. guard connection?.id == connectionId else { return } savedQueries = favorites for favorite in favorites { @@ -497,3 +590,18 @@ final class AIChatViewModel: ObservableObject { } } } + +// A change to any of these means another server, so the list on hand no longer describes it. +struct AIModelListSource: Equatable, Sendable { + let id: UUID + let type: AIProviderType + let presetID: String? + let endpoint: String + + init(config: AIProviderConfig) { + id = config.id + type = config.type + presetID = config.presetID + endpoint = config.endpoint + } +} diff --git a/apple/TablePro/ViewModels/DashboardTileBoard.swift b/apple/TablePro/ViewModels/DashboardTileBoard.swift index 780137070e..4bdaa4c089 100644 --- a/apple/TablePro/ViewModels/DashboardTileBoard.swift +++ b/apple/TablePro/ViewModels/DashboardTileBoard.swift @@ -7,6 +7,7 @@ import Combine import Foundation import os import TableProPluginKit +import TableProSQLGrammar @MainActor final class DashboardTileBoard: ObservableObject { @@ -20,7 +21,8 @@ final class DashboardTileBoard: ObservableObject { var isRunning = false } - typealias Execute = @MainActor (String) async throws -> QueryResult + // Runs the saved query in the scope the gate was asked about, nil when the gate was asked with none. + typealias Execute = @MainActor (String, DatabaseScope?) async throws -> QueryResult struct Environment { var loadCatalog: @MainActor (UUID) async -> DashboardSavedQueryCatalog @@ -32,6 +34,9 @@ final class DashboardTileBoard: ObservableObject { var nullDisplay: @MainActor () -> String var storage: ServerDashboardTileStorage var now: @MainActor () -> Date = Date.init + // Where a tile's query runs, so Read-Only reads an Oracle or Dameng catalog in that schema and the run is pinned + // there. + var runsIn: @MainActor (UUID) -> DatabaseScope? = { _ in nil } @MainActor static func live(services: AppServices) -> Environment { @@ -51,20 +56,18 @@ final class DashboardTileBoard: ObservableObject { ) }, safeModeLevel: { connectionId in - let connectionLevel: SafeModeLevel - switch services.databaseManager.connectionState(connectionId) { - case .live(_, let session): connectionLevel = session.safeModeLevel - case .stored(let connection): connectionLevel = connection.safeModeLevel - case .unknown: connectionLevel = .silent - } - return ManagedPolicyResolver.effectiveSafeModeLevel( - connectionLevel: connectionLevel, - policy: ManagedPolicyReader.shared - ) + ExecutionGateProvider.effectiveSafeModeLevel(for: connectionId, in: services.databaseManager) }, chartsAvailable: { services.featureAccess.isAvailable(.resultCharts) }, nullDisplay: { services.appSettings.dataGrid.nullDisplay }, - storage: ServerDashboardTileStorage() + storage: ServerDashboardTileStorage(), + // A tile runs on the session itself, in the schema it is on when the tile is checked. + runsIn: { connectionId in + guard let browse = services.databaseManager.browseScope(for: connectionId) else { return nil } + let driver = services.databaseManager.driver(for: connectionId) as? SchemaSwitchable + let schema = driver?.currentSchema ?? browse.schema + return DatabaseScope(connectionId: connectionId, database: browse.database, schema: schema) + } ) } } @@ -86,11 +89,17 @@ final class DashboardTileBoard: ObservableObject { private struct GateDecision { let sql: String let safeModeLevel: SafeModeLevel + let runsIn: DatabaseScope? + let decidedAt: Date let denial: String? } - /// One gate decision per tile, statement and Safe Mode level, until Refresh Now. The gate logs - /// every decision, and a tile on a one second cycle must not append one each second. + // Oracle and Dameng read a bare name against a catalog that changes, so their decision is asked again after the + // catalog's own lifetime. + private static let catalogDecisionLifetime: TimeInterval = 60 + + // One gate decision per tile, statement, Safe Mode level and schema, until Refresh Now, or a minute on Oracle and + // Dameng. The gate logs every decision, and a tile on a one second cycle must not append one each second. private var gateDecisions: [UUID: GateDecision] = [:] private var cancellables: Set = [] @@ -253,16 +262,22 @@ final class DashboardTileBoard: ObservableObject { let denial: String? let safeModeLevel = environment.safeModeLevel(connectionId) - if let cached = gateDecisions[tileId], cached.sql == sql, cached.safeModeLevel == safeModeLevel { + let runsIn = environment.runsIn(connectionId) + let readsCatalog = SQLBareNameCalls.engines.contains(databaseType.rawValue) + if let cached = gateDecisions[tileId], cached.sql == sql, cached.safeModeLevel == safeModeLevel, + cached.runsIn == runsIn, + !readsCatalog || (0.. OperationRequest { + private func request(for sql: String, runsIn: DatabaseScope?) -> OperationRequest { OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -331,7 +346,8 @@ final class DashboardTileBoard: ObservableObject { kind: .readQuery, caller: .userInterface, capabilities: [.cannotPrompt], - operationDescription: String(localized: "Run a saved query on the Server Dashboard") + operationDescription: String(localized: "Run a saved query on the Server Dashboard"), + runsIn: runsIn ) } diff --git a/apple/TablePro/ViewModels/ERDiagramViewModel+Editing.swift b/apple/TablePro/ViewModels/ERDiagramViewModel+Editing.swift index 3332224271..3989cce6c4 100644 --- a/apple/TablePro/ViewModels/ERDiagramViewModel+Editing.swift +++ b/apple/TablePro/ViewModels/ERDiagramViewModel+Editing.swift @@ -22,13 +22,160 @@ struct ERDiagramColumnHit: Equatable { } struct ERDiagramForeignKeyDraft: Equatable { + struct ColumnPair: Identifiable, Equatable { + let id: UUID + var column: String + var referencedColumn: String + + init(id: UUID = UUID(), column: String, referencedColumn: String) { + self.id = id + self.column = column + self.referencedColumn = referencedColumn + } + } + + enum Issue: Equatable { + case noPairs + case incompletePair + case repeatedColumn(String) + case repeatedReferencedColumn(String) + + var message: String { + switch self { + case .noPairs, .incompletePair: + String(localized: "Choose a column and a referenced column in every row.") + case .repeatedColumn(let name), .repeatedReferencedColumn(let name): + String(format: String(localized: "“%@” is used more than once."), name) + } + } + } + let table: String - var column: String + var pairs: [ColumnPair] var referencedTable: String - var referencedColumn: String var name: String var onDelete: EditableForeignKeyDefinition.ReferentialAction = .noAction var onUpdate: EditableForeignKeyDefinition.ReferentialAction = .noAction + + var issue: Issue? { + guard !pairs.isEmpty else { return .noPairs } + guard !pairs.contains(where: { $0.column.isEmpty || $0.referencedColumn.isEmpty }) else { return .incompletePair } + if let repeated = Self.firstRepeated(pairs.map(\.column)) { return .repeatedColumn(repeated) } + if let repeated = Self.firstRepeated(pairs.map(\.referencedColumn)) { return .repeatedReferencedColumn(repeated) } + return nil + } + + // Pair order is key order on both sides, which is what the database matches the referenced key by. + var foreignKey: EditableForeignKeyDefinition { + var foreignKey = EditableForeignKeyDefinition.placeholder() + foreignKey.name = name.trimmingCharacters(in: .whitespacesAndNewlines) + foreignKey.columns = pairs.map(\.column) + foreignKey.referencedTable = referencedTable + foreignKey.referencedColumns = pairs.map(\.referencedColumn) + foreignKey.onDelete = onDelete + foreignKey.onUpdate = onUpdate + return foreignKey + } + + // A drop on a member of a composite key offers the whole key, with the dragged pair at that member's position. + static func starting( + table: String, + column: String, + referencedTable: String, + keyColumns: [String], + target: String?, + sourceColumns: [String], + name: String + ) -> ERDiagramForeignKeyDraft { + let pairs: [ColumnPair] + if let target, !keyColumns.contains(target) { + pairs = [ColumnPair(column: column, referencedColumn: target)] + } else if keyColumns.isEmpty { + pairs = [ColumnPair(column: column, referencedColumn: target ?? "")] + } else { + let position = target.flatMap { keyColumns.firstIndex(of: $0) } + pairs = guessedPairs(for: keyColumns, keeping: column, at: position, from: sourceColumns) + } + return ERDiagramForeignKeyDraft(table: table, pairs: pairs, referencedTable: referencedTable, name: name) + } + + func settingColumn(_ column: String, of id: UUID) -> ERDiagramForeignKeyDraft { + var draft = self + guard let index = draft.pairs.firstIndex(where: { $0.id == id }) else { return self } + draft.pairs[index].column = column + return draft + } + + func settingReferencedColumn(_ column: String, of id: UUID) -> ERDiagramForeignKeyDraft { + var draft = self + guard let index = draft.pairs.firstIndex(where: { $0.id == id }) else { return self } + draft.pairs[index].referencedColumn = column + return draft + } + + func addingPair() -> ERDiagramForeignKeyDraft { + var draft = self + draft.pairs.append(ColumnPair(column: "", referencedColumn: "")) + return draft + } + + // The last pair stays: a key needs at least one. + func removingPairs(_ ids: Set) -> ERDiagramForeignKeyDraft { + let kept = pairs.filter { !ids.contains($0.id) } + guard !kept.isEmpty else { return self } + var draft = self + draft.pairs = kept + return draft + } + + func movingPairs(fromOffsets source: IndexSet, toOffset destination: Int) -> ERDiagramForeignKeyDraft { + let moving = source.filter { pairs.indices.contains($0) }.map { pairs[$0] } + var remaining = pairs.enumerated().filter { !source.contains($0.offset) }.map(\.element) + let insertion = destination - source.filter { $0 < destination }.count + remaining.insert(contentsOf: moving, at: min(max(insertion, 0), remaining.count)) + var draft = self + draft.pairs = remaining + return draft + } + + // The referenced side follows the referenced table's key, so both change in one write. + func referencing(_ table: String, keyColumns: [String], sourceColumns: [String]) -> ERDiagramForeignKeyDraft { + var draft = self + draft.referencedTable = table + let first = pairs.first?.column ?? "" + draft.pairs = keyColumns.isEmpty + ? [ColumnPair(column: first, referencedColumn: "")] + : Self.guessedPairs(for: keyColumns, keeping: first, at: nil, from: sourceColumns) + return draft + } + + // Unplaced, the kept column takes a key column no other column is named after, else the last: a table's own id shares the referenced id's name. + private static func guessedPairs( + for keyColumns: [String], + keeping column: String, + at position: Int?, + from sourceColumns: [String] + ) -> [ColumnPair] { + let others = sourceColumns.filter { $0 != column } + func namesake(of key: String, excluding used: Set) -> String? { + others.first { !used.contains($0) && $0.caseInsensitiveCompare(key) == .orderedSame } + } + let slot = position + ?? keyColumns.firstIndex { namesake(of: $0, excluding: []) == nil } + ?? max(keyColumns.count - 1, 0) + var used: Set = [] + return keyColumns.enumerated().map { offset, key in + guard offset != slot else { return ColumnPair(column: column, referencedColumn: key) } + let match = namesake(of: key, excluding: used) ?? "" + if !match.isEmpty { used.insert(match) } + return ColumnPair(column: match, referencedColumn: key) + } + } + + private static func firstRepeated(_ names: [String]) -> String? { + var seen = Set() + return names.first { !seen.insert($0).inserted } + } } enum ERDiagramEditSheet: Identifiable, Equatable { @@ -38,7 +185,7 @@ enum ERDiagramEditSheet: Identifiable, Equatable { var id: String { switch self { - case .addForeignKey(let draft): "fk:\(draft.table).\(draft.column)" + case .addForeignKey(let draft): "fk:\(draft.table).\(draft.pairs.first?.column ?? "")" case .addColumn(let table): "add:\(table)" case .renameColumn(let table, let column): "rename:\(table).\(column)" } @@ -306,12 +453,13 @@ extension ERDiagramViewModel { let referencedTable = target?.tableName ?? tables.first { $0 != table } ?? table - let referencedColumn = target?.columnName ?? defaultReferencedColumn(in: referencedTable) - editSheet = .addForeignKey(ERDiagramForeignKeyDraft( + editSheet = .addForeignKey(.starting( table: table, column: column, referencedTable: referencedTable, - referencedColumn: referencedColumn, + keyColumns: referencedKeyColumns(in: referencedTable), + target: target?.columnName, + sourceColumns: columnNames(of: table), name: defaultForeignKeyName(table: table, column: column) )) } @@ -327,12 +475,14 @@ extension ERDiagramViewModel { return node.columns.filter { $0.stagedEdit != .dropped }.map(\.name) } - func defaultReferencedColumn(in table: String) -> String { + // In table order, which is the key's own order unless it was declared in another; the reader can reorder the pairs. + func referencedKeyColumns(in table: String) -> [String] { guard let id = stagedFullGraph.nodeIndex[table], let node = stagedFullGraph.nodes.first(where: { $0.id == id }) - else { return "" } - let keys = node.columns.filter { $0.isPrimaryKey && $0.stagedEdit != .dropped } - return (keys.count == 1 ? keys.first : node.columns.first { $0.stagedEdit != .dropped })?.name ?? "" + else { return [] } + let kept = node.columns.filter { $0.stagedEdit != .dropped } + let keys = kept.filter(\.isPrimaryKey).map(\.name) + return keys.isEmpty ? kept.prefix(1).map(\.name) : keys } // MySQL wants a foreign key's name unique in the whole database, so it is checked against every key drawn. @@ -360,13 +510,7 @@ extension ERDiagramViewModel { } func stageForeignKey(_ draft: ERDiagramForeignKeyDraft) { - var foreignKey = EditableForeignKeyDefinition.placeholder() - foreignKey.name = draft.name.trimmingCharacters(in: .whitespacesAndNewlines) - foreignKey.columns = [draft.column] - foreignKey.referencedTable = draft.referencedTable - foreignKey.referencedColumns = [draft.referencedColumn] - foreignKey.onDelete = draft.onDelete - foreignKey.onUpdate = draft.onUpdate + let foreignKey = draft.foreignKey stage { try await $0.addForeignKey(foreignKey, to: draft.table) } } diff --git a/apple/TablePro/ViewModels/ERDiagramViewModel.swift b/apple/TablePro/ViewModels/ERDiagramViewModel.swift index 37104d6567..41462c6939 100644 --- a/apple/TablePro/ViewModels/ERDiagramViewModel.swift +++ b/apple/TablePro/ViewModels/ERDiagramViewModel.swift @@ -229,9 +229,8 @@ final class ERDiagramViewModel: ObservableObject { let visibleGraph = makeVisibleGraph() graph = visibleGraph - let layout = await Task.detached { - ERDiagramLayout.compute(graph: visibleGraph) - }.value + // A cancelled load leaves the state loading and unrecorded, so the next appearance runs it again. + guard let layout = try? await ERDiagramLayout.layOut(visibleGraph) else { return } computedLayout = layout loadPersistedPositions() invalidateCachedRects() @@ -483,10 +482,7 @@ final class ERDiagramViewModel: ObservableObject { let currentGraph = graph layoutTask?.cancel() layoutTask = Task { - let layout = await Task.detached { - ERDiagramLayout.compute(graph: currentGraph) - }.value - guard !Task.isCancelled else { return } + guard let layout = try? await ERDiagramLayout.layOut(currentGraph), !Task.isCancelled else { return } computedLayout = layout invalidateCachedRects() } @@ -534,10 +530,7 @@ final class ERDiagramViewModel: ObservableObject { invalidateCachedRects() layoutTask?.cancel() layoutTask = Task { - let layout = await Task.detached { - ERDiagramLayout.compute(graph: visibleGraph) - }.value - guard !Task.isCancelled else { return } + guard let layout = try? await ERDiagramLayout.layOut(visibleGraph), !Task.isCancelled else { return } computedLayout = layout invalidateCachedRects() } diff --git a/apple/TablePro/ViewModels/SchemaEditorViewModel.swift b/apple/TablePro/ViewModels/SchemaEditorViewModel.swift index 3d86e3b1fe..5c20599902 100644 --- a/apple/TablePro/ViewModels/SchemaEditorViewModel.swift +++ b/apple/TablePro/ViewModels/SchemaEditorViewModel.swift @@ -330,7 +330,7 @@ final class SchemaEditorViewModel: ObservableObject { renameCommitted: plan.renamesFirst && committedFirst ) } catch { - failure = error.localizedDescription + failure = GatedOperationFailure(error).message return ApplyOutcome( succeeded: false, committedName: originalName ?? "", renameCommitted: false ) diff --git a/apple/TablePro/ViewModels/ServerDashboardViewModel.swift b/apple/TablePro/ViewModels/ServerDashboardViewModel.swift index 2ad773a298..5bdb0e1791 100644 --- a/apple/TablePro/ViewModels/ServerDashboardViewModel.swift +++ b/apple/TablePro/ViewModels/ServerDashboardViewModel.swift @@ -1,6 +1,7 @@ import Combine import Foundation import os +import TableProPluginKit @MainActor final class ServerDashboardViewModel: ObservableObject { @@ -25,10 +26,7 @@ final class ServerDashboardViewModel: ObservableObject { @Published var refreshInterval: DashboardRefreshInterval = .fiveSeconds { didSet { guard oldValue != refreshInterval else { return } - if refreshTask != nil || refreshInterval != .off { - startAutoRefresh() - } - updateTileActivity() + restartRefreshLoop() } } @@ -66,6 +64,7 @@ final class ServerDashboardViewModel: ObservableObject { nonisolated(unsafe) private var refreshTask: Task? private var isOnScreen = false + private var isWindowVisible = true private let services: AppServices private var providerServerVersion: String? private var hasAdoptedServerVersion = false @@ -122,14 +121,28 @@ final class ServerDashboardViewModel: ObservableObject { // MARK: - Auto Refresh func startAutoRefresh() { - refreshTask?.cancel() isOnScreen = true - updateTileActivity() + restartRefreshLoop() + } - guard refreshInterval != .off else { - refreshTask = nil - return - } + func stopAutoRefresh() { + isOnScreen = false + restartRefreshLoop() + isRefreshing = false + } + + // A hidden, minimized or fully covered window shows nothing, so the server is not asked; coming back refreshes at once. + func windowVisibilityChanged(_ isVisible: Bool) { + guard isWindowVisible != isVisible else { return } + isWindowVisible = isVisible + restartRefreshLoop() + } + + private func restartRefreshLoop() { + refreshTask?.cancel() + refreshTask = nil + updateTileActivity() + guard isOnScreen, isWindowVisible, refreshInterval != .off else { return } refreshTask = Task { [weak self] in while !Task.isCancelled { @@ -144,16 +157,12 @@ final class ServerDashboardViewModel: ObservableObject { } } - func stopAutoRefresh() { - refreshTask?.cancel() - refreshTask = nil - isRefreshing = false - isOnScreen = false - updateTileActivity() + var isPolling: Bool { + refreshTask != nil } private func updateTileActivity() { - tileBoard.isActive = isOnScreen && !isPaused && refreshInterval != .off + tileBoard.isActive = isOnScreen && isWindowVisible && !isPaused && refreshInterval != .off } // MARK: - Data Fetching @@ -174,24 +183,46 @@ final class ServerDashboardViewModel: ObservableObject { await refresh(forcingTiles: true) } - private static func readExecutor(connectionId: UUID, services: AppServices) -> DashboardTileBoard.Execute { + private static func readExecutor( + connectionId: UUID, + services: AppServices + ) -> @MainActor (String) async throws -> QueryResult { { [connectionId, services] query in try await liveDriver(connectionId: connectionId, services: services).executeCatalogRead(query: query) } } - /// A saved query can return a whole table, so a tile stops at the rows it can draw. A driver - /// with no bounded fetch gets `executeRead`, which is also where a scripting engine refuses writes. + // A saved query can return a whole table, so a tile stops at the rows it can draw. A driver with no bounded fetch + // gets `executeRead`, which is also where a scripting engine refuses writes. At Read-Only, a driver that can runs + // the saved query in the engine's read-only mode. It runs in the session's turn, pinned to the scope the gate was + // asked about, as a query tab's run is, so no tab moves the session's schema between the check and the run. private static func tileExecutor(connectionId: UUID, services: AppServices) -> DashboardTileBoard.Execute { - { [connectionId, services] query in - let driver = try liveDriver(connectionId: connectionId, services: services) - let policy = await MainActor.run { - services.databaseManager.session(for: connectionId)?.connection.maskingPolicy ?? .none + { [connectionId, services] query, runsIn in + let manager = services.databaseManager + guard let scope = runsIn ?? manager.browseScope(for: connectionId) else { + throw DatabaseError.connectionFailed(String(localized: "No active connection")) } - if let bounded = try await driver.executeBoundedQuery(query: query, rowCap: DashboardTileTable.rowCap) { - return bounded.masked(by: policy) + let policy = manager.session(for: connectionId)?.connection.maskingPolicy ?? .none + let rowCap = DashboardTileTable.rowCap + let isReadOnly = ExecutionGateProvider.runsAtReadOnly(connectionId, in: manager) + return try await manager.withCancellableRead( + scope: scope, + route: manager.executionRoute(for: scope), + ownsSessionState: false, + readsOnly: true + ) { driver in + if let context = driver.readOnlyRunContext(safeModeIsReadOnly: isReadOnly) { + if let bounded = try await driver.executeBoundedQuery(query: query, rowCap: rowCap, context: context) { + return bounded.masked(by: policy) + } + return try await driver.executeUserQuery(query: query, rowCap: rowCap, parameters: nil, context: context) + .masked(by: policy) + } + if let bounded = try await driver.executeBoundedQuery(query: query, rowCap: rowCap) { + return bounded.masked(by: policy) + } + return try await driver.executeRead(query: query).masked(by: policy) } - return try await driver.executeRead(query: query).masked(by: policy) } } @@ -222,8 +253,11 @@ final class ServerDashboardViewModel: ObservableObject { if provider.supportedPanels.contains(.activeSessions) { do { - sessions = try await provider.fetchSessions(execute: execute) - sessions.sort(using: sessionSortOrder) + var fetched = try await provider.fetchSessions(execute: execute) + fetched.sort(using: sessionSortOrder) + if fetched != sessions { + sessions = fetched + } } catch { Self.logger.warning("Failed to fetch sessions: \(error.localizedDescription)") newPanelErrors[.activeSessions] = error.localizedDescription @@ -232,7 +266,10 @@ final class ServerDashboardViewModel: ObservableObject { if provider.supportedPanels.contains(.serverMetrics) { do { - metrics = try await provider.fetchMetrics(execute: execute) + let fetched = try await provider.fetchMetrics(execute: execute) + if fetched != metrics { + metrics = fetched + } } catch { Self.logger.warning("Failed to fetch metrics: \(error.localizedDescription)") newPanelErrors[.serverMetrics] = error.localizedDescription @@ -241,14 +278,19 @@ final class ServerDashboardViewModel: ObservableObject { if provider.supportedPanels.contains(.slowQueries) { do { - slowQueries = try await provider.fetchSlowQueries(execute: execute) + let fetched = try await provider.fetchSlowQueries(execute: execute) + if !fetched.elementsEqual(slowQueries, by: { $0.hasSameContent(as: $1) }) { + slowQueries = fetched + } } catch { Self.logger.warning("Failed to fetch slow queries: \(error.localizedDescription)") newPanelErrors[.slowQueries] = error.localizedDescription } } - panelErrors = newPanelErrors + if panelErrors != newPanelErrors { + panelErrors = newPanelErrors + } await tileBoard.refresh(force: forcingTiles) lastRefreshDate = Date() } diff --git a/apple/TablePro/ViewModels/SidebarViewModel.swift b/apple/TablePro/ViewModels/SidebarViewModel.swift index 08ecd41429..3dee78e01d 100644 --- a/apple/TablePro/ViewModels/SidebarViewModel.swift +++ b/apple/TablePro/ViewModels/SidebarViewModel.swift @@ -19,14 +19,12 @@ final class SidebarViewModel: ObservableObject { static func shared( connectionId: UUID, databaseType: DatabaseType, - selectedTables: Binding>, pendingTruncates: Binding>, pendingDeletes: Binding>, tableOperationOptions: Binding<[DatabaseTreeTableRef: TableOperationOptions]> ) -> SidebarViewModel { if let existing = registry[connectionId] { existing.updateBindings( - selectedTables: selectedTables, pendingTruncates: pendingTruncates, pendingDeletes: pendingDeletes, tableOperationOptions: tableOperationOptions @@ -34,7 +32,6 @@ final class SidebarViewModel: ObservableObject { return existing } let viewModel = SidebarViewModel( - selectedTables: selectedTables, pendingTruncates: pendingTruncates, pendingDeletes: pendingDeletes, tableOperationOptions: tableOperationOptions, @@ -50,12 +47,10 @@ final class SidebarViewModel: ObservableObject { } func updateBindings( - selectedTables: Binding>, pendingTruncates: Binding>, pendingDeletes: Binding>, tableOperationOptions: Binding<[DatabaseTreeTableRef: TableOperationOptions]> ) { - selectedTablesBinding = selectedTables pendingTruncatesBinding = pendingTruncates pendingDeletesBinding = pendingDeletes tableOperationOptionsBinding = tableOperationOptions @@ -181,13 +176,9 @@ final class SidebarViewModel: ObservableObject { ) } } - @Published var showOperationDialog = false - @Published var pendingOperationType: TableOperationType? - @Published var pendingOperationTables: [DatabaseTreeTableRef] = [] // MARK: - Binding Storage - @Published private var selectedTablesBinding: Binding> @Published private var pendingTruncatesBinding: Binding> @Published private var pendingDeletesBinding: Binding> @Published private var tableOperationOptionsBinding: Binding<[DatabaseTreeTableRef: TableOperationOptions]> @@ -205,11 +196,6 @@ final class SidebarViewModel: ObservableObject { // MARK: - Convenience Accessors - var selectedTables: Set { - get { selectedTablesBinding.wrappedValue } - set { selectedTablesBinding.wrappedValue = newValue } - } - var pendingTruncates: Set { get { pendingTruncatesBinding.wrappedValue } set { pendingTruncatesBinding.wrappedValue = newValue } @@ -233,7 +219,6 @@ final class SidebarViewModel: ObservableObject { // MARK: - Initialization init( - selectedTables: Binding>, pendingTruncates: Binding>, pendingDeletes: Binding>, tableOperationOptions: Binding<[DatabaseTreeTableRef: TableOperationOptions]>, @@ -242,7 +227,6 @@ final class SidebarViewModel: ObservableObject { defaults: UserDefaults = AppStorageEnvironment.shared.defaults, clock: any Clock = ContinuousClock() ) { - self.selectedTablesBinding = selectedTables self.pendingTruncatesBinding = pendingTruncates self.pendingDeletesBinding = pendingDeletes self.tableOperationOptionsBinding = tableOperationOptions @@ -293,9 +277,14 @@ final class SidebarViewModel: ObservableObject { private static func loadInitialExpansion(connectionId: UUID, defaults: UserDefaults) -> ExpansionState { var values: [SidebarObjectKind: Bool] = [:] - for kind in SidebarObjectKind.allCases { + for kind in SidebarObjectKind.builtIn { values[kind] = loadKindExpansion(connectionId: connectionId, kind: kind, defaults: defaults) } + let serverKey = SidebarPersistenceKey.serverKindsExpanded(connectionId: connectionId) + for (id, expanded) in defaults.dictionary(forKey: serverKey) ?? [:] { + guard let expanded = expanded as? Bool, !id.isEmpty else { continue } + values[.server(SidebarServerObjectKind(id: id))] = expanded + } return ExpansionState(values: values) } @@ -338,32 +327,34 @@ final class SidebarViewModel: ObservableObject { } private func persistExpansion(oldValue: ExpansionState) { - for kind in SidebarObjectKind.allCases where oldValue[kind] != expanded[kind] { + for kind in SidebarObjectKind.builtIn where oldValue[kind] != expanded[kind] { defaults.set( expanded[kind], forKey: SidebarPersistenceKey.expanded(connectionId: connectionId, kind: kind) ) } + let serverKinds = Set(oldValue.values.keys).union(expanded.values.keys).compactMap(\.serverKind) + guard serverKinds.contains(where: { oldValue[.server($0)] != expanded[.server($0)] }) else { return } + var stored: [String: Bool] = [:] + for (kind, isExpanded) in expanded.values { + guard let serverKind = kind.serverKind else { continue } + stored[serverKind.id] = isExpanded + } + defaults.set(stored, forKey: SidebarPersistenceKey.serverKindsExpanded(connectionId: connectionId)) } // MARK: - Batch Operations - /// A queued Truncate or Drop carries the row it was raised from, not that row's name. - /// The queue lives on the connection and outlives a database switch, so a name-keyed entry - /// was resolved at Save time against whatever the tab in front pointed at by then. - func batchToggleTruncate(refs: [DatabaseTreeTableRef]? = nil) { - let targets = refs ?? Array(selectedTables) + // Refs, not names: the queue outlives a database switch, so a name would resolve at Save time against whatever tab is in front. + func batchToggleTruncate(refs targets: [DatabaseTreeTableRef], from window: WindowSidebarState) { guard !targets.isEmpty else { return } - /// Unstaging comes first: a queued operation must always be removable, even once the - /// engine can no longer express it. Validating ahead of this left a Redis truncate stuck - /// in the queue after a `SELECT` moved the session to another database. + // Unstaging comes first, so a queued truncate stays removable after a Redis SELECT leaves the engine unable to express it. guard !targets.allSatisfy({ pendingTruncates.contains($0) }) else { unstage(targets, from: &pendingTruncatesBinding.wrappedValue) return } - /// The last gate before the queue, refusing the whole batch the way both menus now do - /// rather than truncating the part of a selection that happens to qualify. + // The whole batch is refused, as both menus do, rather than truncating the part that qualifies. guard TableOperationEligibility.canTruncate(targets) else { Self.logger.warning("Refused to stage a truncate against an object that holds no rows of its own") return @@ -373,29 +364,23 @@ final class SidebarViewModel: ObservableObject { Self.logger.warning("Refused to stage a truncate the engine has no statement for") return } - pendingOperationType = .truncate - pendingOperationTables = targets - showOperationDialog = true + window.requestTableOperation(.truncate, on: targets) } - func batchToggleDelete(refs: [DatabaseTreeTableRef]? = nil) { - let targets = refs ?? Array(selectedTables) + func batchToggleDelete(refs targets: [DatabaseTreeTableRef], from window: WindowSidebarState) { guard !targets.isEmpty else { return } guard !targets.allSatisfy({ pendingDeletes.contains($0) }) else { unstage(targets, from: &pendingDeletesBinding.wrappedValue) return } - /// The same last gate Truncate has. Without it a queued drop the engine cannot express - /// reached Save and was rejected there, after the dialog had already promised it. + // Without this gate a drop the engine cannot express reached Save and failed there, after the prompt promised it. if let eligibility = tableOperationEligibility(for: targets), !TableOperationEligibility.canDrop(targets, context: eligibility) { Self.logger.warning("Refused to stage a drop the engine has no statement for") return } - pendingOperationType = .drop - pendingOperationTables = targets - showOperationDialog = true + window.requestTableOperation(.drop, on: targets) } /// Nil when there is no driver to ask, which is not the same as "refused": with no session @@ -419,20 +404,13 @@ final class SidebarViewModel: ObservableObject { tableOperationOptions = options } - func cancelPendingOperation() { - pendingOperationType = nil - pendingOperationTables = [] - } - - func confirmOperation(options: TableOperationOptions) { - guard let operationType = pendingOperationType else { return } - + func confirmOperation(_ request: PendingTableOperation, options: TableOperationOptions) { var updatedTruncates = pendingTruncates var updatedDeletes = pendingDeletes var updatedOptions = tableOperationOptions - for ref in pendingOperationTables { - if operationType == .truncate { + for ref in request.tables { + if request.operation == .truncate { updatedDeletes.remove(ref) updatedTruncates.insert(ref) } else { @@ -445,16 +423,13 @@ final class SidebarViewModel: ObservableObject { pendingTruncates = updatedTruncates pendingDeletes = updatedDeletes tableOperationOptions = updatedOptions - - pendingOperationType = nil - pendingOperationTables = [] } // MARK: - Clipboard - func copySelectedTableNames() { - guard !selectedTables.isEmpty else { return } - let names = selectedTables.map { $0.table.name }.sorted() + static func copyTableNames(of tables: Set) { + guard !tables.isEmpty else { return } + let names = tables.map { $0.table.name }.sorted() ClipboardService.shared.writeText(names.joined(separator: ",")) } @@ -472,6 +447,7 @@ final class SidebarViewModel: ObservableObject { private var cachedFilteredTriggersFingerprint: (count: Int, generation: Int, query: String)? private var cachedFilteredUserTypes: [UserDefinedTypeInfo] = [] private var cachedFilteredUserTypesFingerprint: (count: Int, generation: Int, query: String)? + private var cachedFilteredServerObjects: [String: (count: Int, generation: Int, query: String, objects: [ServerObjectInfo])] = [:] private var schemaGeneration: Int { SchemaService.shared.generationToken(for: connectionId) @@ -494,7 +470,7 @@ final class SidebarViewModel: ObservableObject { if cachedFilteredByKindFingerprint?.count != fingerprint.count || cachedFilteredByKindFingerprint?.generation != fingerprint.generation || cachedFilteredByKindFingerprint?.query != fingerprint.query { - for tableKind in SidebarObjectKind.allCases where tableKind.category == .table { + for tableKind in SidebarObjectKind.builtIn where tableKind.category == .table { cachedFilteredByKind[tableKind] = applyQuery(query, to: self.tables(of: tableKind, from: tables)) } cachedFilteredByKindFingerprint = fingerprint @@ -550,6 +526,18 @@ final class SidebarViewModel: ObservableObject { return cachedFilteredUserTypes } + func filteredServerObjects(kindId: String, from objects: [ServerObjectInfo]) -> [ServerObjectInfo] { + let query = filterQuery + let generation = schemaGeneration + if let cached = cachedFilteredServerObjects[kindId], + cached.count == objects.count, cached.generation == generation, cached.query == query { + return cached.objects + } + let filtered = DatabaseTreeFilter.filteredServerObjects(objects, searchText: query, database: browsedDatabase) + cachedFilteredServerObjects[kindId] = (objects.count, generation, query, filtered) + return filtered + } + func effectiveExpanded(kind: SidebarObjectKind, hasMatches: Bool) -> Bool { if !filterQuery.isEmpty && hasMatches { return true } return expanded[kind] @@ -660,7 +648,7 @@ final class SidebarViewModel: ObservableObject { private func rebuildKindBuckets(from tables: [TableInfo]) { var buckets: [SidebarObjectKind: [TableInfo]] = [:] - for kind in SidebarObjectKind.allCases { + for kind in SidebarObjectKind.builtIn { buckets[kind] = [] } for table in tables { @@ -675,6 +663,7 @@ final class SidebarViewModel: ObservableObject { cachedFilteredByKindFingerprint = nil cachedFilteredRoutines = [:] cachedFilteredRoutinesFingerprint = nil + cachedFilteredServerObjects = [:] } /// Clearing the field, or typing the first character into an empty one, changes what the list diff --git a/apple/TablePro/ViewModels/UserTypeEditorModel.swift b/apple/TablePro/ViewModels/UserTypeEditorModel.swift index 3d6bc764f8..ae428d77f5 100644 --- a/apple/TablePro/ViewModels/UserTypeEditorModel.swift +++ b/apple/TablePro/ViewModels/UserTypeEditorModel.swift @@ -302,7 +302,7 @@ final class UserTypeEditorModel: ObservableObject { do { outcome = try await UserTypeChangeRunner.run(plan, in: scope, connection: connection, gate: gate) } catch { - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) if let message = failure.message { AlertHelper.showErrorSheet( title: String(localized: "Error Applying Changes"), diff --git a/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift b/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift index d54a6fc8bc..162da203eb 100644 --- a/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift +++ b/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift @@ -290,7 +290,7 @@ extension UsersRolesViewModel { .joined(separator: "\n") await load(forceReload: true) } catch { - applyFailure = error.localizedDescription + applyFailure = GatedOperationFailure(error).message } } diff --git a/apple/TablePro/ViewModels/VersionHistoryViewModel.swift b/apple/TablePro/ViewModels/VersionHistoryViewModel.swift index 6d275ca2b5..d213ba80d1 100644 --- a/apple/TablePro/ViewModels/VersionHistoryViewModel.swift +++ b/apple/TablePro/ViewModels/VersionHistoryViewModel.swift @@ -117,7 +117,8 @@ internal final class VersionHistoryViewModel: ObservableObject { } catch { guard !Task.isCancelled else { return } Self.logger.warning("Version history failed to load: \(error.publicLogShape, privacy: .public)") - let subjectIsGone = (error as? VersionHistoryError) == .subjectNotFound + let failure = error as? VersionHistoryError + let subjectIsGone = failure == .subjectNotFound || failure == .tabClosed guard page.entries.isEmpty || subjectIsGone else { return } page = .empty selection = nil diff --git a/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift b/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift index 9ac471d344..676d4aa587 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift @@ -282,6 +282,21 @@ extension WelcomeViewModel { loadConnections() } + // Leaving sync takes the secrets off iCloud Keychain on every device, so it asks first. + func requestExcludeFromSync(_ ids: [UUID]) { + let targets = ids.compactMap { connectionsById[$0] }.filter(\.participatesInSync) + guard !targets.isEmpty else { return } + connectionsToExcludeFromSync = targets + showExcludeFromSyncConfirmation = true + } + + func confirmExcludeFromSync() { + let ids = connectionsToExcludeFromSync.map(\.id) + connectionsToExcludeFromSync = [] + guard !ids.isEmpty else { return } + setIncludedInSync(ids, included: false) + } + func duplicateConnection(_ id: UUID) { guard let connection = connectionsById[id] else { return } guard let duplicate = storage.duplicateConnection(connection) else { @@ -460,7 +475,11 @@ extension WelcomeViewModel { case .moveToNewGroup(let ids): requestNewGroup(parentId: nil, movingConnectionIds: ids) case .setIncludedInSync(let ids, let included): - setIncludedInSync(ids, included: included) + if included { + setIncludedInSync(ids, included: true) + } else { + requestExcludeFromSync(ids) + } case .deleteConnections(let ids): requestDeleteConnections(ids) case .removeFromRecent(let ids): diff --git a/apple/TablePro/ViewModels/WelcomeViewModel+TeamLibrary.swift b/apple/TablePro/ViewModels/WelcomeViewModel+TeamLibrary.swift index 27c11ca7ba..3250049fca 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel+TeamLibrary.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel+TeamLibrary.swift @@ -7,6 +7,7 @@ // import AppKit +import TableProImport extension WelcomeViewModel { func publishConnectionsToTeamLibrary(_ connectionsToPublish: [DatabaseConnection], teamID: String) { @@ -38,8 +39,12 @@ extension WelcomeViewModel { confirm: @MainActor ([DatabaseConnection]) async -> Bool ) async -> TeamLibraryPublishOutcome { guard await confirm(connections) else { return .declined } + // A pull drops both, so sending them would only put a token typed into a command on the server. let plan = connections.map { - TeamLibraryPublishPlan.Connection(id: $0.id, settings: exporter.portableSettings(for: $0)) + TeamLibraryPublishPlan.Connection( + id: $0.id, + settings: exporter.portableSettings(for: $0).withoutStartupCommands().withoutTunnelCommand() + ) } do throws(TeamLibraryPublishError) { return .finished(try await sync.publishConnections(plan, to: teamID)) diff --git a/apple/TablePro/ViewModels/WelcomeViewModel.swift b/apple/TablePro/ViewModels/WelcomeViewModel.swift index ebaefbff7e..1153d64200 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel.swift @@ -119,6 +119,8 @@ final class WelcomeViewModel: ObservableObject { @Published var presentsWelcomeSheet = false @Published var connectionsToDelete: [DatabaseConnection] = [] @Published var showDeleteConfirmation = false + @Published var connectionsToExcludeFromSync: [DatabaseConnection] = [] + @Published var showExcludeFromSyncConfirmation = false @Published var pendingDeleteHasFavorites = false private var deleteRequestToken = UUID() @Published var showDeleteGroupConfirmation = false diff --git a/apple/TablePro/Views/AIChat/AIChatMessageView.swift b/apple/TablePro/Views/AIChat/AIChatMessageView.swift index 82960082ce..6622c98621 100644 --- a/apple/TablePro/Views/AIChat/AIChatMessageView.swift +++ b/apple/TablePro/Views/AIChat/AIChatMessageView.swift @@ -105,6 +105,13 @@ struct AIChatMessageView: View, Equatable { } } .padding(.horizontal, 8) + + if let limit = message.outputLimit { + Label(limit.note, systemImage: "exclamationmark.triangle") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.horizontal, 8) + } } if let onRetry { @@ -245,3 +252,14 @@ struct ChatTypingIndicatorView: View { .accessibilityLabel(String(localized: "Responding")) } } + +private extension ChatOutputLimit { + var note: String { + switch self { + case .outputTokens: + return String(localized: "The response stopped at the model's output token limit.") + case .contextWindow: + return String(localized: "The response stopped because the conversation filled the model's context window.") + } + } +} diff --git a/apple/TablePro/Views/AIChat/AIChatPanelView.swift b/apple/TablePro/Views/AIChat/AIChatPanelView.swift index a79fa47227..b830982a3b 100644 --- a/apple/TablePro/Views/AIChat/AIChatPanelView.swift +++ b/apple/TablePro/Views/AIChat/AIChatPanelView.swift @@ -29,12 +29,14 @@ struct AIChatPanelView: View { settingsManager.ai.hasActiveProvider } - /// The first call still waiting, in transcript order, which is the only one Return may answer. + // Only a card a turn still waits on may take Return and Escape. private var primaryPendingToolUseId: String? { for turn in viewModel.messages { for block in turn.blocks { guard case .toolUse(let useBlock) = block.kind, - case .pending = useBlock.approvalState else { continue } + case .pending = useBlock.approvalState, + ToolApprovalCenter.shared.isAwaiting(sessionId: viewModel.sessionId, toolUseId: useBlock.id) + else { continue } return useBlock.id } } @@ -73,7 +75,8 @@ struct AIChatPanelView: View { viewModel.connection = connection } .task(id: ModelListTrigger( - providerIDs: settingsManager.ai.providers.map(\.id), + sources: settingsManager.ai.providers.map(AIModelListSource.init(config:)), + savedRevision: viewModel.providerSettingsRevision, isCopilotRunning: copilotService.status == .running )) { await viewModel.loadAvailableModels() @@ -319,14 +322,7 @@ struct AIChatPanelView: View { } } - /// Sized to the model's name, and able to compress below it. - /// - /// Its label used to carry `maxWidth: .infinity`, which spread the button across the window as - /// soon as the conversation had one to spread across. `.fixedSize()` is the other end of the same - /// mistake: measured on macOS 27, a 44-character model name holds the button at 339pt, which - /// overflows the trailing pane's composer row by 85pt at 240pt wide. Unframed it takes the name's - /// width where there is room and truncates where there is not, and the spacer after it is what - /// keeps Send at the trailing edge in both widths. + // No frame and no .fixedSize(): a 44-character model name held a fixed-size button at 339 pt, 85 pt past a 240 pt pane (macOS 27). @ViewBuilder private var modelPicker: some View { let providers = settingsManager.ai.providers @@ -340,6 +336,7 @@ struct AIChatPanelView: View { let label = selectedProvider.map { provider in resolvedModel.isEmpty ? provider.displayName : resolvedModel } ?? String(localized: "Select Model") + let icon = selectedProvider?.icon ?? BrandIcon(slug: nil, fallbackSystemImage: "cpu") Menu { ForEach(providers) { provider in @@ -351,7 +348,7 @@ struct AIChatPanelView: View { } } label: { HStack(spacing: 4) { - Image(systemName: "cpu") + icon.menuImage(pointSize: 12) Text(label) .lineLimit(1) .truncationMode(.middle) @@ -595,9 +592,9 @@ struct AIChatPanelView: View { } } -/// Copilot lists no models until its server runs, and it starts on first use rather than with the -/// panel, so its list is fetched again once it is up. +// Copilot lists no models until its server runs, which starts on first use, not with the panel. private struct ModelListTrigger: Equatable { - let providerIDs: [UUID] + let sources: [AIModelListSource] + let savedRevision: Int let isCopilotRunning: Bool } diff --git a/apple/TablePro/Views/AIChat/AIChatToolUseBlockView.swift b/apple/TablePro/Views/AIChat/AIChatToolUseBlockView.swift index 948fd87f6c..c49084b81c 100644 --- a/apple/TablePro/Views/AIChat/AIChatToolUseBlockView.swift +++ b/apple/TablePro/Views/AIChat/AIChatToolUseBlockView.swift @@ -13,6 +13,7 @@ struct AIChatToolUseBlockView: View { @State private var isExpanded: Bool = false @Environment(\.chatApprovalConnectionName) private var connectionName + @Environment(\.chatApprovalSessionId) private var sessionId internal static let inputFont = Font.system(.caption, design: .monospaced) @@ -21,6 +22,12 @@ struct AIChatToolUseBlockView: View { return false } + // A click on a card no turn waits on is dropped, so its buttons are not drawn. + private var isAwaitingAnswer: Bool { + guard isPending, let sessionId else { return false } + return ToolApprovalCenter.shared.isAwaiting(sessionId: sessionId, toolUseId: block.id) + } + private var shouldShowInput: Bool { hasInput && (isExpanded || (isPending && proposedStatement == nil)) } @@ -79,7 +86,7 @@ struct AIChatToolUseBlockView: View { CodeWell(prettyInput, font: Self.inputFont) } - if case .pending = block.approvalState { + if isAwaitingAnswer { ToolApprovalActionsRow( toolUseId: block.id, toolName: block.name, diff --git a/apple/TablePro/Views/Compare/CompareRowDiffPane.swift b/apple/TablePro/Views/Compare/CompareRowDiffPane.swift index 542c2d52e9..f80cb6f1c3 100644 --- a/apple/TablePro/Views/Compare/CompareRowDiffPane.swift +++ b/apple/TablePro/Views/Compare/CompareRowDiffPane.swift @@ -208,12 +208,27 @@ internal struct CompareRowDiffPane: View { ) } + // Named by side, because the two tables usually share a name and only one database cut its stream. + private func rowCutNotices(for summary: DataDiffSummary, plan: DataComparePlan) -> [String] { + let sides: [(endpoint: DatabaseEndpoint?, cut: Int?)] = [ + (session.source, summary.sourceRowCut), (session.target, summary.targetRowCut) + ] + return sides.compactMap { side in + guard let cut = side.cut else { return nil } + let table = ([side.endpoint?.qualifiedDescription].compactMap { $0 } + [plan.table]).joined(separator: " / ") + return StreamRowCutWarning.text(table: table, limit: cut) + } + } + @ViewBuilder private func notices(for plan: DataComparePlan) -> some View { if let failure = plan.comparisonFailure { notice(failure, systemImage: "exclamationmark.octagon.fill") } if let summary = plan.summary { + ForEach(rowCutNotices(for: summary, plan: plan), id: \.self) { text in + notice(text, systemImage: "exclamationmark.triangle.fill") + } if summary.stoppedAtRowLimit { notice(rowLimitNotice(for: summary), systemImage: "info.circle.fill") } diff --git a/apple/TablePro/Views/Compare/CompareRowGrid.swift b/apple/TablePro/Views/Compare/CompareRowGrid.swift index 80ed3e5c35..9f651d5ba5 100644 --- a/apple/TablePro/Views/Compare/CompareRowGrid.swift +++ b/apple/TablePro/Views/Compare/CompareRowGrid.swift @@ -52,6 +52,7 @@ internal final class CompareRowGridModel: ObservableObject { configuration.supportsValueFilter = false configuration.supportsColumnSort = false configuration.appliesRowCopyFormat = true + configuration.colorsValuesByType = true configuration.commands = [.freezeColumns, .displayAs] return configuration } diff --git a/apple/TablePro/Views/Components/BrandIconView.swift b/apple/TablePro/Views/Components/BrandIconView.swift new file mode 100644 index 0000000000..bfb2e1cb1f --- /dev/null +++ b/apple/TablePro/Views/Components/BrandIconView.swift @@ -0,0 +1,97 @@ +// +// BrandIconView.swift +// TablePro +// + +import AppKit +import SwiftUI + +internal struct BrandIcon: Equatable, Sendable { + internal let slug: String? + internal let fallbackSystemImage: String + + internal init(slug: String?, fallbackSystemImage: String) { + self.slug = slug + self.fallbackSystemImage = fallbackSystemImage + } + + internal static func assetName(for slug: String) -> String { + "\(slug)-icon" + } + + internal var assetName: String? { + slug.map(Self.assetName(for:)) + } + + internal var assetImage: NSImage? { + assetName.flatMap { NSImage(named: $0) } + } + + // An NSMenu draws an asset at its intrinsic size (Kiro's is 1200 pt), so menus get a copy drawn into a fixed square. + internal func nsImage(pointSize: CGFloat = 16, accessibilityDescription: String? = nil) -> NSImage? { + guard let asset = assetImage else { + return NSImage(systemSymbolName: fallbackSystemImage, accessibilityDescription: accessibilityDescription) + } + let image = NSImage(size: NSSize(width: pointSize, height: pointSize), flipped: false) { rect in + asset.draw(in: Self.aspectFit(asset.size, in: rect)) + return true + } + image.isTemplate = asset.isTemplate + image.accessibilityDescription = accessibilityDescription + return image + } + + internal func menuImage(pointSize: CGFloat = 16) -> Image { + guard assetImage != nil, let image = nsImage(pointSize: pointSize) else { + return Image(systemName: fallbackSystemImage) + } + return Image(nsImage: image) + } + + private static func aspectFit(_ size: NSSize, in rect: NSRect) -> NSRect { + guard size.width > 0, size.height > 0 else { return rect } + let scale = min(rect.width / size.width, rect.height / size.height) + let fitted = NSSize(width: size.width * scale, height: size.height * scale) + return NSRect( + x: rect.midX - fitted.width / 2, + y: rect.midY - fitted.height / 2, + width: fitted.width, + height: fitted.height + ) + } +} + +internal struct BrandIconView: View { + private let assetName: String? + private let isTemplate: Bool + private let fallbackSystemImage: String + + internal init(slug: String, fallbackSystemImage: String) { + self.init(icon: BrandIcon(slug: slug, fallbackSystemImage: fallbackSystemImage)) + } + + internal init(icon: BrandIcon) { + if let asset = icon.assetImage { + assetName = icon.assetName + isTemplate = asset.isTemplate + } else { + assetName = nil + isTemplate = false + } + fallbackSystemImage = icon.fallbackSystemImage + } + + internal var body: some View { + Group { + if let assetName { + Image(decorative: assetName) + .renderingMode(isTemplate ? .template : .original) + .resizable() + .scaledToFit() + } else { + Image(systemName: fallbackSystemImage) + } + } + .accessibilityHidden(true) + } +} diff --git a/apple/TablePro/Views/Components/DiagramExporter.swift b/apple/TablePro/Views/Components/DiagramExporter.swift index c7d201013a..becaafb5f7 100644 --- a/apple/TablePro/Views/Components/DiagramExporter.swift +++ b/apple/TablePro/Views/Components/DiagramExporter.swift @@ -21,6 +21,10 @@ enum DiagramExporter { } static func pngData(_ image: NSImage?) -> Data? { + // A bitmap the image already holds encodes as it is, without a TIFF copy and a second decode. + if let rep = image?.representations.first(where: { $0 is NSBitmapImageRep }) as? NSBitmapImageRep { + return rep.representation(using: .png, properties: [:]) + } guard let tiffData = image?.tiffRepresentation, let bitmap = NSBitmapImageRep(data: tiffData) else { return nil } diff --git a/apple/TablePro/Views/Components/TransferProgress.swift b/apple/TablePro/Views/Components/TransferProgress.swift index 9bbda34a00..5f93bbf28f 100644 --- a/apple/TablePro/Views/Components/TransferProgress.swift +++ b/apple/TablePro/Views/Components/TransferProgress.swift @@ -133,6 +133,7 @@ internal struct TransferProgress: Equatable, Sendable { internal var unit: TransferUnit internal var isEstimate: Bool internal var note: String? + internal var fileFraction: Double? internal var isStopping: Bool internal init( @@ -144,6 +145,7 @@ internal struct TransferProgress: Equatable, Sendable { unit: TransferUnit, isEstimate: Bool = false, note: String? = nil, + fileFraction: Double? = nil, isStopping: Bool = false ) { self.job = job @@ -154,6 +156,7 @@ internal struct TransferProgress: Equatable, Sendable { self.unit = unit self.isEstimate = isEstimate self.note = note.flatMap { $0.isEmpty ? nil : $0 } + self.fileFraction = fileFraction self.isStopping = isStopping } @@ -161,8 +164,11 @@ internal struct TransferProgress: Equatable, Sendable { internal var stopPolicy: TransferStopPolicy { job.stopPolicy } - // An estimate stops short of the end, so a bar never reads done while the job still runs. + // An estimate stops short of the end, so a bar never reads done while the job still runs. So does a file read to + // its end: its last statements still run. internal var fraction: Double? { + // A statement count cannot move while one COPY streams its rows, and the file's bytes read can. + if let fileFraction { return min(0.99, max(0.0, fileFraction)) } guard let total else { return nil } let ceiling = isEstimate ? 0.95 : 1.0 return min(ceiling, Double(completed) / Double(total)) diff --git a/apple/TablePro/Views/Components/WindowVisibilityReader.swift b/apple/TablePro/Views/Components/WindowVisibilityReader.swift new file mode 100644 index 0000000000..0685ffcb49 --- /dev/null +++ b/apple/TablePro/Views/Components/WindowVisibilityReader.swift @@ -0,0 +1,58 @@ +import AppKit +import SwiftUI + +// Reports whether any part of the hosting window is on screen. A minimized, hidden or fully covered window, +// a background window tab, and a view taken out of its window all report false; `onDisappear` misses most of these. +struct WindowVisibilityReader: NSViewRepresentable { + var onChange: (Bool) -> Void + + func makeNSView(context: Context) -> WindowVisibilityView { + let view = WindowVisibilityView() + view.onChange = onChange + return view + } + + func updateNSView(_ nsView: WindowVisibilityView, context: Context) { + nsView.onChange = onChange + } + + static func dismantleNSView(_ nsView: WindowVisibilityView, coordinator: ()) { + nsView.stopObserving() + } +} + +final class WindowVisibilityView: NSView { + var onChange: ((Bool) -> Void)? + private var observer: NSObjectProtocol? + private var reported: Bool? + + override func viewDidMoveToWindow() { + super.viewDidMoveToWindow() + stopObserving() + if let window { + observer = NotificationCenter.default.addObserver( + forName: NSWindow.didChangeOcclusionStateNotification, + object: window, + queue: .main + ) { [weak self] _ in + MainActor.assumeIsolated { self?.report() } + } + } + // Moving into a window happens inside a SwiftUI update, which must not publish. + DispatchQueue.main.async { [weak self] in self?.report() } + } + + func stopObserving() { + if let observer { + NotificationCenter.default.removeObserver(observer) + } + observer = nil + } + + private func report() { + let isVisible = window?.occlusionState.contains(.visible) ?? false + guard isVisible != reported else { return } + reported = isVisible + onChange?(isVisible) + } +} diff --git a/apple/TablePro/Views/Connection/PasswordPromptToggle.swift b/apple/TablePro/Views/Connection/PasswordPromptToggle.swift index e93cdc358a..db527b7f46 100644 --- a/apple/TablePro/Views/Connection/PasswordPromptToggle.swift +++ b/apple/TablePro/Views/Connection/PasswordPromptToggle.swift @@ -26,6 +26,7 @@ struct PasswordPromptToggle: View { PasswordSourceTestView( source: source, context: { coordinator.passwordSourceContext() }, + leaseOwner: { coordinator.connectionId }, refusal: { coordinator.storage.storeIsTrusted ? nil : PasswordSourceResolver.ResolutionError.storeNotTrusted } diff --git a/apple/TablePro/Views/Connection/PasswordSourceTest.swift b/apple/TablePro/Views/Connection/PasswordSourceTest.swift index fdf9a8081e..5ad7c12df6 100644 --- a/apple/TablePro/Views/Connection/PasswordSourceTest.swift +++ b/apple/TablePro/Views/Connection/PasswordSourceTest.swift @@ -16,11 +16,11 @@ final class PasswordSourceTestModel: ObservableObject { @Published private(set) var isRunning = false @Published private var latest: (source: PasswordSource, environment: [String: String], outcome: Outcome)? - private let resolve: @Sendable (PasswordSource, ConnectionScriptContext) async throws -> String + private let resolve: @Sendable (PasswordSource, ConnectionScriptContext, UUID?) async throws -> String init( - resolve: @escaping @Sendable (PasswordSource, ConnectionScriptContext) async throws -> String = { source, context in - try await PasswordSourceResolver.test(source, for: context) + resolve: @escaping @Sendable (PasswordSource, ConnectionScriptContext, UUID?) async throws -> String = { source, context, owner in + try await PasswordSourceResolver.test(source, for: context, leaseOwner: owner) } ) { self.resolve = resolve @@ -32,7 +32,7 @@ final class PasswordSourceTestModel: ObservableObject { return latest.outcome } - func run(_ source: PasswordSource, context: ConnectionScriptContext, refusal: (any Error)?) async { + func run(_ source: PasswordSource, context: ConnectionScriptContext, leaseOwner: UUID?, refusal: (any Error)?) async { guard !isRunning else { return } if let refusal { latest = (source, context.environment, .failed(refusal.localizedDescription)) @@ -41,7 +41,7 @@ final class PasswordSourceTestModel: ObservableObject { isRunning = true defer { isRunning = false } do { - let value = try await resolve(source, context) + let value = try await resolve(source, context, leaseOwner) latest = (source, context.environment, .returned(value)) } catch { latest = (source, context.environment, .failed(error.localizedDescription)) @@ -52,6 +52,9 @@ final class PasswordSourceTestModel: ObservableObject { struct PasswordSourceTestView: View { let source: PasswordSource let context: () -> ConnectionScriptContext + // Whose Vault user a database role test signs in as: the profile, or the connection for a source of its own. + // Nil for an unsaved connection, whose test user nothing would reuse. + let leaseOwner: () -> UUID? let refusal: () -> (any Error)? @StateObject private var model = PasswordSourceTestModel() @@ -61,8 +64,9 @@ struct PasswordSourceTestView: View { HStack { Button(String(localized: "Test Source")) { let context = context() + let leaseOwner = leaseOwner() let refusal = refusal() - Task { await model.run(source, context: context, refusal: refusal) } + Task { await model.run(source, context: context, leaseOwner: leaseOwner, refusal: refusal) } } .disabled(model.isRunning) .accessibilityIdentifier("password-source-test") diff --git a/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift b/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift index 029f4797d7..69fa97b307 100644 --- a/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift +++ b/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift @@ -42,6 +42,7 @@ struct CredentialProfilePicker: View { PasswordSourceTestView( source: source, context: { coordinator.passwordSourceContext() }, + leaseOwner: { profile.id }, refusal: { CredentialProfileStorage.shared.storeIsTrusted ? nil : PasswordSourceResolver.ResolutionError.profileStoreNotTrusted diff --git a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift index 551b1e9c6f..d88f931b4e 100644 --- a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift +++ b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator+SafeMode.swift @@ -16,8 +16,7 @@ extension ConnectionFormCoordinator { ) } - /// The level the connection will run at, which is what the form shows. Picking the level - /// already shown keeps the user's saved choice, which comes back once the floor lifts. + // Picking the level already shown keeps the saved choice, which comes back once the floor lifts. var effectiveSafeModeLevel: SafeModeLevel { get { safeModeFloor?.raising(customization.safeModeLevel) ?? customization.safeModeLevel } set { @@ -25,4 +24,15 @@ extension ConnectionFormCoordinator { customization.safeModeLevel = newValue } } + + var isManualCommitAvailable: Bool { ManualCommit.isAvailable(for: network.type) } + + // Off where the engine can't hold, since a stored Manual Commit runs as automatic there. + var effectiveManualCommit: Bool { + get { isManualCommitAvailable && customization.commitMode == .manual } + set { + guard isManualCommitAvailable else { return } + customization.commitMode = newValue ? .manual : .automatic + } + } } diff --git a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift index 3be561720c..2d3dc51202 100644 --- a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift +++ b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift @@ -57,6 +57,11 @@ final class ConnectionFormCoordinator: ObservableObject { @Published var pluginInstallConnection: DatabaseConnection? @Published var pluginDiagnostic: PluginDiagnosticItem? + // A connection that syncs today and is saved Local only has its secrets taken off iCloud Keychain. + var removesSyncedSecretsOnSave: Bool { + advanced.localOnly && originalConnection?.participatesInSync == true + } + @Published var saveError: String? @Published var isChoosingType: Bool = false @@ -371,6 +376,7 @@ final class ConnectionFormCoordinator: ObservableObject { queryTimeoutSeconds: customization.queryTimeoutSeconds, binaryInlineLimitKilobytes: customization.binaryInlineLimitKilobytes, completionIdentifierCase: customization.completionIdentifierCase, + commitMode: customization.commitMode, redisDatabase: advanced.additionalFieldValues[RedisDatabaseIndex.fieldName].map { RedisDatabaseIndex.parse($0) ?? 0 }, @@ -379,7 +385,8 @@ final class ConnectionFormCoordinator: ObservableObject { localOnly: advanced.localOnly, additionalFields: fields, ownedAdditionalFieldIDs: ownedAdditionalFieldIDs(), - maskingRules: maskingRules.savedRules + maskingRules: maskingRules.savedRules, + revokedAIAlwaysAllowedTools: advanced.revokedAIAlwaysAllowedTools ) } diff --git a/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift b/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift index 84c5296869..7107e4813a 100644 --- a/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift +++ b/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift @@ -355,6 +355,8 @@ struct OptionsPaneView: View { private var safetySection: some View { Section { safeModeRow + Toggle(String(localized: "Manual Commit"), isOn: $coordinator.effectiveManualCommit) + .disabled(!coordinator.isManualCommitAvailable) if aiIsEnabled { Picker(String(localized: "AI Policy"), selection: $coordinator.advanced.aiPolicy) { Text(String(localized: "Use Default")) @@ -364,6 +366,9 @@ struct OptionsPaneView: View { .tag(AIConnectionPolicy?.some(policy) as AIConnectionPolicy?) } } + ForEach(coordinator.advanced.aiAlwaysAllowedTools, id: \.self) { tool in + alwaysAllowedRow(tool) + } } Picker(String(localized: "External Clients"), selection: $coordinator.advanced.externalAccess) { ForEach(ExternalAccessLevel.allCases) { level in @@ -378,6 +383,18 @@ struct OptionsPaneView: View { } } + private func alwaysAllowedRow(_ tool: String) -> some View { + LabeledContent { + Button(String(localized: "Forget")) { + coordinator.advanced.forgetAlwaysAllowed(tool) + } + .accessibilityLabel(String(format: String(localized: "Forget %@"), tool)) + } label: { + Text(verbatim: tool) + Text(String(localized: "Always allowed")) + } + } + @ViewBuilder private var safeModeRow: some View { let levels = SafeModeFloor.levels(allowedBy: coordinator.safeModeFloor) @@ -398,9 +415,18 @@ struct OptionsPaneView: View { if let floor = coordinator.safeModeFloor { Text(floor.explanation) } + if coordinator.isManualCommitAvailable { + // swiftlint:disable:next line_length + Text(String(localized: "Your first change opens a transaction, and nothing is saved until you commit. Structure changes, Users & Roles, Copy To, Generate Rows and Shortcuts actions still save at once.")) + } else { + Text(ManualCommit.unavailableExplanation) + } if aiIsEnabled { // swiftlint:disable:next line_length Text(String(localized: "AI Policy controls in-app AI agents. External Clients controls Raycast, Cursor, Claude Desktop, other MCP clients, AppleScript, and Shortcuts. Effective scope is the minimum of the requesting token's scope and the External Clients level.")) + if !coordinator.advanced.aiAlwaysAllowedTools.isEmpty { + Text(String(localized: "Always Allow on an assistant tool card adds the tool here. Forget makes its next call ask again.")) + } } else { // swiftlint:disable:next line_length Text(String(localized: "Controls how external clients (Raycast, Cursor, Claude Desktop, AppleScript, Shortcuts) access this connection. Tokens cannot exceed this level even with full-access scope.")) @@ -441,9 +467,14 @@ struct OptionsPaneView: View { } header: { Text(String(localized: "iCloud Sync")) } footer: { - Text("This connection won't sync to other devices via iCloud.") - .font(.caption) - .foregroundStyle(.secondary) + VStack(alignment: .leading, spacing: 4) { + Text("This connection won't sync to other devices via iCloud.") + if coordinator.removesSyncedSecretsOnSave { + Text("Saving removes its passwords from iCloud Keychain. Your other devices will need them entered again.") + } + } + .font(.caption) + .foregroundStyle(.secondary) } } } diff --git a/apple/TablePro/Views/ConnectionForm/Support/ConnectionFormEdits.swift b/apple/TablePro/Views/ConnectionForm/Support/ConnectionFormEdits.swift index 21fcb4ee5f..904d4b349b 100644 --- a/apple/TablePro/Views/ConnectionForm/Support/ConnectionFormEdits.swift +++ b/apple/TablePro/Views/ConnectionForm/Support/ConnectionFormEdits.swift @@ -35,12 +35,14 @@ struct ConnectionFormEdits: Equatable { var queryTimeoutSeconds: Int? var binaryInlineLimitKilobytes: Int? var completionIdentifierCase: SQLIdentifierCase = .default + var commitMode: ConnectionCommitMode = .automatic var redisDatabase: Int? var startupCommands: String? var localOnly: Bool var additionalFields: [String: String] var ownedAdditionalFieldIDs: Set var maskingRules: [MaskingRule] = [] + var revokedAIAlwaysAllowedTools: Set = [] static let appManagedAdditionalFieldIDs: Set = [ "preConnectScript", @@ -51,6 +53,7 @@ struct ConnectionFormEdits: Equatable { DatabaseConnection.queryTimeoutSecondsKey, DatabaseConnection.binaryInlineLimitKilobytesKey, DatabaseConnection.completionIdentifierCaseKey, + DatabaseConnection.commitModeKey, DatabaseConnection.maskingRulesKey, DatabaseConnection.sshForwardUnixSocketPathKey, MySQLLocalSocket.fieldKey @@ -89,7 +92,10 @@ struct ConnectionFormEdits: Equatable { result.queryTimeoutSeconds = queryTimeoutSeconds result.binaryInlineLimitKilobytes = binaryInlineLimitKilobytes result.completionIdentifierCase = completionIdentifierCase + result.commitMode = commitMode result.maskingRules = maskingRules + // Subtract, never assign: a grant recorded while the form was open stays. + result.aiAlwaysAllowedTools.subtract(revokedAIAlwaysAllowedTools) return result } diff --git a/apple/TablePro/Views/ConnectionForm/ViewModels/AdvancedPaneViewModel.swift b/apple/TablePro/Views/ConnectionForm/ViewModels/AdvancedPaneViewModel.swift index 1b85ee7181..28de5c3540 100644 --- a/apple/TablePro/Views/ConnectionForm/ViewModels/AdvancedPaneViewModel.swift +++ b/apple/TablePro/Views/ConnectionForm/ViewModels/AdvancedPaneViewModel.swift @@ -18,6 +18,8 @@ final class AdvancedPaneViewModel: ObservableObject { @Published var externalAccess: ExternalAccessLevel = .readOnly @Published var localOnly: Bool = false @Published var aiPolicy: AIConnectionPolicy? + @Published private(set) var aiAlwaysAllowedTools: [String] = [] + @Published private(set) var revokedAIAlwaysAllowedTools: Set = [] @Published var coordinator: WeakCoordinatorRef? @@ -139,10 +141,17 @@ final class AdvancedPaneViewModel: ObservableObject { postDisconnectScript = connection.postDisconnectScript ?? "" scriptTimeoutSeconds = connection.scriptTimeoutSeconds aiPolicy = connection.aiPolicy + aiAlwaysAllowedTools = connection.aiAlwaysAllowedTools.sorted() + revokedAIAlwaysAllowedTools = [] externalAccess = connection.externalAccess localOnly = connection.localOnly } + func forgetAlwaysAllowed(_ tool: String) { + aiAlwaysAllowedTools.removeAll { $0 == tool } + revokedAIAlwaysAllowedTools.insert(tool) + } + func write(into fields: inout [String: String]) { for (key, value) in additionalFieldValues { fields[key] = value diff --git a/apple/TablePro/Views/ConnectionForm/ViewModels/CustomizationPaneViewModel.swift b/apple/TablePro/Views/ConnectionForm/ViewModels/CustomizationPaneViewModel.swift index c255b0a0c3..548285cfac 100644 --- a/apple/TablePro/Views/ConnectionForm/ViewModels/CustomizationPaneViewModel.swift +++ b/apple/TablePro/Views/ConnectionForm/ViewModels/CustomizationPaneViewModel.swift @@ -19,6 +19,7 @@ final class CustomizationPaneViewModel: ObservableObject { @Published var queryTimeoutSeconds: Int? @Published var binaryInlineLimitKilobytes: Int? @Published var completionIdentifierCase: SQLIdentifierCase = .default + @Published var commitMode: ConnectionCommitMode = .automatic @Published var coordinator: WeakCoordinatorRef? @@ -52,5 +53,6 @@ final class CustomizationPaneViewModel: ObservableObject { queryTimeoutSeconds = connection.queryTimeoutSeconds binaryInlineLimitKilobytes = connection.binaryInlineLimitKilobytes completionIdentifierCase = connection.completionIdentifierCase + commitMode = connection.commitMode } } diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellAppearance.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellAppearance.swift index bb2faafb71..ec6b958728 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellAppearance.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellAppearance.swift @@ -28,6 +28,7 @@ struct DataGridCellAppearance: Equatable { onEmphasizedSelection: Bool, checkboxMark: DataGridCheckboxMark? = nil, textTone: NSColor? = nil, + valueTone: DataGridValueTone = .plain, revealsInvisibleCharacters: Bool = false ) -> DataGridCellAppearance { let deletedTextColor = state.visualState.isDeleted ? palette.deletedRowText : nil @@ -41,7 +42,12 @@ struct DataGridCellAppearance: Equatable { baseColor = deletedTextColor ?? .systemBlue case .none: font = palette.regularFont - baseColor = deletedTextColor ?? textTone ?? valueTextColor(kind: kind, rawValue: content.rawValue, palette: palette) + baseColor = deletedTextColor ?? textTone ?? valueTextColor( + kind: kind, + rawValue: content.rawValue, + valueTone: valueTone, + palette: palette + ) case .null, .empty, .absent, .unloaded: font = palette.italicFont baseColor = deletedTextColor ?? palette.placeholderText @@ -103,11 +109,12 @@ struct DataGridCellAppearance: Equatable { private static func valueTextColor( kind: DataGridCellKind, rawValue: String?, + valueTone: DataGridValueTone, palette: DataGridCellPalette ) -> NSColor { - guard kind == .boolean, let rawValue, let isTrue = StoredBoolean.value(of: rawValue) else { - return palette.text + if kind == .boolean, let rawValue, let isTrue = StoredBoolean.value(of: rawValue) { + return isTrue ? palette.booleanTrueText : palette.booleanFalseText } - return isTrue ? palette.booleanTrueText : palette.booleanFalseText + return palette.valueColors.color(for: valueTone) ?? palette.text } } diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift index b32be5c107..1f918e38b5 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift @@ -23,16 +23,17 @@ struct DataGridCellContent { static func placeholder( for rawValue: PluginCellValue, isAbsent: Bool = false, + isExpression: Bool = false, unloadedByteCount: Int? = nil, marksEmptyText: Bool = true ) -> DataGridCellPlaceholder? { if let unloadedByteCount { return .unloaded(byteCount: unloadedByteCount) } guard !isAbsent else { return .absent } + if rawValue.isColumnDefault(isExpression: isExpression) { return .defaultMarker } switch rawValue { case .null: return .null case .text(let value): - if value == PluginCellValue.defaultMarkerText { return .defaultMarker } return value.isEmpty && marksEmptyText ? .empty : nil case .bytes: return nil diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellKind.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellKind.swift index 73b673868e..d6f7cbacde 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellKind.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellKind.swift @@ -113,8 +113,9 @@ enum DataGridCellAccessory: Equatable { struct DataGridColumnPresentation: Equatable { let kind: DataGridCellKind let accessory: DataGridCellAccessory + let valueTone: DataGridColumnTone - static let plain = DataGridColumnPresentation(kind: .text, accessory: .none) + static let plain = DataGridColumnPresentation(kind: .text, accessory: .none, valueTone: .plain) static func resolve( columnType: ColumnType?, @@ -123,7 +124,8 @@ struct DataGridColumnPresentation: Equatable { isTypePicker: Bool, isEnumOrSet: Bool, isEditable: Bool, - isCheckbox: Bool = false + isCheckbox: Bool = false, + valueTone: DataGridColumnTone = .plain ) -> DataGridColumnPresentation { let configuredPicker = isDropdown || isTypePicker let picksFromList = configuredPicker || isEnumOrSet || columnType?.isEnumOrSetType == true @@ -149,7 +151,8 @@ struct DataGridColumnPresentation: Equatable { return DataGridColumnPresentation( kind: kind, - accessory: DataGridCellAccessory.reserved(for: kind, isEditable: isEditable) + accessory: DataGridCellAccessory.reserved(for: kind, isEditable: isEditable), + valueTone: valueTone ) } } diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPalette.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPalette.swift index 4f015cde7f..a70264b8c3 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPalette.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPalette.swift @@ -16,6 +16,7 @@ struct DataGridCellPalette: Equatable { let placeholderText: NSColor let booleanTrueText: NSColor let booleanFalseText: NSColor + let valueColors: DataGridValueColors let rowNumberText: NSColor let deletedRowText: NSColor let modifiedColumnTint: NSColor @@ -33,6 +34,7 @@ struct DataGridCellPalette: Equatable { placeholderText: NSColor, booleanTrueText: NSColor, booleanFalseText: NSColor, + valueColors: DataGridValueColors, rowNumberText: NSColor, deletedRowText: NSColor, modifiedColumnTint: NSColor, @@ -49,6 +51,7 @@ struct DataGridCellPalette: Equatable { self.placeholderText = placeholderText self.booleanTrueText = booleanTrueText self.booleanFalseText = booleanFalseText + self.valueColors = valueColors self.rowNumberText = rowNumberText self.deletedRowText = deletedRowText self.modifiedColumnTint = modifiedColumnTint @@ -73,6 +76,36 @@ struct DataGridCellPalette: Equatable { } } +// Stored fields, so a cell's color is a switch and never a lookup. +struct DataGridValueColors: Equatable { + let string: NSColor + let number: NSColor + let dateTime: NSColor + let binary: NSColor + let json: NSColor + let uuid: NSColor + let enumeration: NSColor + + func color(for tone: DataGridValueTone) -> NSColor? { + switch tone { + case .plain: return nil + case .string: return string + case .number: return number + case .dateTime: return dateTime + case .binary: return binary + case .json: return json + case .uuid: return uuid + case .enumeration: return enumeration + } + } +} + +extension DataGridValueColors { + init(all color: NSColor) { + self.init(string: color, number: color, dateTime: color, binary: color, json: color, uuid: color, enumeration: color) + } +} + extension ThemeScope { // The row-state tints are dynamic providers that cost four times a catalog color per fill, so they resolve once here. func dataGridCellPalette( @@ -89,6 +122,15 @@ extension ThemeScope { placeholderText: resolved[.gridNullValue], booleanTrueText: resolved[.gridBoolTrue], booleanFalseText: resolved[.gridBoolFalse], + valueColors: DataGridValueColors( + string: resolved[.gridString], + number: resolved[.gridNumber], + dateTime: resolved[.gridDateTime], + binary: resolved[.gridBinary], + json: resolved[.gridJSON], + uuid: resolved[.gridUUID], + enumeration: resolved[.gridEnum] + ), rowNumberText: resolved[.gridRowNumber], deletedRowText: Self.resolvedTint(.gridDeletedText, of: resolved, in: appearance), modifiedColumnTint: Self.resolvedTint(.gridModified, of: resolved, in: appearance), diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift index 8f7745aac0..1547e807aa 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift @@ -21,6 +21,7 @@ final class DataGridCellPresenter { let scope: ThemeScope let databaseType: DatabaseType? let showsInvisibleCharacters: Bool + let colorsValuesByType: Bool let nullDisplayString: String } @@ -30,11 +31,12 @@ final class DataGridCellPresenter { private let visualIndex = RowVisualIndex() private var visualIndexSource: ObjectIdentifier? private var highlightRuleSet = HighlightRuleSet.empty - private var presentations: [Int: DataGridColumnPresentation] = [:] + // Indexed by data index: the draw path reads it once per cell. + private var presentations: [DataGridColumnPresentation] = [] - init(model: DataGridModel) { + init(model: DataGridModel, valueColors: ValueColorSettings = AppSettingsManager.shared.valueColors) { self.model = model - registry = DataGridCellRegistry(scope: model.themeScope) + registry = DataGridCellRegistry(scope: model.themeScope, valueColors: valueColors) } func adopt(_ model: DataGridModel) { @@ -48,19 +50,22 @@ final class DataGridCellPresenter { guard let model else { return nil } registry.preparePalette(in: appearance) let rows = model.rows + let configuration = model.configuration prepareRowState(rows: rows) return Pass( rows: rows, formats: model.store.displayFormats, selection: model.selection, findMatch: model.find.currentMatch, - isEditable: model.configuration.isEditable, - checkboxColumns: model.configuration.checkboxColumns, - marksEmptyText: model.configuration.marksEmptyText, + isEditable: configuration.isEditable, + checkboxColumns: configuration.checkboxColumns, + marksEmptyText: configuration.marksEmptyText, palette: registry.palette, scope: model.themeScope, - databaseType: model.configuration.traits.databaseType, + databaseType: configuration.traits.databaseType, showsInvisibleCharacters: registry.showsInvisibleCharacters, + colorsValuesByType: configuration.colorsValuesByType + && (registry.colorsValuesByType || configuration.overridesValueColorSetting), nullDisplayString: registry.nullDisplayString ) } @@ -125,22 +130,28 @@ final class DataGridCellPresenter { guard let model else { return [] } let rows = model.rows let configuration = model.configuration - var next: [Int: DataGridColumnPresentation] = [:] + var next: [DataGridColumnPresentation] = [] next.reserveCapacity(rows.columns.count) var changed = IndexSet() for (dataIndex, name) in rows.columns.enumerated() { let type = dataIndex < rows.columnTypes.count ? rows.columnTypes[dataIndex] : nil + let enumValues = rows.columnEnumValues[name] + let valueTone = configuration.colorsValuesByType + ? DataGridColumnTone(columnType: type, declaresEnumLabels: enumValues?.isEmpty == false) + : .plain let presentation = DataGridColumnPresentation.resolve( columnType: type, isForeignKey: rows.columnForeignKeys[name] != nil, isDropdown: configuration.dropdownColumns?.contains(dataIndex) == true, isTypePicker: configuration.typePickerColumns?.contains(dataIndex) == true, - isEnumOrSet: Self.picksFromList(type: type, enumValues: rows.columnEnumValues[name]), + isEnumOrSet: Self.picksFromList(type: type, enumValues: enumValues), isEditable: configuration.isEditable, - isCheckbox: configuration.checkboxColumns.contains(dataIndex) + isCheckbox: configuration.checkboxColumns.contains(dataIndex), + valueTone: valueTone ) - next[dataIndex] = presentation - if presentations[dataIndex] != presentation { + next.append(presentation) + let previous = dataIndex < presentations.count ? presentations[dataIndex] : nil + if previous?.kind != presentation.kind || previous?.accessory != presentation.accessory { changed.insert(dataIndex) } } @@ -149,7 +160,7 @@ final class DataGridCellPresenter { } func presentation(forDataIndex dataIndex: Int) -> DataGridColumnPresentation { - presentations[dataIndex] ?? .plain + presentations.indices.contains(dataIndex) ? presentations[dataIndex] : .plain } private static func picksFromList(type: ColumnType?, enumValues: [String]?) -> Bool { @@ -167,10 +178,6 @@ final class DataGridCellPresenter { // MARK: - Text - func displayText(of row: Row, dataIndex: Int, type: ColumnType?, pass: Pass) -> String { - displayText(of: row, dataIndex: dataIndex, type: type, formats: pass.formats, databaseType: pass.databaseType, columnCount: pass.rows.columns.count) - } - func displayText( of row: Row, dataIndex: Int, @@ -179,19 +186,32 @@ final class DataGridCellPresenter { databaseType: DatabaseType?, columnCount: Int ) -> String { - guard dataIndex >= 0, dataIndex < row.values.count, let cache = model?.display.cache else { return "" } + display(of: row, dataIndex: dataIndex, type: type, formats: formats, databaseType: databaseType, columnCount: columnCount).text + } + + // The text and whether its display format produced it come from one cache entry, written and cleared together. + private func display( + of row: Row, + dataIndex: Int, + type: ColumnType?, + formats: [ValueDisplayFormat?], + databaseType: DatabaseType?, + columnCount: Int + ) -> (text: String, isFormatted: Bool) { + guard dataIndex >= 0, dataIndex < row.values.count, let cache = model?.display.cache else { return ("", false) } let existing = cache.box(forID: row.id) if let existing, dataIndex < existing.values.count, let cached = existing.values[dataIndex] { - return cached + return (cached, existing.isFormatted(at: dataIndex)) } let rawValue = row.values[dataIndex] let format = dataIndex < formats.count ? formats[dataIndex] : nil - let formatted = CellDisplayFormatter.format( + let formatted = CellDisplayFormatter.display( rawValue, columnType: type, displayFormat: format, databaseType: databaseType - ) ?? rawValue.asText ?? "" + ) + let text = formatted.text ?? rawValue.asText ?? "" let neededCount = max(dataIndex + 1, columnCount) let box: RowDisplayBox @@ -203,27 +223,28 @@ final class DataGridCellPresenter { } else { box = RowDisplayBox(ContiguousArray(repeating: nil, count: neededCount)) } - box.values[dataIndex] = formatted + box.setText(text, at: dataIndex, isFormatted: formatted.isFormatted) cache.setBox(box, forID: row.id) - return formatted + return (text, formatted.isFormatted) } // Formats a whole row ahead of the viewport, for the background prewarm. func cacheText(of row: Row, rows: TableRows, formats: [ValueDisplayFormat?], databaseType: DatabaseType?) { guard let cache = model?.display.cache, cache.box(forID: row.id) == nil else { return } let columnCount = rows.columns.count - var values = ContiguousArray(repeating: nil, count: columnCount) + let box = RowDisplayBox(ContiguousArray(repeating: nil, count: columnCount)) for dataIndex in 0..= 0, dataIndex < row.values.count else { return nil } let rawValue = row.values[dataIndex] - let text = displayText(of: row, dataIndex: dataIndex, type: column.type, pass: pass) + let shown = display( + of: row, + dataIndex: dataIndex, + type: column.type, + formats: pass.formats, + databaseType: pass.databaseType, + columnCount: pass.rows.columns.count + ) let isFindMatch = pass.findMatch.map { $0.rowID == row.id && $0.columnID == column.id } ?? false + let presentation = self.presentation(forDataIndex: dataIndex) + let valueTone: DataGridValueTone + if pass.colorsValuesByType { + let format = dataIndex < pass.formats.count ? pass.formats[dataIndex] : nil + valueTone = presentation.valueTone.tone(displayedAs: format, isFormatted: shown.isFormatted) + } else { + valueTone = .plain + } return DataGridCellAppearance.resolve( - kind: presentation(forDataIndex: dataIndex).kind, + kind: presentation.kind, content: DataGridCellContent( - displayText: text, + displayText: shown.text, rawValue: rawValue.asText, placeholder: DataGridCellContent.placeholder( for: rawValue, isAbsent: row.isAbsent(dataIndex), + isExpression: visualState.isExpression(columnIndex: dataIndex), unloadedByteCount: row.unloadedByteCount(dataIndex), marksEmptyText: pass.marksEmptyText ) @@ -266,6 +303,7 @@ final class DataGridCellPresenter { onEmphasizedSelection: onEmphasizedSelection, checkboxMark: checkboxMark(of: row, column: column, checkboxColumns: pass.checkboxColumns), textTone: model?.dataSource?.gridTextTone(for: row.id, column: column), + valueTone: valueTone, revealsInvisibleCharacters: pass.showsInvisibleCharacters ) } @@ -297,6 +335,7 @@ final class DataGridCellPresenter { switch DataGridCellContent.placeholder( for: rawValue, isAbsent: row.isAbsent(dataIndex), + isExpression: model.changeState?.isExpression(rowID: row.id, columnIndex: dataIndex) ?? false, unloadedByteCount: row.unloadedByteCount(dataIndex), marksEmptyText: model.configuration.marksEmptyText ) { diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellRegistry.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellRegistry.swift index 777417659b..fa1d9bf01a 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellRegistry.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellRegistry.swift @@ -9,6 +9,7 @@ import AppKit final class DataGridCellRegistry { private(set) var nullDisplayString: String private(set) var showsInvisibleCharacters: Bool + private(set) var colorsValuesByType: Bool private(set) var palette: DataGridCellPalette private(set) var scope: ThemeScope private var paletteKey: PaletteKey @@ -19,21 +20,26 @@ final class DataGridCellRegistry { let appearance: NSAppearance.Name } - init(scope: ThemeScope = .live) { + init(scope: ThemeScope, valueColors: ValueColorSettings) { self.scope = scope let settings = AppSettingsManager.shared.dataGrid nullDisplayString = settings.nullDisplay showsInvisibleCharacters = settings.showInvisibleCharacters + colorsValuesByType = valueColors.colorsValuesByType let appearance = NSApp?.effectiveAppearance ?? NSAppearance.currentDrawing() paletteKey = Self.paletteKey(for: appearance) palette = scope.dataGridCellPalette(in: appearance) } - func reloadSettings(_ settings: DataGridSettings = AppSettingsManager.shared.dataGrid) { + func reloadSettings(_ settings: DataGridSettings) { nullDisplayString = settings.nullDisplay showsInvisibleCharacters = settings.showInvisibleCharacters } + func reloadValueColors(_ settings: ValueColorSettings) { + colorsValuesByType = settings.colorsValuesByType + } + // Asked by every row as it draws: the tints resolve in the appearance the row draws in, which only the draw knows. func preparePalette(in appearance: NSAppearance) { let key = Self.paletteKey(for: appearance) diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridValueTone.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridValueTone.swift new file mode 100644 index 0000000000..f8a46606f8 --- /dev/null +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridValueTone.swift @@ -0,0 +1,91 @@ +// +// DataGridValueTone.swift +// TablePro +// + +import Foundation + +enum DataGridValueTone: Equatable, Sendable { + case plain + case string + case number + case dateTime + case binary + case json + case uuid + case enumeration + + // A `.text` column is whatever the classifier could not place, so only a character type's name makes it a string. + static func base(of type: ColumnType?, declaresEnumLabels: Bool) -> DataGridValueTone { + guard let type else { return .plain } + switch type { + case .integer, .decimal: + return .number + case .date, .timestamp, .datetime: + return .dateTime + case .blob: + return .binary + case .json: + return .json + case .enumType, .set: + return .enumeration + case .text(let rawType): + if declaresEnumLabels { return .enumeration } + if type.isUUIDType { return .uuid } + guard let name = ColumnType.unwrappedTypeName(rawType), !name.isEmpty, + ColumnTypeSQLQuoting.isCharacterType(.text(rawType: name)) || isCharacterTypeName(name) else { return .plain } + return .string + case .boolean, .spatial, .vector, .array: + return .plain + } + } + + // Character data `isCharacterType` leaves out: it also decides which columns a filter casts before LIKE. + private static let characterTypeNames: Set = [ + "KEYWORD", "WILDCARD", "CONSTANT_KEYWORD", "ASCII", "LONG", "SYSNAME", + ] + + private static func isCharacterTypeName(_ name: String) -> Bool { + characterTypeNames.contains(name.prefix { $0 != "(" }.trimmingCharacters(in: .whitespaces).uppercased()) + } + + static func shown(by format: ValueDisplayFormat) -> DataGridValueTone? { + switch format { + case .raw: + return nil + case .text: + return .string + case .uuid, .uuidSwappedTime: + return .uuid + case .unixTimestamp, .unixTimestampMillis: + return .dateTime + // Each shows a document as text: JSON, PHP serialize or protobuf text format. + case .json, .phpSerialized, .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: + return .json + } + } +} + +struct DataGridColumnTone: Equatable, Sendable { + static let plain = DataGridColumnTone(base: .plain, followsDisplayFormat: false) + + let base: DataGridValueTone + let followsDisplayFormat: Bool + + private init(base: DataGridValueTone, followsDisplayFormat: Bool) { + self.base = base + self.followsDisplayFormat = followsDisplayFormat + } + + init(columnType: ColumnType?, declaresEnumLabels: Bool) { + self.init(base: DataGridValueTone.base(of: columnType, declaresEnumLabels: declaresEnumLabels), followsDisplayFormat: true) + } + + // `isFormatted` comes from the formatter with the cached text: a value the format could not render reads as its column. + func tone(displayedAs format: ValueDisplayFormat?, isFormatted: Bool) -> DataGridValueTone { + guard followsDisplayFormat, isFormatted, let format, let shown = DataGridValueTone.shown(by: format) else { + return base + } + return shown + } +} diff --git a/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift b/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift index 140f135bcc..dc5c2339ca 100644 --- a/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift +++ b/apple/TablePro/Views/DataGrid/Cells/RowVisualIndex.swift @@ -43,7 +43,7 @@ final class RowVisualIndex { } let isInserted = changeManager.insertedRowIDs.contains(rowID) - if let rowChange = changeManager.rowChanges.first(where: { $0.rowID == rowID }) { + if let rowChange = changeManager.pendingChange(forRow: rowID) { states[rowID] = Self.makeState(for: rowChange, inserted: isInserted) return } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift index e27584da1e..95da9f0384 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift @@ -133,6 +133,7 @@ extension DataGridCommands { tableRows: model.rows, displayIDs: model.order.filteredIDs, selectedDisplayIndices: displayIndices, + defaultedCells: model.changeState?.defaultedCells(in: model.rows) ?? [:], columns: projection ) guard output.rowCount > 0 else { return } @@ -218,7 +219,9 @@ extension DataGridCommands { columnTypes: [column.type ?? .text(rawType: nil)], escapeStringLiteral: nil, stringLiteralPrefix: traits.stringLiteralPrefix, - stringEscapes: traits.databaseType.map { SQLLexicalResolver.keptTextStringEscapes(for: $0, session: session) } ?? .standard + stringEscapes: traits.databaseType.map { SQLLexicalResolver.keptTextStringEscapes(for: $0, session: session) } ?? .standard, + booleanLiteralStyle: traits.databaseType.flatMap { try? resolveSQLDialect(for: $0) }?.booleanLiteralStyle ?? .truefalse, + databaseType: traits.databaseType ).generateInClause(rows: values.map { [$0] }) guard !text.isEmpty else { return } ClipboardService.shared.writeText(text) diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift index 33493a31ff..8b1971b7be 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift @@ -176,7 +176,7 @@ extension DataGridCommands { } // The grid's copy says which fields were SQL; any other field is text however it reads. Text from another app is - // read the way a typed cell is. + // read the way a typed cell is. A binary column takes `0x` hex as the bytes it spells, as the hex editor does. func pastedEntry( _ field: String?, of block: DataGridCellPasteBlock, @@ -184,6 +184,10 @@ extension DataGridCommands { column: DataGridColumn ) -> StagedCellEntry? { let rows = model.rows + if !isExpression, let field, BlobFormattingService.hasHexPrefix(field), + rows.editingColumnType(at: column.dataIndex)?.isBlobType == true { + return BlobFormattingService.shared.bytes(fromPrefixedHex: field).map { (value: .bytes($0), isExpression: false) } + } guard block.isGridCopy else { let text = field.flatMap { DataGridCellPasteBlock.pastedText($0, columnType: rows.editingColumnType(at: column.dataIndex)) diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift index cc2d571948..cdfa6e7fa8 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift @@ -78,7 +78,7 @@ extension DataGridCommands { private static func sampleText(ofColumn dataIndex: Int, in rows: TableRows) -> String? { for row in rows.rows where dataIndex < row.values.count { guard let text = row.values[dataIndex].asText, - !text.isEmpty, text != PluginCellValue.defaultMarkerText else { continue } + !text.isEmpty, row.values[dataIndex] != .columnDefault else { continue } return text } return nil @@ -91,7 +91,7 @@ extension DataGridCommands { case .null: return (value: .null, isExpression: false) case .defaultValue: - return (value: .text(PluginCellValue.defaultMarkerText), isExpression: false) + return (value: .columnDefault, isExpression: true) case .expression(let expression): return (value: .text(expression.sql), isExpression: true) case .uuid(let version): diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift index b4124734ad..e56a98cd32 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift @@ -7,35 +7,72 @@ import AppKit import SwiftUI import TableProPluginKit -extension DataGridCommands { - // Every loaded value as the cell shows it, which is what a value filter stores and matches; NULL first. - func distinctValues(for column: DataGridColumn) -> [ColumnDistinctValue] { - let rows = model.rows - let dataIndex = column.dataIndex - guard dataIndex >= 0, dataIndex < rows.columns.count else { return [] } - let format = model.displayFormat(for: column) - let databaseType = configuration.traits.databaseType - var counts: [String: Int] = [:] - var order: [String] = [] - var nullCount = 0 - for row in rows.rows { +// Each loaded value as the cell shows it, counted, in the order first seen. +struct ColumnValueTally: Sendable { + static let cancellationInterval = 4_096 + + var counts: [String: Int] = [:] + var order: [String] = [] + var nullCount = 0 + + // Nil once cancelled, which is asked every 4,096 rows. + static func tally( + _ rows: TableRows, + dataIndex: Int, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer, + isCancelled: () -> Bool + ) -> ColumnValueTally? { + var counted = ColumnValueTally() + guard dataIndex >= 0, dataIndex < rows.columns.count else { return counted } + for (offset, row) in rows.rows.enumerated() { + if offset % cancellationInterval == 0, isCancelled() { return nil } guard dataIndex < row.values.count else { continue } let raw = row.values[dataIndex] if case .null = raw { - nullCount += 1 + counted.nullCount += 1 continue } let display = GridDisplayOrderResolver.displayValue( raw, - columnType: column.type, - displayFormat: format, - databaseType: databaseType + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: dates ) - if counts[display] == nil { - order.append(display) + if counted.counts[display] == nil { + counted.order.append(display) } - counts[display, default: 0] += 1 + counted.counts[display, default: 0] += 1 } + return counted + } + + // A column of unique values costs a format per row and n log n locale-aware comparisons, too slow for the + // main actor. A renderer is never shared across threads, so this one builds its own from the format. + @concurrent + static func distinctValuesOffTheMainActor( + _ rows: TableRows, + dataIndex: Int, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dateFormat: DateFormatOption + ) async -> [ColumnDistinctValue]? { + tally( + rows, + dataIndex: dataIndex, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateDisplayRenderer(format: dateFormat), + isCancelled: { Task.isCancelled } + )?.sorted() + } + + func sorted() -> [ColumnDistinctValue] { var values = order .sorted { $0.localizedStandardCompare($1) == .orderedAscending } .map { ColumnDistinctValue(display: $0, isNull: false, count: counts[$0] ?? 0) } @@ -46,6 +83,21 @@ extension DataGridCommands { } } +extension DataGridCommands { + // Every loaded value as the cell shows it, which is what a value filter stores and matches; NULL first. + func distinctValues(for column: DataGridColumn) -> [ColumnDistinctValue] { + ColumnValueTally.tally( + model.rows, + dataIndex: column.dataIndex, + columnType: column.type, + displayFormat: model.displayFormat(for: column), + databaseType: configuration.traits.databaseType, + dates: DateFormattingService.shared.renderer, + isCancelled: { false } + )?.sorted() ?? [] + } +} + extension DataGridViewController { // Opened from the header menu, and from an active filter's funnel; values still on the server load first. func presentValueFilter(for column: DataGridColumn) { @@ -58,11 +110,52 @@ extension DataGridViewController { guard let self, let current = self.model.columns.column(id: columnID) else { return } self.presentValueFilter(for: current) }) else { return } - guard let anchor = model.anchor(for: column) else { return } + guard model.anchor(for: column) != nil else { return } - let values = gridCommands.distinctValues(for: column) + let columnType = column.type + let displayFormat = model.displayFormat(for: column) + let databaseType = model.configuration.traits.databaseType + let dateFormat = DateFormattingService.shared.currentFormat let initialFilter = model.store.valueFilter.filter(forColumn: dataIndex) + let loadedRowCount = rows.count + let listedModel = model dismissValueFilterPopover() + // A reload, insert or removal while the list builds cancels it, as it would close the open popover. + menuState.valueFilterListing = Task { [weak self] in + let listed = await ColumnValueTally.distinctValuesOffTheMainActor( + rows, + dataIndex: dataIndex, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dateFormat: dateFormat + ) + guard let values = listed, !Task.isCancelled, let self else { return } + self.menuState.valueFilterListing = nil + // A grid that took another tab's model meanwhile shows other rows. + guard self.model === listedModel, + let current = self.model.columns.column(id: columnID), + let anchor = self.model.anchor(for: current) else { return } + // A connection switch unparents the grid without dismantling it, and NSPopover raises on a view with no window. + guard let window = anchor.view.window, window.isVisible else { return } + self.showValueFilterPopover( + for: current, + values: values, + loadedRowCount: loadedRowCount, + initialFilter: initialFilter, + anchor: anchor + ) + } + } + + private func showValueFilterPopover( + for column: DataGridColumn, + values: [ColumnDistinctValue], + loadedRowCount: Int, + initialFilter: ColumnValueFilter?, + anchor: (view: NSView, rect: NSRect) + ) { + let columnID = column.id menuState.valueFilterPopover = PopoverPresenter.show( relativeTo: anchor.rect, of: anchor.view, @@ -71,7 +164,7 @@ extension DataGridViewController { ColumnValueFilterPopover( columnName: column.name, values: values, - loadedRowCount: rows.count, + loadedRowCount: loadedRowCount, initialFilter: initialFilter, // A change closes the popover once applied, in followRowChangesWithPopovers; declining the alert keeps the picks. onApply: { filter in @@ -89,6 +182,8 @@ extension DataGridViewController { } func dismissValueFilterPopover() { + menuState.valueFilterListing?.cancel() + menuState.valueFilterListing = nil guard let popover = menuState.valueFilterPopover else { return } menuState.valueFilterPopover = nil popover.close() diff --git a/apple/TablePro/Views/DataGrid/DataGridFindBar.swift b/apple/TablePro/Views/DataGrid/DataGridFindBar.swift index 2879242aea..2e4752a173 100644 --- a/apple/TablePro/Views/DataGrid/DataGridFindBar.swift +++ b/apple/TablePro/Views/DataGrid/DataGridFindBar.swift @@ -172,11 +172,21 @@ struct DataGridFindBar: View { // MARK: - State private var counterText: String { + Self.counterText(status: status, model: model) + } + + private var showsSearchAllRows: Bool { + Self.showsSearchAllRows(offered: capabilities.offersSearchAllRows, model: model) + } + + static func counterText(status: String?, model: DataGridModel) -> String { + let find = model.find if let status { return status } if let problem = find.problem { return problem } guard model.hasFindTerm else { return "" } // A search that refreshes keeps showing the count it is about to replace. if find.matches.isEmpty { + if find.isAwaitingLoadedRows { return "" } if find.isSearching { return String(localized: "Searching…") } if find.searchStopped { return String(localized: "Search stopped") } } @@ -188,9 +198,10 @@ struct DataGridFindBar: View { } // Only when this page has no match, so the bar stays short while matches are on screen. - private var showsSearchAllRows: Bool { - capabilities.offersSearchAllRows && model.searchAllRowsKeepsTheOptions && model.hasFindTerm - && find.problem == nil && find.matches.isEmpty + static func showsSearchAllRows(offered: Bool, model: DataGridModel) -> Bool { + let find = model.find + return offered && model.searchAllRowsKeepsTheOptions && model.hasFindTerm + && find.problem == nil && find.matches.isEmpty && !find.isAwaitingLoadedRows } // Use Selection for Find searches on its own, so only typing goes through the delay. diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController+Changes.swift b/apple/TablePro/Views/DataGrid/DataGridViewController+Changes.swift index ec17de9e21..8884781785 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController+Changes.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController+Changes.swift @@ -259,11 +259,11 @@ extension DataGridViewController { } else { table.scroll(NSPoint(x: x, y: -headerInset)) } - guard !placement.selectedRows.isEmpty else { return } + guard placement.selection != .empty else { return } // Selecting publishes the model, which a SwiftUI update or layout pass refuses, so it runs on the next turn. DispatchQueue.main.async { [weak self] in guard let self, self.model.mountedView === self else { return } - self.model.selectRows(placement.selectedRows) + self.model.select(placement.selection) guard placement.revealsSelection, let row = self.model.selection.affectedRows.first else { return } self.reveal(row: row, displayColumn: nil) } diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift b/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift index 022f1e79bd..0aba998017 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController+Menus.swift @@ -9,6 +9,8 @@ import AppKit final class DataGridMenuState { let loader = DataGridValueLoader() var valueFilterPopover: NSPopover? + // The value list sorting before its popover opens. + var valueFilterListing: Task? } extension DataGridViewController { diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController+Selection.swift b/apple/TablePro/Views/DataGrid/DataGridViewController+Selection.swift index f3fb692a25..b39acdb327 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController+Selection.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController+Selection.swift @@ -337,6 +337,8 @@ extension DataGridViewController { @discardableResult private func trackDrag(_ drag: GridSelectionDrag, from start: GridCoord) -> Bool { guard let window = view.window else { return false } + isTrackingSelectionDrag = true + defer { isTrackingSelectionDrag = false } var moved = false while let event = window.nextEvent(matching: [.leftMouseDragged, .leftMouseUp]) { if event.type == .leftMouseUp { break } diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController+TableDelegate.swift b/apple/TablePro/Views/DataGrid/DataGridViewController+TableDelegate.swift index 8ad4588ea3..1fd673b3e2 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController+TableDelegate.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController+TableDelegate.swift @@ -154,8 +154,9 @@ extension DataGridViewController: NSTableViewDataSource, NSTableViewDelegate { extension DataGridViewController: NSMenuDelegate { // Built from clickedRow and clickedColumn, so AppKit draws the clicked-row ring and the selection does not move. func menuNeedsUpdate(_ menu: NSMenu) { - menu.removeAllItems() + // The header menu comes built, and has this delegate only so its opening is seen. guard let pane = panes.first(where: { $0.tableView.menu === menu }) else { return } + menu.removeAllItems() let table = pane.tableView let row = table.clickedRow let built: NSMenu? @@ -178,6 +179,16 @@ extension DataGridViewController: NSMenuDelegate { func headerMenu(forTableColumn tableColumn: NSTableColumn, in kind: DataGridPane.Kind) -> NSMenu? { let pane = container.pane(kind) guard let id = pane.columnID(of: tableColumn), let column = model.columns.column(id: id) else { return nil } - return headerMenu(for: column) + let menu = headerMenu(for: column) + menu?.delegate = self + return menu + } + + func menuWillOpen(_ menu: NSMenu) { + isShowingContextMenu = true + } + + func menuDidClose(_ menu: NSMenu) { + isShowingContextMenu = false } } diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController.swift b/apple/TablePro/Views/DataGrid/DataGridViewController.swift index dd7176f7af..88634ef3be 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController.swift @@ -39,6 +39,9 @@ final class DataGridViewController: NSViewController, DataGridMountedView { var isFittingHeightScheduled = false var pendingPageReveal: NSRect? var isRevealingInGrid = false + var isLiveScrolling = false + var isTrackingSelectionDrag = false + var isShowingContextMenu = false private var observers: [NSObjectProtocol] = [] private var cancellables: Set = [] private var themeCancellable: AnyCancellable? @@ -170,6 +173,16 @@ final class DataGridViewController: NSViewController, DataGridMountedView { var isEditing: Bool { isEditingCell } + // An open context menu acts on the row ids it was built from, which a reload hands to other rows. + var holdsTransientUI: Bool { + let state = editingState + return isShowingContextMenu || state.textPopover != nil || state.typedPopover != nil + || state.poppedOutEditor != nil || state.foreignKeyPreview?.isShown == true + || menuState.valueFilterPopover != nil + } + + var isTrackingInteraction: Bool { isLiveScrolling || isTrackingSelectionDrag } + @discardableResult func beginEditing(row: Int, displayColumn: Int) -> Bool { beginEditingCell(row: row, displayColumn: displayColumn) @@ -209,6 +222,10 @@ final class DataGridViewController: NSViewController, DataGridMountedView { .receive(on: RunLoop.main) .sink { [weak self] _ in self?.dataGridSettingsDidChange() } .store(in: &cancellables) + AppEvents.shared.valueColorSettingsChanged + .receive(on: RunLoop.main) + .sink { [weak self] _ in self?.valueColorSettingsDidChange() } + .store(in: &cancellables) AppEvents.shared.systemTimeZoneChanged .receive(on: RunLoop.main) .sink { [weak self] _ in self?.reformatText() } @@ -221,10 +238,16 @@ final class DataGridViewController: NSViewController, DataGridMountedView { }) let scrollView = main.scrollView observers.append(center.addObserver(forName: NSScrollView.willStartLiveScrollNotification, object: scrollView, queue: .main) { [weak self] _ in - MainActor.assumeIsolated { self?.pausePrewarm() } + MainActor.assumeIsolated { + self?.isLiveScrolling = true + self?.pausePrewarm() + } }) observers.append(center.addObserver(forName: NSScrollView.didEndLiveScrollNotification, object: scrollView, queue: .main) { [weak self] _ in - MainActor.assumeIsolated { self?.schedulePrewarmResume() } + MainActor.assumeIsolated { + self?.isLiveScrolling = false + self?.schedulePrewarmResume() + } }) } @@ -286,6 +309,13 @@ final class DataGridViewController: NSViewController, DataGridMountedView { } } + private func valueColorSettingsDidChange() { + let settings = AppSettingsManager.shared.valueColors + guard settings.colorsValuesByType != presenter.registry.colorsValuesByType else { return } + presenter.registry.reloadValueColors(settings) + redrawVisibleRows() + } + // Rebuilds every formatted string: the date format, NULL text or smart detection moved, or the Mac's time zone did. func reformatText() { model.display.cache.removeAll() @@ -388,8 +418,7 @@ final class DataGridViewController: NSViewController, DataGridMountedView { dataColumnsByDisplayPosition: model.columns.columns.map(\.dataIndex), policy: policy, deletedRowIDs: changeState?.deletedRowIDs ?? [], - insertedRowIDs: changeState?.insertedRowIDs ?? [], - modifiedCells: changeState?.modifiedCells ?? [:] + defaultedCells: changeState?.defaultedCells(in: rows) ?? [:] ) } } diff --git a/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift b/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift index 7cdc8a564d..1b6314b1ba 100644 --- a/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift +++ b/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift @@ -176,7 +176,7 @@ struct CreateDatabaseSheet: View { onCreated?(name) dismiss() } catch { - errorMessage = error.localizedDescription + errorMessage = GatedOperationFailure(error).message isCreating = false } } diff --git a/apple/TablePro/Views/ERDiagram/ERDiagramAccessibility.swift b/apple/TablePro/Views/ERDiagram/ERDiagramAccessibility.swift index 5cf49c1b56..2fa50ae688 100644 --- a/apple/TablePro/Views/ERDiagram/ERDiagramAccessibility.swift +++ b/apple/TablePro/Views/ERDiagram/ERDiagramAccessibility.swift @@ -54,6 +54,10 @@ final class ERDiagramNodeElement: NSAccessibilityElement, @unchecked Sendable { builtColumns = nil } + fileprivate func move(to rect: CGRect) { + self.rect = rect + } + fileprivate func rowRect(at index: Int) -> CGRect { CGRect( x: rect.minX, @@ -145,8 +149,25 @@ final class ERDiagramAccessibilityTree { /// yet, so a notification has nowhere to land, and asking for one is itself what builds them. private(set) var hasBeenAsked = false - func invalidate() { - isStale = true + // A drag moves a table on every pointer event and changes nothing else, so the elements a client + // holds take the new rects in place instead of rebuilding every label and relationship. + func sceneDidChange(from old: ERDiagramScene, to new: ERDiagramScene) { + guard !isStale else { return } + guard old.nodes == new.nodes, + old.edges == new.edges, + old.selectedNodeIds == new.selectedNodeIds, + old.nodeRects.count == new.nodeRects.count + else { + isStale = true + return + } + for (id, element) in byNode { + guard let rect = new.nodeRects[id] else { + isStale = true + return + } + element.move(to: rect) + } } func elements(for scene: ERDiagramScene, owner: NSView) -> [ERDiagramNodeElement] { diff --git a/apple/TablePro/Views/ERDiagram/ERDiagramEdgeRenderer.swift b/apple/TablePro/Views/ERDiagram/ERDiagramEdgeRenderer.swift index ba39271269..1a35cdfbad 100644 --- a/apple/TablePro/Views/ERDiagram/ERDiagramEdgeRenderer.swift +++ b/apple/TablePro/Views/ERDiagram/ERDiagramEdgeRenderer.swift @@ -11,13 +11,19 @@ enum ERDiagramEdgeRenderer { } private static let strokeWidth: CGFloat = 1.5 - private static let selectedStrokeWidth: CGFloat = 3 + nonisolated private static let selectedStrokeWidth: CGFloat = 3 private static let dashPattern: [CGFloat] = [6, 4] struct Route { let edge: EREdge let path: CGPath let markers: [CGPath] + + // Padded by the widest stroke, whose round caps reach half of it past the path. + func reaches(_ rect: CGRect) -> Bool { + let bounds = markers.reduce(path.boundingBoxOfPath) { $0.union($1.boundingBoxOfPath) } + return bounds.insetBy(dx: -selectedStrokeWidth, dy: -selectedStrokeWidth).intersects(rect) + } } struct Colors { @@ -26,15 +32,16 @@ enum ERDiagramEdgeRenderer { let dropped: CGColor } - static func drawEdges( - edges: [EREdge], - nodeRects: [UUID: CGRect], - nodeIndex: [String: UUID], + // Every route is computed, since a node's ports are shared out among all its edges; only the + // strokes are limited to the edges that cross the rect being drawn. + static func drawRoutes( + _ routes: [Route], colors: Colors, selectedEdgeId: UUID?, + dirtyRect: CGRect, on canvas: some ERDiagramCanvas ) { - for route in routes(edges: edges, nodeRects: nodeRects, nodeIndex: nodeIndex) { + for route in routes where route.reaches(dirtyRect) { let isSelected = route.edge.id == selectedEdgeId let color: CGColor if isSelected { diff --git a/apple/TablePro/Views/ERDiagram/ERDiagramEditSheets.swift b/apple/TablePro/Views/ERDiagram/ERDiagramEditSheets.swift index 45a9fdc6c5..206e1e56b3 100644 --- a/apple/TablePro/Views/ERDiagram/ERDiagramEditSheets.swift +++ b/apple/TablePro/Views/ERDiagram/ERDiagramEditSheets.swift @@ -5,21 +5,12 @@ import SwiftUI -extension ERDiagramForeignKeyDraft { - // The referenced column follows the referenced table, so both change in one write. - func referencing(_ table: String, column: String) -> ERDiagramForeignKeyDraft { - var draft = self - draft.referencedTable = table - draft.referencedColumn = column - return draft - } -} - struct ERDiagramForeignKeySheet: View { @State private var draft: ERDiagramForeignKeyDraft + @State private var selection: Set = [] private let tables: [String] private let columns: (String) -> [String] - private let defaultReferencedColumn: (String) -> String + private let referencedKeyColumns: (String) -> [String] private let isNameTaken: (String) -> Bool private let dialect: ForeignKeyDialect private let onAdd: (ERDiagramForeignKeyDraft) -> Void @@ -29,7 +20,7 @@ struct ERDiagramForeignKeySheet: View { draft: ERDiagramForeignKeyDraft, tables: [String], columns: @escaping (String) -> [String], - defaultReferencedColumn: @escaping (String) -> String, + referencedKeyColumns: @escaping (String) -> [String], isNameTaken: @escaping (String) -> Bool, dialect: ForeignKeyDialect, onAdd: @escaping (ERDiagramForeignKeyDraft) -> Void, @@ -38,7 +29,7 @@ struct ERDiagramForeignKeySheet: View { _draft = State(initialValue: draft) self.tables = tables self.columns = columns - self.defaultReferencedColumn = defaultReferencedColumn + self.referencedKeyColumns = referencedKeyColumns self.isNameTaken = isNameTaken self.dialect = dialect self.onAdd = onAdd @@ -54,14 +45,18 @@ struct ERDiagramForeignKeySheet: View { } private var canAdd: Bool { - !draft.column.isEmpty && !draft.referencedTable.isEmpty && !draft.referencedColumn.isEmpty - && !trimmedName.isEmpty && !nameIsTaken + draft.issue == nil && !draft.referencedTable.isEmpty && !trimmedName.isEmpty && !nameIsTaken } private var referencedTable: Binding { Binding( get: { draft.referencedTable }, - set: { draft = draft.referencing($0, column: defaultReferencedColumn($0)) } + set: { table in + draft = draft.referencing( + table, keyColumns: referencedKeyColumns(table), sourceColumns: columns(draft.table) + ) + selection = [] + } ) } @@ -74,16 +69,20 @@ struct ERDiagramForeignKeySheet: View { Form { Section { LabeledContent(String(localized: "Table"), value: draft.table) - Picker(String(localized: "Column"), selection: $draft.column) { - ForEach(columns(draft.table), id: \.self) { Text($0).tag($0) } - } } Section(String(localized: "References")) { Picker(String(localized: "Table"), selection: referencedTable) { ForEach(tables, id: \.self) { Text($0).tag($0) } } - Picker(String(localized: "Column"), selection: $draft.referencedColumn) { - ForEach(columns(draft.referencedTable), id: \.self) { Text($0).tag($0) } + } + Section { + pairList + } header: { + Text(String(localized: "Column → Referenced Column")) + } footer: { + if let issue = draft.issue { + Text(issue.message) + .foregroundStyle(.secondary) } } Section { @@ -121,10 +120,137 @@ struct ERDiagramForeignKeySheet: View { } .padding() } - .frame(width: 440) + .frame(width: 480) .accessibilityElement(children: .contain) .accessibilityIdentifier("er-diagram-foreign-key-sheet") } + + // MARK: - Column pairs + + private var pairList: some View { + List(selection: $selection) { + ForEach(draft.pairs) { pair in + ERDiagramColumnPairRow( + column: columnBinding(for: pair.id), + referencedColumn: referencedColumnBinding(for: pair.id), + columns: columns(draft.table), + referencedColumns: columns(draft.referencedTable), + onMoveUp: { move(pair.id, by: -1) }, + onMoveDown: { move(pair.id, by: 1) } + ) + .tag(pair.id) + } + .onMove { source, destination in + draft = draft.movingPairs(fromOffsets: source, toOffset: destination) + } + } + .listStyle(.bordered(alternatesRowBackgrounds: false)) + .frame(height: listHeight) + .onDeleteCommand { removeSelection() } + .safeAreaInset(edge: .bottom, spacing: 0) { + VStack(spacing: 0) { + Divider() + HStack(spacing: 0) { + AddRemoveControlGroup( + addLabel: String(localized: "Add Column Pair"), + removeLabel: String(localized: "Remove Column Pair"), + canRemove: canRemoveSelection, + addIdentifier: "er-diagram-foreign-key-add-pair", + removeIdentifier: "er-diagram-foreign-key-remove-pair", + onAdd: addPair, + onRemove: removeSelection + ) + Spacer() + } + .padding(.horizontal, 4) + .padding(.vertical, 2) + } + .background(.bar) + } + } + + private let rowHeight: CGFloat = 26 + private let buttonBarHeight: CGFloat = 32 + + // No cap: a List in a grouped Form never scrolls, so it shows every pair and the Form scrolls. + private var listHeight: CGFloat { + CGFloat(max(draft.pairs.count, 2)) * rowHeight + buttonBarHeight + 8 + } + + private var canRemoveSelection: Bool { + !selection.isEmpty && selection.count < draft.pairs.count + } + + private func columnBinding(for id: UUID) -> Binding { + Binding( + get: { draft.pairs.first { $0.id == id }?.column ?? "" }, + set: { draft = draft.settingColumn($0, of: id) } + ) + } + + private func referencedColumnBinding(for id: UUID) -> Binding { + Binding( + get: { draft.pairs.first { $0.id == id }?.referencedColumn ?? "" }, + set: { draft = draft.settingReferencedColumn($0, of: id) } + ) + } + + private func addPair() { + draft = draft.addingPair() + if let added = draft.pairs.last?.id { + selection = [added] + } + } + + private func removeSelection() { + guard canRemoveSelection else { return } + draft = draft.removingPairs(selection) + selection = [] + } + + private func move(_ id: UUID, by offset: Int) { + guard let index = draft.pairs.firstIndex(where: { $0.id == id }) else { return } + let target = index + offset + guard draft.pairs.indices.contains(target) else { return } + draft = draft.movingPairs(fromOffsets: IndexSet(integer: index), toOffset: offset > 0 ? target + 1 : target) + } +} + +private struct ERDiagramColumnPairRow: View { + @Binding var column: String + @Binding var referencedColumn: String + let columns: [String] + let referencedColumns: [String] + let onMoveUp: () -> Void + let onMoveDown: () -> Void + + var body: some View { + HStack(spacing: 6) { + picker(String(localized: "Column"), selection: $column, options: columns) + Image(systemName: "arrow.right") + .foregroundStyle(.secondary) + .accessibilityHidden(true) + picker(String(localized: "Referenced Column"), selection: $referencedColumn, options: referencedColumns) + } + .accessibilityElement(children: .contain) + .accessibilityActions { + Button(String(localized: "Move Up"), action: onMoveUp) + Button(String(localized: "Move Down"), action: onMoveDown) + } + } + + // A blank side names nothing until it is chosen, so it gets an item of its own rather than an invalid selection. + private func picker(_ title: String, selection: Binding, options: [String]) -> some View { + Picker(title, selection: selection) { + if selection.wrappedValue.isEmpty { + Text(String(localized: "None")).tag("") + } + ForEach(options, id: \.self) { Text($0).tag($0) } + } + .labelsHidden() + .accessibilityLabel(title) + .frame(maxWidth: .infinity) + } } struct ERDiagramColumnSheet: View { diff --git a/apple/TablePro/Views/ERDiagram/ERDiagramScene.swift b/apple/TablePro/Views/ERDiagram/ERDiagramScene.swift index 167e1ac273..5568765bce 100644 --- a/apple/TablePro/Views/ERDiagram/ERDiagramScene.swift +++ b/apple/TablePro/Views/ERDiagram/ERDiagramScene.swift @@ -34,17 +34,22 @@ enum ERDiagramSceneRenderer { return bounds } - static func draw(_ scene: ERDiagramScene, palette: ERDiagramPalette, dirtyRect: CGRect, on canvas: some ERDiagramCanvas) { - ERDiagramEdgeRenderer.drawEdges( - edges: scene.edges, - nodeRects: scene.nodeRects, - nodeIndex: scene.nodeIndex, + static func draw( + _ scene: ERDiagramScene, + palette: ERDiagramPalette, + dirtyRect: CGRect, + routes: [ERDiagramEdgeRenderer.Route]? = nil, + on canvas: some ERDiagramCanvas + ) { + ERDiagramEdgeRenderer.drawRoutes( + routes ?? ERDiagramEdgeRenderer.routes(edges: scene.edges, nodeRects: scene.nodeRects, nodeIndex: scene.nodeIndex), colors: ERDiagramEdgeRenderer.Colors( stroke: palette.edge, selection: palette.selectionRule, dropped: palette.droppedEdge ), selectedEdgeId: scene.selectedEdgeId, + dirtyRect: dirtyRect, on: canvas ) @@ -114,10 +119,33 @@ enum ERDiagramSceneRenderer { } } + // A copy or a PNG renders on the main actor, so a bigger bitmap costs seconds and gigabytes; PDF and SVG keep full size. + static let maxBitmapSide: CGFloat = 16_384 + static let maxBitmapPixels: CGFloat = 33_554_432 + + static func bitmapScale(for size: CGSize, requested scale: CGFloat) -> CGFloat { + guard size.width > 0, size.height > 0 else { return scale } + let bySide = maxBitmapSide / max(size.width, size.height) + let byArea = (maxBitmapPixels / (size.width * size.height)).squareRoot() + return min(scale, bySide, byArea) + } + static func image(_ scene: ERDiagramScene, appearance: NSAppearance, scale: CGFloat) -> NSImage? { + guard let rep = bitmap(scene, appearance: appearance, scale: scale) else { return nil } + let image = NSImage(size: rep.size) + image.addRepresentation(rep) + return image + } + + static func pngData(_ scene: ERDiagramScene, appearance: NSAppearance, scale: CGFloat) -> Data? { + bitmap(scene, appearance: appearance, scale: scale)?.representation(using: .png, properties: [:]) + } + + private static func bitmap(_ scene: ERDiagramScene, appearance: NSAppearance, scale requested: CGFloat) -> NSBitmapImageRep? { let frame = exportFrame(of: scene) - let pixelsWide = Int((frame.size.width * scale).rounded()) - let pixelsHigh = Int((frame.size.height * scale).rounded()) + let scale = bitmapScale(for: frame.size, requested: requested) + let pixelsWide = Int((frame.size.width * scale).rounded(.down)) + let pixelsHigh = Int((frame.size.height * scale).rounded(.down)) guard pixelsWide > 0, pixelsHigh > 0 else { return nil } guard let rep = NSBitmapImageRep( @@ -136,10 +164,7 @@ enum ERDiagramSceneRenderer { guard let base = NSGraphicsContext(bitmapImageRep: rep) else { return nil } renderExport(scene, frame: frame, appearance: appearance, into: base.cgContext) - - let image = NSImage(size: frame.size) - image.addRepresentation(rep) - return image + return rep } static func pdfData(_ scene: ERDiagramScene, appearance: NSAppearance, title: String) -> Data? { diff --git a/apple/TablePro/Views/ERDiagram/ERDiagramSceneView.swift b/apple/TablePro/Views/ERDiagram/ERDiagramSceneView.swift index f1f70f0913..c763591438 100644 --- a/apple/TablePro/Views/ERDiagram/ERDiagramSceneView.swift +++ b/apple/TablePro/Views/ERDiagram/ERDiagramSceneView.swift @@ -57,8 +57,9 @@ final class ERDiagramSceneView: NSView, NSMenuItemValidation { /// already has. Before anyone asks, the rebuild waits for the first question. var scene = ERDiagramScene() { didSet { + routes = nil needsDisplay = true - accessibilityTree.invalidate() + accessibilityTree.sceneDidChange(from: oldValue, to: scene) if accessibilityTree.hasBeenAsked { _ = accessibilityTree.elements(for: scene, owner: self) } @@ -105,6 +106,8 @@ final class ERDiagramSceneView: NSView, NSMenuItemValidation { private var linkDraft: LinkDraft? private var isShowingHandCursor = false private let accessibilityTree = ERDiagramAccessibilityTree() + // A hover or link redraw keeps the scene, so it reuses the routes the last draw computed. + private var routes: [ERDiagramEdgeRenderer.Route]? private var bakedPalette: ERDiagramPalette? private var themeCancellable: AnyCancellable? @@ -145,7 +148,10 @@ final class ERDiagramSceneView: NSView, NSMenuItemValidation { override func draw(_ dirtyRect: NSRect) { guard let context = NSGraphicsContext.current?.cgContext else { return } let canvas = ERDiagramContextCanvas(context: context) - ERDiagramSceneRenderer.draw(scene, palette: palette, dirtyRect: dirtyRect, on: canvas) + let routes = routes + ?? ERDiagramEdgeRenderer.routes(edges: scene.edges, nodeRects: scene.nodeRects, nodeIndex: scene.nodeIndex) + self.routes = routes + ERDiagramSceneRenderer.draw(scene, palette: palette, dirtyRect: dirtyRect, routes: routes, on: canvas) drawLinkAffordances(on: canvas) } @@ -461,6 +467,7 @@ final class ERDiagramSceneView: NSView, NSMenuItemValidation { } } else if press.isDragging { actions?.endDrag() + announceMove(of: press.nodeId) } else if press.togglesSelection { if let nodeId = press.nodeId { actions?.toggleSelection(nodeId) @@ -616,6 +623,12 @@ final class ERDiagramSceneView: NSView, NSMenuItemValidation { return accessibilityTree.element(for: selected) } + // Once when the drag ends, not on every pointer event the table moved with. + private func announceMove(of nodeId: UUID?) { + guard accessibilityTree.hasBeenAsked, let nodeId, let element = accessibilityTree.element(for: nodeId) else { return } + NSAccessibility.post(element: self, notification: .layoutChanged, userInfo: [.uiElements: [element]]) + } + private func announceSelectionChange(from previous: [UUID]) { guard accessibilityTree.hasBeenAsked, previous != scene.selectedNodeIds else { return } NSAccessibility.post(element: self, notification: .selectedChildrenChanged) diff --git a/apple/TablePro/Views/ERDiagram/ERDiagramView.swift b/apple/TablePro/Views/ERDiagram/ERDiagramView.swift index ef7abaf74e..f8e307b667 100644 --- a/apple/TablePro/Views/ERDiagram/ERDiagramView.swift +++ b/apple/TablePro/Views/ERDiagram/ERDiagramView.swift @@ -71,7 +71,7 @@ struct ERDiagramView: View { draft: draft, tables: viewModel.referenceableTables, columns: { viewModel.columnNames(of: $0) }, - defaultReferencedColumn: { viewModel.defaultReferencedColumn(in: $0) }, + referencedKeyColumns: { viewModel.referencedKeyColumns(in: $0) }, isNameTaken: { viewModel.isForeignKeyNameTaken($0) }, dialect: ForeignKeyDialect.forType(viewModel.editingConnection?.type ?? .mysql), onAdd: { draft in @@ -210,7 +210,11 @@ struct ERDiagramView: View { private func exportData(_ format: ERDiagramExportFormat) -> Data? { switch format { case .png: - return DiagramExporter.pngData(exportImage()) + return ERDiagramSceneRenderer.pngData( + scene, + appearance: viewport.effectiveAppearance, + scale: DiagramExporter.renderScale + ) case .svg: let svg = ERDiagramSceneRenderer.svgDocument(scene, appearance: viewport.effectiveAppearance, title: exportTitle) return Data(svg.utf8) diff --git a/apple/TablePro/Views/Editor/EditorEventRouter.swift b/apple/TablePro/Views/Editor/EditorEventRouter.swift index a6a2ed5293..187d6ffbff 100644 --- a/apple/TablePro/Views/Editor/EditorEventRouter.swift +++ b/apple/TablePro/Views/Editor/EditorEventRouter.swift @@ -2,9 +2,6 @@ // EditorEventRouter.swift // TablePro // -// Registry of the live SQL editors, used to dispatch window-scoped commands -// to the editor in the key window. -// @preconcurrency import AppKit import TableProEditorKit @@ -81,9 +78,7 @@ internal final class EditorEventRouter { // MARK: - Public API - /// Whether the key window has a SQL editor a find can fall back to. A focused editor answers the - /// Find commands itself through the responder chain, so this only decides whether the window's - /// last-resort handler has anything to search once nothing nearer has claimed them. + // A focused editor answers Find itself, so this decides only whether the window's fallback has anything to search. internal var keyWindowHasEditor: Bool { editor(for: NSApp.keyWindow) != nil } @@ -98,8 +93,6 @@ internal final class EditorEventRouter { coordinator.showFindAndReplacePanel() } - /// Whether the key window's editor has a selection to search for, so the menu can disable - /// Use Selection for Find rather than offering a command that would do nothing. internal var keyWindowEditorHasSelectionForFind: Bool { editor(for: NSApp.keyWindow)?.0.hasSelectionForFind ?? false } @@ -149,10 +142,7 @@ internal final class EditorEventRouter { coordinator.moveCursorToStatement(direction) } - /// Runs the statement the caret is in and advances, entirely inside the editor this resolves to. - /// - /// Handing the SQL back for someone else to execute would let the text and the connection come from two different - /// editors, which a window hosting more than one connection can genuinely do. + // Inside the one editor, so the text and the connection cannot come from two editors in a multi-connection window. internal func runStatementAtCursorAndAdvanceForKeyWindow() { guard let (coordinator, _) = editor(for: NSApp.keyWindow) else { return } coordinator.runStatementAtCursorAndAdvance() @@ -173,16 +163,18 @@ internal final class EditorEventRouter { } } - /// Called when the Edit menu's "Clear Selection" item is chosen with the pointer. - /// - /// Only with the pointer. AppKit never dispatches an unmodified Escape as a key equivalent: - /// measured on macOS 26, `NSMenu.performKeyEquivalent` is not called at all for key code 53, - /// which instead reaches `keyDown:` and `cancelOperation(_:)` through the responder chain, as - /// `StandardKeyBinding.dict` maps 0x1B to. The item carries no Escape key equivalent anyway, - /// because `KeyboardSettings.menuKeyEquivalent(for:)` refuses a bare key. Routes the command to - /// the key window's editor so it can dismiss its completion popup, hand the escape to Vim, and - /// restore first responder. Returns whether the editor consumed it so the caller skips its - /// cancelOperation fallback (the data grid's Clear Selection). + // False when the tab has no mounted editor, or it refused the edit: the caller writes the tab instead. + internal func replaceText(forTab tabId: UUID, connectionId: UUID, in range: NSRange, with string: String) -> Bool { + for ref in editors.values { + guard let coordinator = ref.coordinator, let controller = coordinator.controller, + coordinator.tabID == tabId, coordinator.connectionId == connectionId else { continue } + if controller.replaceTextAsOneEdit(in: range, with: string) { return true } + } + return false + } + + // Clear Selection chosen with the pointer: AppKit never sends a bare Escape as a key equivalent (measured, macOS 26). + // True when the editor consumed it, so the caller skips the grid's fallback. @discardableResult internal func handleEscapeFromMenu() -> Bool { guard let (coordinator, _) = editor(for: NSApp.keyWindow) else { return false } @@ -191,10 +183,8 @@ internal final class EditorEventRouter { // MARK: - Lookup - /// Window-scoped on purpose: every caller is a command the window's front content owns rather than - /// whatever holds first responder, and the escape and find routes exist precisely because focus is - /// somewhere else. A tab switch can leave the outgoing editor registered for a moment, and the - /// dictionary has no order, so the focused editor wins before the remaining candidate is used. + // Window-scoped: these routes exist because focus is elsewhere. A switch can leave the outgoing editor registered, + // so the focused one wins. private func editor(for window: NSWindow?) -> (SQLEditorCoordinator, TextView)? { guard let window else { return nil } var unfocused: (SQLEditorCoordinator, TextView)? diff --git a/apple/TablePro/Views/Editor/EditorUndoHistories.swift b/apple/TablePro/Views/Editor/EditorUndoHistories.swift new file mode 100644 index 0000000000..cbfa10f327 --- /dev/null +++ b/apple/TablePro/Views/Editor/EditorUndoHistories.swift @@ -0,0 +1,37 @@ +// +// EditorUndoHistories.swift +// TablePro +// + +import Foundation +import TableProEditorKit + +// What a query tab leaves in the shared editor while another tab is shown. Both parts are offsets into `text`, so +// they come back only while the tab still holds that text. +struct EditorTabHistory { + let text: String + let undo: EditorUndoHistory? + let vimMarks: VimDocumentMarks? +} + +@MainActor +final class EditorUndoHistories { + private var entries: [UUID: EditorTabHistory] = [:] + + func store(_ history: EditorTabHistory?, for tabId: UUID) { + entries[tabId] = history + } + + func take(for tabId: UUID) -> EditorTabHistory? { + entries.removeValue(forKey: tabId) + } + + func forget(_ tabId: UUID) { + entries.removeValue(forKey: tabId) + } + + func forgetAll(except openTabIds: Set) { + guard entries.keys.contains(where: { !openTabIds.contains($0) }) else { return } + entries = entries.filter { openTabIds.contains($0.key) } + } +} diff --git a/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift b/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift index c4d0956510..96f13af9d1 100644 --- a/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift +++ b/apple/TablePro/Views/Editor/Folding/SQLLineFoldProvider.swift @@ -1,18 +1,11 @@ -// -// SQLLineFoldProvider.swift -// TablePro -// - import Foundation import TableProEditorKit import TableProSQLGrammar import TableProTextEngine import TableProUtilities -/// Answers the editor's per-line fold queries from a whole-document scan. -/// -/// The editor restarts its walk at line zero every time the document changes, so the scan is refreshed there and every -/// other line is a dictionary lookup. That keeps the main-thread cost of a pass linear in the document, not quadratic. +// The editor runs `documentFoldScanner()` off the main thread on a copy of the text. The per-line answers below are +// for callers that ask a line at a time; they rescan at line zero, so a pass stays linear in the document. final class SQLLineFoldProvider: LineFoldProvider { var grammar: SQLLexicalGrammar @@ -23,6 +16,15 @@ final class SQLLineFoldProvider: LineFoldProvider { self.grammar = grammar } + func documentFoldScanner() -> LineFoldDocumentScanner? { + let grammar = grammar + return { text in + SQLFoldScanner.scan(text as NSString, grammar: grammar).regions.map { + LineFoldProviderFold(depth: $0.depth, range: $0.range) + } + } + } + func foldLevelAtLine( lineNumber: Int, lineRange: NSRange, diff --git a/apple/TablePro/Views/Editor/QueryEditorBar.swift b/apple/TablePro/Views/Editor/QueryEditorBar.swift index 7418150171..63ff45bc24 100644 --- a/apple/TablePro/Views/Editor/QueryEditorBar.swift +++ b/apple/TablePro/Views/Editor/QueryEditorBar.swift @@ -54,6 +54,8 @@ struct QueryEditorBar: View { schemaName: scope.schemaName, onChange: onContainerChanged ) + // A run moved to another database mid-flight would either land its rows there or lose what it committed. + .disabled(isExecuting) if let vimMode { VimModeIndicatorView(mode: vimMode) diff --git a/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift b/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift index bbafd7d64d..98727fa462 100644 --- a/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift +++ b/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift @@ -1,8 +1,3 @@ -// -// SQLCompletionKind+IconColor.swift -// TablePro -// - import AppKit import TableProAutocomplete @@ -19,6 +14,7 @@ extension SQLCompletionKind { case .alias: return .systemGray case .operator: return .systemIndigo case .favorite: return .systemYellow + case .joinCondition: return .systemBrown } } } diff --git a/apple/TablePro/Views/Editor/SQLEditorController.swift b/apple/TablePro/Views/Editor/SQLEditorController.swift index e62f529488..f313b2d0b0 100644 --- a/apple/TablePro/Views/Editor/SQLEditorController.swift +++ b/apple/TablePro/Views/Editor/SQLEditorController.swift @@ -21,6 +21,7 @@ struct SQLEditorInput { var databaseScope: DatabaseScope? var connectionId: UUID? var tabID: UUID? + var undoHistories: EditorUndoHistories? var sourceFileURL: URL? var claimFocusOnAppear = false /// Called once the editor has latched a focus claim, so the owner's one-shot intent is cleared. @@ -87,6 +88,7 @@ final class SQLEditorController { view.setAccessibilityRole(.group) view.setAccessibilityLabel(String(localized: "SQL query editor")) view.setAccessibilityIdentifier("sql-editor-textview") + restoreTabHistory(of: input) observeEditorSettings() observeVimMode() @@ -100,12 +102,17 @@ final class SQLEditorController { func apply(_ newInput: SQLEditorInput) { guard !isTornDown else { return } let old = input - if newInput.tabID != old.tabID { + let switchesTab = newInput.tabID != old.tabID + if switchesTab { handOverPendingText(to: old.text) + saveTabHistory(of: old) } input = newInput applyInput(newInput) pushEditor() + if switchesTab { + restoreTabHistory(of: newInput) + } if newInput.pendingStatementJump != old.pendingStatementJump, let jump = newInput.pendingStatementJump { coordinator.jumpToStatement(jump) @@ -136,6 +143,7 @@ final class SQLEditorController { guard !isTornDown else { return } isTornDown = true handOverPendingText(to: input.text) + saveTabHistory(of: input) profileTask?.cancel() profileTask = nil revisionSubscription = nil @@ -151,6 +159,28 @@ final class SQLEditorController { Task { @MainActor in binding.wrappedValue = text } } + // One editor serves every query tab, and handing it another tab's text clears its history and Vim marks. + private func saveTabHistory(of input: SQLEditorInput) { + guard let tabID = input.tabID, let histories = input.undoHistories else { return } + let undo = textViewController.takeUndoHistory() + let vimMarks = coordinator.vimDocumentMarks + guard undo != nil || vimMarks != nil else { + histories.store(nil, for: tabID) + return + } + let text = undo?.text ?? textViewController.textView.string + histories.store(EditorTabHistory(text: text, undo: undo, vimMarks: vimMarks), for: tabID) + } + + // A tab whose text changed while it was away gets neither back. + private func restoreTabHistory(of input: SQLEditorInput) { + guard let tabID = input.tabID, let saved = input.undoHistories?.take(for: tabID) else { return } + let matches = saved.undo.map { textViewController.restoreUndoHistory($0) } + ?? (saved.text as NSString).isEqual(to: textViewController.textView.string) + guard matches, let vimMarks = saved.vimMarks else { return } + coordinator.restoreVimDocumentMarks(vimMarks) + } + private func applyInput(_ input: SQLEditorInput) { coordinator.onCloseTab = input.onCloseTab coordinator.onExecuteQuery = input.onExecuteQuery @@ -164,6 +194,7 @@ final class SQLEditorController { coordinator.onAIAction = input.onAIAction coordinator.onSaveAsFavorite = input.onSaveAsFavorite coordinator.schemaProvider = input.schemaProvider + coordinator.databaseScope = input.databaseScope coordinator.databaseType = input.databaseType coordinator.tabID = input.tabID coordinator.connectionId = input.connectionId @@ -226,7 +257,8 @@ final class SQLEditorController { state: state, foldProvider: foldProvider, coordinators: [coordinator], - completionDelegate: completionAdapter + completionDelegate: completionAdapter, + documentID: input.tabID.map { AnyHashable($0) } ) } diff --git a/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift b/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift index 1ec3d48dc9..c912935495 100644 --- a/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift +++ b/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift @@ -35,6 +35,7 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie private let statementRunController = StatementRunController() /// Shared schema provider for inline AI suggestions (avoids duplicate schema fetches) var schemaProvider: SQLSchemaProvider? + var databaseScope: DatabaseScope? private let inlineAccessGate: AIConnectionAccessGate = .live private var contextMenu: AIEditorContextMenu? private var inlineSuggestionManager: InlineSuggestionManager? @@ -111,9 +112,25 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie controller.restoreCollapsedFolds(ranges) } + var vimDocumentMarks: VimDocumentMarks? { + guard let vimEngine else { return pendingVimDocumentMarks } + let marks = vimEngine.documentMarks + return marks.isEmpty ? nil : marks + } + + func restoreVimDocumentMarks(_ marks: VimDocumentMarks) { + guard let vimEngine else { + pendingVimDocumentMarks = marks + return + } + vimEngine.restoreDocumentMarks(marks) + } + /// Vim mode for UI observation @Published private(set) var vimMode: VimMode = .normal private var vimEngine: VimEngine? + // Vim is installed once the editor is in a window, so marks restored before that wait here. + private var pendingVimDocumentMarks: VimDocumentMarks? private var vimKeyInterceptor: VimKeyInterceptor? private var emacsKeyInterceptor: EmacsKeyInterceptor? private var commandHandler = VimCommandLineHandler() @@ -245,12 +262,6 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie self?.vimCursorManager?.updatePosition() } - if !isLargeDocument, !didDestroy, let tabID, let sync = copilotDocumentSync, - resolvedInlineSourceKind == .copilot { - let text = textView.string - Task { await sync.didChangeText(tabID: tabID, newText: text) } - } - frameChangeTask?.cancel() frameChangeTask = Task { [weak controller] in try? await Task.sleep(for: .milliseconds(50)) @@ -265,8 +276,11 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie func textViewDidReplaceDocument(controller: TextViewController) { vimEngine?.invalidateLineCache() + vimEngine?.documentDidReplace() + pendingVimDocumentMarks = nil foldPreview.dismiss() inlineSuggestionManager?.dismissSuggestion() + copilotDocumentSync?.textViewDidReplaceDocument() statementRunController.refreshControls(in: controller) statementRunController.refreshHighlight(in: controller) diagnosticsController.clear(in: controller) @@ -320,10 +334,7 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie removeVimKeyInterceptor() removeEmacsKeyInterceptor() - if let tabID, let sync = copilotDocumentSync { - let id = tabID - Task { await sync.didCloseTab(tabID: id) } - } + copilotDocumentSync?.detach() foldPreview.destroy() inlineSuggestionManager?.uninstall() @@ -620,44 +631,61 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie } private func installCopilotInlineSource() { - let sync = CopilotDocumentSync() + let sync = CopilotDocumentSync(server: LiveCopilotDocumentServer(), lengthLimit: Self.languageServiceLengthLimit) copilotDocumentSync = sync copilotInlineSource = CopilotInlineSource(documentSync: sync) - let capturedTabID = tabID - let capturedText = controller?.textView?.string ?? "" - let capturedSchemaProvider = schemaProvider - let capturedDBType = databaseType - let dbName = connectionId.flatMap { - DatabaseManager.shared.session(for: $0)?.resolvedBrowseDatabase - } ?? "database" + let referenceText: String? + if let textView = controller?.textView, textView.textStorage.length <= Self.languageServiceLengthLimit { + referenceText = textView.string + } else { + referenceText = nil + } + let source = copilotSchemaSource() Task { [weak self] in - if let provider = capturedSchemaProvider, let dbType = capturedDBType { + if let source { await sync.preambleBuilder.buildPreamble( - schemaProvider: provider, - databaseName: dbName, - databaseType: dbType + schemaProvider: source.schemaProvider, + databaseName: source.databaseName, + databaseType: source.databaseType, + referencing: referenceText, + grammar: source.grammar, + maxTables: source.budget ) } guard let self, self.copilotDocumentSync === sync, self.resolvedInlineSourceKind == .copilot else { return } await CopilotService.shared.start() - guard self.copilotDocumentSync === sync else { return } - if let tabID = capturedTabID { - sync.ensureDocumentOpen(tabID: tabID, text: capturedText) - await sync.didActivateTab(tabID: tabID, text: capturedText) - } + guard self.copilotDocumentSync === sync, let textView = self.controller?.textView else { return } + sync.attach( + to: textView, + tabID: { [weak self] in self?.tabID }, + isAllowed: { [weak self] in self?.resolvedInlineSourceKind == .copilot }, + schemaSource: { [weak self] in self?.copilotSchemaSource() } + ) } } + private func copilotSchemaSource() -> CopilotPreambleBuilder.Source? { + guard let schemaProvider, let databaseType else { return nil } + let scopeDatabase = databaseScope.flatMap { $0.isServerScoped ? nil : $0.database } + let databaseName = scopeDatabase ?? connectionId.flatMap { + DatabaseManager.shared.session(for: $0)?.resolvedBrowseDatabase + } ?? "database" + return CopilotPreambleBuilder.Source( + schemaProvider: schemaProvider, + databaseName: databaseName, + databaseType: databaseType, + grammar: lexicalGrammar, + budget: max(AppSettingsManager.shared.ai.maxSchemaTables, 0) + ) + } + private func teardownInlineSources(except kind: InlineSuggestionSourceKind) { if kind != .copilot { - if let tabID, let sync = copilotDocumentSync { - let id = tabID - Task { await sync.didCloseTab(tabID: id) } - } + copilotDocumentSync?.detach() copilotDocumentSync = nil copilotInlineSource = nil } @@ -729,6 +757,10 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie let interceptor = VimKeyInterceptor(engine: engine, inlineSuggestionManager: inlineSuggestionManager) interceptor.install(controller: controller) + if let marks = pendingVimDocumentMarks { + engine.restoreDocumentMarks(marks) + pendingVimDocumentMarks = nil + } self.vimEngine = engine self.vimKeyInterceptor = interceptor self.vimMode = .normal diff --git a/apple/TablePro/Views/Editor/StatementRunController.swift b/apple/TablePro/Views/Editor/StatementRunController.swift index 0ad9853f8e..b7b0acef3c 100644 --- a/apple/TablePro/Views/Editor/StatementRunController.swift +++ b/apple/TablePro/Views/Editor/StatementRunController.swift @@ -15,17 +15,11 @@ enum StatementNavigationDirection { case next } -/// Keeps the editor's per-statement decorations in step with the document. -/// -/// Two things are drawn from one scanner: the run control the gutter puts beside each statement, and the band behind -/// the statement the caret is in. Both come from ``SQLStatementScanner``, which is also what decides the SQL that -/// actually runs, so a control cannot offer to run something different from what it is drawn beside. -/// -/// The two are refreshed differently because they cost differently. The band resolves one statement through -/// ``SQLStatementScanner/locatedStatementAtCursor(in:cursorPosition:grammar:)``, which stops at the caret, so it is -/// the same call the execution path makes and it can run on every keystroke. The controls need every statement in the -/// document, which is a full pass: measured at 31ms on a document at the size limit, well past a frame, so that one is -/// debounced. Nothing is lost by the delay, since the controls only show while the pointer is over the gutter. +// The gutter's run controls and the caret's statement band both come from the scanner execution uses, so a control +// never offers to run something other than what it is drawn beside. The band runs on every caret move, so a SQL +// document resolves it through `SQLStatementCursorIndex`, which rescans only from the last boundary before the caret, +// and a JavaScript or Redis document through a `QueryStatementCaretTable` read once per text. +// The controls need every statement, a full pass of 31ms at the size limit, so that refresh is debounced. @MainActor final class StatementRunController { /// The largest document decorations are computed for, in UTF-16 units. @@ -44,11 +38,32 @@ final class StatementRunController { static let synchronousRefreshLimit = 64_000 var sizeLimit = StatementRunController.defaultSizeLimit - var grammar: SQLLexicalGrammar = .ansi + var grammar: SQLLexicalGrammar = .ansi { + didSet { if grammar != oldValue { caretIndex = nil } } + } /// How the document divides into statements. JavaScript engines do not end a statement at every /// semicolon, so the gutter controls and the caret band have to ask the same splitter execution does. - var statementModel: QueryStatementModel = .sql + var statementModel: QueryStatementModel = .sql { + didSet { + if statementModel != oldValue { + caretIndex = nil + caretTable = nil + } + } + } + + // The band asks on every caret move, so a document keeps its statement boundaries between edits. + private var caretIndex: SQLStatementCursorIndex? + private var caretTable: QueryStatementCaretTable? + private weak var caretIndexStorage: NSTextStorage? + private var caretIndexEdits: StorageEditObservation? + + var caretIndexUnitsScanned: Int { + caretIndex?.unitsScanned ?? 0 + } + + private(set) var caretTableReads = 0 /// Called with the SQL the pressed control stands for, read from the document at the moment of the press, and /// where that SQL starts in the document. @@ -206,6 +221,10 @@ final class StatementRunController { func clear(in controller: TextViewController?) { pendingRefresh?.cancel() pendingRefresh = nil + caretIndex = nil + caretTable = nil + caretIndexStorage = nil + caretIndexEdits = nil controller?.runnableStatements = [] controller?.highlightedStatementRange = nil controller?.onRunStatement = nil @@ -246,11 +265,7 @@ final class StatementRunController { ) } - /// The span the band covers, or `nil` when there is nothing to mark. - /// - /// A non-empty selection runs verbatim rather than being widened to its statement, so the band steps aside and - /// lets the selection speak for itself. Multiple carets resolve through the first one, which is the caret the - /// execution path reads too. + // A selection runs verbatim, so the band steps aside for it. Several carets resolve through the first, as execution does. private func highlightRange(in controller: TextViewController) -> NSRange? { guard isHighlightEnabled, let textView = controller.textView, @@ -262,13 +277,84 @@ final class StatementRunController { let text = textView.string guard (text as NSString).length <= sizeLimit else { return nil } - let statement = QueryStatementScanner.locatedStatementAtCursor( - in: text, - cursorPosition: selection.range.location, - model: statementModel, - grammar: grammar - ) + let statement: SQLStatementScanner.LocatedStatement + if let storage = textView.textStorage { + statement = indexedStatement(at: selection.range.location, in: text, storage: storage) + } else { + statement = QueryStatementScanner.locatedStatementAtCursor( + in: text, + cursorPosition: selection.range.location, + model: statementModel, + grammar: grammar + ) + } guard statement.hasContent, statement.contentRange.length > 0 else { return nil } return statement.contentRange } + + private func indexedStatement( + at cursor: Int, + in text: String, + storage: NSTextStorage + ) -> SQLStatementScanner.LocatedStatement { + if caretIndexStorage !== storage { + observeEdits(on: storage) + } + // Inside an editing batch the text has changed but the storage has not posted its notification yet. + if storage.editedMask.contains(.editedCharacters) { + textDidChange(at: storage.editedRange.location) + } + guard statementModel == .sql else { + if caretTable == nil { + caretTable = QueryStatementCaretTable(text: text, model: statementModel) + caretTableReads += 1 + } + return caretTable?.statement(atCursor: cursor) + ?? QueryStatementScanner.locatedStatementAtCursor( + in: text, cursorPosition: cursor, model: statementModel, grammar: grammar + ) + } + if caretIndex == nil { + caretIndex = SQLStatementCursorIndex(grammar: grammar) + } + return caretIndex?.statementAtCursor(cursor, in: text) + ?? SQLStatementScanner.locatedStatementAtCursor(in: text, cursorPosition: cursor, grammar: grammar) + } + + private func textDidChange(at location: Int) { + caretIndex?.textDidChange(at: location) + caretTable = nil + } + + private func observeEdits(on storage: NSTextStorage) { + caretIndex = nil + caretTable = nil + caretIndexStorage = storage + // Every way the text changes passes through the storage, including the ones that skip the editor's delegate. + let token = NotificationCenter.default.addObserver( + forName: NSTextStorage.didProcessEditingNotification, + object: storage, + queue: nil + ) { [weak self] _ in + MainActor.assumeIsolated { self?.caretIndexStorageDidProcessEditing() } + } + caretIndexEdits = StorageEditObservation(token) + } + + private func caretIndexStorageDidProcessEditing() { + guard let storage = caretIndexStorage, storage.editedMask.contains(.editedCharacters) else { return } + textDidChange(at: storage.editedRange.location) + } +} + +private final class StorageEditObservation { + private let token: any NSObjectProtocol + + init(_ token: any NSObjectProtocol) { + self.token = token + } + + deinit { + NotificationCenter.default.removeObserver(token) + } } diff --git a/apple/TablePro/Views/Import/ImportDialog.swift b/apple/TablePro/Views/Import/ImportDialog.swift index f9627d2e59..51662e3b6d 100644 --- a/apple/TablePro/Views/Import/ImportDialog.swift +++ b/apple/TablePro/Views/Import/ImportDialog.swift @@ -2,8 +2,6 @@ // ImportDialog.swift // TablePro // -// Plugin-aware import dialog. -// import AppKit import Combine @@ -104,7 +102,10 @@ struct ImportDialog: View { } .task { if let initialURL = initialFileURL, fileURL == nil { - await loadFile(initialURL) + // Tracked like a picked file, so a pick made during a long decompression cancels this load. + let load = Task { await loadFile(initialURL) } + loadFileTask = load + await load.value } } .onDisappear { @@ -369,6 +370,7 @@ struct ImportDialog: View { guard let url, case .format(let formatId) = ImportFileFormatResolver.match(url, among: options) else { return } selectedFormatId = formatId + loadFileTask?.cancel() self.loadFileTask = Task { await self.loadFile(url) } @@ -376,6 +378,11 @@ struct ImportDialog: View { @MainActor private func loadFile(_ url: URL) async { + // The previous file's count must not land under this one, even when this one fails before counting. + countStatementsTask?.cancel() + countStatementsTask = nil + statementCount = 0 + isCountingStatements = false cleanupTempFiles() hasPreviewError = false @@ -401,6 +408,13 @@ struct ImportDialog: View { let urlToRead: URL do { urlToRead = try await FileDecompressor.decompressIfNeeded(url) + // A newer pick owns the dialog now; its import must not read this file's copy. + guard !Task.isCancelled else { + if urlToRead != url { + try? FileManager.default.removeItem(at: urlToRead) + } + return + } if urlToRead != url { tempPreviewURL = urlToRead } @@ -423,7 +437,7 @@ struct ImportDialog: View { } let maxPreviewSize = 5 * 1_024 * 1_024 - let previewData = handle.readData(ofLength: maxPreviewSize) + let previewData = try handle.read(upToCount: maxPreviewSize) ?? Data() var decoder = SQLChunkDecoder(encoding: selectedEncoding.encoding) if let preview = decoder.decode(previewData) { @@ -466,14 +480,16 @@ struct ImportDialog: View { statementCount = 0 do { - let encoding = selectedEncoding.encoding - let grammar = sessionGrammar - let parser = SQLFileParser() - let count = try await Task.detached { - try await parser.countStatements(url: url, encoding: encoding, grammar: grammar) - }.value + let count = try await ImportStatementCount.count( + url, + encoding: selectedEncoding.encoding, + grammar: sessionGrammar + ) + // A new file or encoding cancelled this count and started its own, which owns the result. + guard !Task.isCancelled else { return } statementCount = count } catch { + guard !Task.isCancelled else { return } Self.logger.warning("Failed to count statements: \(error.publicLogShape, privacy: .public)") statementCount = 0 } @@ -554,3 +570,22 @@ struct ImportDialog: View { } } } + +enum ImportStatementCount { + // Detached keeps the scan off the main actor but ignores the caller's cancel, so the cancel is passed on. + static func count( + _ url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar, + parser: SQLFileParser = SQLFileParser() + ) async throws -> Int { + let counting = Task.detached { + try await parser.countStatements(url: url, encoding: encoding, grammar: grammar) + } + return try await withTaskCancellationHandler { + try await counting.value + } onCancel: { + counting.cancel() + } + } +} diff --git a/apple/TablePro/Views/Import/ImportProgressView.swift b/apple/TablePro/Views/Import/ImportProgressView.swift index 02aa447db0..939bd6d089 100644 --- a/apple/TablePro/Views/Import/ImportProgressView.swift +++ b/apple/TablePro/Views/Import/ImportProgressView.swift @@ -67,7 +67,8 @@ extension ImportState { total: Int64(estimatedTotalStatements), unit: countsRows ? .rows : .statements, isEstimate: totalIsEstimate, - note: statusMessage + note: statusMessage, + fileFraction: !countsRows && fileBytes > 0 ? Double(fileBytesRead) / Double(fileBytes) : nil ) } } diff --git a/apple/TablePro/Views/Integrations/IntegrationsActivityLogPane.swift b/apple/TablePro/Views/Integrations/IntegrationsActivityLogPane.swift index ae059a6d62..427480ed39 100644 --- a/apple/TablePro/Views/Integrations/IntegrationsActivityLogPane.swift +++ b/apple/TablePro/Views/Integrations/IntegrationsActivityLogPane.swift @@ -1,8 +1,3 @@ -// -// IntegrationsActivityLogPane.swift -// TablePro -// - import AppKit import SwiftUI import UniformTypeIdentifiers @@ -11,8 +6,10 @@ struct IntegrationsActivityLogPane: View { @ObservedObject private var mcpServerManager = MCPServerManager.shared @State private var entries: [AuditEntry] = [] @State private var tokens: [MCPAuthToken] = [] + @State private var clientNames: [String] = [] @State private var connections: [DatabaseConnection] = [] - @State private var selectedTokenId: UUID? + @State private var selectedToken: ActivityTokenFilter? + @State private var selectedClient: String? @State private var selectedCategory: AuditCategory? @State private var selectedRange: ActivityTimeRange = .last7Days @State private var searchText: String = "" @@ -45,7 +42,8 @@ struct IntegrationsActivityLogPane: View { .onReceive(AppEvents.shared.mcpAuditLogChanged) { _ in Task { await reload() } } - .onChange(of: selectedTokenId) { _ in Task { await reload() } } + .onChange(of: selectedToken) { _ in Task { await reload() } } + .onChange(of: selectedClient) { _ in Task { await reload() } } .onChange(of: selectedCategory) { _ in Task { await reload() } } .onChange(of: selectedRange) { _ in Task { await reload() } } .onChange(of: sortOrder) { newValue in @@ -108,16 +106,26 @@ struct IntegrationsActivityLogPane: View { Text(category.displayName).tag(Optional(category)) } } - Picker(String(localized: "Token"), selection: $selectedTokenId) { - Text(String(localized: "All tokens")).tag(UUID?.none) + Picker(String(localized: "Client"), selection: $selectedClient) { + Text(String(localized: "All clients")).tag(String?.none) + ForEach(clientNames, id: \.self) { name in + Text(verbatim: name).tag(Optional(name)) + } + } + Picker(String(localized: "Token"), selection: $selectedToken) { + Text(String(localized: "All tokens")).tag(ActivityTokenFilter?.none) + ForEach(ActivityTokenFilter.ownCredentialNames, id: \.self) { name in + Text(IntegrationsFormatting.displayTokenName(name)).tag(Optional(ActivityTokenFilter.named(name))) + } ForEach(tokens) { token in - Text(IntegrationsFormatting.displayTokenName(token.name)).tag(Optional(token.id)) + Text(IntegrationsFormatting.displayTokenName(token.name)).tag(Optional(ActivityTokenFilter.token(token.id))) } } if hasActiveFilters { Divider() Button(String(localized: "Clear Filters")) { - selectedTokenId = nil + selectedToken = nil + selectedClient = nil selectedCategory = nil selectedRange = .last7Days } @@ -125,8 +133,7 @@ struct IntegrationsActivityLogPane: View { } label: { Label(String(localized: "Filters"), systemImage: ToolbarSymbols.filter(isActive: hasActiveFilters)) } - /// Named here for the reason `WelcomeViewOptionsMenu` gives: a toolbar-hosted `Menu` - /// publishes "chevron.pulldown" from its label alone. + // A toolbar-hosted Menu otherwise publishes "chevron.pulldown" from its label (see WelcomeViewOptionsMenu). .accessibilityLabel(String(localized: "Filters")) .help(String(localized: "Filter activity")) } @@ -154,8 +161,7 @@ struct IntegrationsActivityLogPane: View { .disabled(isLoading) } - /// `sidebar.trailing` is the glyph AppKit's own inspector toggle draws, and unlike - /// `sidebar.right` it mirrors in a right-to-left layout. + // The glyph AppKit's own inspector toggle draws; unlike sidebar.right it mirrors right to left. private var inspectorToggle: some View { Button { showInspector.toggle() @@ -166,11 +172,12 @@ struct IntegrationsActivityLogPane: View { } private var hasActiveFilters: Bool { - selectedTokenId != nil || selectedCategory != nil || selectedRange != .last7Days + selectedToken != nil || selectedClient != nil || selectedCategory != nil || selectedRange != .last7Days } private var hasNoFilters: Bool { - selectedTokenId == nil + selectedToken == nil + && selectedClient == nil && selectedCategory == nil && selectedRange == .all && searchText.isEmpty @@ -184,16 +191,18 @@ struct IntegrationsActivityLogPane: View { let trimmed = searchText.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return entries } let needle = trimmed.lowercased() - return entries.filter { entry in - if entry.action.lowercased().contains(needle) { return true } - if let tokenName = entry.tokenName?.lowercased(), tokenName.contains(needle) { return true } - if let connectionName = connectionName(for: entry.connectionId)?.lowercased(), - connectionName.contains(needle) { - return true - } - if let details = entry.details?.lowercased(), details.contains(needle) { return true } - return false - } + return entries.filter { Self.searchMatches($0, needle: needle, connectionName: connectionName) } + } + + static func searchMatches(_ entry: AuditEntry, needle: String, connectionName: (UUID?) -> String?) -> Bool { + let fields = [ + entry.action, + entry.clientName, + entry.tokenName.map(IntegrationsFormatting.displayTokenName), + connectionName(entry.connectionId), + entry.details + ] + return fields.contains { $0?.lowercased().contains(needle) == true } } private func connectionName(for id: UUID?) -> String? { @@ -213,13 +222,18 @@ struct IntegrationsActivityLogPane: View { } if let store = mcpServerManager.tokenStore { - tokens = await store.list().filter { !$0.isBridgeCredential } + // TablePro mints these afresh every rotation and every agent turn, so they are filtered by name. + tokens = await store.list().filter { !$0.isBridgeCredential && !$0.isClaudeAgentCredential } } connections = ConnectionStorage.shared.loadConnections() - let result = await MCPAuditLogStorage.shared.query( + let storage = MCPAuditLogStorage.shared + clientNames = await storage.clientNames() + let result = await storage.query( category: selectedCategory, - tokenId: selectedTokenId, + tokenId: selectedToken?.tokenId, + tokenName: selectedToken?.tokenName, + clientName: selectedClient, since: selectedRange.startDate, limit: 1_000 ) @@ -246,7 +260,8 @@ struct IntegrationsActivityLogPane: View { private func writeActivityLog(to url: URL) { do { - try csvString(for: filteredEntries).write(to: url, atomically: true, encoding: .utf8) + try Self.csvString(for: filteredEntries, connectionName: connectionName) + .write(to: url, atomically: true, encoding: .utf8) } catch { AlertHelper.showErrorSheet( title: String(localized: "Could not export activity log"), @@ -256,8 +271,8 @@ struct IntegrationsActivityLogPane: View { } } - private func csvString(for entries: [AuditEntry]) -> String { - let header = ["Timestamp", "Category", "Action", "Connection", "Token", "Outcome", "Details"] + static func csvString(for entries: [AuditEntry], connectionName: (UUID?) -> String?) -> String { + let header = ["Timestamp", "Category", "Action", "Client", "Connection", "Token", "Outcome", "Details"] .joined(separator: ",") let timestampFormatter = ISO8601DateFormatter() let rows = entries.map { entry -> String in @@ -265,12 +280,13 @@ struct IntegrationsActivityLogPane: View { timestampFormatter.string(from: entry.timestamp), entry.category.rawValue, entry.action, - connectionName(for: entry.connectionId) ?? "", + entry.clientName ?? "", + connectionName(entry.connectionId) ?? "", entry.tokenName ?? "", entry.outcome, entry.details ?? "" ] - return cells.map(CsvRowConverter.encode).joined(separator: ",") + return cells.map(CsvRowConverter.encodeForSpreadsheet).joined(separator: ",") } return ([header] + rows).joined(separator: "\n") + "\n" } @@ -311,6 +327,11 @@ private struct ActivityLogTable: View { } .width(min: 160, ideal: 220) + TableColumn(String(localized: "Client")) { entry in + clientCell(for: entry) + } + .width(min: 100, ideal: 140) + TableColumn(String(localized: "Token")) { entry in tokenCell(for: entry) } @@ -351,6 +372,15 @@ private struct ActivityLogTable: View { .font(.system(.body, design: .monospaced)) } + @ViewBuilder + private func clientCell(for entry: AuditEntry) -> some View { + if let name = entry.clientName { + Text(verbatim: name) + } else { + Text(verbatim: "-").foregroundStyle(.tertiary) + } + } + @ViewBuilder private func tokenCell(for entry: AuditEntry) -> some View { if let name = entry.tokenName { @@ -394,6 +424,7 @@ private struct ActivityLogTable: View { String(format: String(localized: "Category: %@"), entry.category.displayName), String(format: String(localized: "Action: %@"), entry.action), String(format: String(localized: "Outcome: %@"), outcome), + entry.clientName.map { String(format: String(localized: "Client: %@"), $0) }, tokenLine, connectionLabel(entry.connectionId).map { String(format: String(localized: "Connection: %@"), $0) }, entry.details.map { String(format: String(localized: "Details: %@"), $0) } @@ -438,6 +469,11 @@ private struct ActivityLogInspector: View { } Section(String(localized: "Source")) { + LabeledContent(String(localized: "Client")) { + Text(entry.clientName ?? String(localized: "None")) + .foregroundStyle(entry.clientName == nil ? .tertiary : .primary) + .textSelection(.enabled) + } LabeledContent(String(localized: "Token")) { let tokenText = entry.tokenName.map(IntegrationsFormatting.displayTokenName) ?? String(localized: "None") Text(tokenText) @@ -484,6 +520,24 @@ private struct ActivityLogInspector: View { } } +enum ActivityTokenFilter: Hashable { + case token(UUID) + // TablePro's own credentials, minted afresh each bridge rotation and each Claude Agent turn. + case named(String) + + static let ownCredentialNames = [MCPTokenStore.stdioBridgeTokenName, MCPTokenStore.claudeAgentTokenName] + + var tokenId: UUID? { + if case .token(let id) = self { return id } + return nil + } + + var tokenName: String? { + if case .named(let name) = self { return name } + return nil + } +} + enum ActivityTimeRange: String, CaseIterable, Identifiable { case last24Hours case last7Days @@ -512,9 +566,7 @@ enum ActivityTimeRange: String, CaseIterable, Identifiable { } } -/// `.inspector` and `.inspectorColumnWidth` are macOS 14. Before them the pane is shown beside -/// the log in an `HSplitView`, which is the same arrangement without the system's show/hide -/// animation and its width persistence. +// .inspector is macOS 14; before it the pane sits beside the log in an HSplitView. private struct ActivityLogInspectorPane: ViewModifier { @Binding var isPresented: Bool let entry: AuditEntry? diff --git a/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift b/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift index 6e75c0694c..a7a248cd1b 100644 --- a/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift +++ b/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift @@ -1,38 +1,35 @@ -// -// IntegrationsConnectedClientsPane.swift -// TablePro -// - import SwiftUI +import TableProMCPClients struct IntegrationsConnectedClientsPane: View { @ObservedObject private var manager = MCPServerManager.shared @State private var selection: MCPServerManager.SessionSnapshot.ID? @State private var disconnectCandidate: MCPServerManager.SessionSnapshot? @State private var revocationNotSaved = false - @State private var sortOrder: [KeyPathComparator] = [ - KeyPathComparator(\MCPServerManager.SessionSnapshot.connectedSince, order: .reverse) + @State private var showSetupSheet = false + @State private var detectedClientIDs: Set = [] + @State private var sortOrder: [KeyPathComparator] = [ + KeyPathComparator(\ConnectedClientRow.connectedSince, order: .reverse) ] var body: some View { Group { if manager.connectedClients.isEmpty { - UnavailableStateView( - String(localized: "No clients connected"), - systemImage: "person.2.slash", - description: Text(String(localized: "Clients will appear here while they have an active MCP session.")) - ) + emptyState } else { ConnectedClientsTable( - clients: sortedClients, + rows: sortedRows, selection: $selection, sortOrder: $sortOrder, onDisconnect: { client in disconnectCandidate = client } ) + .onDeleteCommand(perform: disconnectSelection) } } + .task(id: ambiguousClientIDs) { await detectAmbiguousClients() } .navigationTitle(IntegrationsActivitySection.connectedClients.title) .navigationSubtitle(subtitle) + .task { await manager.refreshClientsWhileShown() } .toolbar(content: toolbar) .alert( String(localized: "Disconnect client?"), @@ -46,20 +43,101 @@ struct IntegrationsConnectedClientsPane: View { } message: { Text(String(localized: "TablePro could not save the change to the Keychain. The token works again after TablePro restarts.")) } + .sheet(isPresented: $showSetupSheet) { + MCPClientSetupSheet() + } + } + + private var emptyState: some View { + UnavailableStateView { + Label(emptyStateTitle, systemImage: emptyStateSymbol) + } description: { + Text(emptyStateDescription) + } actions: { + Button(String(localized: "Connect a Client…")) { + showSetupSheet = true + } + .accessibilityIdentifier("connected-clients-connect") + } + } + + private var emptyStateTitle: String { + switch manager.state { + case .running: String(localized: "No clients connected") + case .starting: String(localized: "MCP Server Starting") + case .stopped: String(localized: "MCP Server Off") + case .failed: String(localized: "MCP Server Couldn't Start") + } + } + + private var emptyStateSymbol: String { + switch manager.state { + case .running: "person.2.slash" + case .starting: "hourglass" + case .stopped: "power" + case .failed: "exclamationmark.triangle" + } + } + + private var emptyStateDescription: String { + switch manager.state { + case .running: + String(localized: "Clients will appear here while they have an active MCP session.") + case .starting: + String(localized: "Clients can connect once the MCP server is running.") + case .stopped: + String(localized: "Clients can connect while the MCP server is on. Turn it on in Settings > MCP.") + case .failed: + String(localized: "Clients cannot connect until it runs. Open Settings > MCP to see why and try again.") + } + } + + private func disconnectSelection() { + guard let id = selection, let client = manager.connectedClients.first(where: { $0.id == id }) else { return } + disconnectCandidate = client + } + + private var sortedRows: [ConnectedClientRow] { + manager.connectedClients.map { session in + ConnectedClientRow( + session: session, + name: session.clientDisplayName(detected: detectedClientIDs), + iconSlug: session.catalogIconSlug(detected: detectedClientIDs) + ) + } + .sorted(using: sortOrder) + } + + private var ambiguousClientIDs: Set { + let sessions = manager.connectedClients.map { (clientInfoName: $0.sessionInfoName, tokenName: $0.tokenName) } + return MCPClientCatalogBundle.shared?.ambiguousClientIDs(forSessionsNamed: sessions) ?? [] + } + + private func detectAmbiguousClients() async { + let ids = ambiguousClientIDs + guard !ids.isEmpty, let catalog = MCPClientCatalogBundle.shared else { return } + detectedClientIDs = await Self.detect(ids, scanner: MCPClientScanner(catalog: catalog, context: .live())) } - private var sortedClients: [MCPServerManager.SessionSnapshot] { - manager.connectedClients.sorted(using: sortOrder) + @concurrent + nonisolated private static func detect(_ ids: Set, scanner: MCPClientScanner) async -> Set { + scanner.detectedIDs(among: ids) } @ToolbarContentBuilder private func toolbar() -> some ToolbarContent { + ToolbarItem { + Button { + showSetupSheet = true + } label: { + Label(String(localized: "Connect a Client…"), systemImage: "plus") + } + .help(String(localized: "Set up an app to use TablePro's MCP server")) + .accessibilityIdentifier("connected-clients-toolbar-connect") + } ToolbarItem { Button(role: .destructive) { - if let id = selection, - let client = manager.connectedClients.first(where: { $0.id == id }) { - disconnectCandidate = client - } + disconnectSelection() } label: { Label(String(localized: "Disconnect"), systemImage: ToolbarSymbols.disconnect()) } @@ -84,7 +162,19 @@ struct IntegrationsConnectedClientsPane: View { } private func alertMessage(client: MCPServerManager.SessionSnapshot) -> some View { - Text(String(format: String(localized: "“%@” will be disconnected and any in-flight requests will be cancelled."), client.clientName)) + switch client.transport { + case .http: + Text(String( + format: String(localized: "“%@” will be disconnected and its token revoked. Any in-flight requests will be cancelled."), + client.clientDisplayName(detected: detectedClientIDs) + )) + case .stdioBridge: + Text(String( + // swiftlint:disable:next line_length + format: String(localized: "“%@”'s running requests will be cancelled, but it can send new ones right away. To stop it, remove TablePro from it in Connect a Client, or turn off the MCP server."), + client.clientDisplayName(detected: detectedClientIDs) + )) + } } private var subtitle: String { @@ -104,60 +194,70 @@ struct IntegrationsConnectedClientsPane: View { } } +private struct ConnectedClientRow: Identifiable { + let session: MCPServerManager.SessionSnapshot + let name: String + let iconSlug: String? + + var id: MCPServerManager.SessionSnapshot.ID { session.id } + var connectedSince: Date { session.connectedSince } + var lastActivityAt: Date { session.lastActivityAt } +} + private struct ConnectedClientsTable: View { - let clients: [MCPServerManager.SessionSnapshot] + let rows: [ConnectedClientRow] @Binding var selection: MCPServerManager.SessionSnapshot.ID? - @Binding var sortOrder: [KeyPathComparator] + @Binding var sortOrder: [KeyPathComparator] let onDisconnect: (MCPServerManager.SessionSnapshot) -> Void var body: some View { - Table(of: MCPServerManager.SessionSnapshot.self, + Table(of: ConnectedClientRow.self, selection: $selection, sortOrder: $sortOrder) { - TableColumn(String(localized: "Client"), value: \.clientName) { client in - clientCell(for: client) + TableColumn(String(localized: "Client"), value: \.name) { row in + clientCell(for: row) } .width(min: 160, ideal: 200) - TableColumn(String(localized: "Version")) { client in - versionCell(for: client) + TableColumn(String(localized: "Version")) { row in + versionCell(for: row.session) } .width(min: 70, ideal: 90) - TableColumn(String(localized: "Token")) { client in - tokenCell(for: client) + TableColumn(String(localized: "Token")) { row in + tokenCell(for: row.session) } .width(min: 110, ideal: 140) - TableColumn(String(localized: "Address")) { client in - addressCell(for: client) + TableColumn(String(localized: "Address")) { row in + addressCell(for: row.session) } .width(min: 120, ideal: 160) - TableColumn(String(localized: "Connected"), value: \.connectedSince) { client in - connectedCell(for: client) + TableColumn(String(localized: "Connected"), value: \.connectedSince) { row in + connectedCell(for: row.session) } .width(min: 110, ideal: 130) - TableColumn(String(localized: "Last Activity"), value: \.lastActivityAt) { client in - lastActivityCell(for: client) + TableColumn(String(localized: "Last Activity"), value: \.lastActivityAt) { row in + lastActivityCell(for: row.session) } .width(min: 110, ideal: 130) } rows: { - ForEach(clients) { client in - SwiftUI.TableRow(client) - .contextMenu { contextMenu(for: client) } + ForEach(rows) { row in + SwiftUI.TableRow(row) + .contextMenu { contextMenu(for: row.session) } } } } @ViewBuilder - private func clientCell(for client: MCPServerManager.SessionSnapshot) -> some View { + private func clientCell(for row: ConnectedClientRow) -> some View { Label { - Text(client.clientName) + Text(verbatim: row.name) } icon: { - Image(systemName: "checkmark.circle.fill") - .foregroundStyle(.green) + BrandIconView(icon: BrandIcon(slug: row.iconSlug, fallbackSystemImage: "app")) + .frame(width: 16, height: 16) } } @@ -190,14 +290,20 @@ private struct ConnectedClientsTable: View { @ViewBuilder private func connectedCell(for client: MCPServerManager.SessionSnapshot) -> some View { - Text(client.connectedSince, format: .relative(presentation: .named)) - .help(client.connectedSince.formatted(date: .complete, time: .standard)) + relativeText(client.connectedSince) } @ViewBuilder private func lastActivityCell(for client: MCPServerManager.SessionSnapshot) -> some View { - Text(client.lastActivityAt, format: .relative(presentation: .named)) - .help(client.lastActivityAt.formatted(date: .complete, time: .standard)) + relativeText(client.lastActivityAt) + } + + // The list publishes only when it changes, so the elapsed time redraws on its own schedule. + private func relativeText(_ date: Date) -> some View { + TimelineView(.periodic(from: .now, by: 5)) { _ in + Text(date.formatted(.relative(presentation: .named))) + } + .help(date.formatted(date: .complete, time: .standard)) } @ViewBuilder @@ -209,3 +315,19 @@ private struct ConnectedClientsTable: View { } } } + +extension MCPServerManager.SessionSnapshot { + fileprivate var sessionInfoName: String { + clientInfoName ?? clientName + } + + // A session two catalog rows share keeps its own clientInfo title unless one of them alone is on this Mac. + func clientDisplayName(detected: Set) -> String { + MCPClientCatalogBundle.shared?.client(forSessionNamed: sessionInfoName, tokenName: tokenName, detected: detected)?.name + ?? IntegrationsFormatting.displayClientName(self) + } + + func catalogIconSlug(detected: Set) -> String? { + MCPClientCatalogBundle.shared?.iconSlug(forSessionNamed: sessionInfoName, tokenName: tokenName, detected: detected) + } +} diff --git a/apple/TablePro/Views/Integrations/IntegrationsFormatting.swift b/apple/TablePro/Views/Integrations/IntegrationsFormatting.swift index aa7b9f4354..f64f64a94d 100644 --- a/apple/TablePro/Views/Integrations/IntegrationsFormatting.swift +++ b/apple/TablePro/Views/Integrations/IntegrationsFormatting.swift @@ -1,16 +1,26 @@ -// -// IntegrationsFormatting.swift -// TablePro -// - import AppKit import SwiftUI +import TableProMCP enum IntegrationsFormatting { + // MCPTokenStore keeps both reserved names for TablePro's own tokens, at mint and at load, so a paired + // client cannot borrow these labels. static func displayTokenName(_ name: String) -> String { - name == MCPTokenStore.stdioBridgeTokenName - ? String(localized: "Built-in CLI") - : name + switch name { + case MCPTokenStore.stdioBridgeTokenName: + String(localized: "Built-in CLI") + case MCPTokenStore.claudeAgentTokenName: + AIProviderType.claudeAgent.displayName + default: + name + } + } + + // The tablepro command names itself "Command Line"; the window calls it what Settings calls it. + static func displayClientName(_ client: MCPServerManager.SessionSnapshot) -> String { + client.clientInfoName == CLIClientIdentity.name + ? String(localized: "tablepro command") + : client.clientName } static func outcomeSymbol(_ outcome: AuditOutcome?) -> String { diff --git a/apple/TablePro/Views/Integrations/IntegrationsSetupSheet.swift b/apple/TablePro/Views/Integrations/IntegrationsSetupSheet.swift deleted file mode 100644 index b38124dbb3..0000000000 --- a/apple/TablePro/Views/Integrations/IntegrationsSetupSheet.swift +++ /dev/null @@ -1,151 +0,0 @@ -// -// IntegrationsSetupSheet.swift -// TablePro -// - -import AppKit -import SwiftUI - -struct IntegrationsSetupSheet: View { - let port: Int - - @Environment(\.dismiss) private var dismiss - @State private var selectedClient: IntegrationClient = .claudeDesktop - - var body: some View { - VStack(alignment: .leading, spacing: 0) { - header - - Divider() - - ScrollView { - VStack(alignment: .leading, spacing: 16) { - Picker(String(localized: "Client"), selection: $selectedClient) { - ForEach(IntegrationClient.allCases) { client in - Text(client.displayName).tag(client) - } - } - .pickerStyle(.segmented) - - IntegrationsSetupInstructions(client: selectedClient, port: port) - } - .padding(20) - } - - Divider() - - HStack { - Spacer() - Button(String(localized: "Done")) { - dismiss() - } - .keyboardShortcut(.defaultAction) - } - .padding(16) - } - .frame(width: 580, height: 480) - } - - private var header: some View { - VStack(alignment: .leading, spacing: 4) { - Text(String(localized: "Connect a Client")) - .font(.title2.weight(.semibold)) - Text(String(localized: "Choose your client and follow the steps to connect it to TablePro.")) - .font(.callout) - .foregroundStyle(.secondary) - } - .frame(maxWidth: .infinity, alignment: .leading) - .padding(20) - } -} - -private struct IntegrationsSetupInstructions: View { - let client: IntegrationClient - let port: Int - - var body: some View { - VStack(alignment: .leading, spacing: 12) { - ForEach(Array(steps.enumerated()), id: \.offset) { index, step in - HStack(alignment: .firstTextBaseline, spacing: 8) { - Text("\(index + 1).") - .foregroundStyle(.secondary) - .monospacedDigit() - .frame(width: 20, alignment: .trailing) - Text(step) - .textSelection(.enabled) - } - } - - if let snippet = configSnippet { - CopyableCodeBlock(text: snippet) - } - if let command { - CopyableCodeBlock(text: command) - } - } - .font(.callout) - } - - private var bridgeBinaryPath: String { - Bundle.main.bundleURL.appendingPathComponent("Contents/MacOS/tablepro-mcp").path - } - - private var steps: [String] { - switch client { - case .claudeDesktop: - [ - String(localized: "Open Claude Desktop, go to Settings > Developer"), - String(localized: "Click \"Edit Config\" to open claude_desktop_config.json"), - String(localized: "Add the JSON below inside the file and save"), - String(localized: "Restart Claude Desktop") - ] - case .claudeCode: - [String(localized: "Run the command below in your terminal")] - case .cursor: - [ - String(localized: "Open Cursor, go to Settings > MCP"), - String(localized: "Click \"+ Add new global MCP server\""), - String(localized: "Paste the JSON below and save") - ] - case .zed: - [ - String(localized: "Open Zed and click the Agent Panel icon in the right side of the title bar"), - String(localized: "Click the menu in the Agent Panel header and choose Settings"), - String(localized: "Under MCP Servers click \"+ Add Custom Server\", select the Local tab, paste the JSON below, then click Add Server") - ] - } - } - - private var configSnippet: String? { - switch client { - case .claudeDesktop, .cursor: - return """ - { - "mcpServers": { - "tablepro": { - "command": "\(bridgeBinaryPath)" - } - } - } - """ - case .zed: - return """ - { - "tablepro": { - "command": "\(bridgeBinaryPath)", - "args": [] - } - } - """ - case .claudeCode: - return nil - } - } - - private var command: String? { - switch client { - case .claudeCode: "claude mcp add tablepro -- \(bridgeBinaryPath)" - default: nil - } - } -} diff --git a/apple/TablePro/Views/Integrations/MCPClientSetupSheet.swift b/apple/TablePro/Views/Integrations/MCPClientSetupSheet.swift new file mode 100644 index 0000000000..ea782d00d9 --- /dev/null +++ b/apple/TablePro/Views/Integrations/MCPClientSetupSheet.swift @@ -0,0 +1,349 @@ +import AppKit +import SwiftUI + +struct MCPClientSetupSheet: View { + @Environment(\.dismiss) private var dismiss + @StateObject private var service = MCPClientSetupService() + @ObservedObject private var manager = MCPServerManager.shared + @State private var selection: String? + @State private var removeCandidate: MCPClientRow? + + var body: some View { + VStack(spacing: 0) { + header + Divider() + content + Divider() + footer + } + .frame(width: 680, height: sheetHeight) + .onExitCommand { dismiss() } + .task { await service.refresh() } + // The sheet says " connected" from the client list, which updates only while a view polls it. + .task { await manager.refreshClientsWhileShown() } + .confirmationDialog( + removeTitle, + isPresented: removeBinding, + titleVisibility: .visible, + presenting: removeCandidate + ) { row in + Button(String(localized: "Remove"), role: .destructive) { + selection = row.id + Task { await service.remove(row.id) } + } + Button(String(localized: "Cancel"), role: .cancel) {} + } message: { row in + Text(verbatim: removeMessage(for: row)) + } + } + + private var header: some View { + VStack(alignment: .leading, spacing: 8) { + Text(String(localized: "Connect a Client")) + .font(.title2.weight(.semibold)) + Text(String(localized: "Install adds TablePro to a client on this Mac. Select a client to see its settings and set it up by hand.")) + .font(.callout) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + if service.isAppOnReadOnlyVolume { + NoticeLabel(String(localized: "TablePro is running from a disk image or a temporary location. Move it to the Applications folder first, then open it from there.")) + .font(.callout) + } + if isServerDown { + NoticeLabel(String(localized: "The MCP server is not running, so clients you add cannot connect yet. Turn it on in Settings > MCP.")) + .font(.callout) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(20) + } + + @ViewBuilder + private var content: some View { + if !service.isCatalogAvailable { + UnavailableStateView( + String(localized: "Client List Unavailable"), + systemImage: "exclamationmark.triangle", + description: Text(String(localized: "TablePro could not read its list of clients. Reinstall TablePro to restore it.")) + ) + } else if !service.hasLoaded { + ProgressView() + .controlSize(.small) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } else { + clientList + .frame(maxHeight: .infinity) + Divider() + detail + .frame(height: sheetHeight >= 700 ? 260 : 190) + } + } + + private var clientList: some View { + List(selection: $selection) { + if !foundRows.isEmpty { + Section(String(localized: "On This Mac")) { + ForEach(foundRows) { row in + clientRow(row) + } + } + } + if !otherRows.isEmpty { + Section(String(localized: "Other Clients")) { + ForEach(otherRows) { row in + clientRow(row) + } + } + } + } + .listStyle(.inset(alternatesRowBackgrounds: false)) + .onDeleteCommand { + if let row = selectedRow, row.removeRoute != nil { + removeCandidate = row + } + } + } + + private func clientRow(_ row: MCPClientRow) -> some View { + MCPClientSetupRowView( + row: row, + presentation: service.presentation(for: row), + actionsDisabled: service.isAppOnReadOnlyVolume, + onAction: { action in perform(action, on: row) } + ) + .tag(row.id) + .contextMenu { contextMenu(for: row) } + } + + @ViewBuilder + private func contextMenu(for row: MCPClientRow) -> some View { + if let action = service.presentation(for: row).action, action != .copy { + Button(action.title) { perform(action, on: row) } + .disabled(service.isAppOnReadOnlyVolume) + } + if let snippet = row.snippet { + Button(String(localized: "Copy Configuration")) { copy(snippet) } + } + if let commandLine = row.commandLine { + Button(String(localized: "Copy Command")) { copy(commandLine) } + } + if row.configPath != nil { + Button(String(localized: "Show in Finder")) { reveal(row) } + .disabled(!canReveal(row)) + } + if row.removeRoute != nil { + Divider() + Button(String(localized: "Remove…"), role: .destructive) { removeCandidate = row } + } + } + + @ViewBuilder + private var detail: some View { + if let row = selectedRow { + ScrollView { + VStack(alignment: .leading, spacing: 12) { + detailContent(for: row) + } + .padding(16) + .frame(maxWidth: .infinity, alignment: .leading) + } + } else { + Text(String(localized: "Select a client to see how TablePro connects to it.")) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, maxHeight: .infinity) + } + } + + @ViewBuilder + private func detailContent(for row: MCPClientRow) -> some View { + Text(verbatim: row.name) + .font(.headline) + if let unsupported = row.unsupported { + Text(verbatim: MCPClientDetailText.unsupported(unsupported, name: row.name)) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } else { + if case let .failed(_, message, output)? = service.activities[row.id] { + NoticeLabel(message, severity: .error) + if let output { + CopyableCodeBlock(text: output) + } + } + if let note = MCPClientDetailText.configState(row.config, name: row.name) { + Text(verbatim: note) + .foregroundStyle(.secondary) + .fixedSize(horizontal: false, vertical: true) + } + if let applies = row.applies { + Text(verbatim: MCPClientDetailText.applies(applies, name: row.name)) + .foregroundStyle(.secondary) + } + if let path = row.displayConfigPath { + LabeledContent(String(localized: "Settings file")) { + HStack(spacing: 8) { + Text(verbatim: path) + .font(.system(.callout, design: .monospaced)) + .lineLimit(1) + .truncationMode(.middle) + .textSelection(.enabled) + Button(String(localized: "Show in Finder")) { reveal(row) } + .controlSize(.small) + .disabled(!canReveal(row)) + } + } + } + if let commandLine = row.commandLine { + Text(String(localized: "Command")) + .font(.subheadline.weight(.semibold)) + CopyableCodeBlock(text: commandLine) + } + if let snippet = row.snippet, row.isConfigWritable || row.commandLine == nil { + Text(String(localized: "Configuration")) + .font(.subheadline.weight(.semibold)) + CopyableCodeBlock(text: snippet) + } + if row.removeRoute != nil { + Button(String(localized: "Remove…"), role: .destructive) { removeCandidate = row } + .accessibilityIdentifier("mcp-client-remove") + } + } + } + + private var footer: some View { + HStack { + Spacer() + Button(String(localized: "Done")) { dismiss() } + .keyboardShortcut(.defaultAction) + .accessibilityIdentifier("mcp-client-setup-done") + } + .padding(16) + } + + // A SwiftUI sheet on macOS opens at its minimum size, so the size is fixed here and kept inside a short screen. + private var sheetHeight: CGFloat { + let available = (NSApp.keyWindow?.screen ?? NSScreen.main)?.visibleFrame.height ?? 760 + return min(760, max(480, available - 80)) + } + + private var foundRows: [MCPClientRow] { + service.rows.filter { $0.isInstalled } + } + + private var otherRows: [MCPClientRow] { + service.rows.filter { !$0.isInstalled } + } + + private var selectedRow: MCPClientRow? { + selection.flatMap { service.row(id: $0) } + } + + private var isServerDown: Bool { + switch manager.state { + case .stopped, .failed: true + case .starting, .running: false + } + } + + private func perform(_ action: MCPClientRowAction, on row: MCPClientRow) { + selection = row.id + switch action { + case .install: + Task { await service.install(row.id) } + case .update: + Task { await service.update(row.id) } + case .copy: + if let text = row.copyText { + copy(text) + } + } + } + + private func copy(_ text: String) { + ClipboardService.shared.writeText(text) + AccessibilityAnnouncement.post(String(localized: "Copied to clipboard")) + } + + private func canReveal(_ row: MCPClientRow) -> Bool { + guard let path = row.configPath else { return false } + return row.configFileExists + || FileManager.default.fileExists(atPath: (path as NSString).deletingLastPathComponent) + } + + private func reveal(_ row: MCPClientRow) { + guard let path = row.configPath else { return } + if FileManager.default.fileExists(atPath: path) { + NSWorkspace.shared.activateFileViewerSelecting([URL(fileURLWithPath: path)]) + } else { + NSWorkspace.shared.selectFile(nil, inFileViewerRootedAtPath: (path as NSString).deletingLastPathComponent) + } + } + + private var removeTitle: String { + guard let row = removeCandidate else { return "" } + return String(format: String(localized: "Remove TablePro from %@?"), row.name) + } + + private func removeMessage(for row: MCPClientRow) -> String { + if row.removeRoute == .file, let file = row.configFileName { + return String( + format: String(localized: "%1$@ stops reaching your databases through TablePro. The current %2$@ is kept next to it as %3$@."), + row.name, + file, + file + ".tablepro-backup" + ) + } + return String(format: String(localized: "%@ stops reaching your databases through TablePro."), row.name) + } + + private var removeBinding: Binding { + Binding( + get: { removeCandidate != nil }, + set: { isPresented in + if !isPresented { + removeCandidate = nil + } + } + ) + } +} + +private struct MCPClientSetupRowView: View { + let row: MCPClientRow + let presentation: MCPClientRowPresentation + let actionsDisabled: Bool + let onAction: (MCPClientRowAction) -> Void + + var body: some View { + HStack(spacing: 10) { + BrandIconView(slug: row.iconSlug, fallbackSystemImage: "app") + .frame(width: 18, height: 18) + VStack(alignment: .leading, spacing: 2) { + Text(verbatim: presentation.title) + Text(verbatim: presentation.subtitle) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.middle) + } + Spacer(minLength: 8) + if let status = presentation.status { + Text(verbatim: status) + .font(.callout) + .foregroundStyle(.secondary) + .lineLimit(1) + } + if presentation.isBusy { + ProgressView() + .controlSize(.small) + } else if let action = presentation.action { + Button(action.title) { onAction(action) } + .buttonStyle(.bordered) + .controlSize(.small) + .disabled(actionsDisabled && action != .copy) + .accessibilityLabel(action.accessibilityLabel(clientName: row.name)) + .accessibilityIdentifier("mcp-client-action-\(row.id)") + } + } + .padding(.vertical, 3) + } +} diff --git a/apple/TablePro/Views/Main/Child/ConnectionTransactionActions.swift b/apple/TablePro/Views/Main/Child/ConnectionTransactionActions.swift new file mode 100644 index 0000000000..e3d41788f6 --- /dev/null +++ b/apple/TablePro/Views/Main/Child/ConnectionTransactionActions.swift @@ -0,0 +1,109 @@ +// +// ConnectionTransactionActions.swift +// TablePro +// + +import AppKit + +// One flow for the bar's buttons and the Database menu's commands, so both ask and refuse the same way. +@MainActor +internal enum ConnectionTransactionActions { + // Never gated by Safe Mode: rolling back keeps nothing, so it is the way out at every level. + static func rollBack( + _ hold: SessionTransactionTracker.Hold, + databaseManager: DatabaseManager = .shared, + window: NSWindow? + ) async throws { + let confirmed = await AlertHelper.confirmDestructive( + title: String(localized: "Roll Back the Transaction?"), + message: String(localized: "Its uncommitted changes are discarded."), + confirmButton: String(localized: "Roll Back"), + window: window + ) + guard confirmed else { return } + try await databaseManager.endTransaction(hold, committing: false) + } + + // A COMMIT is a write, so it asks what a typed one asks: nothing at Silent, a confirmation at Alert, Touch ID at Safe Mode. + // False when the user cancelled that question. + @discardableResult + static func commit( + _ hold: SessionTransactionTracker.Hold, + databaseManager: DatabaseManager = .shared, + gate: any ExecutionGate = ExecutionGateProvider.shared + ) async throws -> Bool { + try await commitBeforeEnding([hold], databaseManager: databaseManager, gate: gate) + } + + // Every connection is asked before anything is sent, so a Cancel on any of them commits nothing. + static func commitBeforeEnding( + _ holds: [SessionTransactionTracker.Hold], + databaseManager: DatabaseManager = .shared, + gate: any ExecutionGate = ExecutionGateProvider.shared + ) async throws -> Bool { + var connectionIds: [UUID] = [] + for hold in holds where !connectionIds.contains(hold.connectionId) { + connectionIds.append(hold.connectionId) + } + for connectionId in connectionIds { + guard let session = databaseManager.session(for: connectionId) else { continue } + let decision = await gate.authorize( + OperationRequest( + connectionId: connectionId, + databaseType: session.connection.type, + sql: "COMMIT", + kind: .writeQuery, + caller: .userInterface, + capabilities: .interactiveUser, + operationDescription: String(localized: "Commit the transaction") + ) + ) + switch decision { + case .authorized: + break + case .denied(_, .cancelledByUser): + return false + case .denied(let reason, .policy): + throw SessionTransactionDenial(reason: reason) + } + } + // A session that ended took its transaction with it. + for hold in holds where databaseManager.session(for: hold.connectionId) != nil { + try await databaseManager.endTransaction(hold, committing: true) + } + return true + } + + // The bar, the menu and the close prompts all ask this. The level is the one the gate decides the COMMIT at, the + // managed floor included: the session's own level offered Commit under a managed Read-Only floor the gate then denied. + nonisolated static func offersCommit(_ state: SessionTransactionTracker.State, effectiveLevel: SafeModeLevel) -> Bool { + state == .open && !effectiveLevel.blocksAllWrites + } + + static func offersCommit(_ hold: SessionTransactionTracker.Hold, databaseManager: DatabaseManager = .shared) -> Bool { + offersCommit( + hold.state, + effectiveLevel: ExecutionGateProvider.effectiveSafeModeLevel(for: hold.connectionId, in: databaseManager) + ) + } + + static func perform(window: NSWindow?, _ action: @MainActor () async throws -> Void) async { + do { + try await action() + } catch is CancellationError { + return + } catch { + AlertHelper.showErrorSheet( + title: String(localized: "Could Not End the Transaction"), + message: error.localizedDescription, + window: window + ) + } + } +} + +private struct SessionTransactionDenial: LocalizedError { + let reason: String + + var errorDescription: String? { reason } +} diff --git a/apple/TablePro/Views/Main/Child/ConnectionTransactionBar.swift b/apple/TablePro/Views/Main/Child/ConnectionTransactionBar.swift index 6106744d43..7ea1ea2af6 100644 --- a/apple/TablePro/Views/Main/Child/ConnectionTransactionBar.swift +++ b/apple/TablePro/Views/Main/Child/ConnectionTransactionBar.swift @@ -21,7 +21,7 @@ internal struct ConnectionTransactionBarContent: Equatable { isAborted ? Self.abortedSymbolName : Self.openSymbolName } - init(isAborted: Bool, connectionName: String, database: String?, safeModeLevel: SafeModeLevel) { + init(isAborted: Bool, connectionName: String, database: String?, effectiveSafeModeLevel: SafeModeLevel) { self.isAborted = isAborted let database = database.flatMap { $0.isEmpty ? nil : $0 } if isAborted { @@ -42,7 +42,7 @@ internal struct ConnectionTransactionBarContent: Equatable { accessibilityLabel = database.map { String(format: String(localized: "Transaction open on %1$@, %2$@"), $0, connectionName) } ?? String(format: String(localized: "Transaction open on %@"), connectionName) - offersCommit = !safeModeLevel.blocksAllWrites + offersCommit = ConnectionTransactionActions.offersCommit(.open, effectiveLevel: effectiveSafeModeLevel) } } @@ -51,6 +51,9 @@ internal struct ConnectionTransactionBar: View { let connectionId: UUID @ObservedObject var transactions: SessionTransactionTracker let databaseManager: DatabaseManager + // Passed from the toolbar state the window observes, so the bar redraws when the level changes; the session's own + // level publishes nothing. Commit follows the gate's level, read in `content(for:in:)`. + let safeModeLevel: SafeModeLevel @State private var actingOn: SessionTransactionTracker.Key? @@ -73,7 +76,7 @@ internal struct ConnectionTransactionBar: View { isAborted: hold.state == .aborted, connectionName: session.connection.name, database: hold.key.lane ?? namedDatabase(of: session), - safeModeLevel: session.safeModeLevel + effectiveSafeModeLevel: ExecutionGateProvider.effectiveSafeModeLevel(for: connectionId, in: databaseManager) ) } @@ -109,11 +112,15 @@ internal struct ConnectionTransactionBar: View { Spacer(minLength: 8) Button(String(localized: "Roll Back")) { - act(on: hold) { try await rollBack(hold) } + act(on: hold) { + try await ConnectionTransactionActions.rollBack( + hold, databaseManager: databaseManager, window: NSApp.keyWindow + ) + } } if content.offersCommit { Button(String(localized: "Commit")) { - act(on: hold) { try await commit(hold) } + act(on: hold) { try await ConnectionTransactionActions.commit(hold, databaseManager: databaseManager) } } } } @@ -124,64 +131,11 @@ internal struct ConnectionTransactionBar: View { .paneBarBackground() } - // Never gated by Safe Mode: rolling back keeps nothing, so it is the way out at every level. - private func rollBack(_ hold: SessionTransactionTracker.Hold) async throws { - let confirmed = await AlertHelper.confirmDestructive( - title: String(localized: "Roll Back the Transaction?"), - message: String(localized: "Its uncommitted changes are discarded."), - confirmButton: String(localized: "Roll Back"), - window: NSApp.keyWindow - ) - guard confirmed else { return } - try await databaseManager.endTransaction(hold, committing: false) - } - - // A COMMIT is a write, so it asks what a typed one asks: nothing at Silent, a confirmation at - // Alert, Touch ID at Safe Mode. - private func commit(_ hold: SessionTransactionTracker.Hold) async throws { - guard let session = databaseManager.session(for: connectionId) else { return } - let decision = await ExecutionGateProvider.shared.authorize( - OperationRequest( - connectionId: connectionId, - databaseType: session.connection.type, - sql: "COMMIT", - kind: .writeQuery, - caller: .userInterface, - capabilities: .interactiveUser, - operationDescription: String(localized: "Commit the transaction") - ) - ) - switch decision { - case .authorized: - try await databaseManager.endTransaction(hold, committing: true) - case .denied(_, .cancelledByUser): - return - case .denied(let reason, .policy): - throw SessionTransactionDenial(reason: reason) - } - } - private func act(on hold: SessionTransactionTracker.Hold, _ action: @escaping @MainActor () async throws -> Void) { actingOn = hold.key Task { defer { actingOn = nil } - do { - try await action() - } catch is CancellationError { - return - } catch { - AlertHelper.showErrorSheet( - title: String(localized: "Could Not End the Transaction"), - message: error.localizedDescription, - window: NSApp.keyWindow - ) - } + await ConnectionTransactionActions.perform(window: NSApp.keyWindow, action) } } } - -private struct SessionTransactionDenial: LocalizedError { - let reason: String - - var errorDescription: String? { reason } -} diff --git a/apple/TablePro/Views/Main/Child/DataTabGridAdapter.swift b/apple/TablePro/Views/Main/Child/DataTabGridAdapter.swift index a53466d541..0b608aae63 100644 --- a/apple/TablePro/Views/Main/Child/DataTabGridAdapter.swift +++ b/apple/TablePro/Views/Main/Child/DataTabGridAdapter.swift @@ -105,8 +105,8 @@ final class DataTabGridAdapter: DataGridDataSource, DataGridActions { } func gridDuplicateRows(_ rowIDs: [RowID]) { - guard ownsWindowState, let first = rowIDs.first else { return } - coordinator?.duplicateRow(first) + guard ownsWindowState, !rowIDs.isEmpty else { return } + coordinator?.duplicateRows(rowIDs) } func gridCanPasteRows() -> Bool { diff --git a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift index 86e0de6639..7e2c92da6f 100644 --- a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift +++ b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift @@ -36,7 +36,10 @@ final class DataTabSQLExporter: DataGridSQLExporting { let tableName = tab.tableContext.tableName else { return nil } return Target( tableName: tableName, - schemaName: tab.tableContext.schemaName, + // A query tab's schema is the picker's; the rows name the schema they were read in. + schemaName: tab.tabType == .query + ? tab.display.activeResultSet?.origin?.schemaName ?? tab.tableContext.schemaName + : tab.tableContext.schemaName, primaryKeyColumns: tab.tableContext.primaryKeyColumns, databaseType: coordinator.connection.type, connectionId: coordinator.connectionId, @@ -123,7 +126,9 @@ final class DataTabSQLExporter: DataGridSQLExporting { columnTypes: [column.type ?? .text(rawType: nil)], escapeStringLiteral: driver?.escapeStringLiteral, stringLiteralPrefix: SQLStringLiteralPrefix.forDatabaseType(databaseType), - stringEscapes: SQLLexicalResolver.stringEscapes(for: databaseType, session: driver?.sessionLexicalState) + stringEscapes: SQLLexicalResolver.stringEscapes(for: databaseType, session: driver?.sessionLexicalState), + booleanLiteralStyle: (try? resolveSQLDialect(for: databaseType))?.booleanLiteralStyle ?? .truefalse, + databaseType: databaseType ) return converter.generateInClause(rows: values.map { [$0] }) } @@ -142,7 +147,7 @@ final class DataTabSQLExporter: DataGridSQLExporting { private func sourceRow(_ row: Row, target: Target) -> SQLRowToStatementConverter.SourceRow { var stored = Array(row.values) - let pendingUpdate = target.changeManager.rowChanges.first { $0.rowID == row.id && $0.type == .update } + let pendingUpdate = target.changeManager.pendingUpdate(forRow: row.id) for cellChange in pendingUpdate?.cellChanges ?? [] where stored.indices.contains(cellChange.columnIndex) { stored[cellChange.columnIndex] = cellChange.oldValue } @@ -155,14 +160,10 @@ final class DataTabSQLExporter: DataGridSQLExporting { ) } - // Every marker in a row not saved yet, and in a saved row the cells a pending edit set to Default. + // The cells a pending insert or edit marked as the default. private func stagedDefaults(of row: Row, target: Target) -> Set { - let marker = PluginCellValue.text("__DEFAULT__") - if target.changeManager.insertedRowIDs.contains(row.id) { - return Set(row.values.indices.filter { row.values[$0] == marker }) - } - let pendingUpdate = target.changeManager.rowChanges.first { $0.rowID == row.id && $0.type == .update } - return Set(pendingUpdate?.cellChanges.filter { $0.newValue == marker }.map(\.columnIndex) ?? []) + let marked = target.changeManager.expressionColumns(forRow: row.id) + return Set(marked.filter { row.values.indices.contains($0) && row.values[$0] == .columnDefault }) } private func nonEmpty(_ statements: String) -> String? { diff --git a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift index ad00263ed6..aa10e55631 100644 --- a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift +++ b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift @@ -269,7 +269,7 @@ struct MainEditorContentView: View { switch subject { case .linkedFile(let url): return url.lastPathComponent - case .savedQuery: + case .savedQuery, .queryTab: return "query.sql" } } @@ -478,17 +478,7 @@ struct MainEditorContentView: View { // The tab owns the new container for life, so it moves to that entry and the window follows, like selecting its tab. private func changeContainer(for tab: QueryTab, to name: String) { - let tabId = tab.id - guard tabManager.containerName(of: tab) != name, - tabManager.mutate(tabId: tabId, { tab in - QueryContainerRebinding.rebind(&tab.tableContext, to: name, target: containerSwitchTarget) - }) else { return } - tabManager.markTabRenamed(tabId) - SchemaProviderRegistry.shared.reclaimUnheldProviders(for: connectionId) - guard tabManager.selectedTabId == tabId, - ContainerChangeRerunPolicy.reruns(showing: tabManager.selectedTab?.display.activeResultSet) else { return } - // The statement at the caret may not be the one that produced the rows, so a write asks first. - coordinator.runQuery(viewport: .firstRow, extraCapabilities: .confirmsWrites) + coordinator.changeQueryContainer(of: tab.id, to: name, target: containerSwitchTarget) } private func dismissDiskChangeBanner(tabId: UUID) { @@ -511,7 +501,7 @@ struct MainEditorContentView: View { // By ID: mid-switch, flushTextUpdate() fires for the old tab while selectedTabIndex already points to the new one. guard tabManager.mutate(tabId: tabId, { $0.content.query = newValue }) else { return } - coordinator.scheduleDraftSave() + coordinator.scheduleDraftSave(editedTab: tabId) // Scratch tabs dirty too. Only this tab's window: a background window tab's editor stays mounted and can fire here. guard tabId == tabManager.selectedTabId, @@ -734,6 +724,7 @@ struct MainEditorContentView: View { selectedRowIDs: Set(model.selectedRowIDs), displayIDs: model.displayedRowIDs, deletedRowIDs: changeManager.deletedRowIDs, + defaultedCells: changeManager.defaultedCells(in: rows), valueFilter: tab.valueFilter, dataRevision: coordinator.tabSessionRegistry.session(for: tab.id)?.dataRevision ?? 0, displayRevision: coordinator.gridDisplayRevision, @@ -985,7 +976,14 @@ struct MainEditorContentView: View { resultSetMenu: resultSetMenuModel(for: tab, placedIn: .statusBar), resultSetActions: resultSetActions(for: tab), onToggleFilters: { coordinator.toggleFilterPanel() }, - onFetchAll: { coordinator.fetchAllRows() } + onFetchAll: { coordinator.fetchAllRows() }, + autoRefresh: StatusBarAutoRefreshState( + interval: tab.autoRefreshInterval, + pause: tab.autoRefreshPause, + onChange: { [coordinator, tabId = tab.id] interval in + coordinator.setAutoRefreshInterval(interval, forTab: tabId) + } + ) ) } @@ -1021,7 +1019,7 @@ struct MainEditorContentView: View { isPinned: resultSet.isPinned, isActive: resultSet.id == activeId, ordinal: index + 1, - isComparable: coordinator.isComparable(resultSet, in: tab) + isComparable: coordinator.isComparable(resultSet, in: tab, activeResultSetId: activeId) ) } return ResultSetMenuModel( @@ -1221,6 +1219,7 @@ extension MainEditorContentView { databaseScope: queryScope, connectionId: coordinator.connection.id, tabID: tab.id, + undoHistories: coordinator.editorUndoHistories, sourceFileURL: tab.content.sourceFileURL, claimFocusOnAppear: claimFocus, onFocusClaimed: { diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift index 750e98c33f..8217b45be3 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift @@ -146,13 +146,8 @@ extension MainContentCoordinator { } switch WalkthroughApplyPlan.resolve(beforeSQL: beforeSQL, source: source, tabText: tabText) { case .replace(let tabId, let range): - guard let owner else { return } - owner.tabManager.mutate(tabId: tabId) { tab in - let text = tab.content.query as NSString - guard NSMaxRange(range) <= text.length else { return } - tab.content.query = text.replacingCharacters(in: range, with: afterSQL) - tab.hasUserInteraction = true - } + guard let owner, owner.replaceQueryText(inTab: tabId, range: range, with: afterSQL) else { return } + owner.tabManager.mutate(tabId: tabId) { $0.hasUserInteraction = true } case .insertAsNewQuery: insertQueryFromAI(afterSQL) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift index fda3245d62..6147d54473 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift @@ -61,6 +61,8 @@ extension MainContentCoordinator { /// edits: the table is fetched again with a different column list. case .columnVisibility: return String(localized: "Showing or hiding columns will reload data and discard all unsaved changes.") + case .containerChange: + return String(localized: "Moving this tab to another database or schema will discard all unsaved changes.") } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AutoRefresh.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AutoRefresh.swift new file mode 100644 index 0000000000..474224b02a --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AutoRefresh.swift @@ -0,0 +1,170 @@ +// +// MainContentCoordinator+AutoRefresh.swift +// TablePro +// + +import Combine +import Foundation + +// One per window: only the selected table tab ticks, so a switch, a close or a move ends one wait and starts the next. +@MainActor +final class TableAutoRefreshSchedule { + var clock: any Clock = ContinuousClock() + var visibility: @MainActor (OperationOwner, UUID) -> ResultVisibility = { owner, connectionId in + ResultVisibilityResolver.resolve(owner: owner, connectionId: connectionId) + } + fileprivate(set) var task: Task? + fileprivate var selectionCancellable: AnyCancellable? + + var isRunning: Bool { task != nil } +} + +extension MainContentCoordinator { + func observeSelectionForAutoRefresh() { + autoRefresh.selectionCancellable = tabManager.$selectedTabId + .removeDuplicates() + .dropFirst() + .receive(on: RunLoop.main) + .sink { [weak self] _ in self?.restartAutoRefresh() } + } + + func setAutoRefreshInterval(_ interval: TableAutoRefreshInterval, forTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }), tab.tabType == .table else { return } + if tab.autoRefreshInterval != interval || tab.autoRefreshPause != nil { + tabManager.mutate(tabId: tabId) { tab in + tab.autoRefreshInterval = interval + tab.autoRefreshPause = nil + } + } + guard tabId == tabManager.selectedTabId else { return } + restartAutoRefresh() + } + + func restartAutoRefresh() { + autoRefresh.task?.cancel() + autoRefresh.task = nil + guard !isTearingDown, + let tab = tabManager.selectedTab, + tab.tabType == .table, + tab.autoRefreshInterval.isOn else { return } + let tabId = tab.id + let clock = autoRefresh.clock + // Weak between ticks, so a wait never keeps a closed window's coordinator alive. + autoRefresh.task = Task { [weak self] in + while !Task.isCancelled { + await self?.awaitAutoRefreshLoad(tabId) + guard let interval = self?.autoRefreshInterval(ofTab: tabId), interval.isOn else { return } + do { + try await clock.sleep(for: interval.duration) + } catch { + return + } + guard let self, !Task.isCancelled else { return } + guard await runAutoRefreshTick(tabId) else { return } + } + } + } + + func stopAutoRefresh() { + autoRefresh.task?.cancel() + autoRefresh.task = nil + autoRefresh.selectionCancellable = nil + } + + func autoRefreshGate(forTab tabId: UUID) -> TableAutoRefreshGate { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { + return TableAutoRefreshGate(isTableTab: false) + } + let visibility = autoRefresh.visibility(operationOwner(tabId: tabId), connectionId) + let model = existingGridModel(forTab: tabId) + let database = scope(for: tab)?.database ?? browseDatabaseName + var gate = TableAutoRefreshGate() + gate.isTableTab = tab.tabType == .table && tab.tableContext.tableName != nil + gate.hasLoaded = tab.execution.lastExecutedAt != nil + gate.windowIsVisible = visibility.ownerWindowIsVisible + gate.isSelectedInWindow = visibility.ownerIsSelectedInWindow && tabManager.selectedTabId == tabId + gate.isConnected = services.databaseManager.session(for: connectionId)?.reportedStatus == .connected + gate.viewMode = tab.display.resultsViewMode + // Read only: a read inside the user's transaction sees its snapshot, and a failure aborts it on PostgreSQL. + gate.holdsTransaction = services.databaseManager.sessionTransactions + .transactionState(connectionId: connectionId, container: database) != nil + gate.holdsUnsavedWork = holdsUnsavedWork(tab) + gate.hasSidebarDataWork = commandActions?.hasDataWork ?? false + gate.isEditingCell = model?.isEditing ?? false + gate.gridHoldsTransientUI = model?.holdsTransientUI ?? false + gate.gridIsTrackingInteraction = model?.isTrackingInteraction ?? false + gate.inspectorHoldsValueWindow = trailingPaneState.map { InspectorPopOutRegistry.holdsWindow(openedFrom: $0) } ?? false + gate.load = tabExecution.isBusy(tabId) || tableLoadTasks[tabId] != nil ? .busy : .idle + gate.isCountPending = tab.pagination.isCountPending + gate.isLoadingMore = tab.pagination.isLoadingMore + gate.showsSafeModePrompt = isShowingSafeModePrompt + gate.hasSelection = model.map { !$0.selection.affectedRows.isEmpty } ?? false + gate.hasPrimaryKey = !tab.tableContext.primaryKeyColumns.isEmpty + return gate + } + + func autoRefreshHeldByTransientUI(_ tabId: UUID) -> Bool { + let gate = autoRefreshGate(forTab: tabId) + return gate.gridHoldsTransientUI || gate.inspectorHoldsValueWindow || gate.isEditingCell + } + + private func autoRefreshInterval(ofTab tabId: UUID) -> TableAutoRefreshInterval? { + guard tabManager.selectedTabId == tabId else { return nil } + return tabManager.tabs.first { $0.id == tabId }?.autoRefreshInterval + } + + // Once, not in a loop: a finished task its owner has not retired yet would spin the main actor. + private func awaitAutoRefreshLoad(_ tabId: UUID) async { + if let scheduled = tableLoadTasks[tabId]?.task { + await scheduled.value + } + if let running = queryTasks.task(for: tabId) { + await running.value + } + } + + // False ends the schedule: the tab is gone or no longer selected, or the gate stopped it. + private func runAutoRefreshTick(_ tabId: UUID) async -> Bool { + // A retarget keeps the tab id and turns the interval off while this wait slept. + guard autoRefreshInterval(ofTab: tabId)?.isOn == true else { return false } + switch autoRefreshGate(forTab: tabId).decision { + case .stop: + endAutoRefresh(forTab: tabId) + return false + case .wait(let pause): + recordAutoRefreshPause(pause.help == nil ? nil : pause, forTab: tabId) + return true + case .refresh: + recordAutoRefreshPause(nil, forTab: tabId) + } + // Not runQuery, which commits an open edit, nor reloadTableTab, which retires the count. + if let load = executeTableTabQueryDirectly(trigger: .autoRefresh, viewport: .keepPlace) + ?? queryTasks.task(for: tabId) { + await load.value + } + guard !Task.isCancelled else { return false } + var settled = autoRefreshGate(forTab: tabId) + settled.previousTickFailed = tabManager.tabs.first { $0.id == tabId }?.execution.errorMessage != nil + guard settled.decision != .stop else { + endAutoRefresh(forTab: tabId) + return false + } + return true + } + + // The tab keeps its own inline error; only the interval goes. + private func endAutoRefresh(forTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }), + tab.autoRefreshInterval.isOn || tab.autoRefreshPause != nil else { return } + tabManager.mutate(tabId: tabId) { tab in + tab.autoRefreshInterval = .off + tab.autoRefreshPause = nil + } + } + + // Written only on change: every tab write re-renders the window. + private func recordAutoRefreshPause(_ pause: TableAutoRefreshPause?, forTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }), tab.autoRefreshPause != pause else { return } + tabManager.mutate(tabId: tabId) { $0.autoRefreshPause = pause } + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ContainerChange.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ContainerChange.swift new file mode 100644 index 0000000000..6421fa5ef7 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ContainerChange.swift @@ -0,0 +1,43 @@ +// +// MainContentCoordinator+ContainerChange.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + // Everything the tab was doing belongs to the object it named: its run, its edit in progress and the cached metadata. + internal func retargetTab(_ tabId: UUID) { + existingGridModel(forTab: tabId)?.discardEditing() + flushColumnLayoutPersistence() + supersedeExecution(for: tabId) + releaseRetargetedTabState(for: tabId) + } + + // A rebind is a retarget: rows read from the old container must never land, or stay editable, under the new one. + internal func changeQueryContainer(of tabId: UUID, to name: String, target: ContainerSwitchTarget?) { + guard canChangeContainer(of: tabId, to: name) else { return } + // Asked before the tab moves, so Cancel leaves the staged edits with the database their rows came from. + confirmDiscardRestoringRowsIfNeeded(action: .containerChange) { [weak self] confirmed in + guard confirmed, let self, canChangeContainer(of: tabId, to: name) else { return } + retargetTab(tabId) + guard tabManager.mutate(tabId: tabId, { tab in + QueryContainerRebinding.rebind(&tab.tableContext, to: name, target: target) + // Rows still on screen were read from the old container; Load More would page them from the new one. + tab.pagination.resetLoadMore() + }) else { return } + tabManager.markTabRenamed(tabId) + SchemaProviderRegistry.shared.reclaimUnheldProviders(for: connectionId) + guard ContainerChangeRerunPolicy.reruns(showing: tabManager.selectedTab?.display.activeResultSet) else { return } + // The statement at the caret may not be the one that produced the rows, so a write asks first. + runQuery(viewport: .firstRow, extraCapabilities: .confirmsWrites) + } + } + + // The picker is off while the tab runs: a running tab moved elsewhere would land its rows under the new database. + private func canChangeContainer(of tabId: UUID, to name: String) -> Bool { + guard tabManager.selectedTabId == tabId, !tabExecution.isExecuting(tabId), + let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { return false } + return tabManager.containerName(of: tab) != name + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseFileChanges.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseFileChanges.swift new file mode 100644 index 0000000000..6ca180a9ce --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseFileChanges.swift @@ -0,0 +1,42 @@ +// +// MainContentCoordinator+DatabaseFileChanges.swift +// TablePro +// + +import Foundation + +// Every commit rewrites a database file, so a file event alone says nothing about the catalog. The engine's +// schema token does: the catalog reloads only when it moved, or when the engine has none. +internal struct SchemaChangeTokenGate: Equatable { + // Read before the catalog it stands for, so it never claims a schema newer than the one loaded. + private(set) var loadedToken: String? + + mutating func admits(_ token: String?) -> Bool { + if let token, token == loadedToken { return false } + loadedToken = token + return true + } + + mutating func forget() { + loadedToken = nil + } +} + +extension MainContentCoordinator { + func refreshTablesIfSchemaChanged() async { + let token = try? await services.databaseManager.withBrowseMetadataDriver(connectionId: connectionId) { driver in + try await (driver as? PluginDriverAdapter)?.schemaChangeToken() + } + guard schemaTokenGate.admits(token) else { return } + DatabaseTreeMetadataService.shared.markListsStale(connectionId: connectionId) + schemaColumns.removeAll() + // Never joins a load that began before this write: that load's catalog predates the new token. + let committed = await services.schemaRefreshService.refreshAfterWrite(connection: connection) + // Another window's refresh may have replaced this one; prune against the catalog it leaves. + await services.schemaRefreshService.waitForRefresh(connectionId: connectionId) + pruneStaleSidebarState() + if !committed { + schemaTokenGate.forget() + } + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Discard.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Discard.swift index 9289878f49..d8eb9361dd 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Discard.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Discard.swift @@ -4,10 +4,11 @@ // import Foundation +import TableProPluginKit extension MainContentCoordinator { - func executeSidebarChanges(steps: [DataWriteStep]) async throws { - try await rowEditingCoordinator.executeSidebarChanges(steps: steps) + func executeSidebarChanges(steps: [DataWriteStep], viewTarget: PluginViewWriteTarget?) async throws { + try await rowEditingCoordinator.executeSidebarChanges(steps: steps, viewTarget: viewTarget) } func handleDiscard( diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DraftSave.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DraftSave.swift new file mode 100644 index 0000000000..74c62ffb1b --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DraftSave.swift @@ -0,0 +1,57 @@ +// +// MainContentCoordinator+DraftSave.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + static let periodicSaveInterval: Duration = .seconds(30) + static let draftSaveDebounce: Duration = .seconds(1) + + // Typing changes no tab identity, so without this a scratch query lives only in memory until the 30s timer or quit flush. + // Any window may save: `saveAggregated` writes every window's tabs for the connection. + func scheduleDraftSave(editedTab tabId: UUID? = nil) { + guard !Self.isAppTerminating, !isTearingDown else { return } + if let tabId { draftSaveTabIds.insert(tabId) } + draftSaveTask?.cancel() + let clock = draftSaveClock + draftSaveTask = Task { [weak self] in + try? await clock.sleep(for: Self.draftSaveDebounce) + guard let self, !Task.isCancelled, !Self.isAppTerminating, !self.isTearingDown else { return } + self.recordIdleVersions() + self.persistence.saveAggregated() + } + } + + func startPeriodicSave() { + guard periodicSaveTask == nil else { return } + periodicSaveTask = Task { [weak self] in + while !Task.isCancelled { + try? await Task.sleep(for: Self.periodicSaveInterval) + guard let self, !Task.isCancelled, !Self.isAppTerminating, !self.isTearingDown else { return } + guard self.isFirstCoordinatorForConnection() else { continue } + self.persistence.saveAggregated() + } + } + } + + static func aggregatedSelectedTabId(for connectionId: UUID) -> UUID? { + activeCoordinators.values + .first { $0.connectionId == connectionId && $0.tabManager.selectedTabId != nil }? + .tabManager.selectedTabId + } + + private func isFirstCoordinatorForConnection() -> Bool { + Self.activeCoordinators.values + .first { $0.connectionId == self.connectionId } === self + } + + private func recordIdleVersions() { + let tabIds = draftSaveTabIds + draftSaveTabIds.removeAll() + for tabId in tabIds { + tabVersions.record(tabManager.tabs.first { $0.id == tabId }, connectionId: connectionId, reason: .idle) + } + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+EditorQueryLoading.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+EditorQueryLoading.swift new file mode 100644 index 0000000000..9a1b94e87e --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+EditorQueryLoading.swift @@ -0,0 +1,99 @@ +// +// MainContentCoordinator+EditorQueryLoading.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + @discardableResult + func loadQueryIntoEditor( + _ query: String, + databaseName: String? = nil, + forceNewTab: Bool = false + ) -> WindowTabOpenDisposition { + let targetDatabaseName = databaseName ?? newTabDatabaseName + if !forceNewTab, + let (tab, tabIndex) = tabManager.selectedTabAndIndex, + tab.tabType == .query, + databaseName == nil + || tab.tableContext.resolvedDatabaseName(browsing: browseDatabaseName) == targetDatabaseName { + replaceQueryText(inTab: tab.id, with: query) + tabManager.mutate(at: tabIndex) { $0.hasUserInteraction = true } + return .currentCoordinator + } + + if !forceNewTab, tabManager.tabs.isEmpty { + tabManager.addTab(initialQuery: query, databaseName: targetDatabaseName) + return .currentCoordinator + } + + let payload = EditorTabPayload( + connectionId: connection.id, + tabType: .query, + databaseName: targetDatabaseName, + initialQuery: query + ) + openTabInNewWindow(payload) + return .focusedElsewhere + } + + var aiInsertReusesSelectedQueryTab: Bool { + guard let (tab, _) = tabManager.selectedTabAndIndex, tab.tabType == .query else { return false } + return tab.content.query.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + func insertQueryFromAI(_ query: String) { + if aiInsertReusesSelectedQueryTab, let (tab, tabIndex) = tabManager.selectedTabAndIndex { + replaceQueryText(inTab: tab.id, with: query) + tabManager.mutate(at: tabIndex) { $0.hasUserInteraction = true } + } else if tabManager.tabs.isEmpty { + tabManager.addTab(initialQuery: query, databaseName: newTabDatabaseName) + } else { + let payload = EditorTabPayload( + connectionId: connection.id, + tabType: .query, + initialQuery: query + ) + WindowManager.shared.openTab(payload: payload) + } + } + + func clearSelectedQuery() { + guard let tab = tabManager.selectedTab, tab.tabType == .query else { return } + replaceQueryText(inTab: tab.id, with: "") + } + + // Through the mounted editor the change is one undoable edit; with no editor mounted the model takes the text, + // and the version recorded first is the way back. Nil `reason` is for a caller that recorded already. + @discardableResult + func replaceQueryText( + inTab tabId: UUID, + range: NSRange? = nil, + with text: String, + recording reason: QueryTabVersionReason? = .beforeReplace + ) -> Bool { + flushEditorText(forTab: tabId) + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }), tab.tabType == .query else { return false } + let current = tab.content.query as NSString + let target = range ?? NSRange(location: 0, length: current.length) + guard NSMaxRange(target) <= current.length else { return false } + if let reason { + tabVersions.record(tab, connectionId: connectionId, reason: reason) + } + let replacedInEditor = EditorEventRouter.shared.replaceText( + forTab: tabId, + connectionId: connectionId, + in: target, + with: text + ) + if !replacedInEditor { + let replaced = current.replacingCharacters(in: target, with: text) + tabManager.mutate(tabId: tabId) { $0.content.query = replaced } + } + scheduleDraftSave(editedTab: tabId) + syncQueryToolbarStateForSelectedTab() + refreshUnsavedIndicator() + return true + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift index 067f3496e6..6ee104d4ba 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift @@ -50,7 +50,7 @@ extension MainContentCoordinator { isShowingSafeModePrompt = true let tabId = tab.id - let authorization = authorizationRequest(for: request) + let authorization = authorizationRequest(for: request, runsIn: scope(for: tab)) return Task { defer { isShowingSafeModePrompt = false } switch await executionGate.authorize(authorization) { @@ -66,7 +66,8 @@ extension MainContentCoordinator { // MARK: - Request - private func authorizationRequest(for request: ExplainRequest) -> OperationRequest { + // An Oracle actual plan runs its statement in the tab's schema, where a bare name resolves. + private func authorizationRequest(for request: ExplainRequest, runsIn: DatabaseScope?) -> OperationRequest { OperationRequest( connectionId: connectionId, databaseType: connection.type, @@ -74,7 +75,8 @@ extension MainContentCoordinator { kind: request.operationKind(databaseType: connection.type), caller: .userInterface, capabilities: .interactiveUser, - operationDescription: String(localized: "Execute Query") + operationDescription: String(localized: "Execute Query"), + runsIn: runsIn ) } @@ -115,10 +117,12 @@ extension MainContentCoordinator { let (claim, lease) = beginTabExecution(for: tabId) let conn = connection + let safeModeIsReadOnly = runsAtReadOnly tabManager.mutate(tabId: tabId) { $0.execution.errorMessage = nil } // EXPLAIN ANALYZE and an actual plan run the statement, and a CREATE TABLE AS among them changes the catalog. + // Under Manual Commit such a run is held, as the write itself would be. let ranWrite = request.operationKind(databaseType: conn.type).declaresWrite let failureOutput = ServerOutputBox() let explainTask = Task { [weak self] in @@ -128,16 +132,18 @@ extension MainContentCoordinator { scope: scope, route: services.databaseManager.executionRoute(for: scope), cancellation: .cancellableRead(lease) - ) { [queryExecutor] driver in - try await ServerOutputCapture.running(on: driver, failureOutput: failureOutput) { () async throws -> QueryFetchResult in - if let step = request.driverStep { - return try await QueryExecutor.fetchExplainPlan( - driver: driver, statement: request.subjectSQL, variantId: step.variantId + ) { [queryExecutor, held = manualCommitRun(skipped: !ranWrite)] driver in + try await held.running([request.gatedSQL(databaseType: conn.type)], on: driver) { + try await ServerOutputCapture.running(on: driver, failureOutput: failureOutput) { () async throws -> QueryFetchResult in + if let step = request.driverStep { + return try await QueryExecutor.fetchExplainPlan( + driver: driver, statement: request.subjectSQL, variantId: step.variantId + ) + } + return try await queryExecutor.executeQuery( + driver: driver, sql: request.sql, parameters: nil, rowCap: nil, safeModeIsReadOnly: safeModeIsReadOnly ) } - return try await queryExecutor.executeQuery( - driver: driver, sql: request.sql, parameters: nil, rowCap: nil - ) } } if ranWrite { Self.postStatementRan(request.subjectSQL, on: conn) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift index bec094a2e0..f2876bd7a1 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift @@ -18,13 +18,15 @@ extension MainContentCoordinator { return } - if let (tab, tabIndex) = tabManager.selectedTabAndIndex, - tab.tabType == .query { - let existing = tab.content.query.trimmingCharacters(in: .whitespacesAndNewlines) - if existing.isEmpty { - tabManager.mutate(at: tabIndex) { $0.content.query = favorite.query } + if let selected = tabManager.selectedTab, selected.tabType == .query { + // Flushed first: on a large document the tab's text trails the editor, and the end must be the editor's. + flushEditorText(forTab: selected.id) + let existing = tabManager.selectedTab?.content.query ?? "" + if existing.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + replaceQueryText(inTab: selected.id, with: favorite.query) } else { - tabManager.mutate(at: tabIndex) { $0.content.query += "\n\n" + favorite.query } + let end = NSRange(location: (existing as NSString).length, length: 0) + replaceQueryText(inTab: selected.id, range: end, with: "\n\n" + favorite.query) } } else { runFavoriteInNewTab(favorite) @@ -43,8 +45,7 @@ extension MainContentCoordinator { func openLinkedFavorite(_ favorite: LinkedSQLFavorite) { guard let loaded = FileTextLoader.load(favorite.fileURL) else { return } - /// Any connection's tab, not only this window's selected one: the file can be open in a - /// workspace the window is not showing, and opening it here too made two buffers on one file. + // Any connection's tab: the file can be open in a workspace this window is not showing. if hostedTabRouting.revealTab(editing: favorite.fileURL) { AppActivationPolicyController.shared.activate(ignoringOtherApps: true) return diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift index c9ee39a429..7dbdd9665a 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift @@ -183,6 +183,7 @@ extension MainContentCoordinator { configuration.showRowNumbers = AppSettingsManager.shared.dataGrid.showRowNumbers configuration.appliesRowSortPreferences = true configuration.appliesRowCopyFormat = true + configuration.colorsValuesByType = true configuration.supportsColumnSort = tab.tabType != .table || supportsColumnSort configuration.isEditable = isEditable configuration.editRefusalMessage = isSelected ? activeResultEditRefusal?.message : nil diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridViewport.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridViewport.swift index fffc3eb546..1cbad5583c 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridViewport.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridViewport.swift @@ -41,12 +41,16 @@ extension MainContentCoordinator { let sample = intent == .keepPlace ? viewportSample(of: model) : nil guard sample != nil || keepsSelection else { return .top } + // Mode, columns and active cell too, or a cell cursor comes back as whole rows on column 0. + let selection = model.map { + GridSelectionSnapshot($0.selection, order: $0.order, rows: tableRows, columns: $0.columns) + } let snapshot = GridViewportResolver.snapshot( of: tableRows, displayIDs: model?.displayedRowIDs, firstVisibleDisplayRow: sample?.row ?? 0, firstVisibleOffset: sample?.offset ?? 0, - selectedDisplayRows: keepsSelection ? Array(model?.selection.affectedRows ?? IndexSet()) : [], + selection: keepsSelection ? selection ?? .empty : .empty, keyColumns: keyColumns, isCellModified: { [changeManager] rowID, column in changeManager.isCellModified(rowID: rowID, columnIndex: column) diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ManualCommit.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ManualCommit.swift new file mode 100644 index 0000000000..94593f8a72 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ManualCommit.swift @@ -0,0 +1,18 @@ +// +// MainContentCoordinator+ManualCommit.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + // A table tab's read and an auto-load are the app's own SELECTs, so they never hold anything. + func manualCommitRun(skipped: Bool) -> ManualCommitRun { + ManualCommitRun( + commitMode: skipped ? .automatic : services.databaseManager.commitMode(for: connectionId), + databaseType: connection.type, + grammar: lexicalGrammar, + connectionId: connectionId + ) + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift index d7eab33615..ecd1426d4c 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift @@ -677,10 +677,13 @@ extension MainContentCoordinator { try await dropContainer(target) services.catalogChangeService.record(.containerDropped(target, connectionId: connectionId)) } catch { - navigationLogger.error( - "Failed to drop \(target.id, privacy: .public): \(error.publicLogShape, privacy: .public)" - ) - failures.append((target.name, error.localizedDescription)) + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + navigationLogger.error( + "Failed to drop \(target.id, privacy: .public): \(error.publicLogShape, privacy: .public)" + ) + } + if let message = failure.message { failures.append((target.name, message)) } } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Phase2Launch.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Phase2Launch.swift new file mode 100644 index 0000000000..d3e7b515a4 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Phase2Launch.swift @@ -0,0 +1,41 @@ +// +// MainContentCoordinator+Phase2Launch.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + func launchPhase2AfterApply( + tableName: String?, + isEditable: Bool, + tabId: UUID, + trigger: TableLoadTrigger, + needsMetadataFetch: Bool, + schemaTask: Task?, + claim: TabExecutionClaim + ) { + if isEditable, let tableName, trigger.countsRows { + // Marked pending here: phase 1 clears `isLoading` first, so without it the tab looks settled and offers Count Exactly early. + tabManager.mutate(tabId: tabId) { $0.pagination.isCountPending = true } + launchPhase2( + tableName: tableName, + tabId: tabId, + connectionType: connection.type, + needsMetadataFetch: needsMetadataFetch, + schemaTask: schemaTask + ) + } else if isEditable, let tableName, needsMetadataFetch { + // A tick keeps the count it has; only a manual Refresh recounts. + launchPhase2Work( + tableName: tableName, + tabId: tabId, + connectionType: connection.type, + schemaTask: schemaTask, + countsRows: false + ) + } else if !isEditable || tableName == nil { + clearChangesIfCurrent(claim: claim) + } + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift index 581bd55786..d3946eabbe 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift @@ -20,6 +20,13 @@ extension MainContentCoordinator { ) } + // Stop ends the run before the driver answers, so a rollback the server did for it arrives on a settled claim. It is + // shown on the tab unless the tab already runs something new. + func reportStopRollback(_ error: Error, tabId: UUID) { + guard error is PluginStopRolledBackTransaction, !tabExecution.isExecuting(tabId) else { return } + tabManager.mutate(tabId: tabId) { $0.execution.errorMessage = error.localizedDescription } + } + /// A table tab's SELECT is the app's own, so it may follow a database switch it waited through /// onto a pooled connection. The choice belongs to the tab and never to the statement: an editor /// SELECT can read a temp table or sit inside the user's open transaction, and moving it to @@ -57,6 +64,7 @@ extension MainContentCoordinator { serverOutput: PluginServerOutput = .none ) { guard tabExecution.settle(claim) else { + reportStopRollback(error, tabId: tabId) traceStaleResultDropped(traceToken) return } @@ -73,13 +81,14 @@ extension MainContentCoordinator { reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) + reportStopRollback(error, tabId: tabId) return } if isAutoLoad, services.databaseManager.driver(for: connectionId)?.status != .connected { parkedTableLoad = ParkedTableLoad(tabId: tabId, trigger: trigger) return } - handleQueryExecutionError(error, sql: sql, tabId: tabId, connection: conn, serverOutput: serverOutput) + handleQueryExecutionError(error, sql: sql, tabId: tabId, connection: conn, serverOutput: serverOutput, trigger: trigger) reportQueryOperation( claim: claim, trigger: trigger, outcome: .failed(reason: error.localizedDescription) ) @@ -136,7 +145,9 @@ extension MainContentCoordinator { exportDocuments: [PluginExportDocument]? = nil, absentCells: [Int: Set] = [:], unloadedCells: [Int: [Int: Int]] = [:], - columnOrigins: [PluginColumnOrigin?]? = nil + columnOrigins: [PluginColumnOrigin?]? = nil, + prepared: TableRows.PreparedRows? = nil, + trigger: TableLoadTrigger = .userInitiated ) { queryExecutionCoordinator.applyPhase1Result( tabId: tabId, @@ -164,7 +175,43 @@ extension MainContentCoordinator { exportDocuments: exportDocuments, absentCells: absentCells, unloadedCells: unloadedCells, - columnOrigins: columnOrigins + columnOrigins: columnOrigins, + prepared: prepared, + trigger: trigger + ) + } + + func phase1RowShape(tabId: UUID, tableName: String?, columns: [String], columnTypes: [ColumnType]) -> TableRows.RowShape { + queryExecutionCoordinator.phase1RowShape(tabId: tabId, tableName: tableName, columns: columns, columnTypes: columnTypes) + } + + func preparedPhase1Rows( + _ fetchResult: QueryFetchResult, + tabId: UUID, + tableName: String?, + isTableTab: Bool + ) async -> TableRows.PreparedRows { + await TableRows.preparing( + fetchResult.rows, + rowLocators: isTableTab ? fetchResult.rowLocators : nil, + exportDocuments: fetchResult.exportDocuments, + absentCells: fetchResult.absentCells, + unloadedCells: fetchResult.unloadedCells, + shape: phase1RowShape( + tabId: tabId, tableName: tableName, columns: fetchResult.columns, columnTypes: fetchResult.columnTypes + ) + ) + } + + func extractTableName(from sql: String) -> String? { + tableNameReading.tableName(in: sql) + } + + var tableNameReading: TableNameReading { + TableNameReading( + dialect: sqlDialect, + readings: connection.type.lexicalReadings, + browseSchema: services.databaseManager.session(for: connectionId)?.browseSchema ) } @@ -195,13 +242,15 @@ extension MainContentCoordinator { tableName: String, tabId: UUID, connectionType: DatabaseType, - schemaTask: Task? + schemaTask: Task?, + countsRows: Bool = true ) { queryExecutionCoordinator.launchPhase2Work( tableName: tableName, tabId: tabId, connectionType: connectionType, - schemaTask: schemaTask + schemaTask: schemaTask, + countsRows: countsRows ) } @@ -222,14 +271,16 @@ extension MainContentCoordinator { sql: String, tabId: UUID, connection conn: DatabaseConnection, - serverOutput: PluginServerOutput = .none + serverOutput: PluginServerOutput = .none, + trigger: TableLoadTrigger = .userInitiated ) { queryExecutionCoordinator.handleQueryExecutionError( error, sql: sql, tabId: tabId, connection: conn, - serverOutput: serverOutput + serverOutput: serverOutput, + trigger: trigger ) } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ReadOnlySafeMode.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ReadOnlySafeMode.swift new file mode 100644 index 0000000000..20ef1b8972 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ReadOnlySafeMode.swift @@ -0,0 +1,52 @@ +// +// MainContentCoordinator+ReadOnlySafeMode.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +extension MainContentCoordinator { + // The gate's own answer, read as a run starts, so every read the app sends for the user carries the level its + // approval was given under: a view or a row security policy can call a writing function with no call in the text. + var runsAtReadOnly: Bool { + ExecutionGateProvider.runsAtReadOnly(connectionId, in: services.databaseManager) + } + + // What the tab's query was built from, and what its next page or count runs. + static func browseFilters(_ filterState: TabFilterState) -> [TableFilter] { + filterState.executedFilters + filterState.rerunFilters.filters + } + + static func rawFilterExpressions(_ filters: [TableFilter]) -> [String] { + filters.filter { $0.isEnabled && $0.isRawSQL }.map(\.value) + } + + // A raw filter is text the user typed, inside a query no gate reads below the Full levels. At Read-Only, where the + // driver cannot hold the engine to read-only, one that calls a function goes to the gate, which refuses it, before + // a browse, page turn, sort or count sends it. On Oracle and Dameng a bare name can call one too, and only the + // catalog the gate reads against the whole query can tell it from a column, so every raw filter goes there. + func rawFilterNeedsReadOnlyGate(_ filters: [TableFilter]) -> Bool { + let expressions = Self.rawFilterExpressions(filters) + guard !expressions.isEmpty, runsAtReadOnly, + ExecutionGateProvider.readOnlyFunctionPolicy( + connectionId: connectionId, databaseType: connection.type, in: services.databaseManager + ).functionsCanWrite + else { return false } + if SQLBareNameCalls.engines.contains(connection.type.rawValue) { return true } + return expressions.contains { ReadOnlyFunctionCalls.callsUnlistedFunction($0, databaseType: connection.type) } + } + + func rawFilterRequest(sql: String, canPrompt: Bool, scope: DatabaseScope?) -> OperationRequest { + OperationRequest( + connectionId: connectionId, + databaseType: connection.type, + sql: sql, + kind: .readQuery, + caller: .userInterface, + capabilities: canPrompt ? .interactiveUser : CallerCapabilities.interactiveUser.union(.cannotPrompt), + operationDescription: String(localized: "Execute Query"), + runsIn: scope + ) + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift index e5dcc43095..33edc43c0c 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Refresh.swift @@ -2,8 +2,6 @@ // MainContentCoordinator+Refresh.swift // TablePro // -// Refresh handling operations for MainContentCoordinator -// import AppKit import Combine @@ -102,5 +100,7 @@ extension MainContentCoordinator { tabManager.mutate(at: tabIndex) { $0.pagination.retireDerivedRowCount() } rebuildTableQuery(at: tabIndex) runQuery(viewport: .keepPlace) + // The next tick counts from this load settling, not from the last one. + restartAutoRefresh() } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Rename.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Rename.swift index 868274dbb0..9e8bae0e34 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Rename.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Rename.swift @@ -41,12 +41,15 @@ extension MainContentCoordinator { do { let route = DatabaseManager.shared.executionRoute(for: scope) + let refusesOpenTransaction = !route.isPooled try await DatabaseManager.shared.withScopedDriver( scope: scope, route: route, cancellation: .protectedWrite ) { driver in - try await driver.renameTable( - name: oldName, schema: schema, to: newName, objectType: objectType - ) + try await AppDefinitionCommit.run(on: driver, refusesOpenTransaction: refusesOpenTransaction) { + try await driver.renameTable( + name: oldName, schema: schema, to: newName, objectType: objectType + ) + } } } catch { presentRenameFailure(error, object: ref.id) diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift index 0a683dbc8a..77d770a22a 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultComparison.swift @@ -24,9 +24,10 @@ extension MainContentCoordinator { let left = comparisonSide(of: resultSet, in: tab) else { return } let leftId = resultSet.id let tabId = tab.id + let activeId = tab.display.activeResultSet?.id let partner = Self.defaultComparisonPartner( of: leftId, - among: tab.display.resultSets.filter { isComparable($0, in: tab) }.map(\.id) + among: tab.display.resultSets.filter { isComparable($0, in: tab, activeResultSetId: activeId) }.map(\.id) ) let model = ResultComparisonModel(left: left, preferredRightId: partner) { Self.comparisonCandidates(excluding: leftId, homeTabId: tabId) @@ -55,13 +56,15 @@ extension MainContentCoordinator { for tab in coordinator.tabManager.tabs where tab.tabType == .query { let isHome = tab.id == homeTabId let tabId = tab.id - for resultSet in tab.display.resultSets + let activeId = tab.display.activeResultSet?.id + let titles = ResultSetMenuModel.displayLabels(for: tab.display.resultSets.map(\.label)) + for (index, resultSet) in tab.display.resultSets.enumerated() where resultSet.id != leftId && (isHome || resultSet.isPinned) - && coordinator.isComparable(resultSet, in: tab) { + && coordinator.isComparable(resultSet, in: tab, activeResultSetId: activeId) { let resultId = resultSet.id let candidate = ResultComparisonCandidate( id: resultId, - title: comparisonTitle(of: resultSet, in: tab.display), + title: titles[index], group: isHome ? String(localized: "This Tab") : coordinator.comparisonDetail(for: tab), isPinned: resultSet.isPinned, read: { [weak coordinator] in @@ -87,7 +90,7 @@ extension MainContentCoordinator { func comparisonSide(of resultSet: ResultSet, in tab: QueryTab) -> ResultComparisonSide? { guard !resultSet.isExplainResult else { return nil } - let rows = comparisonRows(of: resultSet, in: tab) + let rows = comparisonRows(of: resultSet, in: tab, activeResultSetId: tab.display.activeResultSet?.id) guard !rows.columns.isEmpty else { return nil } let primaryKey: [String] if let origin = resultSet.origin, origin.keysResolved, origin.tableName?.isEmpty == false { @@ -111,12 +114,18 @@ extension MainContentCoordinator { } func isComparable(_ resultSet: ResultSet, in tab: QueryTab) -> Bool { - !resultSet.isExplainResult && !comparisonRows(of: resultSet, in: tab).columns.isEmpty + isComparable(resultSet, in: tab, activeResultSetId: tab.display.activeResultSet?.id) + } + + // A pass over every result reads the active id once: finding it per result made the pass quadratic. + func isComparable(_ resultSet: ResultSet, in tab: QueryTab, activeResultSetId: UUID?) -> Bool { + !resultSet.isExplainResult + && !comparisonRows(of: resultSet, in: tab, activeResultSetId: activeResultSetId).columns.isEmpty } // The active result's rows live in the tab's shared buffer, which page loads and Fetch All write to. - private func comparisonRows(of resultSet: ResultSet, in tab: QueryTab) -> TableRows { - guard tab.display.activeResultSet?.id == resultSet.id else { return resultSet.tableRows } + private func comparisonRows(of resultSet: ResultSet, in tab: QueryTab, activeResultSetId: UUID?) -> TableRows { + guard activeResultSetId == resultSet.id else { return resultSet.tableRows } return tabSessionRegistry.tableRows(for: tab.id) } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultEditing.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultEditing.swift index 42a1a7eec2..bb38cc8f5a 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultEditing.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ResultEditing.swift @@ -16,10 +16,21 @@ extension MainContentCoordinator { guard tab.tabType == .query else { return tab.tableContext.viewEditRefusal } return ResultEditability.refusal( for: tab.display.activeResultSet?.origin, - currentDatabaseName: scope(for: tab)?.database ?? browseDatabaseName + currentDatabaseName: boundDatabaseName(of: tab), + currentSchemaName: tab.tableContext.schemaName ) } + // Rows read in another database or schema than the tab names now: paging them would read from the new one. + func wasReadElsewhere(_ resultSet: ResultSet, in tab: QueryTab) -> Bool { + guard tab.tabType == .query, let origin = resultSet.origin else { return false } + return origin.wasRead(elsewhereThanDatabase: boundDatabaseName(of: tab), schema: tab.tableContext.schemaName) + } + + private func boundDatabaseName(of tab: QueryTab) -> String { + scope(for: tab)?.database ?? browseDatabaseName + } + /// The one answer to "may the rows on screen be written to". /// /// The grid, the row inspector and the Edit menu each used to decide this for themselves and @@ -27,12 +38,19 @@ extension MainContentCoordinator { /// two. They all read this now. var canEditActiveResult: Bool { guard let tab = tabManager.selectedTab else { return false } - // A claimed execution replaces these rows and clears whatever was staged on them meanwhile, as the data files grid does when busy. + // A claimed execution or a landing save replaces these rows and clears whatever was staged on them meanwhile, as the data + // files grid does when busy. return tab.tableContext.isEditable && tab.tableContext.allowsRowEditing && !safeModeLevel.blocksAllWrites && activeResultEditRefusal == nil && !tabExecution.isExecuting(tab.id) + && !rowEditingCoordinator.isSaving(tabId: tab.id) + } + + // Undo, Redo and Discard wait too: the landing save clears the change manager they would act on. + var isSavingSelectedTab: Bool { + rowEditingCoordinator.isSaving(tabId: tabManager.selectedTabId) } // A query tab learns its result reads a view from the result's origin; a table tab from its context. diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+RowOperations.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+RowOperations.swift index 6f34f12452..1a236f244f 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+RowOperations.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+RowOperations.swift @@ -33,6 +33,10 @@ extension MainContentCoordinator { rowEditingCoordinator.duplicateRow(rowID) } + func duplicateRows(_ rowIDs: [RowID]) { + rowEditingCoordinator.duplicateRows(rowIDs) + } + func handleUndoResult(_ result: UndoResult) { rowEditingCoordinator.handleUndoResult(result) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift new file mode 100644 index 0000000000..92846198f7 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift @@ -0,0 +1,28 @@ +// +// MainContentCoordinator+SafeModePromptRetry.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + // The prompt is one per window, so a first load that meets another run's prompt waits for it to close. + func turnAwayTableLoad(tabId: UUID, trigger: TableLoadTrigger) { + traceNavigationAbandoned(tabId: tabId, outcome: .safeModePromptAlreadyOpen) + tableLoadTurnedAwayByPrompt = ParkedTableLoad(tabId: tabId, trigger: trigger) + declineTableLoad(for: tabId) + } + + func safeModePromptDidChange(wasShowing: Bool) { + guard wasShowing, !isShowingSafeModePrompt, tableLoadTurnedAwayByPrompt != nil else { return } + // Once the run that lowered it has started, so a load it starts itself is not asked for twice. + Task { @MainActor [weak self] in self?.retryTableLoadTurnedAwayByPrompt() } + } + + private func retryTableLoadTurnedAwayByPrompt() { + guard !isShowingSafeModePrompt, !isTearingDown, let turnedAway = tableLoadTurnedAwayByPrompt else { return } + tableLoadTurnedAwayByPrompt = nil + guard turnedAway.tabId == tabManager.selectedTabId else { return } + lazyLoadCurrentTabIfNeeded(trigger: turnedAway.trigger) + } +} diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarSave.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarSave.swift index 6222977189..61f0a3abf2 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarSave.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SidebarSave.swift @@ -13,9 +13,12 @@ extension MainContentCoordinator { let changes = sidebarEditChanges(editedFields: editState.getEditedFields()) guard !changes.isEmpty else { return } // Held to its row counts as a grid save is, so a write that found no row fails instead of saving nothing. - let steps = try changeManager.writeSteps(for: changes, factory: sidebarStatementFactory()) + let viewTarget = activeViewWriteTarget + var factory = try sidebarStatementFactory() + factory.holdsKeyedWritesToOneRow = viewTarget != nil + let steps = try changeManager.writeSteps(for: changes, factory: factory) guard !steps.isEmpty else { return } - try await executeSidebarChanges(steps: steps) + try await executeSidebarChanges(steps: steps, viewTarget: viewTarget) reloadActiveResult() } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabClosing.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabClosing.swift index 857cecd874..c8cdf4bfd3 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabClosing.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabClosing.swift @@ -76,6 +76,8 @@ extension MainContentCoordinator { prune(&erDiagramModels, keeping: openTabIds) discardTableHealthModels(tableHealthModels.keys.filter { !openTabIds.contains($0) }) prune(&navigationHistories, keeping: openTabIds) + // The editor stores the outgoing tab's history on its next update, after the close forgot it. + editorUndoHistories.forgetAll(except: openTabIds) toolbarState.forgetQueryTimings(keeping: openTabIds) for (tabId, session) in structureSessions where !openTabIds.contains(tabId) { session.releaseViewWiring() @@ -112,6 +114,7 @@ extension MainContentCoordinator { erDiagramModels.removeValue(forKey: tab.id) discardTableHealthModels([tab.id]) navigationHistories.removeValue(forKey: tab.id) + editorUndoHistories.forget(tab.id) releaseGridModel(forTab: tab.id) tableMetadataCache.removeValue(forKey: tab.id) guard isSelectedTab(tab) else { return } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabRestore.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabRestore.swift index d79285c440..3d0961cf40 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabRestore.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabRestore.swift @@ -135,7 +135,8 @@ extension MainContentCoordinator { ) } - internal func restoreSavedTabs() async { + // `keepingEntry`: the window was opened on content, which decides the database it shows, even when it names none. + internal func restoreSavedTabs(keepingEntry: Bool = false) async { /// The split view controller owns the window and is wired up before the content view is built, /// unlike the content view's own window, which arrives from `configureWindow`. guard let window = splitViewController?.view.window else { @@ -180,8 +181,8 @@ extension MainContentCoordinator { await applyRestoredGroup( restoredTabs, selectedTabId: result.selectedTabId ?? restoredTabs.first?.id, - activeDatabase: result.lastActiveDatabase, - activeSchema: result.lastActiveSchema, + activeDatabase: keepingEntry ? nil : result.lastActiveDatabase, + activeSchema: keepingEntry ? nil : result.lastActiveSchema, loadTiming: window.isKeyWindow ? .immediate : .deferred ) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TableRowsMutation.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TableRowsMutation.swift index b6fc23496b..27fc922ce3 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TableRowsMutation.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TableRowsMutation.swift @@ -159,8 +159,8 @@ extension MainContentCoordinator { guard tabManager.tabs[tabIdx].tabType == .query else { return } let origin = resultSet.origin tabManager.mutate(at: tabIdx) { tab in + // Not the schema: it is the tab's binding, and a pinned result from another schema would move the next run there. tab.tableContext.tableName = origin?.tableName - tab.tableContext.schemaName = origin?.schemaName tab.tableContext.primaryKeyColumns = origin?.primaryKeyColumns ?? [] tab.tableContext.isEditable = origin?.isEditable ?? false tab.tableContext.isView = origin?.isView ?? false @@ -221,13 +221,15 @@ extension MainContentCoordinator { private func syncLoadMoreState(from resultSet: ResultSet, at tabIdx: Int) { guard tabManager.tabs[tabIdx].tabType == .query else { return } + let readElsewhere = wasReadElsewhere(resultSet, in: tabManager.tabs[tabIdx]) tabManager.mutate(at: tabIdx) { tab in - if resultSet.isTruncated { + if resultSet.isTruncated, !readElsewhere { tab.pagination.hasMoreRows = true tab.pagination.isLoadingMore = false } else { tab.pagination.resetLoadMore() } + guard !readElsewhere else { return } tab.pagination.setBaseQueryForMore( resultSet.baseQuery, parameterValues: resultSet.baseQueryParameterValues diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift index f3bd273ecc..1448717a5a 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+UnloadedValues.swift @@ -17,6 +17,8 @@ extension MainContentCoordinator { !tab.tableContext.primaryKeyColumns.isEmpty, let scope = scope(for: tab) else { return nil } let builder = queryBuilder + // A view can call a function that writes, so the value is read the way the browse was. + let safeModeIsReadOnly = runsAtReadOnly return UnloadedValueReader( tableName: tableName, schemaName: tab.tableContext.schemaName, @@ -31,10 +33,18 @@ extension MainContentCoordinator { isTableTab: true, lease: lease ) { driver in - try await driver.executeUserQuery( + guard let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) else { + return try await driver.executeUserQuery( + query: statement.sql, + rowCap: nil, + parameters: statement.parameters + ).rows + } + return try await driver.executeUserQuery( query: statement.sql, rowCap: nil, - parameters: statement.parameters + parameters: statement.parameters, + context: context ).rows } } @@ -208,7 +218,7 @@ extension MainContentCoordinator { // The server holds the row under its key as last read, not as a pending edit has it. private func storedValue(of row: Row, column: Int) -> PluginCellValue { - let pendingUpdate = changeManager.rowChanges.first { $0.rowID == row.id && $0.type == .update } + let pendingUpdate = changeManager.pendingUpdate(forRow: row.id) if let change = pendingUpdate?.cellChanges.first(where: { $0.columnIndex == column }) { return change.oldValue } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+VersionHistory.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+VersionHistory.swift index 0d8313d732..7d30e1a0f2 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+VersionHistory.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+VersionHistory.swift @@ -17,6 +17,58 @@ extension MainContentCoordinator { openVersionHistory(subject: .linkedFile(url: linked.fileURL), name: linked.fileURL.lastPathComponent) } + func showVersionHistory(ofQueryTab tabId: UUID) { + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }), tab.tabType == .query else { return } + openVersionHistory(subject: .queryTab(id: tabId), name: tab.title) + } + + func revertQueryTab(_ tabId: UUID, toVersion versionId: Int64) { + let provider = queryTabVersionProvider(for: tabId) + Task { + do { + try await provider.prepareRestore(.tabVersion(id: versionId)).apply() + } catch { + Self.versionHistoryLogger.error("Reverting a query tab failed: \(error.publicLogShape, privacy: .public)") + presentError(String(localized: "Couldn't Restore Version"), error.localizedDescription, contentWindow) + } + } + } + + func queryTabVersionProvider(for tabId: UUID) -> QueryTabVersionHistoryProvider { + let recorder = tabVersions + return QueryTabVersionHistoryProvider( + tabId: tabId, + storage: recorder.storage, + liveTab: { [weak self] in self?.liveQueryTab(tabId) }, + write: { [weak self] text in self?.replaceQueryText(inTab: tabId, with: text, recording: nil) ?? false }, + isCapturePaused: recorder.isCapturePaused + ) + } + + // The History tab can outlive a move of the tab it shows, so each call finds the tab's current owner. + static func hostedQueryTabVersionProvider(for tabId: UUID) -> QueryTabVersionHistoryProvider { + QueryTabVersionHistoryProvider( + tabId: tabId, + storage: .shared, + liveTab: { Self.owner(ofQueryTab: tabId)?.liveQueryTab(tabId) }, + write: { text in + Self.owner(ofQueryTab: tabId)?.replaceQueryText(inTab: tabId, with: text, recording: nil) ?? false + } + ) + } + + func liveQueryTab(_ tabId: UUID) -> LiveQueryTab? { + flushEditorText(forTab: tabId) + guard let tab = tabManager.tabs.first(where: { $0.id == tabId }), tab.tabType == .query else { return nil } + return LiveQueryTab(connectionId: connectionId, text: tab.content.query) + } + + private static func owner(ofQueryTab tabId: UUID) -> MainContentCoordinator? { + HostedTabRouting.live.allCoordinators().first { coordinator in + coordinator.tabManager.tabs.contains { $0.id == tabId && $0.tabType == .query } + } + } + func openVersionInEditor(_ content: String) { let payload = EditorTabPayload( connectionId: connectionId, diff --git a/apple/TablePro/Views/Main/Extensions/MainContentView+Bindings.swift b/apple/TablePro/Views/Main/Extensions/MainContentView+Bindings.swift index 3c82d37c5b..69f32d3579 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentView+Bindings.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentView+Bindings.swift @@ -158,7 +158,8 @@ extension MainContentView { resultsViewMode: currentTab?.display.resultsViewMode ?? .data, inspectorRowSourceRevision: coordinator.inspectorRowSourceRevision, gridDisplayRevision: coordinator.gridDisplayRevision, - structureEditor: coordinator.activeStructureEditor.map { ObjectIdentifier($0) } + structureEditor: coordinator.activeStructureEditor.map { ObjectIdentifier($0) }, + isSaving: coordinator.isSavingSelectedTab ) } } @@ -172,6 +173,8 @@ struct InspectorTrigger: Equatable { let gridDisplayRevision: Int // A structure session or Create Table draft appears after its view first mounts, which moves nothing else here. var structureEditor: ObjectIdentifier? + // The fields go read-only while a save of this tab runs, and back when it ends. + var isSaving = false } enum InspectorValueDisplayFormatResolver { diff --git a/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift index 41deb3a3e6..9d2201e6c2 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift @@ -133,7 +133,8 @@ extension MainContentView { case .text(let s): raw = s case .bytes(let data): - raw = BlobFormattingService.shared.format(data, for: .copy) ?? "" + // The grid's form stops at 64 bytes, under the cut below; a full copy would hex the whole value. + raw = BlobFormattingService.shared.format(data, for: .grid) ?? "" } return (raw as NSString).length > 200 ? String(raw.prefix(200)) + "…" : raw } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentView+Setup.swift b/apple/TablePro/Views/Main/Extensions/MainContentView+Setup.swift index 0cef1d7cf8..ab54949fc8 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentView+Setup.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentView+Setup.swift @@ -37,8 +37,10 @@ extension MainContentView { } } - if payload == nil || payload?.intent == .restoreOrDefault { - await coordinator.restoreSavedTabs() + // The merge keeps the founding tab, its selection and its entry in front of the saved ones. + if coordinator.persistence.readsSavedTabs(foundedWith: payload?.intent) { + let foundedOnContent = payload.map { $0.intent != .restoreOrDefault } ?? false + await coordinator.restoreSavedTabs(keepingEntry: foundedOnContent) } // Before the schema load, which can take seconds, so a move waiting on the restore lands now. coordinator.settleTabRestore() diff --git a/apple/TablePro/Views/Main/MainContentCommandActions+Structure.swift b/apple/TablePro/Views/Main/MainContentCommandActions+Structure.swift index e64c645fed..80643e9853 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions+Structure.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions+Structure.swift @@ -43,7 +43,7 @@ extension MainContentCommandActions { guard let tab = coordinator?.tabManager.selectedTab, tab.tabType == .table || tab.tabType == .query else { return nil } if let session = stagedStructureSession { return .structure(session) } - return hasDataWork ? .data : nil + return hasDataWork && coordinator?.isSavingSelectedTab != true ? .data : nil } private func discardDataWorkAfterConfirming() async { diff --git a/apple/TablePro/Views/Main/MainContentCommandActions+UndoState.swift b/apple/TablePro/Views/Main/MainContentCommandActions+UndoState.swift index b3f291f407..c74704fad4 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions+UndoState.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions+UndoState.swift @@ -54,6 +54,7 @@ extension MainContentCommandActions { if isUsersRolesTab { return coordinator?.usersRolesActions?.canUndo() ?? false } if isDiagramTab { return selectedDiagram?.canUndoStagedEdit ?? false } if showsStructureEditor { return structureHistory?.canUndo ?? false } + if coordinator?.isSavingSelectedTab == true { return false } if coordinator?.changeManager.hasCoalescedUndoRun == true { return true } return windowUndoManager?.canUndo ?? false } @@ -62,6 +63,7 @@ extension MainContentCommandActions { if isUsersRolesTab { return coordinator?.usersRolesActions?.canRedo() ?? false } if isDiagramTab { return selectedDiagram?.canRedoStagedEdit ?? false } if showsStructureEditor { return structureHistory?.canRedo ?? false } + if coordinator?.isSavingSelectedTab == true { return false } return windowUndoManager?.canRedo ?? false } @@ -106,6 +108,7 @@ extension MainContentCommandActions { structureHistory?.undo() return } + guard coordinator?.isSavingSelectedTab != true else { return } windowUndoManager?.undo() } @@ -123,6 +126,7 @@ extension MainContentCommandActions { structureHistory?.redo() return } + guard coordinator?.isSavingSelectedTab != true else { return } windowUndoManager?.redo() } } diff --git a/apple/TablePro/Views/Main/MainContentCommandActions.swift b/apple/TablePro/Views/Main/MainContentCommandActions.swift index ec02fa09d7..c7c1904a51 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions.swift @@ -66,6 +66,9 @@ final class MainContentCommandActions: ObservableObject { await RunningWorkApproval.confirm(prompt, window: window) } + // What a window close waits on while another connection's pane mounts. + var paneWaitClock: any Clock = ContinuousClock() + private var notificationTasks: [Task] = [] private var eventCancellables: Set = [] @@ -212,9 +215,12 @@ final class MainContentCommandActions: ObservableObject { let rowIDs = selectedDataRowIDs if !rowIDs.isEmpty { coordinator?.deleteRows(rowIDs) - } else if !selectedTables.wrappedValue.isEmpty { - // Through the sidebar's own path, which asks before it queues a drop. - coordinator?.sidebarViewModel?.batchToggleDelete(refs: Array(selectedTables.wrappedValue)) + } else if !selectedTables.wrappedValue.isEmpty, let coordinator { + // Through the sidebar's own path, which asks this window before it queues a drop. + coordinator.sidebarViewModel?.batchToggleDelete( + refs: Array(selectedTables.wrappedValue), + from: coordinator.windowSidebarState + ) } } @@ -556,12 +562,7 @@ final class MainContentCommandActions: ObservableObject { // Every connection in the window: closing it closes them all, and skipping a background one is silent data loss. internal var hasUnsavedWorkInWindow: Bool { - guard let host = window?.contentViewController as? MainSplitViewController else { - return coordinator?.hasAnyUnsavedWork() ?? false - } - return host.workspaces.workspaces.contains { workspace in - workspace.sessionState?.coordinator.hasAnyUnsavedWork() == true - } + coordinatorsInWindow.contains { $0.hasAnyUnsavedWork() } } // This connection only: closing its tabs leaves another connection's pending work alone. @@ -650,13 +651,16 @@ final class MainContentCommandActions: ObservableObject { return .closed } // Work is stopped only once the close is certain: a cancelled or failed save keeps it running. - let result = await AlertHelper.confirmSaveChanges( - message: RunningWorkPrompt.saveMessage( - String(localized: "Your changes will be lost if you don't save them."), + let keepsTabsOpen = coordinatorsInWindow.contains { hosted in + hosted.tabManager.tabs.contains { hosted.savability(of: $0) == .mountedOnly } + } + let result = await confirmSaveChanges( + RunningWorkPrompt.saveMessage( + Self.unsavedWorkMessage(keepsTabsOpen: keepsTabsOpen), action: .close, alsoStopping: work ), - window: closeAnchorWindow + closeAnchorWindow ) switch result { @@ -775,13 +779,17 @@ final class MainContentCommandActions: ObservableObject { return true } + // The prompt asked about every connection in the window, so Save reaches every tab of each; one it cannot save keeps the window. private func saveAndClose(asBatchSurvivor: Bool?) async -> Bool { - guard let coordinator else { - finish(asBatchSurvivor: asBatchSurvivor) - return true + let host = window?.contentViewController as? MainSplitViewController + let shown = host?.workspaces.selectedConnectionId + for hosted in coordinatorsInWindow where hosted === coordinator || hosted.hasAnyUnsavedWork() { + // A save can raise a sheet, which has to sit over the connection it is about. + host?.workspaces.select(hosted.connectionId) + guard let actions = await shownActions(of: hosted) else { return false } + guard await actions.saveEveryTabsWork() else { return false } } - guard await applyStagedStructureEdits(in: coordinator.tabManager.tabs) else { return false } - guard await saveSelectedTabWork() else { return false } + if let shown { host?.workspaces.select(shown) } finish(asBatchSurvivor: asBatchSurvivor) return true } @@ -809,13 +817,16 @@ final class MainContentCommandActions: ObservableObject { } func copyTableNames() { - coordinator?.sidebarViewModel?.copySelectedTableNames() + SidebarViewModel.copyTableNames(of: selectedTables.wrappedValue) } func truncateTables() { guard coordinator?.admitsSheetCommand() == true else { return } - guard canTruncateSelectedTables else { return } - coordinator?.sidebarViewModel?.batchToggleTruncate() + guard canTruncateSelectedTables, let coordinator else { return } + coordinator.sidebarViewModel?.batchToggleTruncate( + refs: Array(selectedTables.wrappedValue), + from: coordinator.windowSidebarState + ) } // MARK: - Filter Operations (Group A: Called Directly) @@ -1126,14 +1137,7 @@ final class MainContentCommandActions: ObservableObject { } func clearQuery() { - guard let coordinator, - let (tab, tabIndex) = coordinator.tabManager.selectedTabAndIndex, - tab.tabType == .query else { return } - coordinator.tabManager.mutate(at: tabIndex) { $0.content.query = "" } - coordinator.toolbarState.hasQueryText = false - coordinator.scheduleDraftSave() - // Emptying from a command bypasses the editor binding that recomputes the dirty indicator on each keystroke. - coordinator.refreshUnsavedIndicator() + coordinator?.clearSelectedQuery() } var canClearQuery: Bool { @@ -1284,6 +1288,19 @@ final class MainContentCommandActions: ObservableObject { ) } + // Nil off a table tab, which the Database menu's Auto Refresh items dim for. + var tableAutoRefreshInterval: TableAutoRefreshInterval? { + guard let tab = coordinator?.tabManager.selectedTab, + tab.tabType == .table, + tab.tableContext.tableName != nil else { return nil } + return tab.autoRefreshInterval + } + + func setTableAutoRefreshInterval(_ interval: TableAutoRefreshInterval) { + guard let coordinator, let tabId = coordinator.tabManager.selectedTabId else { return } + coordinator.setAutoRefreshInterval(interval, forTab: tabId) + } + private var hasPendingTableOps: Bool { !pendingTruncates.wrappedValue.isEmpty || !pendingDeletes.wrappedValue.isEmpty } @@ -1407,6 +1424,38 @@ final class MainContentCommandActions: ObservableObject { } extension MainContentCommandActions { + // This connection's first. + private var coordinatorsInWindow: [MainContentCoordinator] { + guard let host = window?.contentViewController as? MainSplitViewController else { + return [coordinator].compactMap { $0 } + } + let hosted = host.workspaces.workspaces.compactMap { $0.sessionState?.coordinator } + guard let coordinator else { return hosted } + return [coordinator] + hosted.filter { $0 !== coordinator } + } + + // A connection never shown has no actions until its pane mounts, a few run loop turns after it is selected. + private func shownActions(of hosted: MainContentCoordinator) async -> MainContentCommandActions? { + if hosted === coordinator { return self } + return await Self.actions(of: hosted, waitingOn: paneWaitClock) + } + + // A cancelled close stops waiting at once rather than polling out the rest of the deadline. + private static func actions>( + of hosted: MainContentCoordinator, + waitingOn clock: C + ) async -> MainContentCommandActions? { + let deadline = clock.now.advanced(by: .seconds(2)) + while hosted.commandActions == nil, clock.now < deadline { + do { + try await clock.sleep(for: .milliseconds(20)) + } catch { + return nil + } + } + return hosted.commandActions + } + // Copy with the grid not focused takes the selected rows in the row copy format; a focused grid answers `copy:` itself. func copySelectedRowsInCopyFormat() { guard dataGridOwnsSelection, let model = coordinator?.activeGridModel else { diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 8ec9847488..465beded46 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -26,6 +26,7 @@ enum DiscardAction { case displayOrder case columnVisibility case run + case containerChange } struct DisplayFormatsCacheEntry { @@ -266,7 +267,9 @@ final class MainContentCoordinator: ObservableObject { // MARK: - Published State - @Published var cursorPositions: [CursorPosition] = [] + // Not published: the editor writes it on every caret move, and a publish re-rendered every view observing this + // coordinator. Readers take it when they need it. + var cursorPositions: [CursorPosition] = [] @Published var activeSheet: ActiveSheet? { didSet { releaseImportFileUnlessImporting() } } @@ -288,6 +291,8 @@ final class MainContentCoordinator: ObservableObject { var deferredRestoreLoadTabId: UUID? var displayFormatsCache: [UUID: DisplayFormatsCacheEntry] = [:] + // One editor serves every query tab of the window, so each tab's undo history waits here while another is shown. + let editorUndoHistories = EditorUndoHistories() // Published because the inspector rebuilds its snapshot from it (load, comment refetch, rename). @Published var tableMetadataCache: [UUID: TableMetadataCacheEntry] = [:] // Bumped per Table Info invalidation so an in-flight read can't mark the cache current; per window, as it can precede a tab entry. @@ -330,8 +335,11 @@ final class MainContentCoordinator: ObservableObject { // Not on `QueryTab`: that is the persisted shape, and staying out of its `Equatable` keeps a push from re-publishing tabs. internal var navigationHistories: [UUID: TabNavigationHistory] = [:] internal var redisDatabaseSwitchTask: Task? - private var periodicSaveTask: Task? - private var draftSaveTask: Task? + var periodicSaveTask: Task? + var draftSaveTask: Task? + var draftSaveTabIds: Set = [] + var draftSaveClock: any Clock = ContinuousClock() + var tabVersions = QueryTabVersionRecorder.live private var terminationObserver: NSObjectProtocol? internal var postConnectCancellable: AnyCancellable? // Owned here, not by the view: a connection switch cancels the view's `task`, and a cancelled first load never reran. @@ -363,6 +371,7 @@ final class MainContentCoordinator: ObservableObject { let diskContent: String } internal private(set) var fileWatcher: DatabaseFileWatcher? + var schemaTokenGate = SchemaChangeTokenGate() // Set during handleTabChange to suppress redundant column-change reconfiguration. internal var isHandlingTabSwitch = false @@ -371,7 +380,12 @@ final class MainContentCoordinator: ObservableObject { // Guards re-entrant confirm dialogs (nested run loop during runModal). internal var isShowingConfirmAlert = false - internal var isShowingSafeModePrompt = false + // Lowered by whichever run raised it, so a table load it turned away is retried whatever path asked. + internal var isShowingSafeModePrompt = false { + didSet { safeModePromptDidChange(wasShowing: oldValue) } + } + + var tableLoadTurnedAwayByPrompt: ParkedTableLoad? @Published internal var rewindPlan: RewindPlan? @@ -388,6 +402,7 @@ final class MainContentCoordinator: ObservableObject { var refreshCoalesceTask: Task? var refreshPendingTrailing = false + let autoRefresh = TableAutoRefreshSchedule() // SwiftUI creates and discards coordinators during body re-evaluation; those never activate and need no teardown warning. private let _didActivate = OSAllocatedUnfairLock(initialState: false) @@ -561,44 +576,6 @@ final class MainContentCoordinator: ObservableObject { } } - private static let periodicSaveInterval: Duration = .seconds(30) - private static let draftSaveDebounce: Duration = .seconds(1) - - // Typing changes no tab identity, so without this a scratch query lives only in memory until the 30s timer or quit flush. - func scheduleDraftSave() { - guard !Self.isAppTerminating, !isTearingDown else { return } - draftSaveTask?.cancel() - draftSaveTask = Task { [weak self] in - try? await Task.sleep(for: Self.draftSaveDebounce) - guard let self, !Task.isCancelled, !Self.isAppTerminating, !self.isTearingDown else { return } - guard self.isFirstCoordinatorForConnection() else { return } - self.persistence.saveAggregated() - } - } - - private func startPeriodicSave() { - guard periodicSaveTask == nil else { return } - periodicSaveTask = Task { [weak self] in - while !Task.isCancelled { - try? await Task.sleep(for: Self.periodicSaveInterval) - guard let self, !Task.isCancelled, !Self.isAppTerminating, !self.isTearingDown else { return } - guard self.isFirstCoordinatorForConnection() else { continue } - self.persistence.saveAggregated() - } - } - } - - static func aggregatedSelectedTabId(for connectionId: UUID) -> UUID? { - activeCoordinators.values - .first { $0.connectionId == connectionId && $0.tabManager.selectedTabId != nil }? - .tabManager.selectedTabId - } - - private func isFirstCoordinatorForConnection() -> Bool { - Self.activeCoordinators.values - .first { $0.connectionId == self.connectionId } === self - } - private static let registerTerminationObserver: Void = { NotificationCenter.default.addObserver( forName: NSApplication.willTerminateNotification, @@ -653,10 +630,7 @@ final class MainContentCoordinator: ObservableObject { self?.browseCursorCoordinator.browseContainerDidChange() } tabManager.onTabRetargeted = { [weak self] tabId in - self?.existingGridModel(forTab: tabId)?.discardEditing() - self?.flushColumnLayoutPersistence() - self?.supersedeExecution(for: tabId) - self?.releaseRetargetedTabState(for: tabId) + self?.retargetTab(tabId) } // Synchronous: queue .main satisfies assumeIsolated, and the write finishes before exit, unlike Task-based saves. @@ -704,6 +678,7 @@ final class MainContentCoordinator: ObservableObject { clearedEditsCancellable = resumeWhenEditsClear() gridRepaintCancellable = observeChangeManagerForGridRepaint() maskingRulesCancellable = observeMaskingRuleChanges() + observeSelectionForAutoRefresh() Self.lifecycleLogger.info( "[open] MainContentCoordinator.init done connId=\(connection.id, privacy: .public) elapsedMs=\(Int(Date().timeIntervalSince(initStart) * 1_000))" @@ -749,8 +724,7 @@ final class MainContentCoordinator: ObservableObject { watcher.watch(filePath: filePath, connectionId: connectionId) { [weak self] in guard let self else { return } if case .loading = services.schemaService.state(for: self.connectionId) { return } - DatabaseTreeMetadataService.shared.markListsStale(connectionId: self.connectionId) - Task { await self.refreshTables() } + Task { await self.refreshTablesIfSchemaChanged() } } fileWatcher = watcher } @@ -849,7 +823,7 @@ final class MainContentCoordinator: ObservableObject { connectionId: connectionId, tabType: .query, databaseName: resolved.database, - schemaName: resolved.schema, + schemaName: resolved.isDatabaseScoped ? nil : resolved.schema, initialQuery: source, skipAutoExecute: true, tabTitle: resolved.displayIdentity @@ -893,6 +867,7 @@ final class MainContentCoordinator: ObservableObject { reportEndedExecutions(tabExecution.invalidateAll(reason: .sessionEnded)) refreshCoalesceTask?.cancel() refreshCoalesceTask = nil + stopAutoRefresh() for entry in tableLoadTasks.values { entry.task.cancel() } tableLoadTasks.removeAll() cancelAllRowCountTasks() @@ -1056,6 +1031,7 @@ final class MainContentCoordinator: ObservableObject { let batches = queryExecutionCoordinator.executionBatches(in: sql, sourceOffset: sourceOffset ?? 0) let statements = batches.flatMap(\.statements) guard !statements.isEmpty else { return false } + tabVersions.record(tabManager.tabs[index], connectionId: connectionId, reason: .run) if services.appSettings.editor.queryParametersEnabled, bindsNamedParameters { let combinedSQL = SQLParameterExtractor.parameterSource(of: statements) @@ -1106,16 +1082,18 @@ final class MainContentCoordinator: ObservableObject { } let level = safeModeLevel - if level.appliesToAllQueries && level.requiresConfirmation, - tab.execution.lastExecutedAt == nil - { + let asksBeforeFirstLoad = level.appliesToAllQueries && level.requiresConfirmation + && tab.execution.lastExecutedAt == nil + let gatesRawFilter = rawFilterNeedsReadOnlyGate(Self.browseFilters(tab.filterState)) + if asksBeforeFirstLoad || gatesRawFilter { guard !isShowingSafeModePrompt else { - traceNavigationAbandoned(tabId: tab.id, outcome: .safeModePromptAlreadyOpen) + turnAwayTableLoad(tabId: tab.id, trigger: trigger) return nil } isShowingSafeModePrompt = true let tabId = tab.id let askedContext = tab.tableContext + let askedScope = scope(for: tab) return Task { let decision = await executionGate.authorize( OperationRequest( @@ -1125,7 +1103,8 @@ final class MainContentCoordinator: ObservableObject { kind: .readQuery, caller: .userInterface, capabilities: .interactiveUser, - operationDescription: String(localized: "Execute Query") + operationDescription: String(localized: "Execute Query"), + runsIn: askedScope ) ) isShowingSafeModePrompt = false @@ -1158,65 +1137,6 @@ final class MainContentCoordinator: ObservableObject { return nil } - // MARK: - Editor Query Loading - - @discardableResult - func loadQueryIntoEditor( - _ query: String, - databaseName: String? = nil, - forceNewTab: Bool = false - ) -> WindowTabOpenDisposition { - let targetDatabaseName = databaseName ?? newTabDatabaseName - if !forceNewTab, - let (tab, tabIndex) = tabManager.selectedTabAndIndex, - tab.tabType == .query, - databaseName == nil - || tab.tableContext.resolvedDatabaseName(browsing: browseDatabaseName) == targetDatabaseName { - tabManager.mutate(at: tabIndex) { - $0.content.query = query - $0.hasUserInteraction = true - } - return .currentCoordinator - } - - if !forceNewTab, tabManager.tabs.isEmpty { - tabManager.addTab(initialQuery: query, databaseName: targetDatabaseName) - return .currentCoordinator - } - - let payload = EditorTabPayload( - connectionId: connection.id, - tabType: .query, - databaseName: targetDatabaseName, - initialQuery: query - ) - openTabInNewWindow(payload) - return .focusedElsewhere - } - - var aiInsertReusesSelectedQueryTab: Bool { - guard let (tab, _) = tabManager.selectedTabAndIndex, tab.tabType == .query else { return false } - return tab.content.query.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - } - - func insertQueryFromAI(_ query: String) { - if aiInsertReusesSelectedQueryTab, let (_, tabIndex) = tabManager.selectedTabAndIndex { - tabManager.mutate(at: tabIndex) { mutTab in - mutTab.content.query = query - mutTab.hasUserInteraction = true - } - } else if tabManager.tabs.isEmpty { - tabManager.addTab(initialQuery: query, databaseName: newTabDatabaseName) - } else { - let payload = EditorTabPayload( - connectionId: connection.id, - tabType: .query, - initialQuery: query - ) - WindowManager.shared.openTab(payload: payload) - } - } - internal func executeQueryInternal( _ sql: String, isAutoLoad: Bool = false, @@ -1240,7 +1160,7 @@ final class MainContentCoordinator: ObservableObject { let conn = connection let tabId = tabManager.tabs[index].id - let traceToken = adoptOrBeginExecutionTrace(tabId: tabId) + let traceToken = trigger.tracesLoad ? adoptOrBeginExecutionTrace(tabId: tabId) : nil traceExecutionStarted(traceToken, epoch: claim.epoch, isAutoLoad: isAutoLoad) let statement = resolveStatement(sql: sql, tabType: tab.tabType, bypassLimit: bypassRowLimit) @@ -1270,6 +1190,8 @@ final class MainContentCoordinator: ObservableObject { let isTableTab = tab.tabType == .table let readsView = tab.tableContext.knowsView(named: tableName) let (grammar, lengthColumns) = (lexicalGrammar, unloadedValueLengthColumns(for: tab)) + let safeModeIsReadOnly = runsAtReadOnly + let held = manualCommitRun(skipped: isTableTab || isAutoLoad) let failureOutput = ServerOutputBox() let queryTask = Task { [weak self] in @@ -1287,14 +1209,19 @@ final class MainContentCoordinator: ObservableObject { scope: scope, isTableTab: isTableTab, lease: lease - ) { [queryExecutor] driver in - let fetched = try await queryExecutor.executeQuery( - driver: driver, - sql: statement.sql, - parameters: nil, - rowCap: rowCap, - capturingOutputInto: isTableTab ? nil : failureOutput - ) + ) { [queryExecutor, manager = services.databaseManager] driver in + let fetched = try await held.running([statement.sql], on: driver) { + // Before it runs, so a statement that fails or a connection lost under it is still noted. + if !isAutoLoad { await manager.noteSessionStatements([statement.sql], ranOn: driver, scope: scope) } + return try await queryExecutor.executeQuery( + driver: driver, + sql: statement.sql, + parameters: nil, + rowCap: rowCap, + safeModeIsReadOnly: safeModeIsReadOnly, + capturingOutputInto: isTableTab ? nil : failureOutput + ) + } let edits = isAutoLoad ? nil : await SucceededStatements.single( statement.sql, scope: scope, databaseType: conn.type, grammar: grammar, ranOn: driver ) @@ -1315,6 +1242,7 @@ final class MainContentCoordinator: ObservableObject { : nil // Rows and definition commit together after a definition change, so new columns never sit under old keys or defaults. let definition = needsDefinition ? try? await schemaTask?.value : nil + let prepared = await preparedPhase1Rows(fetchResult, tabId: tabId, tableName: tableName, isTableTab: isTableTab) let applied = await MainActor.run { [weak self] () -> Bool in guard let self else { return false } @@ -1330,6 +1258,8 @@ final class MainContentCoordinator: ObservableObject { traceStaleResultDropped(traceToken) return false } + // A menu or value window opened during the read names rows by position, so a tick's rows wait for the next one. + guard trigger != .autoRefresh || !autoRefreshHeldByTransientUI(tabId) else { return false } toolbarState.recordQueryTiming(fetchResult.resolvedTiming, for: tabId) traceApplyingResult(traceToken, tabId: tabId) @@ -1360,7 +1290,9 @@ final class MainContentCoordinator: ObservableObject { exportDocuments: fetchResult.exportDocuments, absentCells: fetchResult.absentCells, unloadedCells: fetchResult.unloadedCells, - columnOrigins: fetchResult.columnOrigins + columnOrigins: fetchResult.columnOrigins, + prepared: prepared, + trigger: trigger ) scheduleTraceCompletion(traceToken, outcome: .completed) @@ -1373,19 +1305,15 @@ final class MainContentCoordinator: ObservableObject { return } - if isEditable, let tableName { - // Marked pending here: phase 1 clears `isLoading` first, so without it the tab looks settled and offers Count Exactly early. - tabManager.mutate(tabId: tabId) { $0.pagination.isCountPending = true } - launchPhase2( - tableName: tableName, - tabId: tabId, - connectionType: conn.type, - needsMetadataFetch: needsMetadataFetch, - schemaTask: schemaTask - ) - } else if !isEditable || tableName == nil { - clearChangesIfCurrent(claim: claim) - } + launchPhase2AfterApply( + tableName: tableName, + isEditable: isEditable, + tabId: tabId, + trigger: trigger, + needsMetadataFetch: needsMetadataFetch, + schemaTask: schemaTask, + claim: claim + ) } catch { schemaTask?.cancel() if !isAutoLoad { Self.postStatementRan(statement.sql, on: conn) } @@ -1458,17 +1386,6 @@ final class MainContentCoordinator: ObservableObject { return result } - // MARK: - SQL Parsing - - func extractTableName(from sql: String) -> String? { - QuerySqlParser.extractTableName( - from: sql, - dialect: sqlDialect, - readings: connection.type.lexicalReadings, - browseSchema: services.databaseManager.session(for: connectionId)?.browseSchema - ) - } - // MARK: - Sorting func handleSortStateChanged(_ newState: SortState) { diff --git a/apple/TablePro/Views/Main/MainContentView.swift b/apple/TablePro/Views/Main/MainContentView.swift index 82eade5715..c1e5df3ff7 100644 --- a/apple/TablePro/Views/Main/MainContentView.swift +++ b/apple/TablePro/Views/Main/MainContentView.swift @@ -512,7 +512,8 @@ struct MainContentView: View { ConnectionTransactionBar( connectionId: connection.id, transactions: DatabaseManager.shared.sessionTransactions, - databaseManager: DatabaseManager.shared + databaseManager: DatabaseManager.shared, + safeModeLevel: toolbarState.safeModeLevel ) editorContentView } diff --git a/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift b/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift index 3bd997547b..38c7cf6698 100644 --- a/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift +++ b/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift @@ -41,6 +41,11 @@ internal struct CopyObjectsResultView: View { .font(.callout) .foregroundStyle(.secondary) } + ForEach(Array(result.warnings.enumerated()), id: \.offset) { _, warning in + NoticeLabel(warning, severity: .warning) + .font(.callout) + .textSelection(.enabled) + } outcomes(result) } .frame(maxWidth: .infinity, alignment: .leading) diff --git a/apple/TablePro/Views/ObjectSource/ObjectSourceTabView.swift b/apple/TablePro/Views/ObjectSource/ObjectSourceTabView.swift index 289b6e133e..67d6514169 100644 --- a/apple/TablePro/Views/ObjectSource/ObjectSourceTabView.swift +++ b/apple/TablePro/Views/ObjectSource/ObjectSourceTabView.swift @@ -2,7 +2,8 @@ // ObjectSourceTabView.swift // TablePro // -// Tab showing the source of one stored procedure, function, trigger, user-defined type or view. +// Tab showing the source of one stored procedure, function, trigger, user-defined type, view or +// driver-declared server object. // import Combine @@ -73,14 +74,12 @@ final class ObjectSourceLoader: ObservableObject { await load() } - /// One round trip. Re-listing the schema to recover the attributes cost a full catalog scan - /// per open and per reload, for values the sidebar's listing already carried into the ref. A - /// type is the exception: its definition and attributes are read fresh. + // One round trip, with the attributes the listing carried; a type and a server object read theirs fresh. private func fetch() async throws -> Fetched { let scope = DatabaseScope( connectionId: connectionId, database: objectRef.database, - schema: objectRef.schema + schema: objectRef.isDatabaseScoped ? nil : objectRef.schema ) return try await metadata.withMetadataDriver(scope: scope) { [objectRef] driver in switch objectRef.kind { @@ -105,6 +104,15 @@ final class ObjectSourceLoader: ObservableObject { throw PluginObjectSourceError.unsupported(objectRef.name) } return Fetched(source: source, attributes: fetched.attributes) + case .serverObject: + guard let object = objectRef.serverObject else { + throw PluginObjectSourceError.unsupported(objectRef.name) + } + let fetched = try await driver.fetchServerObject(object) + guard let source = fetched.definition, !source.isEmpty else { + throw PluginObjectSourceError.unsupported(objectRef.name) + } + return Fetched(source: source, attributes: fetched.attributes.isEmpty ? objectRef.attributes : fetched.attributes) case .view, .materializedView: let source = try await TableDDLComposer.fetchDDL( for: objectRef.name, diff --git a/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift b/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift index 31526e51b4..8c0baa020c 100644 --- a/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift +++ b/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift @@ -245,11 +245,12 @@ struct CreateUserTypeSheet: View { try await UserTypeCreation.create(definition, in: scope, connection: connection) dismiss() onCreated(definition) - } catch ExecutionGateError.cancelledByUser { - return } catch { - Self.logger.error("Create Type failed: \(error.publicLogShape, privacy: .public)") - failure = error.localizedDescription + let outcome = GatedOperationFailure(error) + if outcome.reportsFailure { + Self.logger.error("Create Type failed: \(error.publicLogShape, privacy: .public)") + } + failure = outcome.message } } } diff --git a/apple/TablePro/Views/Results/BinaryValueViewerView.swift b/apple/TablePro/Views/Results/BinaryValueViewerView.swift index 1e0c45af07..a8b0bff63c 100644 --- a/apple/TablePro/Views/Results/BinaryValueViewerView.swift +++ b/apple/TablePro/Views/Results/BinaryValueViewerView.swift @@ -241,7 +241,7 @@ internal struct BinaryValueViewerView: View { let (decoded, dump) = await Task.detached(priority: .userInitiated) { ( BinaryValueDecoding.document(data, format: format), - String(data: data, encoding: .isoLatin1)?.formattedAsHexDump() ?? "" + data.formattedAsHexDump() ?? "" ) }.value guard !Task.isCancelled else { return } diff --git a/apple/TablePro/Views/Results/CellImageViewer.swift b/apple/TablePro/Views/Results/CellImageViewer.swift index a6e5fc24f0..9117a4677f 100644 --- a/apple/TablePro/Views/Results/CellImageViewer.swift +++ b/apple/TablePro/Views/Results/CellImageViewer.swift @@ -42,6 +42,8 @@ internal struct CellImageViewer: View { let format: CellImageFormat let sourceKind: CellImageSourceKind var onPopOut: (() -> Void)? + // A new one is another value, which opens on the picture. + var documentID: AnyHashable? @ViewBuilder let source: () -> Source @State private var mode: CellImageViewMode = .image @@ -53,6 +55,9 @@ internal struct CellImageViewer: View { content .frame(maxWidth: .infinity, maxHeight: .infinity) } + .onChange(of: documentID) { _ in + mode = .image + } } private var toolbar: some View { diff --git a/apple/TablePro/Views/Results/Comparison/ResultComparisonGrid.swift b/apple/TablePro/Views/Results/Comparison/ResultComparisonGrid.swift index 8804aabd50..e51516aecb 100644 --- a/apple/TablePro/Views/Results/Comparison/ResultComparisonGrid.swift +++ b/apple/TablePro/Views/Results/Comparison/ResultComparisonGrid.swift @@ -34,6 +34,7 @@ internal final class ResultComparisonGridModel: ObservableObject { configuration.supportsValueFilter = false configuration.supportsColumnSort = false configuration.appliesRowCopyFormat = true + configuration.colorsValuesByType = true configuration.commands = [.freezeColumns, .displayAs] return configuration } diff --git a/apple/TablePro/Views/Results/JSONCodeEditor.swift b/apple/TablePro/Views/Results/JSONCodeEditor.swift index 4c78a44aa4..1b310abff9 100644 --- a/apple/TablePro/Views/Results/JSONCodeEditor.swift +++ b/apple/TablePro/Views/Results/JSONCodeEditor.swift @@ -16,6 +16,8 @@ internal struct JSONCodeEditor: View { @ObservedObject private var themeEngine = ThemeEngine.shared @Binding var text: String let isEditable: Bool + // A new one is text replaced from outside: a new undo history, read from the top. + let documentID: AnyHashable? @State private var editorState = SourceEditorState() @State private var configuration: SourceEditorConfiguration @@ -24,9 +26,15 @@ internal struct JSONCodeEditor: View { /// modifier names the representable and never reaches it. @State private var coordinators: [any TextViewCoordinator] - init(text: Binding, isEditable: Bool, accessibilityIdentifier: String? = nil) { + init( + text: Binding, + isEditable: Bool, + accessibilityIdentifier: String? = nil, + documentID: AnyHashable? = nil + ) { self._text = text self.isEditable = isEditable + self.documentID = documentID self._configuration = State(wrappedValue: Self.makeConfiguration(isEditable: isEditable)) self._coordinators = State( wrappedValue: accessibilityIdentifier.map { [EditorAccessibilityIdentifier($0)] } ?? [] @@ -39,9 +47,14 @@ internal struct JSONCodeEditor: View { language: .json, configuration: configuration, state: $editorState, - coordinators: coordinators + coordinators: coordinators, + documentID: documentID ) .frame(maxWidth: .infinity, maxHeight: .infinity) + .onChange(of: documentID) { _ in + editorState.cursorPositions = [CursorPosition(range: NSRange(location: 0, length: 0))] + editorState.scrollPosition = .zero + } .onChange(of: isEditable) { _ in rebuildConfiguration() } diff --git a/apple/TablePro/Views/Results/JSONEditorContentView.swift b/apple/TablePro/Views/Results/JSONEditorContentView.swift index 261f934501..b8540793c4 100644 --- a/apple/TablePro/Views/Results/JSONEditorContentView.swift +++ b/apple/TablePro/Views/Results/JSONEditorContentView.swift @@ -36,8 +36,8 @@ struct JSONEditorContentView: View { onDismiss: onDismiss, onCommit: { newValue in if newValue.isEmpty && initialValue == nil { return } - if newValue != JsonReindenter.normalize(initialValue ?? "") { - onCommit(newValue) + if JsonReindenter.normalize(newValue) != JsonReindenter.normalize(initialValue ?? "") { + onCommit(JsonReindenter.staged(newValue, keepingLayoutOf: initialValue)) } }, onPopOut: onPopOut diff --git a/apple/TablePro/Views/Results/JSONViewerView.swift b/apple/TablePro/Views/Results/JSONViewerView.swift index ea6bdf22c8..43cbdace66 100644 --- a/apple/TablePro/Views/Results/JSONViewerView.swift +++ b/apple/TablePro/Views/Results/JSONViewerView.swift @@ -244,8 +244,9 @@ internal struct JSONViewerView: View { commitAndClose(displayText) } + // Committed as shown: the caller knows the stored value, whose layout decides what is staged. private func commitAndClose(_ value: String) { - onCommit?(JsonReindenter.normalize(value)) + onCommit?(value) onDismiss?() } } diff --git a/apple/TablePro/Views/Results/JSONViewerWindowController.swift b/apple/TablePro/Views/Results/JSONViewerWindowController.swift index 4f7926ab6d..c2536380e6 100644 --- a/apple/TablePro/Views/Results/JSONViewerWindowController.swift +++ b/apple/TablePro/Views/Results/JSONViewerWindowController.swift @@ -55,7 +55,8 @@ internal final class JSONViewerWindowController: ValueViewerWindowController { /// Nil when Save would write back what is stored. A NULL baseline reads as empty text, so an /// empty Save leaves the cell NULL rather than writing an empty string over it. nonisolated static func valueToCommit(saved: String, baseline: String?) -> String? { - saved == JsonReindenter.normalize(baseline ?? "") ? nil : saved + guard JsonReindenter.normalize(saved) != JsonReindenter.normalize(baseline ?? "") else { return nil } + return JsonReindenter.staged(saved, keepingLayoutOf: baseline) } } diff --git a/apple/TablePro/Views/Results/ResultStatusBar.swift b/apple/TablePro/Views/Results/ResultStatusBar.swift index 205be5bc3f..dbea74b74b 100644 --- a/apple/TablePro/Views/Results/ResultStatusBar.swift +++ b/apple/TablePro/Views/Results/ResultStatusBar.swift @@ -3,6 +3,7 @@ // TablePro // +import AppKit import SwiftUI /// The window's bottom bar: what the result is, and the controls that change what of it you see. @@ -50,6 +51,7 @@ struct ResultStatusBar: View { let resultSetActions: ResultSetActions let onToggleFilters: () -> Void let onFetchAll: (() -> Void)? + var autoRefresh: StatusBarAutoRefreshState = .unavailable @State private var showColumnPopover = false @State private var showHighlightPopover = false @@ -290,6 +292,9 @@ struct ResultStatusBar: View { if model.controls.showsHighlightRules { highlightButton } + if model.controls.showsAutoRefresh, presentation.showsAutoRefresh { + autoRefreshMenu(presentation) + } if model.controls.showsFilters { filtersToggle(presentation) } @@ -402,6 +407,51 @@ struct ResultStatusBar: View { return String(format: String(localized: "%d rules"), count) } + // A Menu around an inline Picker: a pop-up button always spells out its choice, so it could not go to a glyph. + private func autoRefreshMenu(_ presentation: ResultStatusPresentation) -> some View { + Menu { + Picker( + String(localized: "Auto Refresh"), + selection: Binding(get: { autoRefresh.interval }, set: { autoRefresh.onChange($0) }) + ) { + ForEach(TableAutoRefreshInterval.allCases) { interval in + Text(interval.title).tag(interval) + } + } + .pickerStyle(.inline) + .labelsHidden() + } label: { + Label { + Text(autoRefresh.interval.shortTitle) + } icon: { + (Self.autoRefreshSymbol(isOn: autoRefresh.interval.isOn).map(Image.init(nsImage:)) + ?? Image(systemName: Self.autoRefreshSymbolName)) + .statusBarControlIcon(besideTitle: presentation.showsControlTitles) + } + } + .menuStyle(.button) + .menuIndicator(presentation.showsControlTitles ? .visible : .hidden) + .statusBarLabelStyle(showsTitle: presentation.showsControlTitles) + .controlSize(.small) + .fixedSize() + .help(autoRefresh.help) + .accessibilityLabel(String(localized: "Auto Refresh")) + .accessibilityValue(autoRefresh.interval.title) + .accessibilityIdentifier("result-status-auto-refresh") + } + + static let autoRefreshSymbolName = "clock.arrow.circlepath" + + // A menu draws a SwiftUI symbol as a template and drops its tint, so the on state is an accent palette image. + static func autoRefreshSymbol(isOn: Bool) -> NSImage? { + let size = NSImage.SymbolConfiguration(pointSize: NSFont.systemFontSize(for: .small), weight: .regular) + let configuration = isOn ? size.applying(NSImage.SymbolConfiguration(paletteColors: [.controlAccentColor])) : size + guard let image = NSImage(systemSymbolName: autoRefreshSymbolName, accessibilityDescription: nil)? + .withSymbolConfiguration(configuration) else { return nil } + image.isTemplate = !isOn + return image + } + private func filtersToggle(_ presentation: ResultStatusPresentation) -> some View { Toggle(isOn: Binding(get: { filterState.isVisible }, set: { _ in onToggleFilters() })) { Label { diff --git a/apple/TablePro/Views/Results/ResultStatusInputs.swift b/apple/TablePro/Views/Results/ResultStatusInputs.swift index af51da33e7..f0382654f5 100644 --- a/apple/TablePro/Views/Results/ResultStatusInputs.swift +++ b/apple/TablePro/Views/Results/ResultStatusInputs.swift @@ -34,6 +34,21 @@ struct StatusBarHighlightState { } } +struct StatusBarAutoRefreshState { + let interval: TableAutoRefreshInterval + let pause: TableAutoRefreshPause? + let onChange: (TableAutoRefreshInterval) -> Void + + static var unavailable: StatusBarAutoRefreshState { + StatusBarAutoRefreshState(interval: .off, pause: nil, onChange: { _ in }) + } + + var help: String { + if interval.isOn, let reason = pause?.help { return reason } + return String(format: String(localized: "Auto Refresh: %@"), interval.title) + } +} + struct StatusBarColumnState { let hidden: Set let columns: [GridColumnEntry] diff --git a/apple/TablePro/Views/Results/ResultsJsonView.swift b/apple/TablePro/Views/Results/ResultsJsonView.swift index d8cc4d3395..6a04df3448 100644 --- a/apple/TablePro/Views/Results/ResultsJsonView.swift +++ b/apple/TablePro/Views/Results/ResultsJsonView.swift @@ -11,6 +11,7 @@ internal struct ResultsJsonView: View { let selectedRowIDs: Set let displayIDs: [RowID]? let deletedRowIDs: Set + let defaultedCells: [RowID: Set] let valueFilter: GridValueFilterState let dataRevision: Int let displayRevision: Int @@ -32,6 +33,7 @@ internal struct ResultsJsonView: View { selectedRowIDs: Set, displayIDs: [RowID]?, deletedRowIDs: Set, + defaultedCells: [RowID: Set] = [:], valueFilter: GridValueFilterState, dataRevision: Int, displayRevision: Int, @@ -41,6 +43,7 @@ internal struct ResultsJsonView: View { self.selectedRowIDs = selectedRowIDs self.displayIDs = displayIDs self.deletedRowIDs = deletedRowIDs + self.defaultedCells = defaultedCells self.valueFilter = valueFilter self.dataRevision = dataRevision self.displayRevision = displayRevision @@ -58,6 +61,7 @@ internal struct ResultsJsonView: View { let displayRevision: Int let selectedRowIDs: Set let deletedRowIDs: Set + let defaultedCells: [RowID: Set] let valueFilter: GridValueFilterState let hiddenColumns: Set let columnOrder: [String]? @@ -69,6 +73,7 @@ internal struct ResultsJsonView: View { displayRevision: displayRevision, selectedRowIDs: selectedRowIDs, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, valueFilter: valueFilter, hiddenColumns: columnLayout.hiddenColumns, columnOrder: columnLayout.columnOrder @@ -202,6 +207,7 @@ internal struct ResultsJsonView: View { let ids = displayIDs let selectedIndices = Set(DataGridRowOrder(filteredIDs: ids, revision: 0).displayIndexes(of: selectedRowIDs, in: snapshot)) let deletedIDs = deletedRowIDs + let defaulted = defaultedCells let layout = columnLayout let result = await Task.detached(priority: .userInitiated) { @@ -210,6 +216,7 @@ internal struct ResultsJsonView: View { displayIDs: ids, selectedIndices: selectedIndices, deletedRowIDs: deletedIDs, + defaultedCells: defaulted, columnLayout: layout ) }.value @@ -245,6 +252,7 @@ internal struct ResultsJsonView: View { displayIDs: [RowID]?, selectedIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columnLayout: ColumnLayoutState ) -> RenderedJson { let output = ResultJsonSerializer.serialize( @@ -252,6 +260,7 @@ internal struct ResultsJsonView: View { displayIDs: displayIDs, selectedDisplayIndices: selectedIndices, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, columns: .fromColumnLayout(columnLayout, columns: tableRows.columns) ) return RenderedJson( diff --git a/apple/TablePro/Views/Results/TextViewerWindowController.swift b/apple/TablePro/Views/Results/TextViewerWindowController.swift index bca006ed2c..b9a073d875 100644 --- a/apple/TablePro/Views/Results/TextViewerWindowController.swift +++ b/apple/TablePro/Views/Results/TextViewerWindowController.swift @@ -8,12 +8,13 @@ import SwiftUI @MainActor internal final class TextViewerWindowController: ValueViewerWindowController { + @discardableResult static func open( text: String?, columnName: String?, isEditable: Bool, onCommit: ((String) -> Void)? - ) { + ) -> TextViewerWindowController { let title: String if let columnName { title = String(format: String(localized: "Text: %@"), columnName) @@ -34,6 +35,7 @@ internal final class TextViewerWindowController: ValueViewerWindowController { onDismiss: dismiss ) } + return controller } } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/BlobHexEditorView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/BlobHexEditorView.swift index 0a3583e812..0522158661 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/BlobHexEditorView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/BlobHexEditorView.swift @@ -29,6 +29,8 @@ internal struct BlobHexEditorView: View { /// committed. A first-appearance flag would not do: this branch is rebuilt when the field leaves /// read-only (a row marked deleted, then restored), and its new `onChange` misses what changed. @State private var draftSource: String? + // Another document reloads the draft even while the field has focus. + @State private var draftDocument: UUID? var body: some View { if context.isReadOnly { @@ -85,9 +87,10 @@ internal struct BlobHexEditorView: View { statusLine } .onAppear(perform: loadDraftIfStale) - .onChange(of: context.value.wrappedValue) { _ in - if !isFocused { - loadDraft() + // The argument, never `context`: an action reads the previous render's. + .onChange(of: DraftSource(documentID: context.documentID, stored: context.value.wrappedValue)) { source in + if source.documentID != draftDocument || !isFocused { + loadDraft(from: source) } } } @@ -113,16 +116,25 @@ internal struct BlobHexEditorView: View { /// One place that formats the stored value for editing, so the draft and the truncation flag /// are always taken from the same read rather than from two. + private struct DraftSource: Equatable { + let documentID: UUID? + let stored: String + } + private func loadDraft() { - let stored = context.value.wrappedValue - let formatted = BlobFormattingService.shared.format(stored, for: .edit) ?? "" + loadDraft(from: DraftSource(documentID: context.documentID, stored: context.value.wrappedValue)) + } + + private func loadDraft(from source: DraftSource) { + let formatted = BlobFormattingService.shared.format(source.stored, for: .edit) ?? "" hexEditText = formatted isTruncated = formatted.hasSuffix("…") - draftSource = stored + draftSource = source.stored + draftDocument = source.documentID } private func loadDraftIfStale() { - guard draftSource != context.value.wrappedValue else { return } + guard draftSource != context.value.wrappedValue || draftDocument != context.documentID else { return } loadDraft() } @@ -132,6 +144,8 @@ internal struct BlobHexEditorView: View { loadDraft() return } + // The bytes the field already holds stage nothing, so leaving it untouched never marks the row edited. + guard !(raw as NSString).isEqual(to: context.value.wrappedValue) else { return } if let commitBytes = context.commitBytes, let data = raw.data(using: .isoLatin1) { commitBytes(data) diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/BooleanPickerView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/BooleanPickerView.swift index ec7100e7ec..6a78e00577 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/BooleanPickerView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/BooleanPickerView.swift @@ -19,6 +19,9 @@ internal struct BooleanPickerView: View { ) Text("true").tag("1") Text("false").tag("0") + if let stored = Self.unrecognizedValue(of: context.valueState) { + Text(verbatim: stored).tag(stored) + } } label: { EmptyView() } @@ -28,9 +31,7 @@ internal struct BooleanPickerView: View { .disabled(context.isReadOnly) } - /// The selection follows `valueState`, so a value the user has just picked on a NULL column - /// shows that value. Reading the stored value instead is what made the control snap back to - /// NULL over a recorded edit. + // The selection follows `valueState`, so a value just picked on a NULL column shows instead of snapping back to NULL. private var selectionBinding: Binding { Binding( get: { @@ -45,11 +46,16 @@ internal struct BooleanPickerView: View { ) } - private static func normalized(_ value: String) -> String { - let lower = value.lowercased() - if lower == "true" || lower == "1" || lower == "t" || lower == "yes" { - return "1" - } - return "0" + internal static func normalized(_ value: String) -> String { + guard let isTrue = StoredBoolean.value(of: value) else { return value.isEmpty ? "0" : value } + return isTrue ? "1" : "0" + } + + // A TINYINT(1) can hold 2, which is neither choice, so the picker shows it rather than reading it as false. + internal static func unrecognizedValue(of state: FieldValueState) -> String? { + guard FieldPickerSentinel.tag(for: state) == nil else { return nil } + let text = state.editableText + guard !text.isEmpty, StoredBoolean.value(of: text) == nil else { return nil } + return text } } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift b/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift index a8e66e7b6b..a3a0441f30 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/FieldEditorContext.swift @@ -27,6 +27,9 @@ internal struct FieldEditorContext { internal let allowsNullAndDefault: Bool internal let isBinaryValue: Bool + // The field's editor id: a new one is another row or a new read, so an editor seeded from the value starts over. + internal let documentID: UUID? + internal init( columnName: String, columnType: ColumnType, @@ -38,7 +41,8 @@ internal struct FieldEditorContext { commitBytes: ((Data) -> Void)? = nil, editor: FieldEditorKind? = nil, allowsNullAndDefault: Bool = true, - isBinaryValue: Bool = false + isBinaryValue: Bool = false, + documentID: UUID? = nil ) { self.columnName = columnName self.columnType = columnType @@ -51,6 +55,7 @@ internal struct FieldEditorContext { self.editor = editor self.allowsNullAndDefault = allowsNullAndDefault self.isBinaryValue = isBinaryValue + self.documentID = documentID } internal var hasMultipleValues: Bool { valueState == .multipleValues } diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/GeometryFieldView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/GeometryFieldView.swift index 0efecf9236..35a5ec6684 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/GeometryFieldView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/GeometryFieldView.swift @@ -29,9 +29,14 @@ internal struct GeometryFieldView: View { Mode(rawValue: storedMode) ?? .map } + // The latch of the row on screen: a mode another row decided is undecided here. + private var rowLatch: ModeLatch { + latch.scoped(to: context.documentID) + } + var body: some View { let resolution = resolvePreview() - let mode = latch.shown(preferred: preferredMode, preview: resolution.preview) + let mode = rowLatch.shown(preferred: preferredMode, preview: resolution.preview) VStack(alignment: .leading, spacing: 4) { toolbar(mode: mode, preview: resolution.preview) switch mode { @@ -43,30 +48,34 @@ internal struct GeometryFieldView: View { } // On a bare container the row's accessibility value lands on the picker and the caption. .accessibilityElement(children: .contain) - .onAppear { onTabStopChange?(mode == .text) } - // No reset on disappear: the lazy list unmounts off-screen rows, and Tab would then stop on a map. - .onChange(of: mode) { onTabStopChange?($0 == .text) } - // Reruns when a read starts or ends, so whichever preview comes first latches the mode. - .task(id: resolution.pending) { + // No reset on disappear: the lazy list unmounts off-screen rows, and Tab would then stop on a map. Keyed by the + // document too, since the registry files the field under an id each row reissues. + .task(id: TabStop(documentID: context.documentID, isText: mode == .text)) { + onTabStopChange?(mode == .text) + } + // Reruns when a read starts or ends, or another row opens, so whichever preview comes first latches the mode. + .task(id: PreviewRead(documentID: context.documentID, pending: resolution.pending)) { + latch = rowLatch guard let pending = resolution.pending else { latch.offer(preferred: preferredMode, preview: resolution.preview) return } - guard let loaded = await previews.load(pending, source: descriptor.source) else { return } + guard let loaded = await previews.load(pending, source: descriptor.source), !Task.isCancelled else { return } + latch = rowLatch latch.offer(preferred: preferredMode, preview: loaded) } } // Text mode reads nothing: the value is parsed only while a map is, or may be, on screen. private func resolvePreview() -> GeometryFieldPreviewStore.Resolution { - guard latch.needsPreview(preferred: preferredMode) else { + guard rowLatch.needsPreview(preferred: preferredMode) else { return GeometryFieldPreviewStore.Resolution(preview: nil, pending: nil) } let request = GeometryFieldPreviewStore.Request( text: context.value.wrappedValue, state: context.valueState ) - return previews.resolve(request, source: descriptor.source) + return previews.resolve(request, source: descriptor.source, documentID: context.documentID) } // MARK: - Toolbar @@ -112,6 +121,7 @@ internal struct GeometryFieldView: View { Binding( get: { mode }, set: { chosen in + latch = rowLatch latch.choose(chosen) storedMode = chosen.rawValue } @@ -221,6 +231,18 @@ internal struct GeometryFieldView: View { // MARK: - Mode +private extension GeometryFieldView { + struct TabStop: Equatable { + let documentID: UUID? + let isText: Bool + } + + struct PreviewRead: Equatable { + let documentID: UUID? + let pending: GeometryFieldPreviewStore.Request? + } +} + internal extension GeometryFieldView { enum Mode: String, Hashable, Sendable { case text @@ -237,9 +259,14 @@ internal extension GeometryFieldView { } } - // Decided once per view: following the text would swap the editor from under the caret once a typed value draws. + // Decided once per row: following the text would swap the editor from under the caret once a typed value draws. struct ModeLatch: Equatable { private(set) var mode: Mode? + private(set) var documentID: UUID? + + func scoped(to documentID: UUID?) -> ModeLatch { + documentID == self.documentID ? self : ModeLatch(mode: nil, documentID: documentID) + } mutating func offer(preferred: Mode, preview: GeometryFieldPreview?) { guard mode == nil else { return } @@ -283,6 +310,8 @@ internal final class GeometryFieldPreviewStore: ObservableObject { internal private(set) var token = 0 private var loaded: Request? private var pending: Request? + // Another row drops the last row's preview, which would otherwise stay up while the new value reads. + private var documentID: UUID? // A parse runs to the end, so a pane re-added mid-read joins it instead of starting a second one. private var inFlight: (request: Request, task: Task)? private let reader: Reader @@ -293,7 +322,13 @@ internal final class GeometryFieldPreviewStore: ObservableObject { self.reader = reader } - internal func resolve(_ request: Request, source: GeometryFieldSource) -> Resolution { + internal func resolve(_ request: Request, source: GeometryFieldSource, documentID: UUID? = nil) -> Resolution { + if documentID != self.documentID { + self.documentID = documentID + loaded = nil + pending = nil + preview = nil + } if request == loaded { return Resolution(preview: preview, pending: nil) } if request == pending { return Resolution(preview: preview, pending: request) } guard GeometryFieldPreview.readsSynchronously(request.text) else { diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/ImageFieldView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/ImageFieldView.swift index c7c997d0dc..ae541ef1c0 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/ImageFieldView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/ImageFieldView.swift @@ -31,6 +31,7 @@ internal struct ImageFieldView: View { columnName: context.columnName ) }, + documentID: context.documentID.map { AnyHashable($0) }, source: { source } ) .frame(height: 220) @@ -55,7 +56,8 @@ internal struct ImageFieldView: View { text: context.value, isEditable: !context.isReadOnly, font: themeEngine.valueFont, - style: .code + style: .code, + documentID: context.documentID.map { AnyHashable($0) } ) case .hex: BlobHexEditorView(context: context, drawsWell: false) diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/JsonEditorView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/JsonEditorView.swift index 509332cab1..3b3b35a614 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/JsonEditorView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/JsonEditorView.swift @@ -11,12 +11,16 @@ internal struct JsonEditorView: View { var isExpanded = false @State private var model: JsonFieldEditingModel + @State private var openedDocument: UUID? + // Moves each time the text is replaced from outside, which gives the editor a new history. + @State private var replacements = 0 init(context: FieldEditorContext, onPopOut: ((String) -> Void)? = nil, isExpanded: Bool = false) { self.context = context self.onPopOut = onPopOut self.isExpanded = isExpanded self._model = State(wrappedValue: JsonFieldEditingModel(storedValue: context.value.wrappedValue)) + self._openedDocument = State(wrappedValue: context.documentID) } // Through the model: binding `$model.displayText` directly would publish the model's own corrections. @@ -44,19 +48,31 @@ internal struct JsonEditorView: View { JSONCodeEditor( text: editorText, isEditable: !context.isReadOnly, - accessibilityIdentifier: "inspector-json-field" + accessibilityIdentifier: "inspector-json-field", + documentID: AnyHashable(replacements) ) .valueWell(clipsContent: true) } } // On a bare container the row's accessibility value lands on the Open in Window button too. .accessibilityElement(children: .contain) - // `newValue`, never `context.value`: the action's context is a render behind and undid every keystroke (#3051). - .onChange(of: context.value.wrappedValue) { newValue in - _ = model.received(newValue) + // The argument, never `context`: the action's context is a render behind and undid every keystroke (#3051). + .onChange(of: JsonFieldSource(documentID: context.documentID, value: context.value.wrappedValue)) { source in + if source.documentID != openedDocument { + openedDocument = source.documentID + model = JsonFieldEditingModel(storedValue: source.value) + replacements &+= 1 + } else if model.received(source.value) { + replacements &+= 1 + } } } + private struct JsonFieldSource: Equatable { + let documentID: UUID? + let value: String + } + private func header(onPopOut: @escaping (String) -> Void) -> some View { HStack(spacing: 8) { Spacer(minLength: 4) diff --git a/apple/TablePro/Views/RowInspector/FieldEditors/MultiLineEditorView.swift b/apple/TablePro/Views/RowInspector/FieldEditors/MultiLineEditorView.swift index 31b682b5ec..9e392e782b 100644 --- a/apple/TablePro/Views/RowInspector/FieldEditors/MultiLineEditorView.swift +++ b/apple/TablePro/Views/RowInspector/FieldEditors/MultiLineEditorView.swift @@ -29,7 +29,8 @@ internal struct MultiLineEditorView: View { font: themeEngine.valueFont, movesFocusOnTab: true, style: .code, - marksInvisibleCharacters: settingsManager.dataGrid.showInvisibleCharacters + marksInvisibleCharacters: settingsManager.dataGrid.showInvisibleCharacters, + documentID: context.documentID.map { AnyHashable($0) } ) .valueWell(clipsContent: true) .overlay(alignment: .topLeading) { placeholder } diff --git a/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift b/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift index 6f8a1f812c..47ca65287f 100644 --- a/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift +++ b/apple/TablePro/Views/RowInspector/InspectorFieldListView.swift @@ -108,7 +108,7 @@ internal struct InspectorFieldListView: View { private func fieldList(_ fields: [FieldEditState]) -> some View { List { - ForEach(fields, id: \.id) { field in + ForEach(fields) { field in row(for: field) .listRowSeparator(.hidden) .listRowInsets(EdgeInsets( @@ -148,9 +148,10 @@ internal struct InspectorFieldListView: View { ? { editState.toggleExpansion(ofColumn: field.columnIndex) } : nil, onPopOut: { onPopOut?(field, $0, kind) }, - onTabStopChange: { [tabStops] in tabStops.record($0, for: field.id) }, + onTabStopChange: { [tabStops] in tabStops.record($0, for: field.editorID) }, focusedField: $focusedField, - fieldID: field.id + fieldID: field.id, + editorID: field.editorID ) } @@ -180,7 +181,8 @@ internal struct InspectorFieldListView: View { : nil, editor: kind, allowsNullAndDefault: offersDatabaseValues, - isBinaryValue: field.isBinaryValue + isBinaryValue: field.isBinaryValue, + documentID: field.editorID ) } @@ -189,7 +191,7 @@ internal struct InspectorFieldListView: View { // A List's key view loop has one stop, measured. Ignored at either end so focus leaves the list instead of wrapping. private func moveFocus(within fields: [FieldEditState], forward: Bool) -> KeyPressResultCompat { guard !fields.isEmpty else { return .ignored } - let order = fields.map(\.id) + let order = fields.map(\.editorID) tabStops.forget(allBut: order) // With Full Keyboard Access on, the controls of a field showing a map are key views. let skipped = NSApplication.shared.isFullKeyboardAccessEnabled ? [] : tabStops.fieldsWithoutTabStop @@ -218,7 +220,7 @@ internal struct InspectorFieldListView: View { guard isEditable, offersDatabaseValues, let focusedField, - let field = editState.fields.first(where: { $0.id == focusedField }), + let field = editState.fields.first(where: { $0.editorID == focusedField }), Self.isFieldEditable(field, kind: FieldEditorResolver.resolve(field: field), rowIsEditable: isEditable), let command = Self.stateShortcutCommand( for: key, offered: editState.setValueCommands(for: field, isWritable: true) @@ -245,17 +247,17 @@ internal struct InspectorFieldListView: View { internal final class InspectorTabStopRegistry: ObservableObject { internal private(set) var fieldsWithoutTabStop: Set = [] - internal func record(_ hasTabStop: Bool, for fieldID: UUID) { + internal func record(_ hasTabStop: Bool, for editorID: UUID) { if hasTabStop { - fieldsWithoutTabStop.remove(fieldID) + fieldsWithoutTabStop.remove(editorID) } else { - fieldsWithoutTabStop.insert(fieldID) + fieldsWithoutTabStop.insert(editorID) } } - // Field ids are reissued on every selection change. - internal func forget(allBut fieldIDs: [UUID]) { - fieldsWithoutTabStop.formIntersection(fieldIDs) + // Editor ids are reissued on every selection change. + internal func forget(allBut editorIDs: [UUID]) { + fieldsWithoutTabStop.formIntersection(editorIDs) } } diff --git a/apple/TablePro/Views/RowInspector/InspectorFieldRow.swift b/apple/TablePro/Views/RowInspector/InspectorFieldRow.swift index 20ca42288f..dc464b9ee2 100644 --- a/apple/TablePro/Views/RowInspector/InspectorFieldRow.swift +++ b/apple/TablePro/Views/RowInspector/InspectorFieldRow.swift @@ -24,7 +24,8 @@ internal struct InspectorFieldRow: View { internal var onTabStopChange: ((Bool) -> Void)? @FocusState.Binding internal var focusedField: UUID? - internal let fieldID: UUID + internal let fieldID: FieldEditID + internal let editorID: UUID var body: some View { row @@ -175,11 +176,23 @@ internal struct InspectorFieldRow: View { isExpanded: isExpanded, onTabStopChange: onTabStopChange ) + .id(editorIdentity) .font(Self.valueFont(for: kind)) - .focused($focusedField, equals: fieldID) + .focused($focusedField, equals: editorID) .accessibilityValue(context.valueState.placeholder ?? context.value.wrappedValue) } + // These follow a new row in place and start over through the context's `documentID`. The others hold a popover + // or a viewer seeded from the value, so they are built again with it. + private var editorIdentity: AnyHashable { + switch kind { + case .singleLine, .boolean, .enumPicker, .multiLine, .json, .blobHex, .image, .geometry: + return AnyHashable(fieldID) + case .phpSerialized, .binaryValue, .setPicker, .arrayElements, .vector: + return AnyHashable(editorID) + } + } + /// Exhaustive, so a new kind is classified instead of inheriting an arm. The structured editors /// opt out: each carries a toolbar beside the value and names the value font on its own text. private static func valueFont(for kind: FieldEditorKind) -> Font? { diff --git a/apple/TablePro/Views/RowInspector/InspectorPopOutRegistry.swift b/apple/TablePro/Views/RowInspector/InspectorPopOutRegistry.swift new file mode 100644 index 0000000000..b62bb99911 --- /dev/null +++ b/apple/TablePro/Views/RowInspector/InspectorPopOutRegistry.swift @@ -0,0 +1,32 @@ +// +// InspectorPopOutRegistry.swift +// TablePro +// + +import Foundation + +// An editable pop-out commits to the row ids it opened on, which a reload hands to other rows, so Auto Refresh waits on it. +@MainActor +internal enum InspectorPopOutRegistry { + private struct Entry { + weak var owner: TrailingPaneState? + weak var window: ValueViewerWindowController? + } + + private static var entries: [Entry] = [] + + // Weak both ways: the window controller lives exactly as long as its window is open. + internal static func record(_ window: ValueViewerWindowController, openedFrom owner: TrailingPaneState) { + prune() + entries.append(Entry(owner: owner, window: window)) + } + + internal static func holdsWindow(openedFrom owner: TrailingPaneState) -> Bool { + prune() + return entries.contains { $0.owner === owner } + } + + private static func prune() { + entries.removeAll { $0.owner == nil || $0.window == nil } + } +} diff --git a/apple/TablePro/Views/RowInspector/RowInspectorView.swift b/apple/TablePro/Views/RowInspector/RowInspectorView.swift index 112c74b661..c0f7a4581e 100644 --- a/apple/TablePro/Views/RowInspector/RowInspectorView.swift +++ b/apple/TablePro/Views/RowInspector/RowInspectorView.swift @@ -138,7 +138,7 @@ internal struct RowInspectorView: View { private func popOut(field: FieldEditState, text: String, kind: FieldEditorKind) { let isEditable = context.isEditable && !context.isRowDeleted && !field.isServerOwned - // Captured, not looked up per keystroke: the field's id is reissued on every selection change. + // Captured, not looked up per keystroke: the window outlives the selection it opened on. let columnIndex = field.columnIndex let rowIDs = state.editState.rowIDs let commit: ((String) -> Void)? = isEditable @@ -149,12 +149,13 @@ internal struct RowInspectorView: View { switch InspectorPopOutWindow.resolve(for: kind) { case .json: - JSONViewerWindowController.open( + let window = JSONViewerWindowController.open( text: text, columnName: field.columnName, isEditable: isEditable, onCommit: commit ) + if commit != nil { InspectorPopOutRegistry.record(window, openedFrom: paneState) } case .php: PhpViewerWindowController.open(text: text, columnName: field.columnName) case .binary(let format): @@ -165,12 +166,13 @@ internal struct RowInspectorView: View { columnName: field.columnName ) case .text: - TextViewerWindowController.open( + let window = TextViewerWindowController.open( text: text, columnName: field.columnName, isEditable: isEditable, onCommit: commit ) + if commit != nil { InspectorPopOutRegistry.record(window, openedFrom: paneState) } } } } diff --git a/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift b/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift index 81c418131c..9916ea38c9 100644 --- a/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift +++ b/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift @@ -19,6 +19,7 @@ struct ServerDashboardView: View { } } .frame(maxWidth: .infinity, maxHeight: .infinity) + .background { WindowVisibilityReader { viewModel.windowVisibilityChanged($0) } } .onAppear { viewModel.startAutoRefresh() } .onDisappear { viewModel.stopAutoRefresh() } .alert( diff --git a/apple/TablePro/Views/Settings/AIProviderDetailSheet.swift b/apple/TablePro/Views/Settings/AIProviderDetailSheet.swift index dde30a684a..2f02da8516 100644 --- a/apple/TablePro/Views/Settings/AIProviderDetailSheet.swift +++ b/apple/TablePro/Views/Settings/AIProviderDetailSheet.swift @@ -24,6 +24,7 @@ struct AIProviderDetailSheet: View { @State private var isFetchingModels = false @State private var modelFetchError: String? @State private var modelFetchTask: Task? + @State private var choseOtherModel = false @State private var isTesting = false @State private var testResult: TestResult? @@ -94,6 +95,9 @@ struct AIProviderDetailSheet: View { } } .onAppear { + // Before any fetch, so a provider whose list cannot load yet still starts on its curated default. + selectInitialModel() + selectModel(draft.model, picked: false) if draft.type == .copilot { Task { await ensureCopilotRunning() @@ -531,18 +535,27 @@ struct AIProviderDetailSheet: View { } private var curatedModels: [CuratedModel] { - descriptor?.curatedModels ?? [] + descriptor?.curatedModels(for: draft.preset) ?? [] } private var fetchedModels: [String] { fetched.map(\.id) } + private func reasoning(forModelID id: String) -> AIReasoningSupport { + descriptor?.reasoning( + forModelID: id, + fetched: fetched.first(where: { $0.id == id }), + preset: draft.preset + ) ?? .unsupported + } + + private var reasoningForCurrentModel: AIReasoningSupport { + reasoning(forModelID: draft.model) + } + private var effortLevelsForCurrentModel: [ReasoningEffort] { - descriptor?.supportedEffortLevels( - forModelID: draft.model, - fetched: fetched.first(where: { $0.id == draft.model }) - ) ?? [] + reasoningForCurrentModel.effortLevels } private var showsReasoningPicker: Bool { @@ -559,7 +572,7 @@ struct AIProviderDetailSheet: View { Section { modelPicker if isCustomModel { - TextField(String(localized: "Model ID"), text: $draft.model) + TextField(String(localized: "Model ID"), text: typedModelBinding) .textFieldStyle(.roundedBorder) } if showsReasoningPicker { @@ -614,34 +627,48 @@ struct AIProviderDetailSheet: View { }, set: { newValue in switch newValue { - case .curated(let id): - draft.model = id - if let curated = curatedModels.first(where: { $0.id == id }) { - if let defaultEffort = curated.defaultEffort, draft.reasoningEffort == nil { - draft.reasoningEffort = defaultEffort - } - let supported = Set(curated.supportedEffortLevels) - if let currentEffort = draft.reasoningEffort, !supported.contains(currentEffort) { - draft.reasoningEffort = curated.defaultEffort - } - } - case .fetched(let id): - draft.model = id - if let effort = draft.reasoningEffort, !effortLevelsForCurrentModel.contains(effort) { - draft.reasoningEffort = nil - } + case .curated(let id), .fetched(let id): + choseOtherModel = false + selectModel(id, picked: true) case .custom: + choseOtherModel = true if curatedModels.contains(where: { $0.id == draft.model }) || fetchedModels.contains(draft.model) { - draft.model = "" + selectModel("", picked: false) } } } ) } + private var typedModelBinding: Binding { + Binding( + get: { draft.model }, + set: { typed in + choseOtherModel = true + selectModel(typed, picked: false) + } + ) + } + + // The model and the effort it allows change together, in one write. + private func selectModel(_ id: String, picked: Bool) { + var next = draft + next.model = id + next.reasoningEffort = picked + ? AIProviderDraftRules.effort( + afterPicking: draft.reasoningEffort, + reasoning: reasoning(forModelID: id), + curated: curatedModels.first(where: { $0.id == id }) + ) + : AIProviderDraftRules.effort(keeping: draft.reasoningEffort, reasoning: reasoning(forModelID: id)) + draft = next + } + private var reasoningPicker: some View { Picker(String(localized: "Reasoning"), selection: $draft.reasoningEffort) { - Text(String(localized: "Off")).tag(ReasoningEffort?.none) + if !reasoningForCurrentModel.isMandatory { + Text(String(localized: "Off")).tag(ReasoningEffort?.none) + } ForEach(effortLevelsForCurrentModel) { effort in Text(effort.displayName).tag(Optional(effort)) } @@ -852,10 +879,10 @@ struct AIProviderDetailSheet: View { } private func selectInitialModel() { - guard draft.model.isEmpty, + guard draft.model.isEmpty, !choseOtherModel, let initial = AIProviderDraftRules.initialModel(curated: curatedModels, fetched: fetched) else { return } - draft.model = initial + selectModel(initial, picked: false) } func testProvider() { diff --git a/apple/TablePro/Views/Settings/AISettingsView.swift b/apple/TablePro/Views/Settings/AISettingsView.swift index b75d3ea480..581c4a9d24 100644 --- a/apple/TablePro/Views/Settings/AISettingsView.swift +++ b/apple/TablePro/Views/Settings/AISettingsView.swift @@ -2,10 +2,8 @@ // AISettingsView.swift // TablePro // -// Single settings tab for AI: providers, active provider, inline suggestions, -// context, and privacy. Modeled after Xcode 26 Intelligence settings. -// +import Combine import SwiftUI struct AISettingsView: View { @@ -47,8 +45,9 @@ struct AISettingsView: View { saveProvider(saved, apiKey: apiKey, isNew: false) editingProviderID = nil }, + // The sheet already asked; the parent alert is for the context menu only. onDelete: { - pendingDeleteID = provider.id + removeProvider(provider.id) editingProviderID = nil }, onCancel: { @@ -101,7 +100,13 @@ struct AISettingsView: View { Picker("Active Provider", selection: $settings.activeProviderID) { Text("None").tag(UUID?.none) ForEach(settings.providers) { provider in - Text(provider.displayName).tag(UUID?.some(provider.id)) + Label { + Text(provider.displayName) + } icon: { + provider.icon.menuImage() + } + .labelStyle(.titleAndIcon) + .tag(UUID?.some(provider.id)) } } .pickerStyle(.menu) @@ -168,8 +173,9 @@ struct AISettingsView: View { } .frame(width: 14) - Image(systemName: provider.symbolName) + BrandIconView(icon: provider.icon) .foregroundStyle(.secondary) + .frame(width: 16, height: 16) .frame(width: 20) VStack(alignment: .leading, spacing: 2) { @@ -200,21 +206,32 @@ struct AISettingsView: View { Button { addingProvider = makeNewProvider(type: type) } label: { - Label(type.displayName, systemImage: type.symbolName) + Label { + Text(type.displayName) + } icon: { + type.icon.menuImage() + } + .labelStyle(.titleAndIcon) } } ForEach(AIProviderPreset.all) { preset in Button { addingProvider = AIProviderConfig(preset: preset) } label: { - Label(preset.displayName, systemImage: preset.symbolName) + Label { + Text(preset.displayName) + } icon: { + preset.icon.menuImage() + } + .labelStyle(.titleAndIcon) } } Divider() Button { addingProvider = makeNewProvider(type: .custom) } label: { - Label(String(localized: "Add Custom Provider…"), systemImage: AIProviderType.custom.symbolName) + Label(String(localized: "Add Custom Provider…"), systemImage: AIProviderType.custom.icon.fallbackSystemImage) + .labelStyle(.titleAndIcon) } } label: { Label(String(localized: "Add Provider…"), systemImage: "plus") @@ -454,6 +471,7 @@ struct AISettingsView: View { if isNew, settings.activeProviderID == nil { settings.activeProviderID = provider.id } + AppEvents.shared.aiProviderSaved.send(provider.id) } private func removeProvider(_ id: UUID) { diff --git a/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift b/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift index b79452e486..18132bc678 100644 --- a/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift +++ b/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift @@ -12,10 +12,10 @@ internal protocol ThemePreviewing: AnyObject { extension ThemeEngine: ThemePreviewing {} -/// One open theme. The draft is saved a moment after it settles and on close, previewed in every -/// window while it is edited, and undone one settled step at a time. The catalog reloads whenever -/// TablePro becomes active, so a file changed or removed elsewhere is noticed here before a save -/// could write over it. +// One open theme. The draft is saved a moment after it settles and on close, previewed in every +// window while it is edited, and undone one settled step at a time. The catalog reloads when its +// folder changes on disk, so a file changed or removed elsewhere is noticed before a save could +// write over it. @MainActor internal final class ThemeEditorModel: ObservableObject { internal enum Alert: Identifiable, Equatable { diff --git a/apple/TablePro/Views/Settings/Appearance/ThemeEditorPreview.swift b/apple/TablePro/Views/Settings/Appearance/ThemeEditorPreview.swift index 6e8bdfaa02..a0053264c7 100644 --- a/apple/TablePro/Views/Settings/Appearance/ThemeEditorPreview.swift +++ b/apple/TablePro/Views/Settings/Appearance/ThemeEditorPreview.swift @@ -162,22 +162,53 @@ private final class ThemePreviewStatementBand: TextViewCoordinator { // MARK: - Grid -// Five rows that put every grid slot on screen: header and row numbers, both stripes, NULL, both booleans, -// an unfocused selection, and a modified, a deleted and an inserted row. +// The preview takes no scroll events, so fixed widths keep every column in the editor window; only the UUID overflows. internal enum ThemePreviewGridSample { internal static let selectedRow = 1 internal static let modifiedRow = 2 - internal static let modifiedColumn = 4 + internal static let modifiedColumn = 2 internal static let deletedRow = 3 internal static let insertedRow = 4 + internal struct Column { + internal let name: String + internal let type: ColumnType + internal let width: CGFloat + } + + internal static let columns: [Column] = [ + Column(name: "id", type: .integer(rawType: "INTEGER"), width: 40), + Column(name: "name", type: .text(rawType: "TEXT"), width: 68), + Column(name: "role", type: .enumType(rawType: "ENUM", values: ["admin", "staff", "guest"]), width: 56), + Column(name: "active", type: .boolean(rawType: "BOOLEAN"), width: 56), + Column(name: "joined", type: .date(rawType: "DATE"), width: 92), + Column(name: "uuid", type: .text(rawType: "UUID"), width: 104), + Column(name: "tags", type: .json(rawType: "JSON"), width: 68), + Column(name: "hash", type: .blob(rawType: "BLOB"), width: 60), + ] + internal static let tableRows: TableRows = { let values: [[PluginCellValue]] = [ - [.text("1"), .text("Ada Lovelace"), .text("ada@example.com"), .text("true"), .text("1250.00")], - [.text("2"), .text("Alan Turing"), .null, .text("false"), .text("980.50")], - [.text("3"), .text("Grace Hopper"), .text("grace@example.com"), .text("true"), .text("3100.75")], - [.text("4"), .text("Edsger Dijkstra"), .text("edsger@example.com"), .text("false"), .text("0.00")], - [.text("5"), .text("Barbara Liskov"), .text("barbara@example.com"), .text("true"), .null], + [ + .text("1"), .text("Ada"), .text("admin"), .text("true"), .text("2023-04-12"), + .text("8f14e45f-ceea-467f-a8f5-2e1c6a3d9b07"), .text(#"["ops"]"#), .bytes(Data([0x9F, 0x3A])), + ], + [ + .text("2"), .text("Alan"), .text("staff"), .text("false"), .text("2023-09-30"), + .text("c9f0f895-fb98-4b91-9c2e-5d7a1e4b3f60"), .null, .bytes(Data([0x4C, 0x21])), + ], + [ + .text("3"), .text("Grace"), .text("admin"), .text("false"), .text("2024-01-08"), + .text("45c48cce-2e2d-4fbd-8a1b-7c3e9d0f6a12"), .text(#"["dev"]"#), .bytes(Data([0x07, 0xE5])), + ], + [ + .text("4"), .text("Edsger"), .text("guest"), .text("true"), .text("2024-06-21"), + .text("d3d94468-02a4-4c3b-b1d6-9e8f7a6c5b43"), .text("[]"), .bytes(Data([0xB2, 0xD0])), + ], + [ + .text("5"), .text("Barbara"), .text("staff"), .text("true"), .text("2025-02-14"), + .text("6512bd43-d9ca-4f6e-9b2a-1c0d8e7f3a95"), .text(#"["ops"]"#), .null, + ], ] let rows = values.enumerated().map { index, row in Row( @@ -187,26 +218,26 @@ internal enum ThemePreviewGridSample { } return TableRows( rows: ContiguousArray(rows), - columns: ["id", "name", "email", "active", "balance"], - columnTypes: [ - .integer(rawType: "INTEGER"), - .text(rawType: "TEXT"), - .text(rawType: "TEXT"), - .boolean(rawType: "BOOLEAN"), - .decimal(rawType: "DECIMAL"), - ] + columns: columns.map(\.name), + columnTypes: columns.map(\.type) ) }() - // A preview: nothing to sort, filter or edit. + // A preview: nothing to sort, filter or edit, and the theme's value colors even while Color values by type is off. internal static var configuration: DataGridConfiguration { var configuration = DataGridConfiguration() configuration.showRowNumbers = true configuration.supportsValueFilter = false configuration.supportsColumnSort = false + configuration.colorsValuesByType = true + configuration.overridesValueColorSetting = true return configuration } + internal static var columnLayout: ColumnLayoutState { + ColumnLayoutState(columnWidths: Dictionary(uniqueKeysWithValues: columns.map { ($0.name, $0.width) })) + } + internal static func visualState(forRow row: Int) -> RowVisualState? { switch row { case modifiedRow: @@ -230,6 +261,7 @@ private struct ThemePreviewGrid: View { self.scope = scope _grid = StateObject(wrappedValue: HeldRowsGridAdapter( configuration: ThemePreviewGridSample.configuration, + columnLayout: ThemePreviewGridSample.columnLayout, themeScope: scope )) } diff --git a/apple/TablePro/Views/Settings/Appearance/ThemeEditorView.swift b/apple/TablePro/Views/Settings/Appearance/ThemeEditorView.swift index e16232a3b6..709db8dc4e 100644 --- a/apple/TablePro/Views/Settings/Appearance/ThemeEditorView.swift +++ b/apple/TablePro/Views/Settings/Appearance/ThemeEditorView.swift @@ -1,8 +1,7 @@ import SwiftUI import TableProTheme -/// The Theme Editor window's content: a live preview over the theme's fields. The window is chrome, -/// so only the preview wears the theme. +// The window is chrome, so only the preview wears the theme. internal struct ThemeEditorView: View { @ObservedObject internal var model: ThemeEditorModel internal let onDuplicate: () -> Void @@ -117,6 +116,8 @@ internal struct ThemeEditorView: View { private func sectionFooter(_ section: ThemeSlotSection) -> some View { if section == .window { Text("With Increase Contrast on, the window uses the system background.") + } else if section == .values { + ThemeValuesSectionFooter(contrastNote: ThemeContrastNote.message(for: section, in: model.draft)) } else if let note = ThemeContrastNote.message(for: section, in: model.draft) { Text(note) .accessibilityIdentifier("theme-editor-contrast-\(section.rawValue)") @@ -163,6 +164,31 @@ internal struct ThemeEditorView: View { } } +// Observes the settings itself, so a settings change re-renders this footer and not the editor or its preview. +private struct ThemeValuesSectionFooter: View { + let contrastNote: String? + @ObservedObject private var settingsManager = AppSettingsManager.shared + + var body: some View { + let settingNote = ThemeValueColorsNote.message( + for: .values, + colorsValuesByType: settingsManager.valueColors.colorsValuesByType + ) + if contrastNote != nil || settingNote != nil { + VStack(alignment: .leading, spacing: 4) { + if let contrastNote { + Text(contrastNote) + .accessibilityIdentifier("theme-editor-contrast-values") + } + if let settingNote { + Text(settingNote) + .accessibilityIdentifier("theme-editor-values-setting-note") + } + } + } + } +} + internal extension ThemeEditorModel.Alert { var title: String { switch self { diff --git a/apple/TablePro/Views/Settings/Appearance/ThemeSlotPresentation.swift b/apple/TablePro/Views/Settings/Appearance/ThemeSlotPresentation.swift index 68a2b3e993..a40a4d5fa6 100644 --- a/apple/TablePro/Views/Settings/Appearance/ThemeSlotPresentation.swift +++ b/apple/TablePro/Views/Settings/Appearance/ThemeSlotPresentation.swift @@ -82,25 +82,69 @@ internal struct ThemeSlotPresentation: Equatable { internal enum ThemeContrastNote { internal static let minimumRatio = 4.5 - internal static func pair(for section: ThemeSlotSection) -> (text: ThemeSlot, background: ThemeSlot)? { + // NULL draws a placeholder, not a value. An unset value color is the grid text, which the Data Grid note checks. + private static func checks( + for section: ThemeSlotSection, + in theme: ThemeDefinition + ) -> (texts: [ThemeSlot], surfaces: [ThemeColor])? { switch section { - case .editor: return (.editorText, .editorBackground) - case .dataGrid: return (.gridText, .gridBackground) - case .window, .syntax, .selection, .changes: return nil + case .editor: + return ([.editorText], [color(.editorBackground, in: theme).opaque]) + case .dataGrid: + return ([.gridText], [color(.gridBackground, in: theme).opaque]) + case .values: + let explicit = ThemeSlotSection.values.slots.filter { $0 != .gridNullValue && theme.hasExplicitColor($0) } + return (explicit, rowSurfaces(in: theme)) + case .window, .syntax, .selection, .changes: + return nil } } - internal static func ratio(for section: ThemeSlotSection, in theme: ThemeDefinition) -> Double? { - guard let pair = pair(for: section) else { return nil } - let background = ThemeSlotPresentation.displayColor(for: pair.background, in: theme).opaque - let text = ThemeSlotPresentation.displayColor(for: pair.text, in: theme).composited(over: background) - return text.contrastRatio(with: background) + // Alternate rows are on by default, and a modified cell or an inserted row washes either row. + private static func rowSurfaces(in theme: ThemeDefinition) -> [ThemeColor] { + let background = color(.gridBackground, in: theme).opaque + let rows = [background, color(.gridAlternateRow, in: theme).composited(over: background)] + let washes = [color(.gridModified, in: theme), color(.gridInserted, in: theme)] + return rows + washes.flatMap { wash in rows.map { wash.composited(over: $0) } } + } + + internal static func lowest(for section: ThemeSlotSection, in theme: ThemeDefinition) -> (slot: ThemeSlot, ratio: Double)? { + guard let checks = checks(for: section, in: theme) else { return nil } + return checks.texts + .map { slot in + let text = color(slot, in: theme) + let ratios = checks.surfaces.map { text.composited(over: $0).contrastRatio(with: $0) } + return (slot: slot, ratio: ratios.min() ?? .infinity) + } + .min { $0.ratio < $1.ratio } + } + + private static func color(_ slot: ThemeSlot, in theme: ThemeDefinition) -> ThemeColor { + ThemeSlotPresentation.displayColor(for: slot, in: theme) } internal static func message(for section: ThemeSlotSection, in theme: ThemeDefinition) -> String? { - guard let ratio = ratio(for: section, in: theme), ratio < minimumRatio else { return nil } - let formatted = ratio.formatted(.number.precision(.fractionLength(1))) - return String(format: String(localized: "Text contrast is %@:1. Aim for at least 4.5:1."), formatted) + guard let lowest = lowest(for: section, in: theme), lowest.ratio < minimumRatio else { return nil } + let formatted = lowest.ratio.formatted(.number.precision(.fractionLength(1))) + switch section { + case .values: + return String( + format: String(localized: "%1$@ contrast is %2$@:1. Aim for at least 4.5:1."), + lowest.slot.label, + formatted + ) + case .window, .editor, .syntax, .dataGrid, .selection, .changes: + return String(format: String(localized: "Text contrast is %@:1. Aim for at least 4.5:1."), formatted) + } + } +} + +internal enum ThemeValueColorsNote { + internal static func message(for section: ThemeSlotSection, colorsValuesByType: Bool) -> String? { + guard section == .values, !colorsValuesByType else { return nil } + return String( + localized: "Except NULL and booleans, these colors show in the grid only while Color values by type is on in Settings > Data." + ) } } diff --git a/apple/TablePro/Views/Settings/Components/CopyableCodeBlock.swift b/apple/TablePro/Views/Settings/Components/CopyableCodeBlock.swift index 414b86118f..9f9ffb0948 100644 --- a/apple/TablePro/Views/Settings/Components/CopyableCodeBlock.swift +++ b/apple/TablePro/Views/Settings/Components/CopyableCodeBlock.swift @@ -2,11 +2,12 @@ import SwiftUI struct CopyableCodeBlock: View { let text: String + var displayText: String? var isSecret = false var body: some View { HStack(alignment: .top) { - Text(text) + Text(verbatim: displayText ?? text) .font(.system(.caption, design: .monospaced)) .textSelection(.enabled) .padding(8) diff --git a/apple/TablePro/Views/Settings/Components/IntegrationClient.swift b/apple/TablePro/Views/Settings/Components/IntegrationClient.swift deleted file mode 100644 index c8b9949adc..0000000000 --- a/apple/TablePro/Views/Settings/Components/IntegrationClient.swift +++ /dev/null @@ -1,19 +0,0 @@ -import Foundation - -enum IntegrationClient: String, CaseIterable, Identifiable, Sendable { - case claudeCode - case claudeDesktop - case cursor - case zed - - var id: String { rawValue } - - var displayName: String { - switch self { - case .claudeCode: return "Claude Code" - case .claudeDesktop: return "Claude Desktop" - case .cursor: return "Cursor" - case .zed: return "Zed" - } - } -} diff --git a/apple/TablePro/Views/Settings/CustomSlashCommandsSection.swift b/apple/TablePro/Views/Settings/CustomSlashCommandsSection.swift index 4c9b2a599c..07487a6fac 100644 --- a/apple/TablePro/Views/Settings/CustomSlashCommandsSection.swift +++ b/apple/TablePro/Views/Settings/CustomSlashCommandsSection.swift @@ -10,6 +10,7 @@ struct CustomSlashCommandsSection: View { @State private var editing: CustomSlashCommand? @State private var isCreating = false @State private var saveError: String? + @State private var pendingDelete: CustomSlashCommand? var body: some View { Section { @@ -74,6 +75,22 @@ struct CustomSlashCommandsSection: View { } message: { message in Text(message) } + .alert( + pendingDelete.map { String(format: String(localized: "Delete “/%@”?"), $0.name) } ?? "", + isPresented: Binding( + get: { pendingDelete != nil }, + set: { if !$0 { pendingDelete = nil } } + ), + presenting: pendingDelete + ) { command in + Button(String(localized: "Delete"), role: .destructive) { + storage.delete(id: command.id) + pendingDelete = nil + } + Button(String(localized: "Cancel"), role: .cancel) { pendingDelete = nil } + } message: { _ in + Text(String(localized: "The command is removed from this Mac and from every Mac that syncs settings. This cannot be undone.")) + } } @ViewBuilder @@ -103,7 +120,7 @@ struct CustomSlashCommandsSection: View { } .controlSize(.small) Button(role: .destructive) { - storage.delete(id: command.id) + pendingDelete = command } label: { Image(systemName: "trash") } diff --git a/apple/TablePro/Views/Settings/DataResultsSettingsView.swift b/apple/TablePro/Views/Settings/DataResultsSettingsView.swift index 1bb0f30e3e..bda0adaed7 100644 --- a/apple/TablePro/Views/Settings/DataResultsSettingsView.swift +++ b/apple/TablePro/Views/Settings/DataResultsSettingsView.swift @@ -7,6 +7,7 @@ import SwiftUI struct DataResultsSettingsView: View { @Binding var dataGrid: DataGridSettings + @Binding var valueColors: ValueColorSettings @Binding var history: HistorySettings @Binding var editor: EditorSettings @Binding var typography: TypographySettings @@ -15,7 +16,7 @@ struct DataResultsSettingsView: View { Form { TypographySection(domain: .dataGrid, settings: $typography) - DataGridSection(settings: $dataGrid) + DataGridSection(settings: $dataGrid, valueColors: $valueColors) Section("JSON Viewer") { Picker("Default view:", selection: $editor.jsonViewerPreferredMode) { @@ -35,6 +36,7 @@ struct DataResultsSettingsView: View { #Preview { DataResultsSettingsView( dataGrid: .constant(.default), + valueColors: .constant(.default), history: .constant(.default), editor: .constant(.default), typography: .constant(.default) diff --git a/apple/TablePro/Views/Settings/GeneralSettingsView.swift b/apple/TablePro/Views/Settings/GeneralSettingsView.swift index d66319fbf1..dd574ad3db 100644 --- a/apple/TablePro/Views/Settings/GeneralSettingsView.swift +++ b/apple/TablePro/Views/Settings/GeneralSettingsView.swift @@ -149,7 +149,7 @@ struct GeneralSettingsView: View { Button(String(localized: "Reset"), role: .destructive) { onResetAll() } Button(String(localized: "Cancel"), role: .cancel) {} } message: { - Text("This will reset all settings across every section to their default values.") + Text("This will reset all settings across every section to their default values. AI providers, their API keys and Enable AI Features are kept.") } .onAppear { if initialLanguage == nil { initialLanguage = settings.language } diff --git a/apple/TablePro/Views/Settings/KeyboardSettingsView.swift b/apple/TablePro/Views/Settings/KeyboardSettingsView.swift index 670755875e..499c040a9d 100644 --- a/apple/TablePro/Views/Settings/KeyboardSettingsView.swift +++ b/apple/TablePro/Views/Settings/KeyboardSettingsView.swift @@ -18,6 +18,7 @@ struct KeyboardSettingsView: View { @State private var systemReservedAlert: ShortcutAction? @State private var reservedAlert: ReservedAlertState? @State private var needsModifierAlert: ShortcutAction? + @State private var confirmsReset = false private var content: some View { VStack(spacing: 0) { @@ -45,7 +46,7 @@ struct KeyboardSettingsView: View { HStack { Spacer() Button(String(localized: "Reset to Defaults")) { - settings = .default + confirmsReset = true } .disabled(settings.shortcuts.isEmpty) } @@ -57,6 +58,14 @@ struct KeyboardSettingsView: View { var body: some View { content + .alert(String(localized: "Reset All Shortcuts?"), isPresented: $confirmsReset) { + Button(String(localized: "Reset"), role: .destructive) { + settings = .default + } + Button(String(localized: "Cancel"), role: .cancel) {} + } message: { + Text(String(localized: "Every custom shortcut goes back to its default.")) + } .alert( String(localized: "Shortcut Conflict"), isPresented: Binding( diff --git a/apple/TablePro/Views/Settings/Plugins/InstalledPluginsView.swift b/apple/TablePro/Views/Settings/Plugins/InstalledPluginsView.swift index 4197393625..a2e78f03f2 100644 --- a/apple/TablePro/Views/Settings/Plugins/InstalledPluginsView.swift +++ b/apple/TablePro/Views/Settings/Plugins/InstalledPluginsView.swift @@ -148,10 +148,12 @@ struct InstalledPluginsView: View { HStack(spacing: 8) { rejectedActionControl(for: plugin, action: action) - Button(String(localized: "Remove")) { - removeRejectedPlugin(plugin) + if pluginManager.canRemoveRejectedPlugin(plugin) { + Button(String(localized: "Remove")) { + removeRejectedPlugin(plugin) + } + .accessibilityLabel(String(format: String(localized: "Remove %@"), plugin.name)) } - .accessibilityLabel(String(format: String(localized: "Remove %@"), plugin.name)) } } .padding(20) @@ -199,20 +201,24 @@ struct InstalledPluginsView: View { } private func removeRejectedPlugin(_ plugin: RejectedPlugin) { - let fm = FileManager.default - if fm.fileExists(atPath: plugin.url.path) { + Task { + let confirmed = await AlertHelper.confirmDestructive( + title: String(format: String(localized: "Move “%@” to the Trash?"), plugin.name), + message: String(localized: "The plugin can't be loaded. You can put it back from the Trash."), + confirmButton: String(localized: "Move to Trash"), + cancelButton: String(localized: "Cancel") + ) + guard confirmed else { return } do { - try fm.removeItem(at: plugin.url) - } catch CocoaError.fileNoSuchFile { + try pluginManager.removeRejectedPlugin(plugin) } catch { errorAlertTitle = String(localized: "Remove Failed") errorAlertMessage = error.localizedDescription showErrorAlert = true return } + if selection == .rejected(plugin.url) { selection = nil } } - pluginManager.removeFromRejected(url: plugin.url) - if selection == .rejected(plugin.url) { selection = nil } } // MARK: - Plugin List diff --git a/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift b/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift index cea90a444d..7d59dadc49 100644 --- a/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift +++ b/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift @@ -29,6 +29,8 @@ struct CredentialProfileEditorSheet: View { @State private var sourceField: String = "" @State private var commandReuseSeconds: Int? @State private var vaultAccess = VaultAccessDraft() + // A new profile keeps this id when saved, so its first connect signs in as the user Test Source leased. + @State private var draftProfileId = UUID() @State private var secureFieldType: DatabaseType = .mysql @State private var secureFieldValues: [String: String] = [:] @@ -176,6 +178,7 @@ struct CredentialProfileEditorSheet: View { PasswordSourceTestView( source: testedSource, context: { testContext }, + leaseOwner: { profileId }, refusal: { Self.testRefusal( for: testedSource, @@ -233,7 +236,10 @@ struct CredentialProfileEditorSheet: View { .font(.caption) .foregroundStyle(.secondary) } - Text("Each connection signs in as a new database user. TablePro renews the lease while connected, revokes it on disconnect, and never saves the generated password.") + Text(String(localized: """ + Connections on this profile share one database user. TablePro renews its lease while in use, revokes it \ + once the last connection disconnects or after 10 minutes unused, and never saves the generated password. + """)) .font(.caption) .foregroundStyle(.secondary) } @@ -253,9 +259,13 @@ struct CredentialProfileEditorSheet: View { return source } + private var profileId: UUID { + existingProfile?.id ?? draftProfileId + } + private var testContext: ConnectionScriptContext { ConnectionScriptContext( - connectionId: existingProfile?.id ?? UUID(), + connectionId: profileId, connectionName: "", host: "", port: 0, @@ -418,7 +428,6 @@ struct CredentialProfileEditorSheet: View { } private func saveProfile() { - let profileId = existingProfile?.id ?? UUID() let liveFields = secureFieldValues.filter { !$0.value.isEmpty } let profile = CredentialProfile( id: profileId, diff --git a/apple/TablePro/Views/Settings/Sections/DataGridSection.swift b/apple/TablePro/Views/Settings/Sections/DataGridSection.swift index 38533e8a2b..9b3c11ec12 100644 --- a/apple/TablePro/Views/Settings/Sections/DataGridSection.swift +++ b/apple/TablePro/Views/Settings/Sections/DataGridSection.swift @@ -7,6 +7,7 @@ import SwiftUI struct DataGridSection: View { @Binding var settings: DataGridSettings + @Binding var valueColors: ValueColorSettings var body: some View { Section { @@ -38,6 +39,9 @@ struct DataGridSection: View { Toggle("Smart value detection", isOn: $settings.enableSmartValueDetection) Toggle("Show invisible characters", isOn: $settings.showInvisibleCharacters) .help(String(localized: "Mark zero-width, control and special space characters in cells and value fields")) + Toggle("Color values by type", isOn: $valueColors.colorsValuesByType) + .help(String(localized: "Draw strings, numbers, dates and other typed values in the colors of the theme's Values section")) + .accessibilityIdentifier("color-values-by-type-toggle") Picker("Default row sort:", selection: $settings.defaultSortBehavior) { ForEach(DefaultSortBehavior.allCases) { behavior in @@ -121,5 +125,18 @@ struct DataGridSection: View { """) } } + + Section { + Picker("Results kept per run:", selection: $settings.resultSetsKeptPerRun) { + Text("No limit").tag(0) + ForEach([10, 20, 50, 100, 200, 500, 1_000], id: \.self) { count in + Text(count.formatted()).tag(count) + } + } + } header: { + Text("Multiple Results") + } footer: { + Text("A run that returns more results with rows keeps only the newest. Pinned results always stay.") + } } } diff --git a/apple/TablePro/Views/Settings/Sections/HistorySection.swift b/apple/TablePro/Views/Settings/Sections/HistorySection.swift index addecec3d7..222f8a544f 100644 --- a/apple/TablePro/Views/Settings/Sections/HistorySection.swift +++ b/apple/TablePro/Views/Settings/Sections/HistorySection.swift @@ -35,7 +35,11 @@ struct HistorySection: View { let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Clear All History?"), message: String( - localized: "This will permanently delete all query history entries, and the saved changes kept for restoring. This action cannot be undone." + localized: """ + This will permanently delete all query history entries, the saved changes kept \ + for restoring, and the earlier versions of your query tabs. This action cannot \ + be undone. + """ ), confirmButton: String(localized: "Clear"), cancelButton: String(localized: "Cancel") @@ -44,6 +48,7 @@ struct HistorySection: View { Task { _ = await QueryHistoryManager.shared.clearEverything() _ = await QueryHistoryManager.shared.clearRewindSnapshots() + _ = await QueryTabVersionStorage.shared.removeAll() } } } diff --git a/apple/TablePro/Views/Settings/Sections/MCPSection.swift b/apple/TablePro/Views/Settings/Sections/MCPSection.swift index 66a2b46c14..96d67c7902 100644 --- a/apple/TablePro/Views/Settings/Sections/MCPSection.swift +++ b/apple/TablePro/Views/Settings/Sections/MCPSection.swift @@ -9,9 +9,9 @@ struct MCPSection: View { @State private var showSetupSheet = false @State private var showCreateSheet = false @State private var showRevealSheet = false - @State private var revealedToken: MCPAuthToken? @State private var revealedPlaintext: String = "" - @State private var revocationNotSaved = false + @State private var tokenAlert: MCPTokenAlert? + @State private var rejectedPort: Int? var body: some View { Section { @@ -67,11 +67,12 @@ struct MCPSection: View { } private var configurationSection: some View { - Section(String(localized: "Server Configuration")) { + Section { LabeledContent(String(localized: "Port")) { - TextField("", value: $settings.port, format: .number.grouping(.never)) + TextField("", value: portBinding, format: .number.grouping(.never)) .frame(width: 80) .multilineTextAlignment(.trailing) + .accessibilityIdentifier("mcp-port-field") } LabeledContent(String(localized: "Default row limit")) { @@ -97,64 +98,113 @@ struct MCPSection: View { } Toggle(String(localized: "Log MCP queries in history"), isOn: $settings.logQueriesInHistory) + } header: { + Text(String(localized: "Server Configuration")) + } footer: { + if let rejectedPort { + NoticeLabel( + String( + format: String(localized: "Port %d cannot be used. Choose a port from %d to %d."), + rejectedPort, + MCPSettings.portRange.lowerBound, + MCPSettings.portRange.upperBound + ), + severity: .error + ) + .font(.caption) + } } } + // An out-of-range port is kept out of the settings, so the server never binds a port the field did not show. + private var portBinding: Binding { + Binding( + get: { settings.port }, + set: { newValue in + if MCPSettings.portRange.contains(newValue) { + rejectedPort = nil + settings.port = newValue + } else { + rejectedPort = newValue + } + } + ) + } + private var authenticationSection: some View { Section(String(localized: "Authentication")) { MCPTokenListView( tokens: tokenList, + isStoreAvailable: manager.tokenStore != nil, onGenerate: { showCreateSheet = true }, - onRevoke: { id in - Task { - if !(await manager.revokeToken(id: id)) { - revocationNotSaved = true - } - await refreshTokens() - } - }, - onDelete: { id in Task { await manager.tokenStore?.delete(tokenId: id); await refreshTokens() } } + onRevoke: { id in revoke(id) }, + onDelete: { ids in delete(ids) } ) } - .task { await refreshTokens() } + .task(id: manager.tokenStore.map { ObjectIdentifier($0) }) { await refreshTokens() } .sheet(isPresented: $showCreateSheet) { - MCPTokenCreateSheet(onGenerate: handleGenerate) + MCPTokenCreateSheet { name, permissions, connectionIds, expiresAt in + await generate(name: name, permissions: permissions, connectionIds: connectionIds, expiresAt: expiresAt) + } } .sheet(isPresented: $showRevealSheet) { - if let revealedToken { + if !revealedPlaintext.isEmpty { MCPTokenRevealSheet( - token: revealedToken, plaintext: revealedPlaintext, - port: manager.listeningPort ?? settings.port + port: manager.listeningPort ?? settings.port, + configuredPort: settings.port ) } } - .alert(String(localized: "Couldn't Save the Revocation"), isPresented: $revocationNotSaved) { + .alert(tokenAlert?.title ?? "", isPresented: tokenAlertBinding, presenting: tokenAlert) { _ in Button(String(localized: "OK")) {} - } message: { - Text(String(localized: "TablePro could not save the change to the Keychain. The token works again after TablePro restarts.")) + } message: { alert in + Text(verbatim: alert.message) } } + private var tokenAlertBinding: Binding { + Binding( + get: { tokenAlert != nil }, + set: { isPresented in + if !isPresented { + tokenAlert = nil + } + } + ) + } + private var helpSection: some View { Section { Button(String(localized: "Connect a Client…")) { showSetupSheet = true } + .accessibilityIdentifier("mcp-connect-client") Button(String(localized: "View Activity…")) { WindowOpener.shared.openIntegrationsActivity() } + .accessibilityIdentifier("mcp-view-activity") } .sheet(isPresented: $showSetupSheet) { - IntegrationsSetupSheet(port: manager.listeningPort ?? settings.port) + MCPClientSetupSheet() } } - private func handleGenerate(name: String, permissions: TokenPermissions, connectionIds: Set?, expiresAt: Date?) { - Task { - guard let store = manager.tokenStore else { return } - let access: ConnectionAccess = connectionIds.map { .limited($0) } ?? .all - guard let result = try? await store.generate( + private func generate( + name: String, + permissions: TokenPermissions, + connectionIds: Set?, + expiresAt: Date? + ) async -> MCPTokenCreateFailure? { + guard let store = manager.tokenStore else { + return MCPTokenCreateFailure( + message: String(localized: "The MCP server is not running, so TablePro cannot save a token now. Start the server, then try again."), + endsSheet: true + ) + } + let access: ConnectionAccess = connectionIds.map { .limited($0) } ?? .all + do { + let result = try await store.generate( name: name, permissions: permissions, connectionAccess: access, @@ -162,21 +212,88 @@ struct MCPSection: View { isBridgeCredential: false, isClaudeAgentCredential: false, extraScopes: [] - ) else { return } - revealedToken = result.token + ) revealedPlaintext = result.plaintext showCreateSheet = false showRevealSheet = true await refreshTokens() + return nil + } catch { + return MCPTokenCreateFailure(message: Self.generateFailureMessage(error), endsSheet: false) + } + } + + private static func generateFailureMessage(_ error: Error) -> String { + switch error as? MCPTokenStoreError { + case .storedTokensUnreadable?: + String(localized: "TablePro could not read the tokens saved in the Keychain, so it will not save a new one over them. Restart TablePro and try again.") + case .randomGenerationFailed?: + String(localized: "macOS could not generate a secure token. Try again.") + case .keychainUnavailable?, .notSaved?, nil: + String(localized: "TablePro could not save the token to the Keychain, so it was not created. Try again.") + } + } + + private func revoke(_ id: UUID) { + Task { + guard manager.tokenStore != nil else { + tokenAlert = .serverNotRunning + return + } + if !(await manager.revokeToken(id: id)) { + tokenAlert = .revocationNotSaved + } + await refreshTokens() + } + } + + private func delete(_ ids: Set) { + Task { + guard let store = manager.tokenStore else { + tokenAlert = .serverNotRunning + return + } + var allSaved = true + for id in ids { + let saved = await store.delete(tokenId: id) + allSaved = allSaved && saved + } + if !allSaved { + tokenAlert = .deletionNotSaved + } + await refreshTokens() } } private func refreshTokens() async { - guard let store = MCPServerManager.shared.tokenStore else { return } + guard let store = manager.tokenStore else { return } tokenList = await store.list().filter { !$0.isBridgeCredential } } } +private enum MCPTokenAlert: Equatable { + case serverNotRunning + case revocationNotSaved + case deletionNotSaved + + var title: String { + switch self { + case .serverNotRunning: String(localized: "MCP Server Not Running") + case .revocationNotSaved: String(localized: "Couldn't Save the Revocation") + case .deletionNotSaved: String(localized: "Couldn't Save the Deletion") + } + } + + var message: String { + switch self { + case .serverNotRunning: + String(localized: "The MCP server is not running, so TablePro cannot change its tokens now. Start the server, then try again.") + case .revocationNotSaved, .deletionNotSaved: + String(localized: "TablePro could not save the change to the Keychain. The token works again after TablePro restarts.") + } + } +} + private struct MCPStatusRow: View { @ObservedObject private var manager = MCPServerManager.shared let configuredPort: Int diff --git a/apple/TablePro/Views/Settings/Sections/MCPTokenCreateSheet.swift b/apple/TablePro/Views/Settings/Sections/MCPTokenCreateSheet.swift index 7bd91997bc..7e8b296c7b 100644 --- a/apple/TablePro/Views/Settings/Sections/MCPTokenCreateSheet.swift +++ b/apple/TablePro/Views/Settings/Sections/MCPTokenCreateSheet.swift @@ -1,12 +1,18 @@ import SwiftUI +struct MCPTokenCreateFailure: Equatable, Sendable { + let message: String + // Nothing in this sheet can fix a server that is not running, so the alert closes it. + let endsSheet: Bool +} + struct MCPTokenCreateSheet: View { private enum Field: Hashable { case name } @Environment(\.dismiss) private var dismiss - let onGenerate: (String, TokenPermissions, Set?, Date?) -> Void + let onGenerate: (String, TokenPermissions, Set?, Date?) async -> MCPTokenCreateFailure? @State private var tokenName = "" @State private var permissions: TokenPermissions = .readOnly @@ -15,6 +21,8 @@ struct MCPTokenCreateSheet: View { @State private var expirationOption: ExpirationOption = .never @State private var customExpirationDate = Calendar.current.date(byAdding: .day, value: 30, to: .now) ?? .now @State private var connections: [DatabaseConnection] = [] + @State private var isGenerating = false + @State private var failure: MCPTokenCreateFailure? @FocusState private var focused: Field? var body: some View { @@ -36,11 +44,36 @@ struct MCPTokenCreateSheet: View { .task { connections = ConnectionStorage.shared.loadConnections() } + .alert( + String(localized: "Couldn't Create the Token"), + isPresented: failureBinding, + presenting: failure + ) { failure in + Button(String(localized: "OK")) { + self.failure = nil + if failure.endsSheet { + dismiss() + } + } + } message: { failure in + Text(verbatim: failure.message) + } + } + + private var failureBinding: Binding { + Binding( + get: { failure != nil }, + set: { isPresented in + if !isPresented { + failure = nil + } + } + ) } private var nameSection: some View { Section(String(localized: "Token Name")) { - TextField(String(localized: "e.g., Claude Code on VPS"), text: $tokenName) + TextField(String(localized: "e.g., Cursor on my Mac"), text: $tokenName) .focused($focused, equals: .name) } } @@ -120,10 +153,15 @@ struct MCPTokenCreateSheet: View { Button(String(localized: "Generate")) { let connectionIds: Set? = connectionAccess == .selected ? selectedConnectionIds : nil - onGenerate(tokenName, permissions, connectionIds, resolvedExpirationDate) + isGenerating = true + Task { + failure = await onGenerate(tokenName, permissions, connectionIds, resolvedExpirationDate) + isGenerating = false + } } .keyboardShortcut(.defaultAction) - .disabled(tokenName.trimmingCharacters(in: .whitespaces).isEmpty + .disabled(isGenerating + || tokenName.trimmingCharacters(in: .whitespaces).isEmpty || (connectionAccess == .selected && selectedConnectionIds.isEmpty)) } } diff --git a/apple/TablePro/Views/Settings/Sections/MCPTokenListView.swift b/apple/TablePro/Views/Settings/Sections/MCPTokenListView.swift index e949d0b88c..8f1e0f9ad9 100644 --- a/apple/TablePro/Views/Settings/Sections/MCPTokenListView.swift +++ b/apple/TablePro/Views/Settings/Sections/MCPTokenListView.swift @@ -4,12 +4,13 @@ import TableProMCP struct MCPTokenListView: View { let tokens: [MCPAuthToken] + let isStoreAvailable: Bool let onGenerate: () -> Void let onRevoke: (UUID) -> Void - let onDelete: (UUID) -> Void + let onDelete: (Set) -> Void @State private var selection: Set = [] - @State private var deleteCandidate: MCPAuthToken? + @State private var deleteCandidates: [MCPAuthToken] = [] var body: some View { VStack(alignment: .leading, spacing: 8) { @@ -36,22 +37,28 @@ struct MCPTokenListView: View { Label(String(localized: "Generate Token"), systemImage: "plus") } .buttonStyle(.borderedProminent) + .disabled(!isStoreAvailable) .accessibilityLabel(String(localized: "Generate token")) Spacer() } } - .alert(deleteAlertTitle, isPresented: deleteAlertBinding, presenting: deleteCandidate) { token in + .alert( + deleteAlertTitle, + isPresented: deleteAlertBinding, + presenting: deleteCandidates.isEmpty ? nil : deleteCandidates + ) { candidates in Button(String(localized: "Cancel"), role: .cancel) { - deleteCandidate = nil + deleteCandidates = [] } Button(String(localized: "Delete"), role: .destructive) { - onDelete(token.id) - selection.remove(token.id) - deleteCandidate = nil + let ids = Set(candidates.map(\.id)) + onDelete(ids) + selection.subtract(ids) + deleteCandidates = [] } - } message: { token in - Text(String(format: String(localized: "“%@” will be permanently deleted. External clients using this token will lose access immediately."), token.name)) + } message: { candidates in + Text(verbatim: Self.deleteAlertMessage(for: candidates)) } } @@ -60,9 +67,11 @@ struct MCPTokenListView: View { Image(systemName: "key") .font(.title) .foregroundStyle(.tertiary) - Text(String(localized: "No tokens created")) + Text(isStoreAvailable ? String(localized: "No tokens created") : String(localized: "Tokens are not available")) .foregroundStyle(.secondary) - Text(String(localized: "Generate a token so external clients can connect with their own credentials.")) + Text(isStoreAvailable + ? String(localized: "Generate a token so external clients can connect with their own credentials.") + : String(localized: "Tokens load while the MCP server is running.")) .font(.caption) .foregroundStyle(.tertiary) .multilineTextAlignment(.center) @@ -87,15 +96,19 @@ struct MCPTokenListView: View { } Divider() Button(role: .destructive) { - deleteCandidate = token + // A right-click inside a multiple selection acts on all of it, as in Finder. + deleteCandidates = selection.contains(token.id) ? selectedTokens : [token] } label: { Label(String(localized: "Delete…"), systemImage: "trash") } } + private var selectedTokens: [MCPAuthToken] { + tokens.filter { selection.contains($0.id) } + } + private func deleteSelectionFromKeyboard() { - guard let id = selection.first, let token = tokens.first(where: { $0.id == id }) else { return } - deleteCandidate = token + deleteCandidates = selectedTokens } private func copyTokenId(_ id: UUID) { @@ -104,15 +117,27 @@ struct MCPTokenListView: View { } private var deleteAlertTitle: String { - String(localized: "Delete token?") + deleteCandidates.count > 1 + ? String(format: String(localized: "Delete %d tokens?"), deleteCandidates.count) + : String(localized: "Delete token?") + } + + private static func deleteAlertMessage(for candidates: [MCPAuthToken]) -> String { + guard candidates.count > 1 else { + return String( + format: String(localized: "“%@” will be permanently deleted. External clients using this token will lose access immediately."), + candidates.first?.name ?? "" + ) + } + return String(localized: "These tokens will be permanently deleted. External clients using them will lose access immediately.") } private var deleteAlertBinding: Binding { Binding( - get: { deleteCandidate != nil }, + get: { !deleteCandidates.isEmpty }, set: { isPresented in if !isPresented { - deleteCandidate = nil + deleteCandidates = [] } } ) diff --git a/apple/TablePro/Views/Settings/Sections/MCPTokenRevealSheet.swift b/apple/TablePro/Views/Settings/Sections/MCPTokenRevealSheet.swift index 9c2c59ab94..7dd6d41913 100644 --- a/apple/TablePro/Views/Settings/Sections/MCPTokenRevealSheet.swift +++ b/apple/TablePro/Views/Settings/Sections/MCPTokenRevealSheet.swift @@ -2,14 +2,24 @@ import AppKit import SwiftUI struct MCPTokenRevealSheet: View { - let token: MCPAuthToken let plaintext: String let port: Int + let configuredPort: Int @Environment(\.dismiss) private var dismiss @State private var isTokenRevealed = false - @State private var selectedClient: IntegrationClient = .claudeCode - @FocusState private var doneFocused: Bool + @State private var selectedClientID: String + + private let setup: MCPHTTPClientSetup + + init(plaintext: String, port: Int, configuredPort: Int) { + self.plaintext = plaintext + self.port = port + self.configuredPort = configuredPort + let setup = MCPHTTPClientSetup(port: port) + self.setup = setup + _selectedClientID = State(initialValue: setup.choices.first?.id ?? MCPHTTPClientSetup.otherClientID) + } var body: some View { VStack(spacing: 0) { @@ -28,7 +38,6 @@ struct MCPTokenRevealSheet: View { Spacer() Button(String(localized: "Done")) { dismiss() } .keyboardShortcut(.defaultAction) - .focused($doneFocused) } .padding() } @@ -46,7 +55,7 @@ struct MCPTokenRevealSheet: View { .font(.headline) HStack(spacing: 8) { - Text(isTokenRevealed ? plaintext : maskedToken) + Text(verbatim: displayedToken) .font(.system(.body, design: .monospaced)) .textSelection(.enabled) .lineLimit(1) @@ -71,74 +80,84 @@ struct MCPTokenRevealSheet: View { .clipShape(RoundedRectangle(cornerRadius: 6)) Button { - copyToClipboard(plaintext) + ClipboardService.shared.writeSecretText(plaintext) + AccessibilityAnnouncement.post(String(localized: "Copied to clipboard")) } label: { Label(String(localized: "Copy Token"), systemImage: "doc.on.doc") } - .buttonStyle(.borderedProminent) .accessibilityLabel(String(localized: "Copy token")) } } - private var maskedToken: String { - String(plaintext.prefix(8)) + String(repeating: "\u{2022}", count: 24) - } - private var setupInstructions: some View { VStack(alignment: .leading, spacing: 12) { Text(String(localized: "Setup Instructions")) .font(.headline) - Picker(String(localized: "Client"), selection: $selectedClient) { - ForEach(IntegrationClient.allCases) { client in - Text(client.displayName).tag(client) + Picker(String(localized: "Client"), selection: $selectedClientID) { + ForEach(setup.choices) { choice in + Text(verbatim: choice.name).tag(choice.id) } } - .labelsHidden() - .pickerStyle(.segmented) + .pickerStyle(.menu) + .fixedSize() + .accessibilityIdentifier("mcp-token-client-picker") + + if port != configuredPort { + NoticeLabel(String( + format: String(localized: "Port %d was in use, so this address changes when TablePro restarts."), + configuredPort + )) + } - CopyableCodeBlock(text: configSnippet(for: selectedClient), isSecret: true) + if selectedClientID == MCPHTTPClientSetup.otherClientID { + otherClientInstructions + } else { + clientInstructions + } } } - private var baseURL: String { - let scheme = "http" - return "\(scheme)://127.0.0.1:\(port)/mcp" - } - - private func configSnippet(for client: IntegrationClient) -> String { - switch client { - case .claudeCode: - return "claude mcp add tablepro --transport http \(baseURL) --header \"Authorization: Bearer \(plaintext)\"" - case .claudeDesktop, .cursor: - return """ - { - "mcpServers": { - "tablepro": { - "url": "\(baseURL)", - "headers": { - "Authorization": "Bearer \(plaintext)" - } - } - } + @ViewBuilder + private var clientInstructions: some View { + if let copied = setup.render(clientID: selectedClientID, token: plaintext), + let shown = setup.render(clientID: selectedClientID, token: displayedToken) { + if let commandLine = copied.commandLine { + Text(String(localized: "Run this in Terminal:")) + CopyableCodeBlock(text: commandLine, displayText: shown.commandLine, isSecret: true) } - """ - case .zed: - return """ - { - "tablepro": { - "url": "\(baseURL)", - "headers": { - "Authorization": "Bearer \(plaintext)" + if let snippet = copied.snippet { + if let path = copied.displayConfigPath { + Text(String(format: String(localized: "Add this to %@:"), path)) } - } + CopyableCodeBlock(text: snippet, displayText: shown.snippet, isSecret: true) } - """ + } else { + otherClientInstructions + } + } + + private var otherClientInstructions: some View { + VStack(alignment: .leading, spacing: 8) { + Text(String(localized: "Give your client this address and header:")) + CopyableCodeBlock(text: setup.url) + CopyableCodeBlock( + text: header(token: plaintext), + displayText: header(token: displayedToken), + isSecret: true + ) } } - private func copyToClipboard(_ text: String) { - ClipboardService.shared.writeSecretText(text) - AccessibilityAnnouncement.post(String(localized: "Copied to clipboard")) + private func header(token: String) -> String { + "Authorization: " + MCPHTTPClientSetup.authorization(token: token) + } + + private var displayedToken: String { + isTokenRevealed ? plaintext : maskedToken + } + + private var maskedToken: String { + String(plaintext.prefix(8)) + String(repeating: "\u{2022}", count: 24) } } diff --git a/apple/TablePro/Views/Settings/Sections/PairingApprovalSheet.swift b/apple/TablePro/Views/Settings/Sections/PairingApprovalSheet.swift index 08ed74db2d..501dd6e6b9 100644 --- a/apple/TablePro/Views/Settings/Sections/PairingApprovalSheet.swift +++ b/apple/TablePro/Views/Settings/Sections/PairingApprovalSheet.swift @@ -56,12 +56,7 @@ struct PairingApprovalSheet: View { expirySection } .formStyle(.grouped) - /// The one greedy child, and the reason the sheet needs a bound at all. A grouped - /// `Form` is a scroll container, so it answers any proposal with the whole proposed - /// height: unbounded, it measured `CGFloat.greatestFiniteMagnitude`, and the window - /// sized from that raised. Capping it here rather than fixing the sheet's own height - /// keeps the rest of the sheet free to grow with the text size. The sections scroll - /// inside the cap. (#2930) + // A grouped Form answers any proposal with the whole height, so it alone is capped (#2930). .frame(maxHeight: 420) Divider() @@ -81,8 +76,7 @@ struct PairingApprovalSheet: View { private var header: some View { VStack(alignment: .leading, spacing: 6) { - /// The name is whatever the request asked to be called, so it is bounded here rather - /// than trusted to be short enough to lay out. + // The name is whatever the request asked to be called, so its length is bounded here. Text(String(format: String(localized: "Allow %@ to access TablePro?"), request.clientName)) .font(.headline) .lineLimit(2) @@ -97,10 +91,7 @@ struct PairingApprovalSheet: View { .padding() } - /// Where the one-time code is delivered, which is the only part of the request that says who - /// actually receives the grant. The client name above is a label the request chose for itself, - /// so approving on that alone approves a destination nobody was shown. Truncated in the middle - /// rather than the tail, because the scheme and the host are what a reader checks. + // The client name is the request's own label; this says who receives the grant, so it keeps the scheme and host. private var destinationLabel: some View { HStack(spacing: 6) { Image(systemName: "arrow.up.forward.app") @@ -122,7 +113,7 @@ struct PairingApprovalSheet: View { .foregroundStyle(isExpired ? .red : Color(nsColor: .secondaryLabelColor)) .imageScale(.small) .accessibilityHidden(true) - Text(countdownText) + countdownText .font(.caption) .monospacedDigit() .foregroundStyle(.secondary) @@ -139,11 +130,11 @@ struct PairingApprovalSheet: View { remainingSeconds <= 0 } - private var countdownText: String { + private var countdownText: Text { if isExpired { - return String(localized: "Code expired") + return Text(String(localized: "Code expired")) } - return String(format: String(localized: "Code expires in %d seconds"), remainingSeconds) + return Text("Code expires in ^[\(remainingSeconds) second](inflect: true)") } private var permissionsSection: some View { @@ -234,8 +225,7 @@ struct PairingApprovalSheet: View { request.scopeRequest.optionalGrants.contains(.connectionsDisplay) } - /// Starts off even when the setting allows it: an agent with a shell can open a pairing link to - /// its own loopback redirect, and one click on Approve must not hand it these connections. + // Off even when allowed: an agent with a shell can pair to its own redirect, and one Approve must not hand it these. private var hiddenConnectionsSection: some View { Section(String(localized: "Connections Hidden from AI")) { if allowsHiddenConnectionListing { diff --git a/apple/TablePro/Views/Settings/Sections/SyncSection.swift b/apple/TablePro/Views/Settings/Sections/SyncSection.swift index 5d665fa927..ba96e48ba2 100644 --- a/apple/TablePro/Views/Settings/Sections/SyncSection.swift +++ b/apple/TablePro/Views/Settings/Sections/SyncSection.swift @@ -18,14 +18,6 @@ struct SyncSection: View { var body: some View { Section { Toggle("Sync this Mac with iCloud", isOn: $settingsManager.sync.enabled) - .onChange(of: settingsManager.sync.enabled) { newValue in - updatePasswordSyncFlag() - if newValue { - syncCoordinator.enableSync() - } else { - syncCoordinator.disableSync() - } - } .help("Syncs connections, table favorites, settings, and SSH profiles across your Macs via iCloud.") .disabled(!isProAvailable) @@ -92,15 +84,11 @@ struct SyncSection: View { .onChange(of: settingsManager.sync.syncConnections) { newValue in if !newValue, settingsManager.sync.syncPasswords { settingsManager.sync.syncPasswords = false - onPasswordSyncChanged(false) } } if settingsManager.sync.syncConnections { Toggle("Passwords", isOn: $settingsManager.sync.syncPasswords) - .onChange(of: settingsManager.sync.syncPasswords) { newValue in - onPasswordSyncChanged(newValue) - } .help("Syncs passwords via iCloud Keychain (end-to-end encrypted).") .padding(.leading, 20) @@ -121,17 +109,4 @@ struct SyncSection: View { Toggle("Saved Queries", isOn: $settingsManager.sync.syncSQLFavorites) } } - - // MARK: - Helpers - - private func onPasswordSyncChanged(_ enabled: Bool) { - let effective = settingsManager.sync.enabled && settingsManager.sync.syncConnections && enabled - AppStorageEnvironment.shared.defaults.set(effective, forKey: KeychainHelper.passwordSyncEnabledKey) - } - - private func updatePasswordSyncFlag() { - let sync = settingsManager.sync - let effective = sync.enabled && sync.syncConnections && sync.syncPasswords - AppStorageEnvironment.shared.defaults.set(effective, forKey: KeychainHelper.passwordSyncEnabledKey) - } } diff --git a/apple/TablePro/Views/Settings/Sections/TrustedExternalConnectionsSection.swift b/apple/TablePro/Views/Settings/Sections/TrustedExternalConnectionsSection.swift index 6a7d0de63b..e0a1dc3802 100644 --- a/apple/TablePro/Views/Settings/Sections/TrustedExternalConnectionsSection.swift +++ b/apple/TablePro/Views/Settings/Sections/TrustedExternalConnectionsSection.swift @@ -21,11 +21,16 @@ struct TrustedExternalConnectionsSection: View { .foregroundStyle(.secondary) } else { ForEach(entries) { entry in - LabeledContent(entry.key.displayDescription) { + LabeledContent { Button("Forget") { store.revoke(entry.key) refresh() } + } label: { + Text(entry.key.displayDescription) + if let name = savedConnectionName(of: entry) { + Text(String(format: String(localized: "Saved connection “%@”"), name)) + } } } @@ -42,6 +47,11 @@ struct TrustedExternalConnectionsSection: View { .onAppear(perform: refresh) } + // Two saved connections can share type, host, database and user, so the name tells their entries apart. + private func savedConnectionName(of entry: TrustedExternalConnection) -> String? { + entry.key.savedConnectionId.flatMap { ConnectionStorage.shared.loadConnection(id: $0)?.name } + } + private func refresh() { entries = store.entries().sorted { $0.trustedAt > $1.trustedAt } } diff --git a/apple/TablePro/Views/Settings/SettingsWindowController.swift b/apple/TablePro/Views/Settings/SettingsWindowController.swift index aa23cfa15c..d4a54f321f 100644 --- a/apple/TablePro/Views/Settings/SettingsWindowController.swift +++ b/apple/TablePro/Views/Settings/SettingsWindowController.swift @@ -7,9 +7,7 @@ import AppKit import os import SwiftUI -/// Hosts the settings panes in an AppKit window so the app no longer needs a SwiftUI `Settings` -/// scene. One instance is kept for the app's lifetime, matching the single-window scene it -/// replaces. +// An AppKit window, so the app needs no SwiftUI Settings scene. @MainActor internal final class SettingsWindowController: NSWindowController { private static var shared: SettingsWindowController? @@ -35,9 +33,7 @@ internal final class SettingsWindowController: NSWindowController { internal convenience init(registryClient: RegistryClient) { let panes = SettingsPaneTabViewController(registryClient: registryClient) - /// `NSWindow(contentViewController:)` binds the window title to the content view - /// controller's own title, so the selected pane publishes the title and nothing writes - /// `window.title` directly. + // NSWindow(contentViewController:) binds the title to the panes controller's, so nothing writes window.title. let window = NSWindow(contentViewController: panes) window.identifier = NSUserInterfaceItemIdentifier(WindowIdentifier.settings) window.keepsKeyViewLoopCurrent() @@ -47,8 +43,7 @@ internal final class SettingsWindowController: NSWindowController { window.contentMinSize = SettingsPaneTabViewController.paneSize window.setContentSize(SettingsPaneTabViewController.paneSize) window.applyAutosaveName(WindowIdentifier.settings) - /// `contentMinSize` only constrains a user drag, so a frame saved before the window had - /// a minimum is grown back to the pane minimum on restore. + // contentMinSize only limits a drag, so a frame saved before the window had a minimum is grown back here. window.setContentSize( NSSize( width: max(window.contentLayoutRect.width, SettingsPaneTabViewController.paneSize.width), @@ -60,8 +55,7 @@ internal final class SettingsWindowController: NSWindowController { } internal final class SettingsPaneTabViewController: NSTabViewController { - /// Wide enough for twelve items in a `.preference` toolbar. At 720 the last of them - /// falls into an overflow chevron, which is where a pane nobody can find comes from. + // Fits twelve items in a .preference toolbar; at 720 the last one hides behind an overflow chevron. internal static let paneSize = NSSize(width: 800, height: 500) internal static let paneOrder: [SettingsPane] = SettingsPane.allCases @@ -85,8 +79,7 @@ internal final class SettingsPaneTabViewController: NSTabViewController { } internal func select(_ pane: SettingsPane?) { - /// `loadViewIfNeeded()` is macOS 14. Reading `view` is what it does: the getter loads - /// the view when it has not been loaded yet. + // Before macOS 14, reading view loads it, which is all loadViewIfNeeded() does. if #available(macOS 14.0, *) { loadViewIfNeeded() } else { @@ -103,8 +96,7 @@ internal final class SettingsPaneTabViewController: NSTabViewController { override func viewDidLoad() { super.viewDidLoad() tabStyle = .toolbar - /// Adding the first item selects it, which persists `.general` over the pane a caller - /// asked for, so the pane to restore is read while no item exists yet. + // Adding the first item selects and persists .general, so the saved pane is read before any item exists. let restored = persistedPane for pane in Self.paneOrder { addTabViewItem(makeTabViewItem(for: pane)) @@ -129,9 +121,7 @@ internal final class SettingsPaneTabViewController: NSTabViewController { ) .environment(\.appServices, .live) let hosting = NSHostingController(rootView: content) - /// A tab child publishes no size to the window, which owns its minimum through - /// `contentMinSize`. Its title does travel: `NSTabViewController` mirrors the selected - /// child's title into its own, and the window's title is bound to that. + // The window owns the minimum size; the title still reaches it, as NSTabViewController mirrors the selected child's. hosting.sizingOptions = [] hosting.title = pane.title @@ -174,6 +164,7 @@ private struct SettingsPaneContent: View { case .data: DataResultsSettingsView( dataGrid: $settingsManager.dataGrid, + valueColors: $settingsManager.valueColors, history: $settingsManager.history, editor: $settingsManager.editor, typography: $settingsManager.typography diff --git a/apple/TablePro/Views/Settings/SyncSettingsView.swift b/apple/TablePro/Views/Settings/SyncSettingsView.swift index 6e4dceb9f4..c71657a438 100644 --- a/apple/TablePro/Views/Settings/SyncSettingsView.swift +++ b/apple/TablePro/Views/Settings/SyncSettingsView.swift @@ -89,7 +89,7 @@ private struct SyncNoticeSection: View { Task { await SyncCoordinator.shared.uploadAgain() } } case .turnOffSync: - // Through the setting, so the switch's own change handler also settles password sync. + // Through the setting, which also turns password sync off and stops the coordinator. Button(action.title) { AppSettingsManager.shared.sync.enabled = false } diff --git a/apple/TablePro/Views/Shared/FieldEditors/TextValueEditor.swift b/apple/TablePro/Views/Shared/FieldEditors/TextValueEditor.swift index 93bf43bc6c..5a3997dd4c 100644 --- a/apple/TablePro/Views/Shared/FieldEditors/TextValueEditor.swift +++ b/apple/TablePro/Views/Shared/FieldEditors/TextValueEditor.swift @@ -22,6 +22,8 @@ internal struct TextValueEditor: NSViewRepresentable { var textContainerInset = NSSize(width: 4, height: 6) var style: Style = .plain var marksInvisibleCharacters = false + // A new one is another document: the view shows it from the top, with no undo left from the one before. + var documentID: AnyHashable? func makeNSView(context: Context) -> NSScrollView { let scrollView = NSTextView.scrollableTextView() @@ -35,6 +37,7 @@ internal struct TextValueEditor: NSViewRepresentable { textView.string = text context.coordinator.lastAppliedText = text + context.coordinator.documentID = documentID return scrollView } @@ -42,6 +45,11 @@ internal struct TextValueEditor: NSViewRepresentable { guard let textView = scrollView.documentView as? NSTextView else { return } applyConfiguration(to: scrollView, textView: textView, coordinator: context.coordinator) + if context.coordinator.documentID != documentID { + context.coordinator.documentID = documentID + openDocument(in: textView, coordinator: context.coordinator) + return + } guard context.coordinator.lastAppliedText != text else { return } context.coordinator.lastAppliedText = text @@ -51,6 +59,21 @@ internal struct TextValueEditor: NSViewRepresentable { textView.setSelectedRange(NSRange(location: clamped, length: 0)) } + // The typing undo indexes into the text being replaced, and undoing it afterwards raises (measured). NSTextView files + // it under its text storage, so only this view's history goes. + private func openDocument(in textView: NSTextView, coordinator: Coordinator) { + textView.breakUndoCoalescing() + if let storage = textView.textStorage { + textView.undoManager?.removeAllActions(withTarget: storage) + } + if coordinator.lastAppliedText != text { + coordinator.lastAppliedText = text + textView.string = text + } + textView.setSelectedRange(NSRange(location: 0, length: 0)) + textView.scroll(.zero) + } + func makeCoordinator() -> Coordinator { Coordinator(text: $text, movesFocusOnTab: movesFocusOnTab) } @@ -123,6 +146,7 @@ internal struct TextValueEditor: NSViewRepresentable { /// moved" without bridging the whole NSString back, which costs 45ms on a 1 MB value and /// would run on every SwiftUI update pass. var lastAppliedText: String = "" + var documentID: AnyHashable? var themeStyle: ThemeTextViewStyle? var invisibleCharacters: InvisibleCharacterTextDisplay? diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeCellView.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeCellView.swift index e30599e652..32fbe661df 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeCellView.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeCellView.swift @@ -40,6 +40,8 @@ final class DatabaseTreeCellView: RenamableSidebarCellView return metadata.isSystemDatabase ? "gearshape" : "cylinder" case .schema, .tableFolder: return "folder" + case .serverObject(let ref): + return ref.kind.iconName case .routine, .trigger, .userType, .status, .recentSection, .temporarySection, .foldersSection, .objectKindSection, .containerObjectKindSection, .hierarchicalSchemaSection, .redisKeysSection, .redisNode: return "tablecells" diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeDoubleClickIntent.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeDoubleClickIntent.swift index b1808d59ad..dae589edb4 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeDoubleClickIntent.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeDoubleClickIntent.swift @@ -23,9 +23,7 @@ internal enum DatabaseTreeDoubleClickIntent: Equatable { } internal enum DatabaseTreeDoubleClickResolver { - /// Being a table wins over being expandable. A partitioned table is both, and the HIG puts - /// disclosure on the triangle and the Right and Left arrows, so the row's own double-click is - /// free to mean what it means everywhere else: open the thing you clicked. + // Being a table wins over being expandable: the HIG puts disclosure on the triangle and the arrow keys. internal static func resolve(node: DatabaseTreeNode) -> DatabaseTreeDoubleClickIntent { if let ref = DatabaseTreeSelection.tableRef(of: node) { return .openPermanently(ref) @@ -37,6 +35,8 @@ internal enum DatabaseTreeDoubleClickResolver { return .openObjectSource(ref.objectRef) case .userType(let ref): return .openObjectSource(ref.objectRef) + case .serverObject(let ref): + return .openObjectSource(ref.objectRef) default: return node.isExpandable ? .toggleDisclosure : .ignore } diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeFilter.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeFilter.swift index adbf4d9282..a280e732b0 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeFilter.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeFilter.swift @@ -37,6 +37,7 @@ struct DatabaseTreeBucketSource: Equatable { let routines: SameRows let triggers: SameRows let userTypes: SameRows + let serverObjects: [String: SameRows] let searchText: String let database: String? @@ -45,6 +46,7 @@ struct DatabaseTreeBucketSource: Equatable { routines: [RoutineInfo], triggers: [TriggerInfo], userTypes: [UserDefinedTypeInfo], + serverObjects: [String: [ServerObjectInfo]] = [:], searchText: String, database: String? ) { @@ -52,6 +54,7 @@ struct DatabaseTreeBucketSource: Equatable { self.routines = SameRows(rows: routines) self.triggers = SameRows(rows: triggers) self.userTypes = SameRows(rows: userTypes) + self.serverObjects = serverObjects.mapValues { SameRows(rows: $0) } self.searchText = searchText self.database = database } @@ -62,6 +65,7 @@ struct DatabaseTreeBucketSource: Equatable { routines: routines.rows, triggers: triggers.rows, userTypes: userTypes.rows, + serverObjects: serverObjects.mapValues(\.rows), searchText: searchText, database: database ) @@ -87,25 +91,27 @@ struct DatabaseTreeBucketMemo { } } -/// One filtered pass over a container's objects, split by kind. The container row needs the counts -/// and each group row under it needs one bucket, so both read this rather than filtering the whole -/// list again per group. +// One filtered pass per container, read by the container row and every group under it. struct DatabaseTreeObjectBuckets { let tables: [SidebarObjectKind: [TableInfo]] let routines: [SidebarObjectKind: [RoutineInfo]] let triggers: [TriggerInfo] let userTypes: [UserDefinedTypeInfo] + // Keyed by the declared kind's id. + let serverObjects: [String: [ServerObjectInfo]] init( tables: [SidebarObjectKind: [TableInfo]], routines: [SidebarObjectKind: [RoutineInfo]], triggers: [TriggerInfo], - userTypes: [UserDefinedTypeInfo] = [] + userTypes: [UserDefinedTypeInfo] = [], + serverObjects: [String: [ServerObjectInfo]] = [:] ) { self.tables = tables self.routines = routines self.triggers = triggers self.userTypes = userTypes + self.serverObjects = serverObjects } var isEmpty: Bool { @@ -113,8 +119,10 @@ struct DatabaseTreeObjectBuckets { && routines.values.allSatisfy(\.isEmpty) && triggers.isEmpty && userTypes.isEmpty + && serverObjects.values.allSatisfy(\.isEmpty) } + // A declared kind compares by id, so the key built here finds the declared descriptor. var itemCounts: [SidebarObjectKind: Int] { var counts = tables.mapValues(\.count) for (kind, list) in routines { @@ -124,6 +132,9 @@ struct DatabaseTreeObjectBuckets { if !userTypes.isEmpty { counts[.type, default: 0] += userTypes.count } + for (kindId, list) in serverObjects where !list.isEmpty { + counts[.server(SidebarServerObjectKind(id: kindId)), default: 0] += list.count + } return counts } } @@ -230,11 +241,21 @@ enum DatabaseTreeFilter { return deduplicated(matched, by: \.id) } + static func filteredServerObjects( + _ objects: [ServerObjectInfo], + searchText: String, + database: String? = nil + ) -> [ServerObjectInfo] { + let matched = SidebarSearch(searchText).ranked(objects, name: { $0.name }, database: database, schema: { $0.schema }) + return deduplicated(matched, by: \.id) + } + static func objectBuckets( tables: [TableInfo], routines: [RoutineInfo], triggers: [TriggerInfo], userTypes: [UserDefinedTypeInfo] = [], + serverObjects: [String: [ServerObjectInfo]] = [:], searchText: String, database: String? = nil ) -> DatabaseTreeObjectBuckets { @@ -250,7 +271,10 @@ enum DatabaseTreeFilter { tables: tableBuckets, routines: routineBuckets, triggers: filteredTriggers(triggers, searchText: searchText, database: database), - userTypes: filteredUserTypes(userTypes, searchText: searchText, database: database) + userTypes: filteredUserTypes(userTypes, searchText: searchText, database: database), + serverObjects: serverObjects.mapValues { + filteredServerObjects($0, searchText: searchText, database: database) + } ) } @@ -310,6 +334,7 @@ enum DatabaseTreeFilter { routines: service.routines(for: connectionId, schema: schema), triggers: service.triggers(for: connectionId, schema: schema), userTypes: service.userDefinedTypes(for: connectionId, schema: schema), + serverObjects: service.serverObjectsByKind(for: connectionId, schema: schema), searchText: searchText, database: database )), @@ -324,14 +349,14 @@ enum DatabaseTreeFilter { return kept } - /// A schema the search matched by name shows everything inside it. Filtering its objects by the - /// same query leaves the matched schema reporting no items. + // A schema matched by name shows everything inside it, or it would report no items. static func hierarchicalBucketSource( schema: String, tables: [TableInfo], routines: [RoutineInfo], triggers: [TriggerInfo], userTypes: [UserDefinedTypeInfo], + serverObjects: [String: [ServerObjectInfo]] = [:], searchText: String, database: String? = nil ) -> DatabaseTreeBucketSource { @@ -341,6 +366,7 @@ enum DatabaseTreeFilter { routines: routines, triggers: triggers, userTypes: userTypes, + serverObjects: serverObjects, searchText: matchesSchema ? "" : searchText, database: database ) @@ -557,11 +583,15 @@ enum DatabaseTreeFilter { !service.isTablesCurrent(connectionId: connectionId, database: database, schema: schema) { tables = listingMatches.rowsBySchema[schema] ?? [] } + let serverKinds = service.serverKinds(connectionId: connectionId, database: database, schema: schema) return objectBuckets( tables: tables, routines: service.routines(connectionId: connectionId, database: database, schema: schema), triggers: service.triggers(connectionId: connectionId, database: database, schema: schema), userTypes: service.userDefinedTypes(connectionId: connectionId, database: database, schema: schema), + serverObjects: Dictionary(uniqueKeysWithValues: serverKinds.map { kind in + (kind.id, service.serverObjects(connectionId: connectionId, database: database, schema: schema, kindId: kind.id)) + }), searchText: searchText, database: database ) diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeNode.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeNode.swift index 0a8aaa293c..a8957add7d 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeNode.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeNode.swift @@ -5,16 +5,16 @@ import Foundation -/// Which groups a container shows. A group stands for objects the container holds, and a container -/// with nothing to list answers with its own status row instead. +// A group stands for objects the container holds; a container with nothing to list shows its own status row. internal enum DatabaseTreeObjectGroupResolver { internal static func groups( database: String, schema: String?, itemCounts: [SidebarObjectKind: Int], - keeping: Set = [] + keeping: Set = [], + declared: [SidebarServerObjectKind] = [] ) -> [DatabaseTreeObjectGroup] { - SidebarObjectKind.visible(itemCounts: itemCounts, keeping: keeping) + SidebarObjectKind.visible(itemCounts: itemCounts, keeping: keeping, declared: declared) .map { DatabaseTreeObjectGroup(database: database, schema: schema, kind: $0) } } } @@ -31,36 +31,48 @@ internal extension DatabaseTreeNode.Status { } } -/// Where each side kind's fetch stands for one container. A kind the engine does not declare has no -/// phase, because a fetch the engine never runs stays idle and would read as loading forever. +// An undeclared kind has no phase: a fetch the engine never runs stays idle and would read as loading forever. internal struct DatabaseTreeSidePhases: Equatable { let routines: MetadataLoadPhase let triggers: MetadataLoadPhase? let types: MetadataLoadPhase? + // Keyed by kind id, holding only the declared kinds this container lists. + let serverObjects: [String: MetadataLoadPhase] + + init( + routines: MetadataLoadPhase, + triggers: MetadataLoadPhase?, + types: MetadataLoadPhase?, + serverObjects: [String: MetadataLoadPhase] = [:] + ) { + self.routines = routines + self.triggers = triggers + self.types = types + self.serverObjects = serverObjects + } var all: [MetadataLoadPhase] { - [routines] + [triggers, types].compactMap { $0 } + [routines] + [triggers, types].compactMap { $0 } + serverObjects.keys.sorted().compactMap { serverObjects[$0] } } - func phase(for category: SidebarObjectCategory) -> MetadataLoadPhase { - switch category { + func phase(for kind: SidebarObjectKind) -> MetadataLoadPhase { + switch kind.category { case .table: return .loaded case .routine: return routines case .trigger: return triggers ?? .loaded case .type: return types ?? .loaded + case .server: return kind.serverKind.flatMap { serverObjects[$0.id] } ?? .loaded } } - /// The declared kinds whose fetch failed. Each keeps its group, empty of rows, so the failure shows - /// where the rows would be: hiding the group would make a list the server refused read as empty. + // A failed kind keeps its group, so a list the server refused never reads as empty. func failedKinds(among declared: Set) -> Set { - declared.filter { phase(for: $0.category).failureMessage != nil } + declared.filter { phase(for: $0).failureMessage != nil } } - /// The row a group with no rows shows: the failure, or nothing, since a group without rows is - /// listed only to show a failure or to take filed objects back out of their folders. + // A group without rows is listed only for a failure or to take filed objects back out of their folders. func placeholder(for kind: SidebarObjectKind) -> DatabaseTreeNode.Status? { - phase(for: kind.category).failureMessage.map(DatabaseTreeNode.Status.error) + phase(for: kind).failureMessage.map(DatabaseTreeNode.Status.error) } } @@ -94,6 +106,7 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { case routine(DatabaseTreeRoutineRef) case trigger(DatabaseTreeTriggerRef) case userType(DatabaseTreeUserTypeRef) + case serverObject(DatabaseTreeServerObjectRef) case status(Status) /// Flat shape: one collapsible section per object kind. @@ -128,7 +141,7 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { case .redisNode(let node): guard case .namespace = node else { return false } return true - case .recentTable, .temporaryTable, .routine, .trigger, .userType, .status: + case .recentTable, .temporaryTable, .routine, .trigger, .userType, .serverObject, .status: return false } } @@ -156,19 +169,14 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { return nil } - /// A source-list group row: chrome the app invented to bucket objects, not an object the - /// database has. AppKit draws these itself once `isGroupItem` says so, and it stops indenting - /// their children, which is what puts a table at the same depth as a database in the tree. - /// - /// A schema is deliberately not one. It is a real object with its own menu and its own - /// children, so it stays an ordinary container row the way a folder does in Xcode's navigator. + // Chrome the app invented to bucket objects; a schema is a real object, so it stays an ordinary container row. var isGroupRow: Bool { switch kind { case .recentSection, .temporarySection, .foldersSection, .objectKindSection, .redisKeysSection: return true case .database, .schema, .containerObjectKindSection, .tableFolder, .hierarchicalSchemaSection, .recentTable, .temporaryTable, .table, .partition, - .routine, .trigger, .userType, .status, .redisNode: + .routine, .trigger, .userType, .serverObject, .status, .redisNode: return false } } @@ -178,7 +186,7 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { case .database, .schema: return true case .recentSection, .recentTable, .temporarySection, .temporaryTable, .foldersSection, .tableFolder, - .table, .partition, .routine, .trigger, .userType, .status, .objectKindSection, + .table, .partition, .routine, .trigger, .userType, .serverObject, .status, .objectKindSection, .containerObjectKindSection, .hierarchicalSchemaSection, .redisKeysSection, .redisNode: return false } @@ -191,7 +199,7 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { case .schema(let database, let schema): return .schema(database: database, schema: schema, isSystem: systemSchemas.contains(schema)) case .recentSection, .recentTable, .temporarySection, .temporaryTable, .foldersSection, .tableFolder, - .table, .partition, .routine, .trigger, .userType, .status, .objectKindSection, + .table, .partition, .routine, .trigger, .userType, .serverObject, .status, .objectKindSection, .containerObjectKindSection, .hierarchicalSchemaSection, .redisKeysSection, .redisNode: return nil } @@ -256,6 +264,7 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { static func routineId(_ ref: DatabaseTreeRoutineRef) -> String { "routine\u{1}\(ref.id)" } static func triggerId(_ ref: DatabaseTreeTriggerRef) -> String { "trigger\u{1}\(ref.id)" } static func userTypeId(_ ref: DatabaseTreeUserTypeRef) -> String { "usertype\u{1}\(ref.id)" } + static func serverObjectId(_ ref: DatabaseTreeServerObjectRef) -> String { "serverobject\u{1}\(ref.id)" } static func statusId(parentId: String, status: Status) -> String { switch status { case .loading: return "\(parentId)\u{1}status.loading" @@ -265,9 +274,9 @@ final class DatabaseTreeNode: NSObject, SidebarOutlineNode { } } - static func objectKindSectionId(_ kind: SidebarObjectKind) -> String { "kindSection\u{1}\(kind.rawValue)" } + static func objectKindSectionId(_ kind: SidebarObjectKind) -> String { "kindSection\u{1}\(kind.persistenceName)" } static func containerObjectKindSectionId(_ group: DatabaseTreeObjectGroup) -> String { - "containerKindSection\u{1}\(group.database)\u{1}\(group.schema ?? "")\u{1}\(group.kind.rawValue)" + "containerKindSection\u{1}\(group.database)\u{1}\(group.schema ?? "")\u{1}\(group.kind.persistenceName)" } static func hierarchicalSchemaSectionId(_ schema: String) -> String { "hschema\u{1}\(schema)" } static let redisKeysSectionId = "redis-keys-section" diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift index 3d05770683..dd17344ce9 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift @@ -89,11 +89,13 @@ extension DatabaseTreeOutlineCoordinator { } case .truncateTables(let targets, let ref): activateThen(ref) { [weak self] in - self?.viewModel?.batchToggleTruncate(refs: targets) + guard let self, let windowState = self.windowState else { return } + self.viewModel?.batchToggleTruncate(refs: targets, from: windowState) } case .dropTables(let targets, let ref): activateThen(ref) { [weak self] in - self?.viewModel?.batchToggleDelete(refs: targets) + guard let self, let windowState = self.windowState else { return } + self.viewModel?.batchToggleDelete(refs: targets, from: windowState) } case .beginRenameTable(let ref, let isRecentRow): activateThen(ref) { [weak self] in diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Expansion.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Expansion.swift index c161f342db..e591b4ba23 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Expansion.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Expansion.swift @@ -87,6 +87,7 @@ extension DatabaseTreeOutlineCoordinator { restoreObjectGroupExpansion(under: schemaNode) } } + restoreObjectGroupExpansion(under: databaseNode) } } @@ -182,24 +183,21 @@ extension DatabaseTreeOutlineCoordinator { } } - /// The cascade only re-asserts disclosure the store already holds, so it runs behind the flag - /// `applyDesiredExpansion` uses. Recording each row it opens rewrote the whole expansion - /// snapshot to `UserDefaults`, and re-entered this restore once per row. + // Behind the `applyDesiredExpansion` flag: recording each row rewrote the expansion snapshot and re-entered this. internal func restoreDescendantExpansion(afterExpanding node: DatabaseTreeNode) { let wasApplyingExpansion = isApplyingExpansion isApplyingExpansion = true defer { isApplyingExpansion = wasApplyingExpansion } switch node.kind { - case .database where !supportsSchemaLevel: - restoreObjectGroupExpansion(under: node) - case .schema, .hierarchicalSchemaSection: + // A schema engine's database holds its `.database` kind groups beside the schemas. + case .database, .schema, .hierarchicalSchemaSection: restoreObjectGroupExpansion(under: node) case .foldersSection: restoreFolderExpansion(under: node) case .objectKindSection, .containerObjectKindSection, .tableFolder, .table, .partition: restorePartitionExpansion(under: node) - case .recentSection, .recentTable, .temporarySection, .temporaryTable, .database, .routine, .trigger, - .userType, .status, .redisKeysSection, .redisNode: + case .recentSection, .recentTable, .temporarySection, .temporaryTable, .routine, .trigger, + .userType, .serverObject, .status, .redisKeysSection, .redisNode: break } } @@ -257,7 +255,7 @@ extension DatabaseTreeOutlineCoordinator { case .partition(let ref): guard let tableRef = ref.tableRef else { break } recordTableExpansion(tableRef, expanded: expanded) - case .recentTable, .temporaryTable, .routine, .trigger, .userType, .status, .redisNode: + case .recentTable, .temporaryTable, .routine, .trigger, .userType, .serverObject, .status, .redisNode: break } } @@ -269,6 +267,7 @@ extension DatabaseTreeOutlineCoordinator { if service.schemaListNeedsFetch(connectionId: connectionId, database: metadata.name) { Task { await service.loadSchemas(connectionId: connectionId, database: metadata.name) } } + loadDatabaseServerObjects(database: metadata.name) loadExternalSchemaNames(database: metadata.name) } else { loadObjects(database: metadata.name, schema: nil) @@ -288,12 +287,18 @@ extension DatabaseTreeOutlineCoordinator { else { return } loadHierarchicalSchemaObjects(schema) case .recentSection, .recentTable, .temporarySection, .temporaryTable, .foldersSection, .tableFolder, - .routine, .trigger, .userType, .status, .objectKindSection, .containerObjectKindSection, + .routine, .trigger, .userType, .serverObject, .status, .objectKindSection, .containerObjectKindSection, .redisKeysSection, .redisNode: break } } + private func loadDatabaseServerObjects(database: String) { + guard service.serverObjectsNeedFetch(connectionId: connectionId, database: database, schema: nil) else { return } + let connectionId = connectionId + Task { await service.loadServerObjects(connectionId: connectionId, database: database, schema: nil) } + } + /// A schema the user has open keeps reading the lists a change overtook while a search runs; one /// the search opened shows the listing's tables instead. internal func isOpenedByUser(database: String, schema: String) -> Bool { diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+FolderNodes.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+FolderNodes.swift index 74b3fa8fa7..a7560cd73c 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+FolderNodes.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+FolderNodes.swift @@ -13,7 +13,7 @@ import TableProPluginKit /// its children. Either way an object sits in one row only, so a filed table is drawn inside its /// folder and left out of its kind section. extension DatabaseTreeOutlineCoordinator { - internal static let tableKinds = SidebarObjectKind.allCases.filter { $0.category == .table } + internal static let tableKinds = SidebarObjectKind.builtIn.filter { $0.category == .table } internal func folderScope(for container: TableFolderContainer) -> DatabaseScope? { switch container { diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Menu.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Menu.swift index d48854232f..b0bbb8a0b4 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Menu.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Menu.swift @@ -173,7 +173,7 @@ extension DatabaseTreeOutlineCoordinator: NSMenuDelegate { private func objectKindTitles() -> [SidebarObjectKind: String] { let tableEntityName = PluginManager.shared.tableEntityName(for: databaseType) var titles: [SidebarObjectKind: String] = [:] - for kind in SidebarObjectKind.allCases { + for kind in SidebarObjectKind.ordered(declared: declaredServerKinds) { titles[kind] = kind.title(tableEntityName: tableEntityName) } return titles diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Nodes.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Nodes.swift index 665e162416..191aa1ff86 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Nodes.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Nodes.swift @@ -58,7 +58,7 @@ extension DatabaseTreeOutlineCoordinator { return folderMemberNodes(of: ref) case .database(let metadata): return supportsSchemaLevel - ? schemaNodes(database: metadata.name) + ? databaseLevelNodes(database: metadata.name) : objectNodes(database: metadata.name, schema: nil) case .schema(let database, let schema): return objectNodes(database: database, schema: schema) @@ -78,7 +78,7 @@ extension DatabaseTreeOutlineCoordinator { return redisChildren(of: nil) case .redisNode(let redisNode): return redisChildren(of: redisNode) - case .recentTable, .temporaryTable, .routine, .trigger, .userType, .status: + case .recentTable, .temporaryTable, .routine, .trigger, .userType, .serverObject, .status: return [] } } @@ -208,16 +208,17 @@ extension DatabaseTreeOutlineCoordinator { .map { node(id: DatabaseTreeNode.schemaId(database: database, schema: $0), kind: .schema(database: database, schema: $0)) } } - /// A torn-down sidebar has no view model and lists nothing at all. Counting its way to a root of - /// failed sections would fill a pane that is on its way out. + // A torn-down sidebar has no view model and lists nothing, rather than a root of failed sections. private func visibleObjectKinds() -> [SidebarObjectKind] { guard viewModel != nil else { return [] } - let itemCounts = SidebarObjectKind.allCases.reduce(into: [SidebarObjectKind: Int]()) { + let declared = flatServerKinds + let itemCounts = SidebarObjectKind.ordered(declared: declared).reduce(into: [SidebarObjectKind: Int]()) { $0[$1] = flatItemCount(for: $1) } return SidebarObjectKind.visible( itemCounts: itemCounts, - keeping: flatPlan().filedKinds.union(flatSidePhases.failedKinds(among: declaredObjectKinds)) + keeping: flatPlan().filedKinds.union(flatSidePhases.failedKinds(among: declaredObjectKinds)), + declared: declared ) } @@ -228,10 +229,29 @@ extension DatabaseTreeOutlineCoordinator { return .emptyContainer(sideStates: flatSidePhases.all) } - /// What the engine says it fetches. It decides which fetches a container waits on before saying - /// it is empty, and which failures keep a section; it never lists a section on its own. + // Decides which fetches a container waits on and which failures keep a section; it never lists one on its own. internal var declaredObjectKinds: Set { - databaseType.declaredObjectKinds + databaseType.declaredObjectKinds.union(declaredServerKinds.map { SidebarObjectKind.server($0) }) + } + + // Asked of the live driver, which answers per server flavor and version. + internal var declaredServerKinds: [SidebarServerObjectKind] { + SidebarServerObjectKind.declared(by: DatabaseManager.shared.driver(for: connectionId)) + } + + // A hierarchical engine lists its kinds per schema, never in the browsed catalog. + internal var flatServerKinds: [SidebarServerObjectKind] { + PluginManager.shared.databaseGroupingStrategy(for: databaseType) == .hierarchicalSchema ? [] : declaredServerKinds + } + + private var containerServerKinds: [SidebarServerObjectKind] { + declaredServerKinds.filter { $0.scope == .container } + } + + // The tree reads the kinds of the container's own level; a schema-grouped shape lists `.container` kinds alone. + private func serverKinds(database: String, schema: String?) -> [SidebarServerObjectKind] { + if rootShape == .hierarchicalSchema { return containerServerKinds } + return service.serverKinds(connectionId: connectionId, database: database, schema: schema) } internal func flatItemCount(for kind: SidebarObjectKind) -> Int { @@ -245,6 +265,12 @@ extension DatabaseTreeOutlineCoordinator { return viewModel.filteredTriggers(from: schemaService.triggers(for: connectionId)).count case .type: return viewModel.filteredUserTypes(from: schemaService.userDefinedTypes(for: connectionId)).count + case .server: + guard let serverKind = kind.serverKind else { return 0 } + return viewModel.filteredServerObjects( + kindId: serverKind.id, + from: schemaService.serverObjects(for: connectionId, kindId: serverKind.id) + ).count } } @@ -258,14 +284,16 @@ extension DatabaseTreeOutlineCoordinator { return [statusNode(parentId: DatabaseTreeNode.objectKindSectionId(kind), status: placeholder)] } - /// Only the kinds the engine declares take part, the rule `sideLoadStates` follows for a tree - /// container, so a fetch the engine never runs cannot hold the root on a spinner. + // Declared kinds only, as `sideLoadStates`, so a fetch the engine never runs cannot hold the root on a spinner. private var flatSidePhases: DatabaseTreeSidePhases { let declared = declaredObjectKinds return DatabaseTreeSidePhases( routines: schemaService.routinesLoadState(for: connectionId).erased, triggers: declared.contains(.trigger) ? schemaService.triggersLoadState(for: connectionId).erased : nil, - types: declared.contains(.type) ? schemaService.userDefinedTypesLoadState(for: connectionId).erased : nil + types: declared.contains(.type) ? schemaService.userDefinedTypesLoadState(for: connectionId).erased : nil, + serverObjects: Dictionary(uniqueKeysWithValues: flatServerKinds.map { kind in + (kind.id, schemaService.serverObjectsLoadState(for: connectionId, kindId: kind.id).erased) + }) ) } @@ -295,6 +323,13 @@ extension DatabaseTreeOutlineCoordinator { let ref = DatabaseTreeUserTypeRef(database: database, schema: type.schema, type: type) return node(id: DatabaseTreeNode.userTypeId(ref), kind: .userType(ref)) } + case .server: + guard let serverKind = kind.serverKind else { return [] } + let objects = viewModel.filteredServerObjects( + kindId: serverKind.id, + from: schemaService.serverObjects(for: connectionId, kindId: serverKind.id) + ) + return serverObjectNodes(objects, kind: serverKind, database: database, schema: { $0.schema }) } } @@ -414,7 +449,10 @@ extension DatabaseTreeOutlineCoordinator { : nil, types: declared.contains(.type) ? schemaService.userDefinedTypesLoadState(for: connectionId, schema: schema).erased - : nil + : nil, + serverObjects: Dictionary(uniqueKeysWithValues: containerServerKinds.map { kind in + (kind.id, schemaService.serverObjectsLoadState(for: connectionId, schema: schema, kindId: kind.id).erased) + }) ) } @@ -435,6 +473,9 @@ extension DatabaseTreeOutlineCoordinator { routines: shown(schemaService.routines(for: connectionId, schema: schema)), triggers: shown(schemaService.triggers(for: connectionId, schema: schema)), userTypes: shown(schemaService.userDefinedTypes(for: connectionId, schema: schema)), + serverObjects: Dictionary(uniqueKeysWithValues: containerServerKinds.map { kind in + (kind.id, shown(schemaService.serverObjects(for: connectionId, schema: schema, kindId: kind.id))) + }), searchText: searchText, database: browsingDatabase ) @@ -536,6 +577,30 @@ extension DatabaseTreeOutlineCoordinator { DatabaseTreeMetadataService.listsTablesPerSchema(PluginManager.shared.databaseGroupingStrategy(for: databaseType)) } + // A schema engine's database lists its schemas, then the kinds it declares for the whole database. + private func databaseLevelNodes(database: String) -> [DatabaseTreeNode] { + let schemas = schemaNodes(database: database) + let groups = databaseScopedGroupNodes(database: database) + guard !groups.isEmpty else { return schemas } + if schemas.count == 1, case .status(.empty) = schemas[0].kind { return groups } + return schemas + groups + } + + private func databaseScopedGroupNodes(database: String) -> [DatabaseTreeNode] { + let kinds = serverKinds(database: database, schema: nil) + guard !kinds.isEmpty else { return [] } + let phases = sideLoadStates(database: database, schema: nil) + return DatabaseTreeObjectGroupResolver.groups( + database: database, + schema: nil, + itemCounts: objectBuckets(database: database, schema: nil).itemCounts, + keeping: phases.failedKinds(among: Set(kinds.map { SidebarObjectKind.server($0) })), + declared: kinds + ) + .filter { $0.kind.category == .server } + .map { node(id: DatabaseTreeNode.containerObjectKindSectionId($0), kind: .containerObjectKindSection($0)) } + } + private func schemaNodes(database: String) -> [DatabaseTreeNode] { let parentId = DatabaseTreeNode.databaseId(database) switch service.schemaListState(connectionId: connectionId, database: database) { @@ -632,6 +697,14 @@ extension DatabaseTreeOutlineCoordinator { func shown(_ items: [Item], isCurrent: Bool) -> [Item] { showsOvertaken || isCurrent ? items : [] } + let serverObjects = Dictionary(uniqueKeysWithValues: serverKinds(database: database, schema: schema).map { kind in + (kind.id, shown( + service.serverObjects(connectionId: connectionId, database: database, schema: schema, kindId: kind.id), + isCurrent: service.isServerObjectsCurrent( + connectionId: connectionId, database: database, schema: schema, kindId: kind.id + ) + )) + }) return DatabaseTreeBucketSource( tables: tables, routines: shown( @@ -646,6 +719,7 @@ extension DatabaseTreeOutlineCoordinator { service.userDefinedTypes(connectionId: connectionId, database: database, schema: schema), isCurrent: service.isTypesCurrent(connectionId: connectionId, database: database, schema: schema) ), + serverObjects: serverObjects, searchText: searchText, database: database ) @@ -666,9 +740,7 @@ extension DatabaseTreeOutlineCoordinator { return objectBuckets(database: database, schema: schema) } - /// A fetch the engine never runs stays idle for good, and idle is not loaded: counting it - /// would hold every empty container on a spinner for a list that is never coming. So only the - /// kinds this engine declares take part in deciding between "empty" and "loading". + // Declared kinds only: an idle fetch the engine never runs would hold every empty container on a spinner. private func sideLoadStates(database: String, schema: String?) -> DatabaseTreeSidePhases { let declared = declaredObjectKinds return DatabaseTreeSidePhases( @@ -678,7 +750,12 @@ extension DatabaseTreeOutlineCoordinator { : nil, types: declared.contains(.type) ? service.typesLoadState(connectionId: connectionId, database: database, schema: schema).erased - : nil + : nil, + serverObjects: Dictionary(uniqueKeysWithValues: serverKinds(database: database, schema: schema).map { kind in + (kind.id, service.serverObjectsLoadState( + connectionId: connectionId, database: database, schema: schema, kindId: kind.id + ).erased) + }) ) } @@ -712,7 +789,8 @@ extension DatabaseTreeOutlineCoordinator { database: database, schema: schema, itemCounts: looseItemCounts(buckets.itemCounts, plan: plan), - keeping: plan.filedKinds.union(sidePhases.failedKinds(among: declaredObjectKinds)) + keeping: plan.filedKinds.union(sidePhases.failedKinds(among: declaredObjectKinds)), + declared: serverKinds(database: database, schema: schema) ) guard !groups.isEmpty || !folders.isEmpty else { return [statusNode(parentId: parentId, status: .emptyContainer(sideStates: sidePhases.all))] @@ -728,8 +806,7 @@ extension DatabaseTreeOutlineCoordinator { private func containerObjectNodes(for group: DatabaseTreeObjectGroup) -> [DatabaseTreeNode] { let buckets = objectBuckets(database: group.database, schema: group.schema) let emptyId = DatabaseTreeNode.containerObjectKindSectionId(group) - /// A schema-grouped group carries no database, and a reference naming an empty one would - /// switch the session to a database called "" the moment the row opened. + // An empty database name would switch the session to a database called "" the moment the row opened. let database: String? = group.database.isEmpty ? nil : group.database let placeholder = sidePhases(for: group).placeholder(for: group.kind).map { [statusNode(parentId: emptyId, status: $0)] @@ -756,6 +833,25 @@ extension DatabaseTreeOutlineCoordinator { let ref = DatabaseTreeUserTypeRef(database: database, schema: group.schema, type: type) return node(id: DatabaseTreeNode.userTypeId(ref), kind: .userType(ref)) } + case .server: + guard let serverKind = group.kind.serverKind else { return placeholder } + let objects = buckets.serverObjects[serverKind.id] ?? [] + guard !objects.isEmpty else { return placeholder } + // A `.database` kind's object keeps its own schema, which is not the group's. + let fixedSchema = group.schema + return serverObjectNodes(objects, kind: serverKind, database: database, schema: { fixedSchema ?? $0.schema }) + } + } + + private func serverObjectNodes( + _ objects: [ServerObjectInfo], + kind: SidebarServerObjectKind, + database: String?, + schema: (ServerObjectInfo) -> String? + ) -> [DatabaseTreeNode] { + objects.map { object in + let ref = DatabaseTreeServerObjectRef(database: database, schema: schema(object), kind: kind, object: object) + return node(id: DatabaseTreeNode.serverObjectId(ref), kind: .serverObject(ref)) } } diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift index c5afbe0e81..7c40f92912 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift @@ -561,7 +561,14 @@ final class DatabaseTreeOutlineCoordinator: NSObject, NSTextFieldDelegate { private func refreshDatabase(_ database: String) { if supportsSchemaLevel { - Task { await service.refreshSchemas(connectionId: connectionId, database: database) } + let connectionId = connectionId + Task { + async let schemas: Void = service.refreshSchemas(connectionId: connectionId, database: database) + async let serverObjects: Void = service.refreshServerObjects( + connectionId: connectionId, database: database, schema: nil + ) + _ = await (schemas, serverObjects) + } } else { Task { await service.refreshObjects(connectionId: connectionId, database: database, schema: nil) } } @@ -648,8 +655,7 @@ final class DatabaseTreeOutlineCoordinator: NSObject, NSTextFieldDelegate { } } - /// A table kind is refreshed as a catalog change, as Cmd+R is, so the partition lists the tree - /// holds under the section are marked stale along with the browsed list. + // A table kind refreshes as a catalog change, as Cmd+R does, so the partition lists under it go stale too. internal func refreshObjectKind(_ kind: SidebarObjectKind) { guard let mainCoordinator else { return } switch kind { @@ -662,6 +668,18 @@ final class DatabaseTreeOutlineCoordinator: NSObject, NSTextFieldDelegate { database: mainCoordinator.browseDatabaseName, kinds: .tables ))) + case .server(let serverKind): + let connectionId = connectionId + let manager = DatabaseManager.shared + let scope = manager.browseScope(for: connectionId) + let kindId = serverKind.id + Task { [schemaService] in + try? await manager.withBrowseMetadataDriver(connectionId: connectionId) { driver in + _ = await schemaService.reloadServerObjects( + connectionId: connectionId, driver: driver, scope: scope, kindId: kindId + ) + } + } } } @@ -697,6 +715,13 @@ final class DatabaseTreeOutlineCoordinator: NSObject, NSTextFieldDelegate { database: group.database, schema: group.schema ) + case .server: + await service.refreshServerObjects( + connectionId: connectionId, + database: group.database, + schema: group.schema, + kindId: group.kind.serverKind?.id + ) } } } diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeRenderInputs.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeRenderInputs.swift index 6dbbfc1048..b7c9f74179 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeRenderInputs.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeRenderInputs.swift @@ -10,6 +10,7 @@ import Foundation struct DatabaseTreeRenderInputs: Equatable { typealias DatabaseKey = DatabaseTreeMetadataService.DatabaseKey typealias ObjectsKey = DatabaseTreeMetadataService.ObjectsKey + typealias ServerObjectsKey = DatabaseTreeMetadataService.ServerObjectsKey struct Rows: Equatable { let phase: MetadataLoadPhase @@ -77,7 +78,9 @@ struct DatabaseTreeRenderInputs: Equatable { let routines: [ObjectsKey: Rows] let triggers: [ObjectsKey: Rows] let types: [ObjectsKey: Rows] + let serverObjects: [ServerObjectsKey: Rows] let currency: [ObjectsKey: Currency] + let serverObjectsCurrency: [ServerObjectsKey: Bool] let partitions: [DatabaseTreeMetadataService.PartitionsKey: Rows] let listings: [DatabaseKey: Listing] let listsRevision: Int @@ -87,6 +90,7 @@ struct DatabaseTreeRenderInputs: Equatable { let catalogRoutines: Rows let catalogTriggers: Rows let catalogTypes: Rows + let catalogServerObjects: [String: Rows] let temporaryTables: [TemporaryTableStore.Key: TemporaryTables] let sidebar: Sidebar? @@ -105,6 +109,15 @@ struct DatabaseTreeRenderInputs: Equatable { let routines = Self.slice(treeMetadata.routinesState, where: ownsObjects) { Rows($0) } let triggers = Self.slice(treeMetadata.triggersState, where: ownsObjects) { Rows($0) } let types = Self.slice(treeMetadata.typesState, where: ownsObjects) { Rows($0) } + let serverObjects = Self.slice(treeMetadata.serverObjectsState, where: { $0.connectionId == connectionId }) { + Rows($0) + } + var serverObjectsCurrency: [ServerObjectsKey: Bool] = [:] + for key in serverObjects.keys { + serverObjectsCurrency[key] = treeMetadata.isServerObjectsCurrent( + connectionId: connectionId, database: key.database, schema: key.schema, kindId: key.kindId + ) + } var currency: [ObjectsKey: Currency] = [:] for key in Set(tables.keys).union(routines.keys).union(triggers.keys).union(types.keys) { currency[key] = Currency( @@ -131,7 +144,9 @@ struct DatabaseTreeRenderInputs: Equatable { self.routines = routines self.triggers = triggers self.types = types + self.serverObjects = serverObjects self.currency = currency + self.serverObjectsCurrency = serverObjectsCurrency self.partitions = Self.slice(treeMetadata.partitionsState, where: { $0.connectionId == connectionId }) { Rows($0) } self.listings = listings self.listsRevision = treeMetadata.listsRevision(for: connectionId) @@ -140,6 +155,7 @@ struct DatabaseTreeRenderInputs: Equatable { self.catalogRoutines = Rows(schemaService.routinesLoadState(for: connectionId)) self.catalogTriggers = Rows(schemaService.triggersLoadState(for: connectionId)) self.catalogTypes = Rows(schemaService.userDefinedTypesLoadState(for: connectionId)) + self.catalogServerObjects = schemaService.serverObjectStates(for: connectionId).mapValues { Rows($0) } self.temporaryTables = Self.slice(temporaryTables.listings, where: { $0.connectionId == connectionId }) { TemporaryTables(tables: SameRows(rows: $0.tables), session: $0.session) } diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeRowView.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeRowView.swift index ea0b5d864a..85da448667 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeRowView.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeRowView.swift @@ -125,6 +125,8 @@ struct DatabaseTreeRowView: View { TriggerRowView(trigger: ref.trigger) case .userType(let ref): UserTypeRowView(type: ref.type) + case .serverObject(let ref): + ServerObjectRowView(object: ref.object, kind: ref.kind) case .status(let status): statusRow(status) case .objectKindSection(let kind): diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeSelection.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeSelection.swift index 85fa896b1f..a85dc4ca84 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeSelection.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeSelection.swift @@ -18,7 +18,7 @@ internal enum DatabaseTreeSelection { case .status, .recentSection, .temporarySection, .foldersSection, .objectKindSection, .redisKeysSection: return false case .database, .schema, .hierarchicalSchemaSection, .table, .partition, .routine, .trigger, .userType, - .recentTable, .temporaryTable, .containerObjectKindSection, .tableFolder, .redisNode: + .serverObject, .recentTable, .temporaryTable, .containerObjectKindSection, .tableFolder, .redisNode: return true } } diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeTypeSelect.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeTypeSelect.swift index 55f3ed21e8..2e1bd67a02 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeTypeSelect.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeTypeSelect.swift @@ -72,6 +72,8 @@ internal enum DatabaseTreeTypeSelect { return ref.trigger.name case .userType(let ref): return ref.type.name + case .serverObject(let ref): + return ref.object.name case .hierarchicalSchemaSection(let schema): return schema case .redisNode(let node): diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeView.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeView.swift index e1aaf317a4..01010cd35d 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeView.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeView.swift @@ -48,6 +48,21 @@ struct DatabaseTreeUserTypeRef: Identifiable, Equatable { } } +struct DatabaseTreeServerObjectRef: Identifiable, Equatable { + let database: String? + let schema: String? + let kind: SidebarServerObjectKind + let object: ServerObjectInfo + + var id: String { + IdentityPath.joined([database ?? "", schema ?? "", object.id], separator: "|") + } + + var objectRef: DatabaseObjectRef { + DatabaseObjectRef(serverObject: object, kind: kind, database: database ?? "") + } +} + struct DatabaseTreeView: View { @ObservedObject private var databaseManager = DatabaseManager.shared @ObservedObject private var treeService = DatabaseTreeMetadataService.shared diff --git a/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift b/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift index 11ae7629f4..c72f6f12ea 100644 --- a/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift +++ b/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift @@ -188,7 +188,7 @@ struct EditSequenceSheet: View { try await SequenceEditing.alter(details, to: settings, target: target, connection: connection) dismiss() } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing(details) } } diff --git a/apple/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift b/apple/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift index 39d1569256..3318610d63 100644 --- a/apple/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift +++ b/apple/TablePro/Views/Sidebar/Menu/DatabaseTreeMenuSpec.swift @@ -7,12 +7,7 @@ import Foundation import TableProPluginKit internal enum DatabaseTreeMenuSpec { - /// A menu is a list of groups, and every group here is one intent: reach the object, note it, - /// move its data, change it. The HIG asks for a small number of items in about three groups and - /// for commands relevant to the clicked object only, so nothing connection-wide belongs on a - /// row. View Options used to be appended to every menu including every object row, and View ER - /// Diagram sat in the middle of a table row's clipboard and export commands; both now live on - /// the empty-area menu, where the thing they act on is the sidebar rather than an object. + // One intent per group, and only commands for the clicked object; connection-wide ones live on the empty-area menu. internal static func sections(for context: DatabaseTreeMenuContext) -> [DatabaseTreeMenuSection] { guard let clicked = context.clicked else { return backgroundSections(context) } switch clicked { @@ -46,6 +41,8 @@ internal enum DatabaseTreeMenuSpec { return triggerSections(ref) case .userType(let ref): return userTypeSections(ref, context: context) + case .serverObject(let ref): + return serverObjectSections(ref) case .objectKindSection(let kind): return objectKindSections(kind, context: context) case .containerObjectKindSection(let group): @@ -336,6 +333,14 @@ internal enum DatabaseTreeMenuSpec { ] } + // List plus DDL only: the row offers nothing that writes. + private static func serverObjectSections(_ ref: DatabaseTreeServerObjectRef) -> [DatabaseTreeMenuSection] { + [ + DatabaseTreeMenuSection([.command(String(localized: "Copy Name"), .copyText(ref.object.name))]), + DatabaseTreeMenuSection([.command(String(localized: "Show DDL"), .showObjectSource(ref.objectRef))]) + ] + } + private static func userTypeSections( _ ref: DatabaseTreeUserTypeRef, context: DatabaseTreeMenuContext diff --git a/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift b/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift index bdc4397a00..b73407bedf 100644 --- a/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift +++ b/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift @@ -144,10 +144,8 @@ struct ObjectCommentSheet: View { do { try await ObjectCommentEditing.setComment(draft.commentToSave, on: target, connection: connection) dismiss() - } catch ExecutionGateError.cancelledByUser { - phase = .editing } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing } } diff --git a/apple/TablePro/Views/Sidebar/ServerObjectRowView.swift b/apple/TablePro/Views/Sidebar/ServerObjectRowView.swift new file mode 100644 index 0000000000..cd9b583400 --- /dev/null +++ b/apple/TablePro/Views/Sidebar/ServerObjectRowView.swift @@ -0,0 +1,79 @@ +// +// ServerObjectRowView.swift +// TablePro +// + +import SwiftUI + +enum ServerObjectRowLogic { + static func displayName(for object: ServerObjectInfo, kind: SidebarServerObjectKind) -> String { + kind.scope == .database ? object.name : IdentityPath.qualified(name: object.name, schema: object.schema) + } + + static func accessibilityLabel(for object: ServerObjectInfo, kind: SidebarServerObjectKind) -> String { + let base = String(format: String(localized: "%1$@: %2$@"), kind.displayName, object.name) + guard let detail = object.detail, !detail.isEmpty else { return base } + return base + ", " + detail + } + + // The driver states the object's state in its attributes, in its own words, so it is not repeated here. + static func tooltip(for object: ServerObjectInfo, kind: SidebarServerObjectKind) -> String { + var lines: [String] = [displayName(for: object, kind: kind)] + if let detail = object.detail, !detail.isEmpty { lines.append(detail) } + lines.append(contentsOf: object.attributes.map { "\($0.label): \($0.value)" }) + return lines.joined(separator: "\n") + } +} + +// Plain drawn values: `ServerObjectInfo` compares by id, so a refreshed version or state would read as unchanged. +struct ServerObjectRowView: View { + @ObservedObject private var settingsManager = AppSettingsManager.shared + let name: String + let detail: String? + let iconName: String + let tooltip: String + let accessibilityLabel: String + + init(name: String, detail: String?, iconName: String, tooltip: String, accessibilityLabel: String) { + self.name = name + self.detail = detail + self.iconName = iconName + self.tooltip = tooltip + self.accessibilityLabel = accessibilityLabel + } + + init(object: ServerObjectInfo, kind: SidebarServerObjectKind) { + self.init( + name: object.name, + detail: object.detail, + iconName: kind.iconName, + tooltip: ServerObjectRowLogic.tooltip(for: object, kind: kind), + accessibilityLabel: ServerObjectRowLogic.accessibilityLabel(for: object, kind: kind) + ) + } + + var body: some View { + Label { + HStack(spacing: 6) { + Text(name) + .lineLimit(1) + .truncationMode(.tail) + if let detail, !detail.isEmpty { + Text(detail) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(1) + .truncationMode(.tail) + } + } + } icon: { + Image(systemName: iconName) + .selectionAwareTint(Color.accentColor) + .frame(width: 16) + } + .sidebarRowIcon(visible: settingsManager.general.showObjectIcons) + .accessibilityElement(children: .combine) + .accessibilityLabel(accessibilityLabel) + .help(tooltip) + } +} diff --git a/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift b/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift index e63e765d73..d970311881 100644 --- a/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift +++ b/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift @@ -176,7 +176,7 @@ struct SetSequenceValueSheet: View { try await SequenceEditing.setNextValue(nextValue, of: details, target: target, connection: connection) dismiss() } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing(details) } } diff --git a/apple/TablePro/Views/Sidebar/SidebarPersistenceKey.swift b/apple/TablePro/Views/Sidebar/SidebarPersistenceKey.swift index 98ac303f63..17d3e1d637 100644 --- a/apple/TablePro/Views/Sidebar/SidebarPersistenceKey.swift +++ b/apple/TablePro/Views/Sidebar/SidebarPersistenceKey.swift @@ -42,12 +42,17 @@ enum SidebarPersistenceKey { "sidebar.layout.\(connectionId.uuidString)" } + // Built-in kinds only; a declared kind's state lives under `serverKindsExpanded`. static func expanded(connectionId: UUID, kind: SidebarObjectKind) -> String { - "sidebar.\(connectionId.uuidString).\(kind.rawValue).expanded" + "sidebar.\(connectionId.uuidString).\(kind.persistenceName).expanded" } - /// Every per-connection key this type can produce. A key added above and forgotten here outlives - /// the connection it belongs to for the life of the install. + // One key for every declared kind, since their ids are open-ended and `all` has to name each key. + static func serverKindsExpanded(connectionId: UUID) -> String { + "sidebar.\(connectionId.uuidString).serverKinds.expanded" + } + + // A key missing here outlives its connection for the life of the install. static func all(connectionId: UUID) -> [String] { [ tablesExpanded(connectionId: connectionId), @@ -58,8 +63,9 @@ enum SidebarPersistenceKey { selectedTab(connectionId: connectionId), selectedFavorite(connectionId: connectionId), favoriteDatabaseEnvironmentFilter(connectionId: connectionId), - layout(connectionId: connectionId) - ] + SidebarObjectKind.allCases.map { expanded(connectionId: connectionId, kind: $0) } + layout(connectionId: connectionId), + serverKindsExpanded(connectionId: connectionId) + ] + SidebarObjectKind.builtIn.map { expanded(connectionId: connectionId, kind: $0) } } static func removeAll( diff --git a/apple/TablePro/Views/Sidebar/SidebarView.swift b/apple/TablePro/Views/Sidebar/SidebarView.swift index ec55eab1ab..09dfc708e4 100644 --- a/apple/TablePro/Views/Sidebar/SidebarView.swift +++ b/apple/TablePro/Views/Sidebar/SidebarView.swift @@ -47,7 +47,8 @@ struct SidebarView: View { } private var hasAnyMatch: Bool { - SidebarObjectKind.allCases.contains { kind in + let declared = SidebarServerObjectKind.declared(by: DatabaseManager.shared.driver(for: connectionId)) + return SidebarObjectKind.ordered(declared: declared).contains { kind in countFor(kind: kind) > 0 } || !otherSchemaMatches.isEmpty || !attachedSchemaMatches.isEmpty || hasTemporaryMatch } @@ -148,20 +149,14 @@ struct SidebarView: View { self.windowState = windowState _pendingTruncates = pendingTruncates _pendingDeletes = pendingDeletes - let selectedBinding = Binding( - get: { windowState.selectedTables }, - set: { windowState.selectTables($0) } - ) let vm = SidebarViewModel.shared( connectionId: connectionId, databaseType: databaseType, - selectedTables: selectedBinding, pendingTruncates: pendingTruncates, pendingDeletes: pendingDeletes, tableOperationOptions: tableOperationOptions ) - /// Nothing observable is written here. This initializer runs on every evaluation of the - /// parent's body, and the view model already seeds its own filter and watches the field. + // Nothing observable is written here: this runs on every evaluation of the parent's body. _viewModel = StateObject(wrappedValue: vm) self.connectionId = connectionId self.coordinator = coordinator @@ -213,34 +208,30 @@ struct SidebarView: View { // A refresh can put the catalog back to loading before the frame, and a stale true skips the next hold. CommittedTurn.after(in: coordinator?.contentWindow) { showsLoadedCatalog = isCatalogLoaded } } - .onChange(of: viewModel.showOperationDialog) { isPresented in - guard isPresented else { return } - presentOperationAlert() + .onChange(of: windowState.pendingTableOperation?.id) { _ in + guard let request = windowState.pendingTableOperation else { return } + presentOperationAlert(for: request) } } - private func presentOperationAlert() { - guard let operationType = viewModel.pendingOperationType, - let firstTable = viewModel.pendingOperationTables.first - else { - viewModel.showOperationDialog = false + private func presentOperationAlert(for request: PendingTableOperation) { + let windowState = windowState + guard let firstTable = request.tables.first else { + windowState.resolveTableOperation(request) return } let prompt = TableOperationPrompt( - operationType: operationType, + operationType: request.operation, tableName: firstTable.table.name, - tableCount: viewModel.pendingOperationTables.count, + tableCount: request.tables.count, cascadeSupported: PluginManager.shared.supportsCascadeDrop(for: viewModel.databaseType), foreignKeyDisableSupported: PluginManager.shared.supportsForeignKeyDisable(for: viewModel.databaseType) ) let model = viewModel TableOperationAlert.present(prompt: prompt, window: coordinator?.contentWindow) { options in - model.showOperationDialog = false - guard let options else { - model.cancelPendingOperation() - return - } - model.confirmOperation(options: options) + windowState.resolveTableOperation(request) + guard let options else { return } + model.confirmOperation(request, options: options) } } @@ -493,6 +484,12 @@ struct SidebarView: View { case .routine: return viewModel.filteredRoutines(of: kind, from: routines).count case .trigger: return viewModel.filteredTriggers(from: triggers).count case .type: return viewModel.filteredUserTypes(from: userDefinedTypes).count + case .server: + guard let serverKind = kind.serverKind else { return 0 } + return viewModel.filteredServerObjects( + kindId: serverKind.id, + from: schemaService.serverObjects(for: connectionId, kindId: serverKind.id) + ).count } } } diff --git a/apple/TablePro/Views/Structure/ClickHousePartsView.swift b/apple/TablePro/Views/Structure/ClickHousePartsView.swift index 6f37691102..f95d50739e 100644 --- a/apple/TablePro/Views/Structure/ClickHousePartsView.swift +++ b/apple/TablePro/Views/Structure/ClickHousePartsView.swift @@ -8,6 +8,56 @@ import SwiftUI import TableProPluginKit import TableProSQLGrammar +internal struct ClickHousePartsAlert: Equatable { + let title: String + let message: String +} + +// A failed or refused action never replaces the list it acted on; only a load with nothing shown does. +internal struct ClickHousePartsPaneState { + internal enum Display: Equatable { + case loading + case loadFailed(String) + case empty + case parts + } + + private(set) var parts: [ClickHousePartInfo] = [] + private(set) var isLoading = true + private(set) var loadError: String? + var alert: ClickHousePartsAlert? + + var display: Display { + guard parts.isEmpty else { return .parts } + if let loadError { return .loadFailed(loadError) } + return isLoading ? .loading : .empty + } + + mutating func beginLoad() { + isLoading = true + loadError = nil + } + + mutating func finishLoad(_ parts: [ClickHousePartInfo]) { + self.parts = parts + isLoading = false + } + + mutating func failLoad(_ message: String) { + isLoading = false + guard parts.isEmpty else { + alert = ClickHousePartsAlert(title: String(localized: "Couldn't Read the Parts"), message: message) + return + } + loadError = message + } + + mutating func actionEnded(_ failure: GatedOperationFailure, title: String) { + guard let message = failure.message else { return } + alert = ClickHousePartsAlert(title: title, message: message) + } +} + struct ClickHousePartsView: View { private static let logger = Logger(subsystem: "com.TablePro", category: "ClickHousePartsView") @@ -17,31 +67,31 @@ struct ClickHousePartsView: View { let connection: DatabaseConnection let reloadToken: Int - @State private var parts: [ClickHousePartInfo] = [] - @State private var isLoading = true - @State private var errorMessage: String? + @State private var pane = ClickHousePartsPaneState() @State private var loadedToken: Int? @StateObject private var grid = StructureListGridModel(accessibilityLabel: StructurePane.parts.displayName) @StateObject private var selection = StructureListSelection() var body: some View { VStack(spacing: 0) { - if isLoading { + switch pane.display { + case .loading: ProgressView() .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else if let error = errorMessage { - VStack(spacing: 8) { - Image(systemName: "exclamationmark.triangle") - .font(.largeTitle) - .foregroundStyle(.orange) - .accessibilityHidden(true) - RevealedTextView(error) - .foregroundStyle(.secondary) + case .loadFailed(let message): + UnavailableStateView { + Label(String(localized: "Couldn't Read the Parts"), systemImage: "exclamationmark.triangle") + } description: { + RevealedTextView(message) + } actions: { + Button(String(localized: "Try Again")) { + Task { await loadParts() } + } } .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else if parts.isEmpty { + case .empty: VStack(spacing: 8) { Image(systemName: "tray") .font(.largeTitle) @@ -52,11 +102,20 @@ struct ClickHousePartsView: View { } .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else { + case .parts: partsToolbar partsTable } } + .alert( + pane.alert?.title ?? "", + isPresented: Binding(get: { pane.alert != nil }, set: { if !$0 { pane.alert = nil } }), + presenting: pane.alert + ) { _ in + Button(String(localized: "OK"), role: .cancel) {} + } message: { alert in + Text(verbatim: RevealedText(alert.message).plainText) + } .task(id: reloadToken) { await loadPartsIfNeeded() } } @@ -92,7 +151,7 @@ struct ClickHousePartsView: View { columns: Self.columnTitles, defaultWidths: Self.columnWidths, minimumWidths: Self.columnMinimumWidths, - rows: parts.map(Self.gridRow(for:)), + rows: pane.parts.map(Self.gridRow(for:)), databaseType: connection.type, selectedIDs: selection.ids, select: { selection.ids = $0 }, @@ -130,6 +189,20 @@ struct ClickHousePartsView: View { ) } + // Column order is ClickHousePartStatements.parts. + internal static func part(from row: [PluginCellValue]) -> ClickHousePartInfo? { + guard let name = row[safe: 2]?.asText else { return nil } + return ClickHousePartInfo( + partition: row[safe: 0]?.asText ?? "", + partitionId: row[safe: 1]?.asText ?? "", + name: name, + rows: row[safe: 3]?.asText.flatMap { UInt64($0) } ?? 0, + bytesOnDisk: row[safe: 4]?.asText.flatMap { UInt64($0) } ?? 0, + modificationTime: row[safe: 5]?.asText ?? "", + active: row[safe: 6]?.asText == "1" + ) + } + // MARK: - Actions private func optimizeTable() { @@ -140,19 +213,20 @@ struct ClickHousePartsView: View { database: scope.database, table: tableName, quote: driver.quoteIdentifier ), description: String(localized: "Optimize Table"), + failureTitle: String(localized: "Couldn't Optimize Table"), kind: .maintenance ) } } private func dropSelectedPartition() { - guard let partitionValue = selectedPartitionValue() else { return } + guard let part = selectedPart() else { return } Task { @MainActor in let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Drop Partition?"), message: String( format: String(localized: "This will permanently delete all data in partition '%@'."), - partitionValue + part.partition ), confirmButton: String(localized: "Drop"), cancelButton: String(localized: "Cancel") @@ -160,27 +234,31 @@ struct ClickHousePartsView: View { guard confirmed else { return } guard let driver = DatabaseManager.shared.driver(for: scope.connectionId) else { return } - let sql = ClickHousePartStatements.dropPartition( + let failureTitle = String(localized: "Couldn't Drop Partition") + guard let sql = ClickHousePartStatements.dropPartition( database: scope.database, table: tableName, - partition: partitionValue, - quote: driver.quoteIdentifier, - escape: driver.escapeStringLiteral - ) - guard await run(sql, description: String(localized: "Drop Partition"), kind: .destructiveQuery) - else { return } + partitionId: part.partitionId, + quote: driver.quoteIdentifier + ) else { + pane.alert = ClickHousePartsAlert(title: failureTitle, message: Self.unusablePartitionIdMessage) + return + } + guard await run( + sql, description: String(localized: "Drop Partition"), failureTitle: failureTitle, kind: .destructiveQuery + ) else { return } selection.ids.removeAll() } } private func detachSelectedPartition() { - guard let partitionValue = selectedPartitionValue() else { return } + guard let part = selectedPart() else { return } Task { @MainActor in let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Detach Partition?"), message: String( format: String(localized: "This will detach partition '%@'. Data will be preserved but inaccessible until re-attached."), - partitionValue + part.partition ), confirmButton: String(localized: "Detach"), cancelButton: String(localized: "Cancel") @@ -188,22 +266,30 @@ struct ClickHousePartsView: View { guard confirmed else { return } guard let driver = DatabaseManager.shared.driver(for: scope.connectionId) else { return } - let sql = ClickHousePartStatements.detachPartition( + let failureTitle = String(localized: "Couldn't Detach Partition") + guard let sql = ClickHousePartStatements.detachPartition( database: scope.database, table: tableName, - partition: partitionValue, - quote: driver.quoteIdentifier, - escape: driver.escapeStringLiteral - ) - guard await run(sql, description: String(localized: "Detach Partition"), kind: .destructiveQuery) - else { return } + partitionId: part.partitionId, + quote: driver.quoteIdentifier + ) else { + pane.alert = ClickHousePartsAlert(title: failureTitle, message: Self.unusablePartitionIdMessage) + return + } + guard await run( + sql, description: String(localized: "Detach Partition"), failureTitle: failureTitle, kind: .destructiveQuery + ) else { return } selection.ids.removeAll() } } + private static var unusablePartitionIdMessage: String { + String(localized: "The server reported a partition ID that cannot be used in a statement, so nothing was run.") + } + @MainActor @discardableResult - private func run(_ sql: String, description: String, kind: OperationKind) async -> Bool { + private func run(_ sql: String, description: String, failureTitle: String, kind: OperationKind) async -> Bool { let scope = scope do { let decision = await ExecutionGateProvider.shared.authorize( @@ -217,10 +303,7 @@ struct ClickHousePartsView: View { operationDescription: description ) ) - guard case .authorized = decision else { - errorMessage = decision.deniedReason - return false - } + if let denial = decision.denialError { throw denial } _ = try await DatabaseManager.shared.withScopedDriver( scope: scope, route: DatabaseManager.shared.schemaChangeRoute(for: scope), @@ -228,30 +311,30 @@ struct ClickHousePartsView: View { ) { driver in try await driver.execute(query: sql) } - if kind != .maintenance { - CatalogChangeService.post( - .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) - ) - } - await loadParts() - return true } catch { - Self.logger.error("\(description, privacy: .public) failed: \(error.publicLogShape, privacy: .public)") - errorMessage = error.localizedDescription + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + Self.logger.error("\(description, privacy: .public) failed: \(error.publicLogShape, privacy: .public)") + } + pane.actionEnded(failure, title: failureTitle) return false } + if kind != .maintenance { + CatalogChangeService.post( + .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) + ) + } + await loadParts() + return true } - private func selectedPartitionValue() -> String? { - Self.partition(of: selection.ids, in: parts) + private func selectedPart() -> ClickHousePartInfo? { + Self.selectedPart(of: selection.ids, in: pane.parts) } - internal static func partition(of selectedIDs: Set, in parts: [ClickHousePartInfo]) -> String? { - guard selectedIDs.count == 1, - let selectedId = selectedIDs.first, - let part = parts.first(where: { $0.id == selectedId }) - else { return nil } - return part.partition + internal static func selectedPart(of selectedIDs: Set, in parts: [ClickHousePartInfo]) -> ClickHousePartInfo? { + guard selectedIDs.count == 1, let selectedId = selectedIDs.first else { return nil } + return parts.first { $0.id == selectedId } } // MARK: - Data Loading @@ -263,8 +346,7 @@ struct ClickHousePartsView: View { private func loadParts() async { let token = reloadToken - isLoading = true - errorMessage = nil + pane.beginLoad() let scope = scope let tableName = tableName @@ -276,29 +358,12 @@ struct ClickHousePartsView: View { } guard !Task.isCancelled else { return } selection.ids = [] - parts = result.rows.compactMap { row -> ClickHousePartInfo? in - guard let name = row[safe: 1]?.asText else { return nil } - let partition = row[safe: 0]?.asText ?? "" - let rows = row[safe: 2]?.asText.flatMap { UInt64($0) } ?? 0 - let bytesOnDisk = row[safe: 3]?.asText.flatMap { UInt64($0) } ?? 0 - let modTime = row[safe: 4]?.asText ?? "" - let active = row[safe: 5]?.asText == "1" - return ClickHousePartInfo( - partition: partition, - name: name, - rows: rows, - bytesOnDisk: bytesOnDisk, - modificationTime: modTime, - active: active - ) - } + pane.finishLoad(result.rows.compactMap(Self.part(from:))) loadedToken = token } catch { guard !Task.isCancelled else { return } Self.logger.error("Failed to load parts: \(error.publicLogShape, privacy: .public)") - errorMessage = error.localizedDescription + pane.failLoad(error.localizedDescription) } - - isLoading = false } } diff --git a/apple/TablePro/Views/Structure/CreateTableDraft.swift b/apple/TablePro/Views/Structure/CreateTableDraft.swift index bffd4f2653..854b5fea41 100644 --- a/apple/TablePro/Views/Structure/CreateTableDraft.swift +++ b/apple/TablePro/Views/Structure/CreateTableDraft.swift @@ -469,8 +469,7 @@ internal final class CreateTableDraft: ObservableObject { coordinator: coordinator ) } catch { - Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") - reportFailure(error.localizedDescription, on: coordinator) + reportFailure(error: error, on: coordinator) } } } @@ -508,8 +507,7 @@ internal final class CreateTableDraft: ObservableObject { } catch let formError as PluginCreateTableFormError { form?.recordSubmissionError(formError) } catch { - Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") - reportFailure(error.localizedDescription, on: coordinator) + reportFailure(error: error, on: coordinator) } } } @@ -560,6 +558,15 @@ internal final class CreateTableDraft: ObservableObject { ) } + private func reportFailure(error: any Error, on coordinator: MainContentCoordinator) { + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") + } + guard let message = failure.message else { return } + reportFailure(message, on: coordinator) + } + private func reportFailure(_ message: String, on coordinator: MainContentCoordinator) { AlertHelper.showErrorSheet( title: String(localized: "Create Table Failed"), diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift index b760819838..15048571f6 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift @@ -31,52 +31,13 @@ internal enum StructureSaveOutcome: Equatable { } } -/// How a save that did not apply ends, read from the error alone so it can be tested without -/// presenting anything. -/// -/// The gate's sheet is the one confirmation a save gets, so Cancel there is the ordinary way to -/// back out, and it ends quietly the way closing any confirmation does. Answering it with an -/// "Error Applying Changes" sheet put a second dialog in front of the user for the choice they had -/// just made. -internal enum StructureApplyFailure: Equatable { - case cancelledByUser - case refused(String) - case failed(String) - - internal init(_ error: any Error) { - guard let gateError = error as? ExecutionGateError else { - self = .failed(error.localizedDescription) - return - } - switch gateError { - case .cancelledByUser: - self = .cancelledByUser - case .denied(let reason): - self = .refused(reason) - } - } - - internal var outcome: StructureSaveOutcome { +internal extension GatedOperationFailure { + var outcome: StructureSaveOutcome { switch self { case .cancelledByUser, .refused: .refused case .failed(let message): .failed(message) } } - - /// What the error sheet says. Nil for a Cancel, which shows nothing. - internal var message: String? { - switch self { - case .cancelledByUser: nil - case .refused(let reason), .failed(let reason): reason - } - } - - /// A refusal and a Cancel both stop at the gate, before the first statement, so neither is a - /// failed operation to report or a reason to reload the catalog. - internal var reportsFailure: Bool { - if case .failed = self { return true } - return false - } } internal extension StructureEditingSession { @@ -159,7 +120,7 @@ internal extension StructureEditingSession { // A failed save that may have written; its error waits for the table to be read again. private struct AlterFailure { - let failure: StructureApplyFailure + let failure: GatedOperationFailure let startedAt: ContinuousClock.Instant } @@ -240,7 +201,7 @@ internal extension StructureEditingSession { report(.succeeded(OperationSummary()), startedAt: operationStart, coordinator: coordinator) return (.applied, nil) } catch { - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) // A gate refusal and a driver's refusal both stop before the first statement. guard failure.reportsFailure, !(error is SchemaOperationRefusedError) else { present(failure, startedAt: operationStart, coordinator: coordinator) @@ -251,7 +212,7 @@ internal extension StructureEditingSession { } private func present( - _ failure: StructureApplyFailure, + _ failure: GatedOperationFailure, note: String? = nil, startedAt: ContinuousClock.Instant, coordinator: MainContentCoordinator? @@ -340,7 +301,7 @@ internal extension StructureEditingSession { } catch { changeManager.releaseHold(hold, written: false) isApplying = false - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) if failure.reportsFailure { CatalogChangeService.post( .changed(CatalogChange(connectionId: connection.id, database: prepared.scope.database, kinds: .tables)) diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift b/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift index 30c1ff93cc..47f269457e 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift @@ -65,7 +65,7 @@ internal extension StructureEditingSession { ) await finishColumnReorder(prepared, clearTarget: clearTarget) } catch { - if StructureApplyFailure(error).reportsFailure { + if GatedOperationFailure(error).reportsFailure { CatalogChangeService.post( .changed( CatalogChange( @@ -115,7 +115,7 @@ internal extension StructureEditingSession { } private func reportColumnReorderFailure(_ error: any Error) { - guard let message = StructureApplyFailure(error).message else { return } + guard let message = GatedOperationFailure(error).message else { return } AlertHelper.showErrorSheet( title: String(localized: "Column Reorder Failed"), message: message, diff --git a/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift b/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift index 4524bc9ac7..8eaa7cd55f 100644 --- a/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift +++ b/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift @@ -146,14 +146,12 @@ enum StructureRebuildPlanRunner { throw RunError.schemaChanged } - /// Only the transaction this plan opened is ever rolled back, and none is opened over a - /// transaction the session already holds: on the same shared session an app-owned - /// `COMMIT` commits the user's pending work, and MySQL's `START TRANSACTION` commits it - /// implicitly. A plan that cannot open one falls to its own compensation statements, - /// which is what an engine whose DDL commits as it runs already relies on. + // Never opens a transaction over one the session holds; a plan that cannot open one falls to its compensation statements. + // `.automatic`: the epilogue runs inside the transaction, where SQLite ignores `PRAGMA foreign_keys = on`. let owner = WriteTransactionOwner.resolve( supportsTransactions: driver.supportsTransactions, - sessionState: await driver.heldSessionTransactionState() + sessionState: await driver.heldSessionTransactionState(), + commitMode: .automatic ) // Inside a transaction SQLite ignores `PRAGMA foreign_keys = off`, so the rebuild's DROP TABLE cascades. if !plan.prologue.isEmpty, plan.isTransactional, owner == .session { @@ -164,7 +162,7 @@ enum StructureRebuildPlanRunner { _ = try? await driver.execute(query: sql) } - let usesTransaction = plan.isTransactional && owner.opensTransaction + let usesTransaction = plan.isTransactional && owner.sendsBegin && owner.commitsAtEnd if usesTransaction { try await driver.beginTransaction(mode: .readWrite) } diff --git a/apple/TablePro/Views/Structure/TriggerDetailView.swift b/apple/TablePro/Views/Structure/TriggerDetailView.swift index a0885aac43..34a9be327b 100644 --- a/apple/TablePro/Views/Structure/TriggerDetailView.swift +++ b/apple/TablePro/Views/Structure/TriggerDetailView.swift @@ -277,7 +277,7 @@ struct TriggerDetailView: View { name: trigger.name ) } catch { - actionError = error.localizedDescription + actionError = GatedOperationFailure(error).message } } } diff --git a/apple/TablePro/Views/Structure/TriggerEditorView.swift b/apple/TablePro/Views/Structure/TriggerEditorView.swift index 6dee7b596e..ef9b441168 100644 --- a/apple/TablePro/Views/Structure/TriggerEditorView.swift +++ b/apple/TablePro/Views/Structure/TriggerEditorView.swift @@ -29,7 +29,12 @@ struct TriggerEditorView: View { @State private var isApplying = false @State private var errorMessage: String? @State private var droppedDefinition: String? - @AppStorage("structureCodeFontSize", store: AppStorageEnvironment.shared.defaults) private var fontSize: Double = 13 + @AppStorage(TriggerEditorView.fontSizeKey, store: AppStorageEnvironment.shared.defaults) + private var fontSize = TriggerEditorView.defaultFontSize + + // The Structure tab's DDL pane writes its zoom here too. + static let fontSizeKey = "structureCodeFontSize" + static let defaultFontSize = 13.0 init( scope: DatabaseScope, @@ -45,7 +50,14 @@ struct TriggerEditorView: View { self.mode = mode self.onClose = onClose self._sql = State(wrappedValue: initialSQL) - self._editorConfiguration = State(wrappedValue: Self.makeConfiguration(fontSize: 13)) + self._editorConfiguration = State(wrappedValue: Self.initialConfiguration()) + } + + // The State is seeded once, here, so it reads the store; `fontSize` only follows later zooms. + static func initialConfiguration( + in defaults: UserDefaults = AppStorageEnvironment.shared.defaults + ) -> SourceEditorConfiguration { + makeConfiguration(fontSize: defaults.object(forKey: fontSizeKey) as? Double ?? defaultFontSize) } var body: some View { @@ -145,7 +157,7 @@ struct TriggerEditorView: View { if let dropped = (error as? TriggerEditingError)?.droppedDefinition { droppedDefinition = dropped } - errorMessage = error.localizedDescription + errorMessage = GatedOperationFailure(error).message } } } diff --git a/apple/TablePro/Views/TableListing/TableListingTabView.swift b/apple/TablePro/Views/TableListing/TableListingTabView.swift index 44fd926e3b..a5b15db1b4 100644 --- a/apple/TablePro/Views/TableListing/TableListingTabView.swift +++ b/apple/TablePro/Views/TableListing/TableListingTabView.swift @@ -75,10 +75,10 @@ internal struct TableListingTabView: View { private var rowsGrid: some View { DataGridView(model: model.grid.model) - .onAppear { model.grid.model.configuration = configuration } + .onAppear { model.grid.model.configuration = Self.gridConfiguration(databaseType: databaseType) } } - private var configuration: DataGridConfiguration { + static func gridConfiguration(databaseType: DatabaseType) -> DataGridConfiguration { var configuration = DataGridConfiguration() configuration.traits = DataGridEngineTraits(databaseType: databaseType) configuration.showRowNumbers = true diff --git a/apple/TablePro/Views/VersionHistory/VersionHistoryFactory.swift b/apple/TablePro/Views/VersionHistory/VersionHistoryFactory.swift index cd08b4b162..17070e411d 100644 --- a/apple/TablePro/Views/VersionHistory/VersionHistoryFactory.swift +++ b/apple/TablePro/Views/VersionHistory/VersionHistoryFactory.swift @@ -36,6 +36,15 @@ internal enum VersionHistoryFactory { confirmButton: String(localized: "Restore") ) } + case .queryTab(let id): + viewModel = VersionHistoryViewModel( + subject: subject, + provider: MainContentCoordinator.hostedQueryTabVersionProvider(for: id), + refreshSignal: AppEvents.shared.queryTabVersionsDidUpdate + .filter { $0 == nil || $0 == id } + .map { _ in () } + .eraseToAnyPublisher() + ) } viewModel.reportRestoreFailure = { message in AlertHelper.showErrorSheet( diff --git a/apple/TablePro/Views/VersionHistory/VersionHistoryFormatting.swift b/apple/TablePro/Views/VersionHistory/VersionHistoryFormatting.swift index 4663c05bcc..d1855d56b7 100644 --- a/apple/TablePro/Views/VersionHistory/VersionHistoryFormatting.swift +++ b/apple/TablePro/Views/VersionHistory/VersionHistoryFormatting.swift @@ -71,9 +71,22 @@ internal enum VersionHistoryFormatting { return String(format: String(localized: "Keeps the last %d versions on this Mac."), count) case .showsLatestCommits(let count): return String(format: String(localized: "Showing the %d most recent commits."), count) + case .keepsRecentVersions(let count, let days): + return String( + format: String(localized: "Keeps up to %1$d versions from the last %2$d days on this Mac."), + count, + days + ) } } + // The Revert To menu lists only past versions, so each row leads with its date. + static func menuTitle(for entry: VersionHistoryEntry) -> String { + guard let date = dateText(for: entry) else { return title(for: entry) } + guard let summary = entry.summary, !summary.isBlank else { return date } + return String(format: String(localized: "%1$@ (%2$@)"), date, summary) + } + private static func dateText(for entry: VersionHistoryEntry) -> String? { entry.date?.formatted(date: .abbreviated, time: .shortened) } diff --git a/apple/TablePro/Views/Welcome/WelcomePresentations.swift b/apple/TablePro/Views/Welcome/WelcomePresentations.swift index 4a1379516b..3913fea692 100644 --- a/apple/TablePro/Views/Welcome/WelcomePresentations.swift +++ b/apple/TablePro/Views/Welcome/WelcomePresentations.swift @@ -137,6 +137,19 @@ private struct WelcomeDeletionAlerts: ViewModifier { } message: { deletionMessage } + .alert( + String(localized: "Exclude from iCloud Sync?"), + isPresented: $vm.showExcludeFromSyncConfirmation + ) { + Button(String(localized: "Exclude"), role: .destructive) { + vm.confirmExcludeFromSync() + } + Button(String(localized: "Cancel"), role: .cancel) { + vm.connectionsToExcludeFromSync = [] + } + } message: { + Text("Saved passwords are removed from iCloud Keychain, and your other devices will need them entered again.") + } } @ViewBuilder diff --git a/apple/TableProMobile/TableProMobile/AppState.swift b/apple/TableProMobile/TableProMobile/AppState.swift index 859465654b..fc2c1244aa 100644 --- a/apple/TableProMobile/TableProMobile/AppState.swift +++ b/apple/TableProMobile/TableProMobile/AppState.swift @@ -5,6 +5,7 @@ import TableProConnectionLibrary import TableProDatabase import TableProLogRedaction import TableProModels +import TableProSyncTransport @MainActor @Observable final class AppState { @@ -40,6 +41,7 @@ final class AppState { let backgroundRelease: BackgroundReleaseCoordinator let queryActivities = QueryActivityController() let catalogChanges = CatalogChangeBoard() + let sessionModels = ConnectionSessionModels() let syncCoordinator: IOSSyncCoordinator @ObservationIgnored private var automaticPresentationOwner: UUID? @@ -72,7 +74,7 @@ final class AppState { driverFactory injectedDriverFactory: (any DriverFactory)? = nil ) { self.sampleInstaller = sampleInstaller - self.libraryPublisher = libraryPublisher ?? .live() + self.libraryPublisher = libraryPublisher ?? .live(defaults: defaults) self.localDatabaseFiles = localDatabaseFiles localDatabaseFiles.container.recordCurrentContainer() onboarding = OnboardingPreferences(defaults: defaults) @@ -117,7 +119,7 @@ final class AppState { guard !TestRuntime.isActive else { return } if loadStatus == .ready { - KeychainSecureStore.cleanOrphanedCredentials(validConnectionIds: Set(connections.map(\.id))) + removeWhatDeletedConnectionsLeft() Task { publishLibrary() } @@ -126,6 +128,20 @@ final class AppState { connectSync() } + // The Keychain and certificate folder listings block, so they run off the main actor at utility + // priority, against the library as it stands once the Keychain listing returns, not at launch. + private func removeWhatDeletedConnectionsLeft() { + Task.detached(priority: .utility) { [self] in + await KeychainSecureStore.cleanOrphanedCredentials { self.loadedConnectionIds } + guard let kept = await loadedConnectionIds else { return } + CertificateMaterializer().releaseAll(except: kept) + } + } + + private var loadedConnectionIds: Set? { + loadStatus == .ready ? Set(connections.map(\.id)) : nil + } + // Separate from init, which skips it under tests, so a test can run the wiring the app runs. func connectSync() { syncCoordinator.onConnectionsChanged = { [weak self] merged in @@ -186,6 +202,7 @@ final class AppState { private func sessionEnded(_ connectionId: UUID, driver: any DatabaseDriver) { sessionTransactions.sessionEnded(connectionId, driver: driver) + sessionModels.existing(connectionId)?.sessionEnded(driver) bumpSessionRevision(connectionId) } @@ -204,6 +221,18 @@ final class AppState { publishLibrary() } + // A background wake can come while the device is locked and the library unreadable, so nothing runs until it loads. + func syncFromBackground(_ trigger: SyncTrigger) async -> Bool { + guard onboarding.isCloudSyncEnabled else { return false } + retryLoadIfFailed() + guard loadStatus == .ready else { + Self.logger.warning("Background sync skipped: persistence load not ready (likely device locked)") + return false + } + await syncCoordinator.sync(trigger) + return true + } + private func refuseWriteIfNotReady() -> Bool { guard isLibraryWritable else { Self.logger.error("Refusing a library write while the stored library is not loaded") @@ -225,6 +254,7 @@ final class AppState { let removedIds = Set(connections.map(\.id)).subtracting(merged.map(\.id)) guard persist(connections: merged, onlyIfSaved: true) else { return false } localState.purge(removedIds) + sessionModels.purge(removedIds) savedQueries.removeRecords(ofConnections: removedIds, tombstoning: false) publishLibrary() return true @@ -441,6 +471,7 @@ final class AppState { guard !removed.isEmpty else { return } persist(connections: connections.filter { !ids.contains($0.id) }) localState.purge(Set(removed.map(\.id))) + sessionModels.purge(Set(removed.map(\.id))) let syncedIds = Set(removed.filter(\.participatesInSync).map(\.id)) savedQueries.removeRecords(ofConnections: syncedIds, tombstoning: true) savedQueries.removeRecords(ofConnections: Set(removed.map(\.id)).subtracting(syncedIds), tombstoning: false) diff --git a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift index e7c5b14de0..aaee312be2 100644 --- a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift +++ b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift @@ -7,6 +7,7 @@ import TableProPluginKit // Nil when there is no session to read from: nothing is read and nothing is kept. typealias QueryColumnReader = @Sendable (_ table: String, _ schema: String?) async throws -> [ColumnInfo]? typealias QuerySchemaTableReader = @Sendable (_ schema: String) async throws -> [TableInfo]? +typealias QueryForeignKeyReader = @Sendable (_ table: String, _ schema: String?) async throws -> [ForeignKeyInfo]? actor QueryCompletionCatalog: SQLCompletionCatalog { private static let logger = Logger(subsystem: "com.TablePro", category: "QueryCompletionCatalog") @@ -21,27 +22,52 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { let columns: [ColumnInfo] } - private var tables: [TableInfo] + nonisolated private struct CachedForeignKeys: Sendable { + let generation: Int + let keys: [SQLCompletionForeignKey] + } + + private var tables: [TableInfo] { + didSet { + cachedTableItems = nil + cachedTableColumnItems = nil + } + } + // Every table-slot request reads all of them, so they are built once per table list. + private var cachedTableItems: [SQLCompletionItem]? private let databaseType: DatabaseType private let schema: String? private let readColumns: QueryColumnReader private let readCatalogColumns: QueryColumnReader? private var columnReadsAllowed = true private var columnGeneration = 0 - private var columns: [ColumnKey: CachedColumns] = [:] + private var columns: [ColumnKey: CachedColumns] = [:] { + didSet { cachedTableColumnItems = nil } + } + // Every column request before a FROM reads all of them, so they are built once per column cache, table list and + // namespace list. + private var cachedTableColumnItems: [SQLCompletionItem]? + private(set) var tableColumnItemsBuildCount = 0 private var reads: [ColumnKey: Task<[ColumnInfo]?, any Error>] = [:] private var sessionReads: [ColumnKey: Task<[ColumnInfo]?, any Error>] = [:] private let readSchemaTables: QuerySchemaTableReader? private let readCatalogSchemaTables: QuerySchemaTableReader? private var sessionSchemaReads: [String: Task<[TableInfo]?, any Error>] = [:] private var sessionLocalTables: Set = [] - private var schemas: [String] - private var databases: [String] + private var schemas: [String] { + didSet { cachedTableColumnItems = nil } + } + private var databases: [String] { + didSet { cachedTableColumnItems = nil } + } private var schemaTables: [String: (generation: Int, tables: [TableInfo])] = [:] private var schemaTableReads: [String: Task<[TableInfo]?, any Error>] = [:] + private let readCatalogForeignKeys: QueryForeignKeyReader? + private var cachedForeignKeys: [ColumnKey: CachedForeignKeys] = [:] + private var foreignKeyReads: [ColumnKey: Task<[SQLCompletionForeignKey]?, any Error>] = [:] // `readCatalogColumns` reads on a connection beside the session and answers nil when there is none, - // so only `readColumns`, the session's own, waits for runs. + // so only `readColumns`, the session's own, waits for runs. Foreign keys are read on that connection only. init( tables: [TableInfo], databaseType: DatabaseType, @@ -50,6 +76,7 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { readCatalogColumns: QueryColumnReader? = nil, readSchemaTables: QuerySchemaTableReader? = nil, readCatalogSchemaTables: QuerySchemaTableReader? = nil, + readCatalogForeignKeys: QueryForeignKeyReader? = nil, schemas: [String] = [], databases: [String] = [] ) { @@ -60,6 +87,7 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { self.readCatalogColumns = readCatalogColumns self.readSchemaTables = readSchemaTables self.readCatalogSchemaTables = readCatalogSchemaTables + self.readCatalogForeignKeys = readCatalogForeignKeys self.schemas = schemas self.databases = databases } @@ -123,19 +151,21 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } func tableCompletionItems() -> [SQLCompletionItem] { - tables.map { table in - let isView = table.type == .view || table.type == .materializedView - return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) - } + if let cachedTableItems { return cachedTableItems } + let items = tables.map { tableItem($0) } + cachedTableItems = items + return items + } + + private func tableItem(_ table: TableInfo) -> SQLCompletionItem { + let isView = table.type == .view || table.type == .materializedView + return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) } func tableCompletionItems(inSchema schema: String) async -> [SQLCompletionItem] { guard isKnownSchema(schema) else { return [] } guard !isActiveSchema(schema) else { return tableCompletionItems() } - return await tables(inSchema: schema).map { table in - let isView = table.type == .view || table.type == .materializedView - return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) - } + return await tables(inSchema: schema).map { tableItem($0) } } func schemaCompletionItems() -> [SQLCompletionItem] { @@ -317,6 +347,14 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } func allColumnsFromCachedTables() -> [SQLCompletionItem] { + if let cachedTableColumnItems { return cachedTableColumnItems } + let items = buildTableColumnItems() + tableColumnItemsBuildCount += 1 + cachedTableColumnItems = items + return items + } + + private func buildTableColumnItems() -> [SQLCompletionItem] { // Sorted, because the ranker keeps emission order between equal scores and a dictionary's order is per process. let cached = columns.sorted { lhs, rhs in (lhs.key.schema ?? "", lhs.key.table) < (rhs.key.schema ?? "", rhs.key.table) @@ -343,6 +381,76 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } } + // Kept and marked stale with the columns. A failed read keeps the last answer for this generation, so a catalog + // that refuses it is not asked on every key. + func foreignKeys(of tableName: String, schema: String?) async -> [SQLCompletionForeignKey] { + guard let readCatalogForeignKeys else { return [] } + let key = columnKey(tableName, schema: schema) + let known = cachedForeignKeys[key] + if let known, known.generation == columnGeneration { return known.keys } + if let pending = foreignKeyReads[key] { + return (try? await pending.value).flatMap { $0 } ?? known?.keys ?? [] + } + + let generation = columnGeneration + let namespace = schema ?? self.schema + let read = Task<[SQLCompletionForeignKey]?, any Error> { + try await readCatalogForeignKeys(key.table, key.schema).map { Self.completionForeignKeys($0, within: namespace) } + } + foreignKeyReads[key] = read + defer { + if foreignKeyReads[key] == read { foreignKeyReads[key] = nil } + } + do { + guard let keys = try await read.value else { return known?.keys ?? [] } + if generation == columnGeneration { + cachedForeignKeys[key] = CachedForeignKeys(generation: generation, keys: keys) + } + return keys + } catch { + Self.logger.warning("Reading foreign keys for completion failed: \(error.localizedDescription, privacy: .private)") + if generation == columnGeneration { + cachedForeignKeys[key] = CachedForeignKeys(generation: generation, keys: known?.keys ?? []) + } + return known?.keys ?? [] + } + } + + // A key into another schema or database would pair with a same-named table of this one. + nonisolated private static func completionForeignKeys( + _ rows: [ForeignKeyInfo], + within namespace: String? + ) -> [SQLCompletionForeignKey] { + let local = rows.filter { row in + guard let namespace, let target = row.referencedSchema, !target.isEmpty else { return true } + return target.caseInsensitiveCompare(namespace) == .orderedSame + } + return SQLCompletionForeignKey.grouping(local.map { + SQLCompletionForeignKey.Row( + name: $0.name, + column: $0.column, + referencedTable: $0.referencedTable, + referencedSchema: $0.referencedSchema, + referencedColumn: $0.referencedColumn + ) + }) + } + + // Nothing spells the items after ranking here, so a condition's names are quoted as the other items are. + nonisolated func writtenText(of identifier: SQLCompletionIdentifier) -> String? { + let type = databaseType + return identifier.parts.map { part -> String in + switch part { + case .catalog(let name): + return CompletionIdentifierQuoting.quoted(name, for: type) + case .statement(let text, let isQuoted): + return isQuoted + ? SQLBuilder.quoteIdentifier(text, for: type) + : CompletionIdentifierQuoting.quotedAsWritten(text, for: type) + } + }.joined(separator: ".") + } + private func columns(of tableName: String, schema: String?) async -> [ColumnInfo] { let key = columnKey(tableName, schema: schema) let known = columns[key] diff --git a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift index a2cf6873c8..26993d44cf 100644 --- a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift +++ b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift @@ -80,7 +80,8 @@ final class QueryCompletionController { readColumns: @escaping QueryColumnReader, readCatalogColumns: QueryColumnReader? = nil, readSchemaTables: QuerySchemaTableReader? = nil, - readCatalogSchemaTables: QuerySchemaTableReader? = nil + readCatalogSchemaTables: QuerySchemaTableReader? = nil, + readCatalogForeignKeys: QueryForeignKeyReader? = nil ) { let previous = scope guard previous != newScope else { return } @@ -145,6 +146,7 @@ final class QueryCompletionController { readCatalogColumns: readCatalogColumns, readSchemaTables: readSchemaTables, readCatalogSchemaTables: readCatalogSchemaTables, + readCatalogForeignKeys: readCatalogForeignKeys, schemas: newScope.schemas, databases: newScope.databases ) @@ -216,7 +218,7 @@ final class QueryCompletionController { ) show([], answer: nil) return edit - case .keyword, .operator: + case .keyword, .operator, .joinCondition: return Self.keywordEdit(for: item, replacing: answer.range, in: source) case .function, .favorite: let insertion = SQLCompletionInsertion.resolve(for: item) diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift index 8555d70ad1..7e72b5f3a9 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift @@ -4,39 +4,46 @@ import os import SwiftUI import TableProConnectionLibrary import TableProDatabase -import TableProLogRedaction import TableProModels import TableProSQLGrammar +// One window's view of a connection. The session, its lists and its lifecycle are the connection's +// `ConnectionSessionModel`, shared by every window; this keeps what is the window's own and asks the model for the rest. @MainActor @Observable final class ConnectionCoordinator { - private(set) var connection: DatabaseConnection - - private(set) var session: ConnectionSession? - private(set) var phase: ConnectionPhase = .connecting - private(set) var tables: [TableInfo] = [] { - didSet { selectedTable = TableSelectionResolver.keeping(selectedTable, in: tables) } - } - private(set) var databases: [String] = [] - private(set) var schemas: [String] = [] - private(set) var activeDatabase: String = "" - private(set) var activeSchema: String = "public" + private let model: ConnectionSessionModel + private let appState: AppState - private(set) var isSwitching = false { - didSet { resumeFollowIfPending() } - } + // False until the window connects, and again once it walks away: it then shows no session, whatever the model + // still holds for other windows. + private(set) var isAttached = false + // What the window showed when the session moved under its unsaved edit; it stays there until the edit ends. + private var pin: ConnectionSnapshot? + + var connection: DatabaseConnection { model.record } + var session: ConnectionSession? { isAttached ? shownState.session : nil } + var phase: ConnectionPhase { isAttached ? shownState.phase : .notConnected } + var isReconnecting: Bool { isAttached && shownState.isReconnecting } + var tables: [TableInfo] { shownCatalog.tables } + var databases: [String] { shownCatalog.databases } + var schemas: [String] { shownCatalog.schemas } + var activeDatabase: String { shownCatalog.placement.database } + var activeSchema: String { shownCatalog.placement.schema } + + private var shownState: ConnectionSessionState { pin?.state ?? model.state } + private var shownCatalog: ConnectionCatalog { pin?.catalog ?? model.catalog } + + private(set) var isSwitching = false private(set) var switchTarget: String? - private(set) var isReconnecting = false { - didSet { resumeFollowIfPending() } - } - // Another window moved the session while this one was busy; it follows once it is free. - @ObservationIgnored private var followPending = false - // A follow that changes the database rebuilds the table screen, which would drop an unsaved row edit. + // A move of the session would rebuild the table screen, which would drop an unsaved row edit. var isHeldByEditor = false { didSet { guard isHeldByEditor != oldValue else { return } appState.setEditorHold(isHeldByEditor, by: self, on: connection.id) - resumeFollowIfPending() + guard !isHeldByEditor, pin != nil else { return } + pin = nil + selectedTable = TableSelectionResolver.keeping(selectedTable, in: tables) + resumedFollow = Task { await followLiveSession() } } } @@ -44,19 +51,8 @@ final class ConnectionCoordinator { appState.holdsEditor(connection.id, besides: self) } - // One follow at a time, so an older one's late tables never land over a newer session's. - @ObservationIgnored private var isFollowing = false { - didSet { resumeFollowIfPending() } - } @ObservationIgnored private(set) var resumedFollow: Task? - @ObservationIgnored private var sessionCheck: Task? - @ObservationIgnored private var statementFailedDuringCheck = false - @ObservationIgnored private var metadataRefused = false - @ObservationIgnored private var seenCatalogGeneration: Int - @ObservationIgnored private var isReloadingCatalog = false - @ObservationIgnored private var catalogReloadRequested = false - // A redial that failed over an open transaction; the user hears about it once a later dial lands. - @ObservationIgnored private var unsettledLoss: SessionLoss? + @ObservationIgnored private var joiners = AttemptJoiners() private(set) var failureAlertTitle: String? var failureAlertMessage: String? var showFailureAlert = false @@ -71,12 +67,11 @@ final class ConnectionCoordinator { var selectedTable: TableInfo? private(set) var queryHistory: [QueryHistoryItem] = [] + private(set) var keepsDamagedHistoryCopy = false + @ObservationIgnored private var historyGeneration = 0 private var historyStorage: QueryHistoryStorage { appState.queryHistory } - private let appState: AppState - var connectionManager: ConnectionManager { appState.connectionManager } - private static let logger = Logger(subsystem: "com.TablePro", category: "ConnectionCoordinator") enum ConnectionPhase: Sendable { case notConnected @@ -123,28 +118,14 @@ final class ConnectionCoordinator { } } - var displayName: String { - connection.name.isEmpty ? connection.host : connection.name - } - - var supportsDatabaseSwitching: Bool { - connection.type == .mysql || connection.type == .mariadb || - connection.type == .tidb || connection.type == .oceanbase || - connection.type == .postgresql || connection.type == .redshift || - connection.type == .mssql || connection.type == .clickhouse || - connection.type == .mongodb || connection.type == .cloudflareD1 - } - - var supportsSchemas: Bool { - connection.type == .postgresql || connection.type == .redshift || - connection.type == .mssql || connection.type == .duckdb || - connection.type == .oracle - } + var displayName: String { model.displayName } + var supportsDatabaseSwitching: Bool { model.supportsDatabaseSwitching } + var supportsSchemas: Bool { model.supportsSchemas } init(connection: DatabaseConnection, appState: AppState) { - self.connection = connection self.appState = appState - seenCatalogGeneration = appState.catalogChanges.generation(for: connection.id) + model = appState.sessionModels.model(for: connection, appState: appState) + model.register(self) } // MARK: - Persisted State @@ -155,87 +136,59 @@ final class ConnectionCoordinator { let tab = ConnectedTab(rawValue: savedTab) { selectedTab = tab } - activeDatabase = defaults.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) ?? "" - activeSchema = defaults.string(forKey: ConnectionDefaultsKey.lastSchema.name(for: connection.id)) ?? "public" + model.restorePersistedPlacement() } // MARK: - Connection Lifecycle - // The attempt allowed to write `session` and `phase`. Cancelling mints a new one. - private var attemptToken = UUID() - private var connectTask: Task? { - didSet { resumeFollowIfPending() } - } - - private var joiners = AttemptJoiners() - let prompts = ConnectionPromptQueue() - // Returning early without touching `phase` is what left the connecting screen up for good. + // The last caller to go away walks the window away, so a screen that closed mid-open leaves nothing waiting. func connect() async { prompts.expectScreen() - if let inFlight = connectTask { - await join(inFlight) - return - } - - let token = UUID() - attemptToken = token - phase = .connecting - - let task = Task { [weak self] in - guard let self else { return } - await self.runAttempt(token: token) - } - connectTask = task - await join(task) - if connectTask == task { connectTask = nil } - } - - // The last caller to go away abandons the attempt, so the next screen never joins it and waits forever. - private func join(_ task: Task) async { + isAttached = true let joiner = joiners.join() await withTaskCancellationHandler { - await task.value + await model.connect(from: self) _ = joiners.leave(joiner) } onCancel: { Task { @MainActor [weak self] in - guard let self else { return } - guard joiners.leave(joiner), connectTask == task else { return } + guard let self, joiners.leave(joiner), case .connecting = phase else { return } cancelConnect() } } + await model.runHeldBackWork(from: self) } func adopt(_ record: DatabaseConnection) { - guard record.id == connection.id, record != connection else { return } - connection = record + model.adopt(record) } func retire() { - attemptToken = UUID() - connectTask?.cancel() - connectTask = nil + isAttached = false + pin = nil prompts.cancelAll() if appState.sessionRedialer.leave(connection.id, consumer: self) { appState.connectionManager.invalidateAttempt(for: connection.id) } appState.setEditorHold(false, by: self, on: connection.id) - session = nil + model.leave(self) + } + + // The record changed how it dials, so no window keeps the session. + func dropSession() { + model.reset() } // Never waits on the driver, which ignores `Task.cancel()`. The questions go first: a reconnect or a - // database switch asks without owning `connectTask`, and a suspended one holds its tunnel open. - // A lost session's redial is the session check, not `connectTask`; retiring mints a new attempt token, so a dial - // that returns after Cancel is never adopted. + // database switch asks without being the open, and a suspended one holds its tunnel open. func cancelConnect() { prompts.cancelAll() - guard connectTask != nil || (isLost && isReconnecting) else { return } + guard isAttached, model.state.isOpening || (isLost && isReconnecting) else { return } retire() - phase = .notConnected } - // A lost session redials where this window was, so the screen of a row edit in progress is not rebuilt. + // A lost session redials where it was, so the screen of a row edit in progress is not rebuilt. func reconnect() async { if isLost { await restoreSession(after: .whileIdle) @@ -244,111 +197,6 @@ final class ConnectionCoordinator { } } - private func runAttempt(token: UUID) async { - // Another window's dial is shared once its schema is back, never torn down mid-dial. - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token else { return } - if let existing = appState.connectionManager.session(for: connection.id) { - do { - let existingTables = try await existing.driver.fetchTables(schema: nil) - guard attemptToken == token else { return } - session = existing - tables = existingTables - await loadDatabases() - await loadSchemas() - await refreshTransaction() - guard attemptToken == token else { return } - phase = .connected - appState.libraryPreferences.recordConnected(connection.id) - reportDroppedTransaction(unsettledLoss) - return - } catch { - guard attemptToken == token else { return } - if await keepsRefusingSession(existing, token: token, refusal: error) { return } - guard attemptToken == token else { return } - session = nil - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token else { return } - if let live = appState.connectionManager.session(for: connection.id), live.driver !== existing.driver { - await runAttempt(token: token) - return - } - await appState.connectionManager.disconnect(connection.id) - } - } - - guard attemptToken == token else { return } - await connectFresh(token: token) - } - - // `allowSignIn` is false on the retry that follows a sign-in, so a connection that keeps - // failing cannot put the prompt up again and again. That retry dials with the new credentials - // rather than join a dial another window started without them. - private func connectFresh(token: UUID, allowSignIn: Bool = true) async { - Telemetry.shared.connectionAttempted(databaseType: connection.type.rawValue) - - do { - let outcome = try await appState.sessionRedialer.redial( - connection, - schema: nil, - prompter: prompts, - joining: allowSignIn, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ) - let newSession = outcome.session - let newTables = try await newSession.driver.fetchTables(schema: nil) - guard attemptToken == token else { return } - if !outcome.dialedHere { - adoptPlacement(of: newSession, includingDatabase: true) - } - session = newSession - tables = newTables - if outcome.dialedHere { - appState.bumpSessionRevision(connection.id) - } - await loadDatabases() - await loadSchemas() - await refreshTransaction() - guard attemptToken == token else { return } - phase = .connected - appState.libraryPreferences.recordConnected(connection.id) - Telemetry.shared.connectionSucceeded(databaseType: connection.type.rawValue) - navigateToPendingTable() - reportDroppedTransaction(unsettledLoss) - } catch { - guard attemptToken == token else { return } - var failure = error - // A sign-in that expired is recoverable, so offer it once and retry rather than - // leaving the user on an error screen whose only button repeats the same failure. - if allowSignIn, - let method = SignInOffer.method(for: error, connection: connection, secureStore: appState.secureStore) { - switch await SignInOffer.offer(method, prompts: prompts) { - case .signedIn: - guard attemptToken == token else { return } - await connectFresh(token: token, allowSignIn: false) - return - case .failed(let signInError): - failure = signInError - case .declined: - break - } - } - guard attemptToken == token else { return } - phase = .failed(classify(failure, operation: "connect")) - } - } - - private func classify(_ error: any Error, operation: String) -> AppError { - ErrorClassifier.classify(error, context: ErrorContext( - operation: operation, - databaseType: connection.type, - host: connection.host, - sshEnabled: connection.sshEnabled, - signsInWithIAM: connection.iamSignIn != nil - )) - } - func reconnectIfNeeded() async { await restoreSession(after: .whileIdle) } @@ -369,265 +217,35 @@ final class ConnectionCoordinator { // True when the redial dropped a held transaction and the outcome is already settled for the user. @discardableResult private func restoreSession(after loss: SessionLoss) async -> Bool { - // One check at a time, and a statement that failed while it ran makes the loss ambiguous. - if let sessionCheck { - if loss == .duringWork { statementFailedDuringCheck = true } - return await sessionCheck.value - } - statementFailedDuringCheck = false - let check = Task { - let settled = await checkSession(after: loss) - sessionCheck = nil - return settled - } - sessionCheck = check - return await check.value - } - - private func checkSession(after loss: SessionLoss) async -> Bool { - let held = session - // A failed redial dropped the session, so there is nothing to ping and this trigger dials again. - guard held != nil || isLost, !isSwitching, !isReconnecting, connectTask == nil else { return false } - let token = attemptToken - var workWasRunning = appState.sessionTransactions.isRunningStatements(connection.id) - - switch await newerSession(than: held, token: token) { - case .taken: - return held == nil && reportDroppedTransaction(unsettledLoss) - case .deferred: - return false - case .none: - break - } - if let held { - if appState.connectionManager.session(for: connection.id) != nil, await Self.answersPing(held.driver) { - return false - } - guard attemptToken == token else { return false } - // Another window can finish a whole redial while this ping is out. - if await newerSession(than: held, token: token) != .none { return false } - } - - workWasRunning = workWasRunning || appState.sessionTransactions.isRunningStatements(connection.id) - let ended = appState.sessionTransactions.endedHold(for: connection.id, on: held?.driver) - workWasRunning = workWasRunning || ended?.duringWork == true - let heldTransaction = transactionHold != nil || ended != nil - let released = ended.flatMap { releasedLoss($0, on: held, asked: loss) } - isReconnecting = true - defer { isReconnecting = false } - do { - let outcome = try await appState.sessionRedialer.redial( - dialTarget, - schema: supportsSchemas ? activeSchema : nil, - prompter: prompts, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ) - guard attemptToken == token else { return false } - await takeSession(outcome.session, token: token, dialedHere: outcome.dialedHere) - guard attemptToken == token else { return false } - let dropped = heldTransaction ? released ?? settledLoss(loss, workWasRunning: workWasRunning) : unsettledLoss - return reportDroppedTransaction(dropped) - } catch { - guard attemptToken == token else { return false } - if heldTransaction { - unsettledLoss = released ?? settledLoss(loss, workWasRunning: workWasRunning) - } - phase = .lost(classify(error, operation: "reconnect")) - self.session = nil - return false - } - } - - private func settledLoss(_ loss: SessionLoss, workWasRunning: Bool) -> SessionLoss { - workWasRunning || statementFailedDuringCheck ? .duringWork : loss - } - - // The release closed the file once its running statement ended, so a later failure on the closed file decides nothing. - private func releasedLoss( - _ ended: SessionTransactionMonitor.EndedHold, - on held: ConnectionSession?, - asked loss: SessionLoss - ) -> SessionLoss? { - guard held?.driver.holdsSuspensionBlockingResource == true else { return nil } - if ended.duringWork { return .inBackgroundAfterStatement } - return loss == .duringRollBack ? .duringRollBack : .inBackground - } - - // True when the transaction a redial ended is settled for the user. - @discardableResult - private func reportDroppedTransaction(_ loss: SessionLoss?) -> Bool { - unsettledLoss = nil - guard let loss else { return false } - appState.sessionTransactions.noteRolledBack(connection.id) - if let message = Self.lostTransactionMessage(loss, connectionName: displayName) { - presentFailure(title: nil, message: message) - } - return true - } - - private enum NewerSession { - case none - case taken - // Another window holds it on another database or schema, and this window's unsaved edit belongs where it is. - case deferred - } - - private func newerSession(than held: ConnectionSession?, token: UUID) async -> NewerSession { - // A redial another window started is not ready to share until its schema is back. - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token, - let live = appState.connectionManager.session(for: connection.id), - live.driver !== held?.driver else { - return .none - } - guard !editPins(against: live) else { - followPending = true - return .deferred - } - return await takeSession(live, token: token, dialedHere: false) ? .taken : .deferred - } - - // An unsaved edit belongs to the placement it was made in, so a session elsewhere waits until the edit ends; - // taking it would rebuild the editor's screen, or point its save at a same-named table in another database. - private func editPins(against live: ConnectionSession) -> Bool { - guard isHeldByEditor else { return false } - let moved = movedPlacement(of: live, includingDatabase: true) - return moved.database != nil || moved.schema != nil - } - - private var dialTarget: DatabaseConnection { - guard supportsDatabaseSwitching, !activeDatabase.isEmpty else { return connection } - var target = connection - target.database = activeDatabase - return target - } - - @discardableResult - private func takeSession( - _ newSession: ConnectionSession, - token: UUID, - dialedHere: Bool, - yieldingToEdits: Bool = false - ) async -> Bool { - let placementAsked = movedPlacement(of: newSession, includingDatabase: !dialedHere) - let fetched: [TableInfo]? - do { - fetched = try await newSession.driver.fetchTables(schema: nil) - } catch { - Self.logger.warning("Failed to load tables on the new session: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - fetched = nil - } - guard attemptToken == token else { return false } - if !dialedHere, editPins(against: newSession) { - followPending = true - return false - } - // A row edit started while the tables loaded lives on a screen this would rebuild, and another switch that - // landed meanwhile made these tables the wrong database's. - if yieldingToEdits, - !canFollowLiveSession || movedPlacement(of: newSession, includingDatabase: !dialedHere) != placementAsked { - followPending = true - return false - } - // The dial restored this window's schema, or another window's; a failed restore leaves the default. - let movedPlacement = adoptPlacement(of: newSession, includingDatabase: !dialedHere) - if let fetched { - tables = fetched - } else { - metadataRefused = true - if movedPlacement { tables = [] } - } - session = newSession - phase = .connected - if dialedHere { - appState.bumpSessionRevision(connection.id) + guard isAttached else { return false } + let settled = await model.restoreSession(after: loss, from: self) + await model.runHeldBackWork(from: self) + return settled + } + + // An unsaved edit belongs to the placement it was made in, so while it is open the window keeps what it showed + // when the session moved elsewhere: taking it would rebuild the editor's screen, or point its save at a same-named + // table in another database. A move this window asked for itself reaches it. + func sessionModelChanged(from old: ConnectionSnapshot) { + guard pin == nil else { return } + if isHeldByEditor, !isSwitching, model.moves(from: old) { + // An open or a switch in flight is not where the edit was made; the loss or session before it is. + pin = ConnectionSnapshot(state: old.state.atRest, catalog: old.catalog) + return } - await refreshTransaction() - return true - } - - // A session another window dialed or moved sits on that window's database and schema, so this one - // follows it before any write can name a table from the old list. - @discardableResult - private func adoptPlacement(of taken: ConnectionSession, includingDatabase: Bool) -> Bool { - let moved = movedPlacement(of: taken, includingDatabase: includingDatabase) - if let database = moved.database { activeDatabase = database } - if let schema = moved.schema { activeSchema = schema } - return moved.database != nil || moved.schema != nil + guard old.catalog.tables != model.catalog.tables else { return } + selectedTable = TableSelectionResolver.keeping(selectedTable, in: model.catalog.tables) } - private func movedPlacement(of taken: ConnectionSession, includingDatabase: Bool) -> (database: String?, schema: String?) { - var database: String? - if includingDatabase { - let candidate = sessionDatabase(of: taken) - if supportsDatabaseSwitching, !candidate.isEmpty, candidate != activeDatabase { database = candidate } - } - var schema: String? - if supportsSchemas, let current = taken.driver.currentSchema, current != activeSchema { schema = current } - return (database, schema) + // It shows the model's session and no edit holds it, so it can hear what the model does for every window. + var showsLiveSession: Bool { + isAttached && pin == nil && !isHeldByEditor } - // Another window replaced the shared session, or switched its database or schema. + // The manager's session moved, or another party replaced it. func followLiveSession() async { - guard canFollowLiveSession, !isFollowing else { - followPending = true - return - } - // A window whose redial failed takes the session another window dialed. - if isLost { - guard appState.connectionManager.session(for: connection.id) != nil else { return } - await restoreSession(after: .whileIdle) - return - } - isFollowing = true - defer { isFollowing = false } - let token = attemptToken - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token else { return } - guard canFollowLiveSession else { - followPending = true - return - } - guard let held = session, let live = appState.connectionManager.session(for: connection.id) else { return } - if live.driver !== held.driver { - guard await takeSession(live, token: token, dialedHere: false, yieldingToEdits: true), - attemptToken == token else { return } - await loadSchemas() - return - } - let moved = movedPlacement(of: live, includingDatabase: true) - guard moved.database != nil || moved.schema != nil else { return } - let fetched: [TableInfo]? - do { - fetched = try await live.driver.fetchTables(schema: supportsSchemas ? moved.schema ?? activeSchema : nil) - } catch { - Self.logger.warning("Failed to load tables where another window moved the session: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - fetched = nil - } - guard attemptToken == token, session?.driver === live.driver else { return } - // A row edit started while the tables loaded lives on a screen the new placement would rebuild, and another - // switch that landed meanwhile made these tables the wrong database's. - guard canFollowLiveSession, movedPlacement(of: live, includingDatabase: true) == moved else { - followPending = true - return - } - // Placement and tables change in one step, so a row of the old list never pairs with the new placement. - if let database = moved.database { activeDatabase = database } - if let schema = moved.schema { activeSchema = schema } - session = live - if let fetched { - tables = fetched - } else { - metadataRefused = true - tables = [] - } - // SQL Server lists another database's schemas. - if moved.database != nil { - await loadSchemas() - guard attemptToken == token else { return } - } - await refreshTransaction() + guard isAttached, pin == nil else { return } + await model.follow(from: self) } // Where an unqualified table name resolves for this window. Another window's switch can move it under an open @@ -642,47 +260,29 @@ final class ConnectionCoordinator { TablePlacement( driver: session.map { ObjectIdentifier($0.driver) }, database: activeDatabase, - schema: supportsSchemas ? activeSchema : nil + schema: tableQualifier ) } - static var editorHeldElsewhereMessage: String { - String(localized: "Another window has unsaved changes on this connection. Save or discard them before switching database.") + // What the Tables tab qualifies a table with. MySQL's `db`.`table` names the database where other engines name a + // schema, so a `USE` run in the Query tab never moves where a browse, a save or a drop lands. + var tableQualifier: String? { + if supportsSchemas { return activeSchema } + guard model.qualifiesTablesWithDatabase, !activeDatabase.isEmpty else { return nil } + return activeDatabase } - private var canFollowLiveSession: Bool { - (session != nil || isLost) && !isSwitching && !isReconnecting && connectTask == nil && !isHeldByEditor + // SQL Server names it in front of the schema; other engines ignore it. + var tableDatabase: String? { + activeDatabase.isEmpty ? nil : activeDatabase } - private func resumeFollowIfPending() { - guard followPending, canFollowLiveSession, !isFollowing else { return } - followPending = false - resumedFollow = Task { await followLiveSession() } - } - - // A transaction the server aborted refuses catalog reads, yet the session still holds the user's work. - private func keepsRefusingSession(_ existing: ConnectionSession, token: UUID, refusal: any Error) async -> Bool { - guard await Self.answersPing(existing.driver), attemptToken == token else { return false } - await refreshTransaction() - guard attemptToken == token, - appState.sessionTransactions.hold(for: connection.id, on: existing.driver) != nil else { return false } - Self.logger.info("Kept a session that refused its catalog: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") - let database = sessionDatabase(of: existing) - if supportsDatabaseSwitching, !database.isEmpty { - activeDatabase = database - } - metadataRefused = true - session = existing - phase = .connected - appState.libraryPreferences.recordConnected(connection.id) - return true + static var editorHeldElsewhereMessage: String { + String(localized: "Another window has unsaved changes on this connection. Save or discard them before switching database.") } - private func sessionDatabase(of taken: ConnectionSession) -> String { - guard let live = appState.connectionManager.session(for: connection.id), live.driver === taken.driver else { - return taken.activeDatabase - } - return live.activeDatabase + static var editorHeldHereMessage: String { + String(localized: "This window has unsaved changes on this connection, so the session stayed on its database. Save or discard them before switching database.") } static func lostTransactionMessage(_ loss: SessionLoss, connectionName: String) -> String? { @@ -720,37 +320,18 @@ final class ConnectionCoordinator { } } - private static func answersPing(_ driver: any DatabaseDriver) async -> Bool { - do { - _ = try await driver.ping() - return true - } catch { - return false - } - } - // MARK: - Transactions var transactionHold: SessionTransactionMonitor.Hold? { appState.sessionTransactions.hold(for: connection.id, on: session?.driver) } - // PostgreSQL, Redshift and SQL Server keep DDL inside the transaction that ran it, where only the session sees it; - // a session that cannot say whether it holds one counts as holding one. var catalogSeesSessionDDL: Bool { - guard Self.transactionalDDLTypes.contains(connection.type) else { return true } - return appState.sessionTransactions.isKnownIdle(connection.id, on: session?.driver) + model.catalogSeesSessionDDL(on: session?.driver) } - private static let transactionalDDLTypes: Set = [.postgresql, .redshift, .mssql] - - // Completion reads beside the session only while both see the same tables: same DDL, same database, no table - // only a procedure on the session made. var catalogMatchesSession: Bool { - guard catalogSeesSessionDDL else { return false } - return !appState.catalogChanges.sessionLeftCatalog( - for: connection.id, driver: session.map { ObjectIdentifier($0.driver) } - ) + model.catalogMatchesSession(on: session?.driver) } var transactionToken: SessionTransactionMonitor.HoldToken? { @@ -777,7 +358,7 @@ final class ConnectionCoordinator { if heldBefore, !lossSettled, rolledBack(value) { appState.sessionTransactions.noteRolledBack(connection.id) } - await reloadMetadataIfReleased() + await model.reloadMetadataIfReleased(from: self) return value } catch { await restoreSession(after: loss) @@ -809,9 +390,11 @@ final class ConnectionCoordinator { guard await !restoreSession(after: .duringWork) else { return } throw error } - await reloadMetadataIfReleased() + await model.reloadMetadataIfReleased(from: self) // A catalog connection sees DDL the transaction ran only once it commits. noteCatalogChanged([.tables, .schemas, .databases]) + // A `SET LOCAL search_path` ends with the transaction. + await followSessionSchema() } func rollBackTransaction(as token: SessionTransactionMonitor.HoldToken) async throws { @@ -821,224 +404,169 @@ final class ConnectionCoordinator { guard await !restoreSession(after: .duringRollBack) else { return } throw error } - await reloadMetadataIfReleased() + await model.reloadMetadataIfReleased(from: self) // The lists may show a schema or a database the rolled-back transaction created. noteCatalogChanged([.schemas, .databases]) } func refreshTransaction() async { - await appState.sessionTransactions.refresh(connection.id) + await model.refreshTransaction() } - // A Roll Back from any window can undo DDL; a failed read keeps the list it has. + // A Roll Back from any window can undo DDL. func reloadAfterRollBack() async { - guard !metadataRefused else { - await reloadMetadataIfReleased() - return - } - guard let session else { return } - do { - let reloaded = try await session.driver.fetchTables(schema: supportsSchemas ? activeSchema : nil) - guard self.session?.driver === session.driver else { return } - tables = reloaded - } catch { - Self.logger.warning("Failed to reload tables after a Roll Back: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - } - // With autocommit off, a MySQL read opens a transaction. - await refreshTransaction() + guard isAttached else { return } + await model.reloadAfterRollBack(from: self) } - private func reloadMetadataIfReleased() async { - guard metadataRefused, let session, - appState.sessionTransactions.hold(for: connection.id, on: session.driver) == nil else { return } - do { - let reloaded = try await session.driver.fetchTables(schema: nil) - guard self.session?.driver === session.driver else { return } - tables = reloaded - metadataRefused = false - } catch { - Self.logger.warning("Tables are still refused: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - return - } - await loadDatabases() - await loadSchemas() + // MARK: - Database / Schema Switching + + // What a run may have moved: a `USE` moves the database, and a search path, a role or a transaction's end the schema. + struct SessionMove { + var database = false + var schema = false } - // MARK: - Database / Schema Switching + // A read that costs a statement replaces ROW_COUNT() and the warnings, so it is sent only after a run that can have + // moved the session. The work runs in a task of its own, since Stop may have cancelled the run that asks. + func followSessionPlacement(after move: SessionMove) async { + await followSessionDatabase(after: move) + if move.schema { await followSessionSchema() } + } - // A MongoDB `use` moves the session itself, so the lists and the picker follow it there, once every request queued - // before has returned: a request Stop ended lands its `use` only then. The switch runs outside the run that asks, - // which Stop may have cancelled. While another window holds unsaved edits the session goes back instead, so they - // save where the picker says. - func followSessionDatabase() async { - guard supportsDatabaseSwitching, let driver = session?.driver as? PluginBackedDriver else { return } + private func followSessionDatabase(after move: SessionMove) async { + guard supportsDatabaseSwitching, let driver = session?.driver as? any SessionDatabaseReporting, + move.database || driver.reportsDatabaseWithoutStatement else { return } await Task { - guard let used = try? await driver.settledCurrentDatabase(), used != activeDatabase, - session?.driver === driver else { return } - guard !anotherWindowHoldsEditor else { - try? await driver.switchDatabase(to: activeDatabase) - presentFailure(title: Self.switchDatabaseFailedTitle, message: Self.editorHeldElsewhereMessage) + let used = try? await driver.sessionDatabase() + // With autocommit off, a MySQL read opens a transaction. + if !driver.reportsDatabaseWithoutStatement { await refreshTransaction() } + guard let used, !used.isEmpty, used != activeDatabase, session?.driver === driver else { return } + // An unsaved edit holds the session where it was made, as it holds the picker. + if isHeldByEditor || anotherWindowHoldsEditor, !activeDatabase.isEmpty { + let back = activeDatabase + let failure = await whileSwitching(to: back) { + do { + try await driver.switchDatabase(to: back) + return nil + } catch { + return ConnectionFailure(error: error, title: Self.switchDatabaseFailedTitle, redialRepeats: false) + } + } + guard failure != nil else { + presentFailure( + title: Self.switchDatabaseFailedTitle, + message: isHeldByEditor ? Self.editorHeldHereMessage : Self.editorHeldElsewhereMessage + ) + return + } + // The lists follow the session, and every window with an edit open keeps the database it was made in, + // this one too, since it did not ask for the move. + model.beginSwitch() + _ = await model.switchDatabase(to: used, from: self, inside: nil) + model.endSwitch() + await model.loadSchemasAfterSwitch(to: used, from: self) + await model.runHeldBackWork(from: self) + // Every Tables tab write names its database, and a plugin driver runs the app's requests in the menu's. + presentFailure( + title: Self.switchDatabaseFailedTitle, + message: String( + format: String(localized: "The query moved the session to “%@” and it could not go back to “%@”. Unsaved changes still save to “%@”."), + used, back, back + ) + ) return } await switchDatabase(to: used) }.value } + // The Tables tab names its schema, so only completion and the picker read where the session is. + private func followSessionSchema() async { + guard supportsSchemas, let driver = session?.driver as? any SessionSchemaReporting else { return } + await driver.settleCurrentSchema() + guard session?.driver === driver, let current = driver.currentSchema, current != activeSchema else { return } + appState.bumpSessionRevision(connection.id) + await followLiveSession() + } + + // From the start of a run with a `USE` until the picker follows it, another run's statements could go out around the + // move, so no window runs one. The Tables tab names its database, so its writes go on. A driver that reports its + // database without a statement keeps the app's requests where the picker says. + func beginDatabaseMove(ifRunning text: String) -> Bool { + guard supportsDatabaseSwitching, let driver = session?.driver as? any SessionDatabaseReporting, + !driver.reportsDatabaseWithoutStatement, + SessionLocalStatements.effect(of: [text], databaseType: connection.type).usesDatabase else { return false } + model.beginDatabaseMove() + return true + } + + func endDatabaseMove() { + model.endDatabaseMove() + } + + var isDatabaseMoving: Bool { model.isDatabaseMoving } + + static var databaseMovingRefusal: AppError { + AppError( + category: .query, + title: String(localized: "Database Switch Running"), + message: String(localized: "A query that switches the database is running on this connection. Try again once it ends."), + recovery: nil, + underlying: nil + ) + } + func switchDatabase(to name: String) async { - guard session != nil, name != activeDatabase, !isSwitching else { return } + await switchDatabase(to: name, inside: nil) + } + + // `open` is the open this switch puts back where the connection was; the switch runs as a step of it. + func switchDatabase(to name: String, inside open: ConnectionAttempt?) async { + guard isAttached, session != nil || model.isSwitchingDatabase, name != activeDatabase, !isSwitching else { return } guard !anotherWindowHoldsEditor else { presentFailure(title: Self.switchDatabaseFailedTitle, message: Self.editorHeldElsewhereMessage) return } let failure = await whileSwitching(to: name) { - let failure: Failure? - if connection.type == .postgresql || connection.type == .redshift { - await reconnectWithDatabase(name) - failure = nil - } else { - failure = await useDatabase(name) - } - await refreshTransaction() - appState.bumpSessionRevision(connection.id) - return failure + await model.switchDatabase(to: name, from: self, inside: open) } + await model.loadSchemasAfterSwitch(to: name, from: self) guard let failure else { return } await report(failure, operation: "switchDatabase") } - private func useDatabase(_ name: String) async -> Failure? { - do { - try await appState.connectionManager.switchDatabase(connection.id, to: name) - } catch { - return Failure(error: error, title: Self.switchDatabaseFailedTitle, redialRepeats: false) - } - if let freshSession = appState.connectionManager.session(for: connection.id) { - self.session = freshSession - } - UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) - do { - let fetched = try await session?.driver.fetchTables(schema: nil) ?? [] - // Placement and tables change in one step, so a row of the old list never pairs with the new database. - activeDatabase = name - tables = fetched - return nil - } catch { - // The session is on the new database either way, so its list empties until a reload can read it. - activeDatabase = name - metadataRefused = true - tables = [] - return Failure(error: error, title: Self.refreshTablesFailedTitle, redialRepeats: true) - } - } - - // Lets another window's redial land before the teardown that would cancel it, and dials where other windows join. - private func reconnectWithDatabase(_ database: String) async { - let token = attemptToken - await appState.sessionRedialer.settled(connection.id) - // A window closed during that wait must not tear down the session the other windows now use. - guard attemptToken == token else { return } - let previous = dialTarget - await appState.connectionManager.disconnect(connection.id) - self.session = nil - - var newConnection = connection - newConnection.database = database - - do { - let newSession = try await appState.sessionRedialer.redial( - newConnection, - schema: nil, - prompter: prompts, - joining: false, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ).session - guard attemptToken == token else { return } - let fetched = try await newSession.driver.fetchTables(schema: nil) - guard attemptToken == token else { return } - self.session = newSession - self.tables = fetched - activeDatabase = database - UserDefaults.standard.set(database, forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) - await loadSchemas() - } catch { - guard attemptToken == token else { return } - Self.logger.error("Failed to switch to database \(database, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - // Back to the database the picker still shows, not the saved one, so the picker stays true. - do { - let fallback = try await appState.sessionRedialer.redial( - previous, - schema: supportsSchemas ? activeSchema : nil, - prompter: prompts, - joining: false, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ) - guard attemptToken == token else { return } - await takeSession(fallback.session, token: token, dialedHere: fallback.dialedHere) - guard attemptToken == token else { return } - presentFailure( - title: Self.switchDatabaseFailedTitle, - message: classify(error, operation: "switchDatabase").message - ) - } catch { - guard attemptToken == token else { return } - phase = .lost(classify(error, operation: "switchDatabase")) - self.session = nil - } - } - } - func switchSchema(to name: String) async { guard let session, name != activeSchema, !isSwitching else { return } let failure = await whileSwitching(to: name) { - do { - try await session.driver.switchSchema(to: name) - } catch { - return Failure(error: error, title: Self.switchSchemaFailedTitle, redialRepeats: false) - } - UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastSchema.name(for: connection.id)) - appState.bumpSessionRevision(connection.id) - do { - let fetched = try await session.driver.fetchTables(schema: name) - activeSchema = name - tables = fetched - return nil - } catch { - activeSchema = name - metadataRefused = true - tables = [] - return Failure(error: error, title: Self.refreshTablesFailedTitle, redialRepeats: true) - } + await model.switchSchema(to: name, on: session) } guard let failure else { return } await report(failure, operation: "switchSchema") } - private func whileSwitching(to target: String, _ work: () async -> Failure?) async -> Failure? { + private func whileSwitching( + to target: String, + _ work: () async -> ConnectionFailure? + ) async -> ConnectionFailure? { switchTarget = target isSwitching = true - defer { - isSwitching = false - switchTarget = nil - } - return await work() - } - - private struct Failure { - let error: any Error - let title: String - // A failed read that a redial repeats needs no alert once the redial lands. - let redialRepeats: Bool + model.beginSwitch() + let failure = await work() + model.endSwitch() + isSwitching = false + switchTarget = nil + await model.runHeldBackWork(from: self) + return failure } // A failure on a dead session redials first. The alert then says why, unless the session is lost, which its own // screen says. - private func report(_ failure: Failure, operation: String) async { + func report(_ failure: ConnectionFailure, operation: String) async { let failedOn = session?.driver if await restoreSession(after: .whileIdle) { return } guard !isLost, !failure.redialRepeats || session?.driver === failedOn else { return } - presentFailure(title: failure.title, message: classify(failure.error, operation: operation).message) + presentFailure(title: failure.title, message: model.classify(failure.error, operation: operation).message) } func presentFailure(title: String?, message: String) { @@ -1054,13 +582,9 @@ final class ConnectionCoordinator { // MARK: - Tables func refreshTables() async { - guard let session else { return } - do { - let schema = supportsSchemas ? activeSchema : nil - self.tables = try await session.driver.fetchTables(schema: schema) - } catch { - Self.logger.warning("Failed to refresh tables: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - await report(Failure(error: error, title: Self.refreshTablesFailedTitle, redialRepeats: true), operation: "refreshTables") + guard session != nil else { return } + if let failure = await model.refreshTables() { + await report(failure, operation: "refreshTables") } // With autocommit off, a MySQL read opens a transaction. await refreshTransaction() @@ -1070,130 +594,86 @@ final class ConnectionCoordinator { // `reloadedHere`: the caller already read the session, the one place a transaction's own DDL shows. func noteCatalogChanged(_ kinds: SQLCatalogObjectKinds, reloadedHere: Bool = false) { - let board = appState.catalogChanges - let caughtUp = board.generation(for: connection.id) == seenCatalogGeneration - board.post(kinds, for: connection.id) - if reloadedHere, caughtUp { - seenCatalogGeneration = board.generation(for: connection.id) - } + model.noteCatalogChanged(kinds, reloadedHere: reloadedHere) } - // A statement a window ran may have created or dropped what the lists show; failed reads keep the lists. - // One reload at a time: a later post waits for it, so an older answer never lands over a newer one. func reloadChangedCatalog() async { - guard !isReloadingCatalog else { - catalogReloadRequested = true - return - } - isReloadingCatalog = true - defer { isReloadingCatalog = false } - repeat { - catalogReloadRequested = false - await reloadChangedCatalogOnce() - } while catalogReloadRequested + guard isAttached else { return } + await model.reloadChangedCatalog() } - private func reloadChangedCatalogOnce() async { - let board = appState.catalogChanges - let generation = board.generation(for: connection.id) - guard generation != seenCatalogGeneration, let session, !isSwitching else { return } - var wanted = board.kinds(for: connection.id, since: seenCatalogGeneration) - if !supportsSchemas { wanted.remove(.schemas) } - if !supportsDatabaseSwitching { wanted.remove(.databases) } - let kinds = wanted - let database = supportsDatabaseSwitching ? activeDatabase : "" - let schema = supportsSchemas ? activeSchema : nil - let place = CatalogPlace(driver: ObjectIdentifier(session.driver), database: activeDatabase, schema: activeSchema) - let monitor = appState.sessionTransactions - - var snapshot: CatalogSnapshot? - // On the session a read would replace ROW_COUNT() and the warnings, so it is only where the DDL is visible, or - // where a USE or a role moved the session off what the catalog connection sees, as Refresh reads it there too. - if catalogMatchesSession { - do { - snapshot = try await appState.connectionManager.readCatalog(for: connection.id, database: database) { - try await Self.readCatalog(kinds, schema: schema, from: $0) - } - } catch { - Self.logger.warning("Reloading the catalog after a change failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - seenCatalogGeneration = generation - return - } - } - if snapshot == nil { - // A server that has a catalog connection but could not open one keeps its lists: a read on the session would - // replace what the user's last statement left there. Only its own uncommitted DDL sends the read there. - if catalogMatchesSession, appState.connectionManager.opensCatalogConnections(for: connection.id) { - seenCatalogGeneration = generation - return - } - // A stream on the session would end under this read, so it waits for the session to go idle. - guard !monitor.isBusy(connection.id) else { return } - board.beginSessionRead(for: connection.id) - snapshot = try? await monitor.track(connection.id, reading: true) { - try await Self.readCatalog(kinds, schema: schema, from: session.driver) - } - board.endSessionRead(for: connection.id) - } - seenCatalogGeneration = generation - guard let snapshot, place == CatalogPlace( - driver: self.session.map { ObjectIdentifier($0.driver) }, database: activeDatabase, schema: activeSchema - ) else { return } - if let reloaded = snapshot.tables { tables = reloaded } - if let reloaded = snapshot.schemas { schemas = reloaded } - if let reloaded = snapshot.databases { databases = reloaded } - } + // MARK: - Query History - private struct CatalogPlace: Equatable { - let driver: ObjectIdentifier? - let database: String - let schema: String + // The file is read and written off the main actor, so a run never waits on it. + func loadHistory() async { + let read = await Self.readHistory(for: connection.id, in: historyStorage) + show(read.listed) + keepsDamagedHistoryCopy = read.keepsDamagedCopy } - private struct CatalogSnapshot: Sendable { - var tables: [TableInfo]? - var schemas: [String]? - var databases: [String]? + func addHistoryItem(_ item: QueryHistoryItem) async { + guard let listed = await Self.record(item, in: historyStorage) else { return } + show(listed) } - // The table list is the read that matters: its failure fails the reload, so a dead connection is dropped. - nonisolated private static func readCatalog( - _ kinds: SQLCatalogObjectKinds, - schema: String?, - from driver: any DatabaseDriver - ) async throws -> CatalogSnapshot { - var snapshot = CatalogSnapshot() - if kinds.contains(.tables) { - snapshot.tables = try await driver.fetchTables(schema: schema) - } - if kinds.contains(.schemas) { - snapshot.schemas = try? await driver.fetchSchemas() - } - if kinds.contains(.databases) { - snapshot.databases = try? await driver.fetchDatabases() - } - return snapshot + // Saves finish in any order; a list read before a later write never replaces what that write left. + private func show(_ listed: QueryHistoryStorage.Listed) { + guard listed.generation >= historyGeneration else { return } + historyGeneration = listed.generation + queryHistory = listed.items } - // MARK: - Query History - - func loadHistory() { - queryHistory = historyStorage.load(for: connection.id) + @concurrent + nonisolated private static func readHistory( + for connectionId: UUID, + in storage: QueryHistoryStorage + ) async -> (listed: QueryHistoryStorage.Listed, keepsDamagedCopy: Bool) { + (storage.listed(for: connectionId), storage.hasSetAsideCopies) } - func addHistoryItem(_ item: QueryHistoryItem) { - guard historyStorage.save(item) else { return } - loadHistory() + @concurrent + nonisolated private static func record( + _ item: QueryHistoryItem, + in storage: QueryHistoryStorage + ) async -> QueryHistoryStorage.Listed? { + storage.record(item) } - func deleteHistoryItem(_ id: UUID) { - historyStorage.delete(id) + // The row leaves the list at once, as a swipe expects, and the stored list, with any run saved meanwhile, follows. + @discardableResult + func deleteHistoryItem(_ id: UUID) -> Task { queryHistory.removeAll { $0.id == id } + let connectionId = connection.id + let storage = historyStorage + return Task { + let read = await Self.changeHistory(for: connectionId, in: storage) { $0.delete(id) } + show(read.listed) + } } - func clearHistory() { - historyStorage.clearAll(for: [connection.id]) + @discardableResult + func clearHistory() -> Task { + let removingSetAside = keepsDamagedHistoryCopy + let connectionId = connection.id + let storage = historyStorage queryHistory = [] + return Task { + let read = await Self.changeHistory(for: connectionId, in: storage) { storage in + storage.clearAll(for: [connectionId], removingSetAsideCopies: removingSetAside) + } + show(read.listed) + keepsDamagedHistoryCopy = read.keepsDamagedCopy + } + } + + @concurrent + nonisolated private static func changeHistory( + for connectionId: UUID, + in storage: QueryHistoryStorage, + _ change: @Sendable (QueryHistoryStorage) -> Void + ) async -> (listed: QueryHistoryStorage.Listed, keepsDamagedCopy: Bool) { + change(storage) + return (storage.listed(for: connectionId), storage.hasSetAsideCopies) } func navigateToPendingTable() { @@ -1202,47 +682,4 @@ final class ConnectionCoordinator { selectedTab = .tables selectedTable = table } - - // MARK: - Private Helpers - - private func loadDatabases() async { - guard let session, supportsDatabaseSwitching else { return } - do { - databases = try await session.driver.fetchDatabases() - if !activeDatabase.isEmpty, databases.contains(activeDatabase) { - let sessionDB = appState.connectionManager.session(for: connection.id)?.activeDatabase ?? connection.database - if activeDatabase != sessionDB { - let target = activeDatabase - activeDatabase = sessionDB - await switchDatabase(to: target) - } - } else { - activeDatabase = appState.connectionManager.session(for: connection.id)?.activeDatabase ?? connection.database - // A database the connection names by its id is listed by its name, which the driver reports. - if !databases.contains(activeDatabase), let reported = session.driver.currentDatabase, - databases.contains(reported) { - await switchDatabase(to: reported) - } - } - } catch { - Self.logger.warning("Failed to load databases: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - } - } - - private func loadSchemas() async { - guard let session, supportsSchemas else { return } - do { - schemas = try await session.driver.fetchSchemas() - let currentSchema = session.driver.currentSchema ?? "public" - if schemas.contains(activeSchema), activeSchema != currentSchema { - let target = activeSchema - activeSchema = currentSchema - await switchSchema(to: target) - } else if !schemas.contains(activeSchema) { - activeSchema = currentSchema - } - } catch { - Self.logger.warning("Failed to load schemas: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - } - } } diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift index 7960b26091..1368652e4e 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift @@ -72,9 +72,11 @@ final class ConnectionCoordinatorStore { } private func rebuild(_ id: UUID, droppingSession: Bool) { - coordinators.removeValue(forKey: id)?.retire() + let retired = coordinators.removeValue(forKey: id) + retired?.retire() generations[id, default: 0] += 1 guard droppingSession else { return } + retired?.dropSession() disconnect(id) } @@ -83,6 +85,7 @@ final class ConnectionCoordinatorStore { if let retired = coordinators.removeValue(forKey: id) { removedRecords[id] = retired.connection retired.retire() + retired.dropSession() } disconnect(id) } diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift new file mode 100644 index 0000000000..1291979f5d --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift @@ -0,0 +1,1416 @@ +import Foundation +import Observation +import os +import TableProConnectionLibrary +import TableProDatabase +import TableProLogRedaction +import TableProModels +import TableProPluginKit +import TableProSQLGrammar + +// What a window shows of its connection: the model's, or what it showed when the session moved under an unsaved edit. +struct ConnectionSnapshot { + let state: ConnectionSessionState + let catalog: ConnectionCatalog +} + +struct ConnectionFailure { + let error: any Error + let title: String + // A failed read that a redial repeats needs no alert once the redial lands. + let redialRepeats: Bool +} + +// One per connection, kept while a window holds it. +final class ConnectionSessionModels { + private struct WeakModel { + weak var model: ConnectionSessionModel? + } + + private var models: [UUID: WeakModel] = [:] + + func model(for connection: DatabaseConnection, appState: AppState) -> ConnectionSessionModel { + if let existing = models[connection.id]?.model { + existing.adopt(connection) + return existing + } + let created = ConnectionSessionModel(record: connection, appState: appState) + models[connection.id] = WeakModel(model: created) + return created + } + + func existing(_ connectionId: UUID) -> ConnectionSessionModel? { + models[connectionId]?.model + } + + func purge(_ connectionIds: Set) { + for connectionId in connectionIds { + models.removeValue(forKey: connectionId) + } + } +} + +// The session every window of one connection shows: one lifecycle, one table list, one check, one catalog reload. +// A window asks through its `ConnectionCoordinator`, which keeps what is the window's own. +@MainActor @Observable +final class ConnectionSessionModel { + private(set) var record: DatabaseConnection + + // Written only by `enter(_:)`, which refuses a step of an attempt the state no longer holds. + private(set) var state: ConnectionSessionState = .notConnected { + didSet { scenesSaw(ConnectionSnapshot(state: oldValue, catalog: catalog)) } + } + private(set) var catalog = ConnectionCatalog() { + didSet { scenesSaw(ConnectionSnapshot(state: state, catalog: oldValue)) } + } + + // A transaction lost with no window told yet; the next transition that lands a session reports it. + @ObservationIgnored private var pendingLoss: SessionLoss? + @ObservationIgnored private var seenCatalogGeneration: Int + @ObservationIgnored private var isReloadingCatalog = false + @ObservationIgnored private var catalogReloadRequested = false + @ObservationIgnored private var isReloadingAfterRollBack = false + @ObservationIgnored private var rollBackReloadRequested = false + @ObservationIgnored private var catalogReloadHeldBack = false + @ObservationIgnored private var isFollowing = false + @ObservationIgnored private var followRequested = false + @ObservationIgnored private weak var followRequester: ConnectionCoordinator? + // Each request that moves the placement bumps it, so only the newest one lands. + @ObservationIgnored private var placementEpoch = 0 + @ObservationIgnored private var switchesInFlight = 0 + @ObservationIgnored private var placementRestored = false + // Why the last dial into the remembered database failed with no fallback; the next open starts from the saved one. + @ObservationIgnored private var rememberedDialFailure: (any Error)? + @ObservationIgnored private var scenes: [WeakScene] = [] + + private let appState: AppState + private static let logger = Logger(subsystem: "com.TablePro", category: "ConnectionSessionModel") + + typealias SessionLoss = ConnectionCoordinator.SessionLoss + private typealias Placement = ConnectionCatalog.Placement + + private struct WeakScene { + weak var scene: ConnectionCoordinator? + } + + init(record: DatabaseConnection, appState: AppState) { + self.record = record + self.appState = appState + seenCatalogGeneration = appState.catalogChanges.generation(for: record.id) + } + + var connectionId: UUID { record.id } + + var session: ConnectionSession? { state.session } + + var activeDatabase: String { catalog.placement.database } + var activeSchema: String { catalog.placement.schema } + + private var isLost: Bool { + if case .lost = state.phase { return true } + return false + } + + var displayName: String { + record.name.isEmpty ? record.host : record.name + } + + var supportsDatabaseSwitching: Bool { + record.type == .mysql || record.type == .mariadb || + record.type == .tidb || record.type == .oceanbase || + record.type == .postgresql || record.type == .redshift || + record.type == .mssql || record.type == .clickhouse || + record.type == .mongodb || record.type == .cloudflareD1 + } + + var supportsSchemas: Bool { + record.type == .postgresql || record.type == .redshift || + record.type == .mssql || record.type == .duckdb || + record.type == .oracle + } + + var qualifiesTablesWithDatabase: Bool { + record.type == .mysql || record.type == .mariadb || record.type == .tidb || record.type == .oceanbase + } + + private var redialsToSwitchDatabase: Bool { + record.type == .postgresql || record.type == .redshift + } + + // MySQL, MariaDB and TiDB sign in apart from the database; SQL Server signs contained users in to the named one. + private var dialsRememberedDatabase: Bool { + redialsToSwitchDatabase || record.type == .mysql || record.type == .mariadb || record.type == .tidb + } + + func adopt(_ record: DatabaseConnection) { + guard record.id == self.record.id, record != self.record else { return } + self.record = record + } + + // The first window restores where the connection was; a later one finds the model there already. + func restorePersistedPlacement() { + guard !placementRestored else { return } + placementRestored = true + let defaults = UserDefaults.standard + catalog.placement = Placement( + database: defaults.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connectionId)) ?? "", + schema: defaults.string(forKey: ConnectionDefaultsKey.lastSchema.name(for: connectionId)) ?? "public" + ) + } + + func classify(_ error: any Error, operation: String) -> AppError { + ErrorClassifier.classify(error, context: ErrorContext( + operation: operation, + databaseType: record.type, + host: record.host, + sshEnabled: record.sshEnabled, + signsInWithIAM: record.iamSignIn != nil + )) + } + + // MARK: - Windows + + func register(_ scene: ConnectionCoordinator) { + scenes.removeAll { $0.scene == nil } + scenes.append(WeakScene(scene: scene)) + } + + private func scenesSaw(_ old: ConnectionSnapshot) { + for scene in scenes.compactMap(\.scene) { + scene.sessionModelChanged(from: old) + } + } + + private func consumers(of attempt: ConnectionAttempt) -> [ConnectionCoordinator] { + let live = scenes.compactMap(\.scene) + return attempt.consumers.compactMap { id in live.first { ObjectIdentifier($0) == id } } + } + + // The window that asked, or the next one still waiting: it answers the attempt's questions and hears its outcome. + private func lead(of attempt: ConnectionAttempt) -> ConnectionCoordinator? { + consumers(of: attempt).first + } + + // True when the session the model now holds sits somewhere else than `old` showed, so an unsaved edit made there + // would land on another table. + func moves(from old: ConnectionSnapshot) -> Bool { + if catalog.placement != old.catalog.placement { return true } + guard let live = state.session, live.driver !== old.state.session?.driver else { return false } + let moved = movedPlacement(of: live, from: old.catalog.placement, includingDatabase: true) + return moved.database != nil || moved.schema != nil + } + + // MARK: - Lifecycle + + @discardableResult + private func enter(_ event: ConnectionSessionState.Event) -> Bool { + guard let next = state.applying(event) else { return false } + state = next + return true + } + + private func isCurrent(_ attempt: ConnectionAttempt) -> Bool { + state.attempt === attempt + } + + func connect(from scene: ConnectionCoordinator) async { + switch state { + case .connecting(let open, _): + await wait(on: open, from: scene) + case .checking(let running, _), .reconnecting(let running, _): + await wait(on: running, from: scene) + if case .connected = state { await attach() } + case .connected(let held): + if appState.connectionManager.session(for: connectionId)?.driver === held.driver { + await attach() + } else { + // The session ended under every window, as the background release does to a file. + await restoreSession(after: .whileIdle, from: scene) + } + case .notConnected, .failed, .lost: + await open(from: scene) + } + } + + // A window that joins answers the dial's questions too, and keeps the dial when the window that asked goes. + private func wait(on attempt: ConnectionAttempt, from scene: ConnectionCoordinator) async { + attempt.join(scene) + await appState.sessionRedialer.settled(connectionId, consumer: scene, prompter: scene.prompts) + _ = await attempt.task?.value + } + + private func open(from scene: ConnectionCoordinator) async { + let attempt = ConnectionAttempt(.open, origin: state.openOrigin) + attempt.join(scene) + let wanted = catalog.placement + placementRestored = true + enter(.open(attempt)) + attempt.task = Task { [weak self] in + await self?.runAttempt(attempt, restoring: wanted) + return false + } + _ = await attempt.task?.value + } + + // A window that opens on a connected session reads the lists itself, which covers every change posted before. + private func attach() async { + guard let session else { return } + let generation = appState.catalogChanges.generation(for: connectionId) + var load = await loadTables(in: supportsSchemas ? activeSchema : nil, on: session) + if supportsDatabaseSwitching { load.databases = try? await session.driver.fetchDatabases() } + if supportsSchemas { load.schemas = try? await session.driver.fetchSchemas() } + if load.failure == nil, commit(load, onFailure: .keep), generation > seenCatalogGeneration { + seenCatalogGeneration = generation + } + await refreshTransaction() + appState.libraryPreferences.recordConnected(connectionId) + } + + // The last window to leave abandons what is in flight, so the next one never joins it and waits forever. + func leave(_ scene: ConnectionCoordinator) { + guard let attempt = state.attempt, attempt.leave(scene) else { return } + enter(.abandon(attempt)) + if attempt.purpose == .open { attempt.task?.cancel() } + leaveDial() + } + + // The record changed how it dials, so the next window opens it again. + func reset() { + let attempt = state.attempt + // What failed was the old record's dial. + if !redialsToSwitchDatabase { + rememberedDialFailure = nil + } + enter(.cancel) + if attempt?.purpose == .open { attempt?.task?.cancel() } + leaveDial() + } + + private func leaveDial() { + if appState.sessionRedialer.leave(connectionId, consumer: self) { + appState.connectionManager.invalidateAttempt(for: connectionId) + } + } + + private func runAttempt(_ attempt: ConnectionAttempt, restoring wanted: Placement) async { + // Another dial is shared once its schema is back, never torn down mid-dial. + await appState.sessionRedialer.settled(connectionId, consumer: self, prompter: lead(of: attempt)?.prompts) + guard isCurrent(attempt) else { return } + if let existing = appState.connectionManager.session(for: connectionId) { + let load = await loadTables(in: nil, on: existing) + guard isCurrent(attempt) else { return } + if load.failure == nil { + let placement = adoptedPlacement(of: existing, includingDatabase: true) + enter(.adopt(existing, by: attempt)) + commit(load, at: placement, onFailure: .keep) + await finishOpen(attempt, restoring: wanted) + return + } + if await keepsRefusingSession(existing, refused: load, by: attempt) { return } + guard isCurrent(attempt) else { return } + await appState.sessionRedialer.settled(connectionId, consumer: self, prompter: lead(of: attempt)?.prompts) + guard isCurrent(attempt) else { return } + if let live = appState.connectionManager.session(for: connectionId), live.driver !== existing.driver { + await runAttempt(attempt, restoring: wanted) + return + } + await appState.connectionManager.disconnect(connectionId) + } + + guard isCurrent(attempt) else { return } + await connectFresh(attempt, restoring: wanted) + } + + // `allowSignIn` is false on the retry that follows a sign-in, so a connection that keeps + // failing cannot put the prompt up again and again. That retry dials with the new credentials + // rather than join a dial started without them. + private func connectFresh(_ attempt: ConnectionAttempt, restoring wanted: Placement, allowSignIn: Bool = true) async { + Telemetry.shared.connectionAttempted(databaseType: record.type.rawValue) + + do { + let saved = record + // A remembered database that never answered may hang again, so the next open starts from the saved one and + // switches after, as opens did before. A USE engine does so after any failure it could not fall back from. + let failedBefore = rememberedDialFailure + rememberedDialFailure = nil + let target = failedBefore == nil ? openTarget(restoring: wanted) : record + let landed: OpenLanding + var restoring = wanted + var refused: RefusedDatabase? + // The saved database's list then says whether the remembered one is gone, as after a refused dial. + if let failedBefore, !redialsToSwitchDatabase, !wanted.database.isEmpty, wanted.database != saved.database { + refused = RefusedDatabase(name: wanted.database, error: failedBefore, reportedBySwitch: true) + } + do { + landed = try await dialForOpen( + target, schema: schemaRestored(by: target, restoring: wanted), by: attempt, joining: allowSignIn + ) + } catch let thrown { + let listRefused = thrown as? TableListRefused + let error = listRefused?.error ?? thrown + if target.database != saved.database, + startsNextOpenFromSaved(after: error, listRefused: listRefused != nil, by: attempt) { + rememberedDialFailure = error + } + // A sign-in is the connection's, not the database's: the saved database would fail the same way. + guard target.database != saved.database, isCurrent(attempt), + listRefused != nil || blamesDatabase(error), + SignInOffer.method(for: error, connection: saved, secureStore: appState.secureStore) == nil + else { throw error } + Self.logger.warning("Failed to open remembered database \(target.database, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + // A dial that landed and then refused its table list leaves a session to tear down first. + await appState.connectionManager.disconnect(connectionId) + guard isCurrent(attempt) else { return } + landed = try await dialForOpen(saved, schema: nil, by: attempt, joining: false) + // A USE engine still moves there with USE, which says why it fails, as its open always did. + if redialsToSwitchDatabase { + restoring.database = saved.database + } + refused = RefusedDatabase(name: target.database, error: error, reportedBySwitch: !redialsToSwitchDatabase) + } + guard isCurrent(attempt) else { return } + let newSession = landed.outcome.session + let placement = adoptedPlacement(of: newSession, includingDatabase: true) + enter(.adopt(newSession, by: attempt)) + commit(landed.load, at: placement, onFailure: .keep) + if landed.outcome.dialedHere { + appState.bumpSessionRevision(connectionId) + } + guard await finishOpen( + attempt, + restoring: restoring, + listedSchemas: landed.outcome.schemas, + refused: refused + ) else { return } + Telemetry.shared.connectionSucceeded(databaseType: record.type.rawValue) + for scene in consumers(of: attempt) { + scene.navigateToPendingTable() + } + } catch let thrown { + let error = (thrown as? TableListRefused)?.error ?? thrown + guard isCurrent(attempt) else { return } + var failure = error + // A sign-in that expired is recoverable, so offer it once and retry rather than + // leaving the user on an error screen whose only button repeats the same failure. + if allowSignIn, let prompts = lead(of: attempt)?.prompts, + let method = SignInOffer.method(for: error, connection: record, secureStore: appState.secureStore) { + switch await SignInOffer.offer(method, prompts: prompts) { + case .signedIn: + guard isCurrent(attempt) else { return } + await connectFresh(attempt, restoring: wanted, allowSignIn: false) + return + case .failed(let signInError): + failure = signInError + case .declined: + break + } + } + guard isCurrent(attempt) else { return } + enter(.fail(classify(failure, operation: "connect"), attempt)) + } + } + + // Where the connection was, so the first table list is already the right one. + private func openTarget(restoring wanted: Placement) -> DatabaseConnection { + guard dialsRememberedDatabase, !wanted.database.isEmpty else { return record } + var target = record + target.database = wanted.database + return target + } + + // A remembered schema belongs to the remembered database, so the dial restores it only when it lands there. + private func schemaRestored(by target: DatabaseConnection, restoring wanted: Placement) -> String? { + guard supportsSchemas else { return nil } + guard !supportsDatabaseSwitching || wanted.database.isEmpty || target.database == wanted.database else { + return nil + } + return wanted.schema + } + + private struct OpenLanding { + let outcome: SessionRedialer.Outcome + let load: CatalogLoad + } + + // The database the open could not go back to, and why. + private struct RefusedDatabase { + let name: String + let error: any Error + // The open's own USE tried it again and said why that failed. + let reportedBySwitch: Bool + } + + nonisolated private struct TableListRefused: Error { + let error: any Error + } + + // A PostgreSQL dial that failed on the way, or on the credentials, fails the same way for the saved database, and + // a second one would ask the same host key question or code again, wait out another timeout or count another failed + // login. Those come typed, or as libpq's own English text; anything else the server answered, in whatever language + // it speaks, may be about the database, so the open tries the saved one. + nonisolated static func mayBlameDatabase(_ error: any Error) -> Bool { + if error is CancellationError || error is SSHTunnelError || error is SSLHandshakeError + || error is LocalNetworkPermissionError || error is ConnectionError || error is URLError + || error is POSIXError { + return false + } + if case PostgreSQLError.unreachable = error { return false } + let message = error.localizedDescription.lowercased() + return !notAboutTheDatabase.contains { message.contains($0) } + } + + // The MySQL driver types a refused database by its code. + private func blamesDatabase(_ error: any Error) -> Bool { + if redialsToSwitchDatabase { return Self.mayBlameDatabase(error) } + if case MySQLError.databaseRefused = error { return true } + return false + } + + // A USE engine's other failure may still be the database's on a server that numbers it its own way. + private func startsNextOpenFromSaved(after error: any Error, listRefused: Bool, by attempt: ConnectionAttempt) -> Bool { + if redialsToSwitchDatabase { + if case PostgreSQLError.unreachable = error { return true } + return false + } + return isCurrent(attempt) && !listRefused && !blamesDatabase(error) && !(error is CancellationError) + } + + private nonisolated static let notAboutTheDatabase = [ + "password authentication failed", "authentication failed", "fe_sendauth", "no password supplied", + "could not connect", "connection refused", "timeout expired", "timed out", "could not translate host name", + "network is unreachable", "no route to host", "host is down", "software caused connection abort", + "server closed the connection", "connection reset", + "lost connection", "ssl error", "ssl syscall", "server does not support ssl", + ] + + // A table list the new session refuses fails the dial, as it fails a database switch. + private func dialForOpen( + _ target: DatabaseConnection, + schema: String?, + by attempt: ConnectionAttempt, + joining: Bool + ) async throws -> OpenLanding { + let outcome = try await appState.sessionRedialer.redial( + target, + schema: schema, + onlyIfListed: true, + prompter: lead(of: attempt)?.prompts, + joining: joining, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(attempt) == true } + ) + let load = await loadTables(in: nil, on: outcome.session) + if let refusal = load.failure { throw TableListRefused(error: refusal) } + return OpenLanding(outcome: outcome, load: load) + } + + // The open landed where the session is; this puts it back where the connection was, through the asking window. + // A fresh open dialed there already, so this only lists, unless that dial fell back to the saved database. + @discardableResult + private func finishOpen( + _ attempt: ConnectionAttempt, + restoring wanted: Placement, + listedSchemas: [String]? = nil, + refused: RefusedDatabase? = nil + ) async -> Bool { + let landedOn = (driver: session.map { ObjectIdentifier($0.driver) }, database: activeDatabase) + let databases = await loadDatabases(restoring: wanted.database, from: lead(of: attempt), inside: attempt) + if let refused { + settleRefusedDatabase(refused, listed: databases, reportingTo: lead(of: attempt)) + } + // The dial's schema list belongs to the session it read, which a database switch above replaces or moves. + let stayed = session.map { ObjectIdentifier($0.driver) } == landedOn.driver && activeDatabase == landedOn.database + await loadSchemas(restoring: wanted.schema, from: lead(of: attempt), listed: stayed ? listedSchemas : nil) + await refreshTransaction() + // A database restore that failed both ways left the open lost, and it stays lost. + guard enter(.opened(attempt)) else { return false } + appState.libraryPreferences.recordConnected(connectionId) + reportDroppedTransaction(pendingLoss, to: lead(of: attempt)) + return true + } + + // A database the server still lists failed to open, so the alert says why, as a failed switch does. One it no + // longer lists is gone: the open stays quiet, as it always did for one, and forgets it so the next open skips it. + private func settleRefusedDatabase( + _ refused: RefusedDatabase, + listed: [String]?, + reportingTo scene: ConnectionCoordinator? + ) { + guard let listed, !listed.contains(refused.name) else { + guard !refused.reportedBySwitch else { return } + scene?.presentFailure( + title: ConnectionCoordinator.switchDatabaseFailedTitle, + message: classify(refused.error, operation: "switchDatabase").message + ) + return + } + let key = ConnectionDefaultsKey.lastDB.name(for: connectionId) + if UserDefaults.standard.string(forKey: key) == refused.name { + UserDefaults.standard.removeObject(forKey: key) + } + } + + // A transaction the server aborted refuses catalog reads, yet the session still holds the user's work. + private func keepsRefusingSession( + _ existing: ConnectionSession, + refused load: CatalogLoad, + by attempt: ConnectionAttempt + ) async -> Bool { + guard let refusal = load.failure, await Self.answersPing(existing.driver), isCurrent(attempt) else { return false } + await refreshTransaction() + guard isCurrent(attempt), + appState.sessionTransactions.hold(for: connectionId, on: existing.driver) != nil else { return false } + Self.logger.info("Kept a session that refused its catalog: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + var placement = catalog.placement + let database = sessionDatabase(of: existing) + if supportsDatabaseSwitching, !database.isEmpty { + placement.database = database + } + enter(.adopt(existing, by: attempt)) + commit(load, at: placement, onFailure: .refuse) + enter(.opened(attempt)) + appState.libraryPreferences.recordConnected(connectionId) + return true + } + + // Another party ended the session the model holds, as the background release does to a file. The next + // transition that lands a session reports what it held, whichever window asks. + func sessionEnded(_ driver: any DatabaseDriver) { + guard case .connected(let held) = state, held.driver === driver, + let ended = appState.sessionTransactions.endedHold(for: connectionId, on: driver) else { return } + if driver.holdsSuspensionBlockingResource { + pendingLoss = ended.duringWork ? .inBackgroundAfterStatement : .inBackground + } else { + pendingLoss = ended.duringWork ? .duringWork : .whileIdle + } + } + + // MARK: - Session Check + + // True when the redial dropped a held transaction and the outcome is already settled for the user. + @discardableResult + func restoreSession(after loss: SessionLoss, from scene: ConnectionCoordinator) async -> Bool { + // One check per connection, and a statement that failed while it ran makes the loss ambiguous. + if let running = state.check, let task = running.task { + if loss == .duringWork { running.ambiguous = true } + running.join(scene) + return await task.value + } + guard !scene.isSwitching else { return false } + let check = ConnectionAttempt(.redial) + check.join(scene) + guard enter(.check(check)) else { return false } + let task = Task { await checkSession(check, after: loss) } + check.task = task + return await task.value + } + + private func checkSession(_ check: ConnectionAttempt, after loss: SessionLoss) async -> Bool { + let held = session + var workWasRunning = appState.sessionTransactions.isRunningStatements(connectionId) + + switch await newerSession(than: held, during: check) { + case .taken: + return reportDroppedTransaction(pendingLoss, to: lead(of: check)) + case .refused: + enter(.settle(check)) + return false + case .none: + break + } + if let held { + if appState.connectionManager.session(for: connectionId) != nil, await Self.answersPing(held.driver) { + enter(.settle(check)) + return false + } + guard isCurrent(check) else { return false } + // Another dial can land a whole session while this ping is out. + switch await newerSession(than: held, during: check) { + case .taken: + return reportDroppedTransaction(pendingLoss, to: lead(of: check)) + case .refused: + enter(.settle(check)) + return false + case .none: + break + } + } + + workWasRunning = workWasRunning || appState.sessionTransactions.isRunningStatements(connectionId) + let ended = appState.sessionTransactions.endedHold(for: connectionId, on: held?.driver) + workWasRunning = workWasRunning || ended?.duringWork == true + let heldTransaction = appState.sessionTransactions.hold(for: connectionId, on: held?.driver) != nil + || ended != nil + let released = ended.flatMap { releasedLoss($0, on: held, asked: loss) } + guard enter(.dial(check)) else { return false } + do { + let outcome = try await appState.sessionRedialer.redial( + dialTarget, + schema: supportsSchemas ? activeSchema : nil, + prompter: lead(of: check)?.prompts, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(check) == true } + ) + guard isCurrent(check), + await takeSession(outcome.session, dialedHere: outcome.dialedHere, by: check) else { return false } + let dropped = heldTransaction + ? released ?? Self.settledLoss(loss, workWasRunning: workWasRunning, ambiguous: check.ambiguous) + : pendingLoss + return reportDroppedTransaction(dropped, to: lead(of: check)) + } catch { + guard isCurrent(check) else { return false } + if heldTransaction { + pendingLoss = released + ?? Self.settledLoss(loss, workWasRunning: workWasRunning, ambiguous: check.ambiguous) + } + enter(.lose(classify(error, operation: "reconnect"), check)) + return false + } + } + + private static func settledLoss(_ loss: SessionLoss, workWasRunning: Bool, ambiguous: Bool) -> SessionLoss { + workWasRunning || ambiguous ? .duringWork : loss + } + + // The release closed the file once its running statement ended, so a later failure on the closed file decides nothing. + private func releasedLoss( + _ ended: SessionTransactionMonitor.EndedHold, + on held: ConnectionSession?, + asked loss: SessionLoss + ) -> SessionLoss? { + guard held?.driver.holdsSuspensionBlockingResource == true else { return nil } + if ended.duringWork { return .inBackgroundAfterStatement } + return loss == .duringRollBack ? .duringRollBack : .inBackground + } + + // True when the transaction a redial ended is settled for the user. + @discardableResult + private func reportDroppedTransaction(_ loss: SessionLoss?, to scene: ConnectionCoordinator?) -> Bool { + pendingLoss = nil + guard let loss else { return false } + appState.sessionTransactions.noteRolledBack(connectionId) + if let message = ConnectionCoordinator.lostTransactionMessage(loss, connectionName: displayName) { + scene?.presentFailure(title: nil, message: message) + } + return true + } + + private enum NewerSession { + case none + case taken + case refused + } + + private func newerSession(than held: ConnectionSession?, during check: ConnectionAttempt) async -> NewerSession { + // A dial in flight is not ready to share until its schema is back. + await appState.sessionRedialer.settled(connectionId, consumer: self, prompter: lead(of: check)?.prompts) + guard isCurrent(check), + let live = appState.connectionManager.session(for: connectionId), + live.driver !== held?.driver else { + return .none + } + return await takeSession(live, dialedHere: false, by: check) ? .taken : .refused + } + + private var dialTarget: DatabaseConnection { + guard supportsDatabaseSwitching, !activeDatabase.isEmpty else { return record } + var target = record + target.database = activeDatabase + return target + } + + // `lander` is the attempt that dialed or found the session; a follow takes it with none. + @discardableResult + private func takeSession( + _ newSession: ConnectionSession, + dialedHere: Bool, + by lander: ConnectionAttempt?, + yieldingToBusy: Bool = false + ) async -> Bool { + let placementAsked = movedPlacement(of: newSession, includingDatabase: !dialedHere) + let load = await loadTables(in: nil, on: newSession) + if let refusal = load.failure { + Self.logger.warning("Failed to load tables on the new session: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + } + if let lander, !isCurrent(lander) { return false } + // Another switch that landed while the tables loaded made them the wrong database's. + if yieldingToBusy, + !canFollow || movedPlacement(of: newSession, includingDatabase: !dialedHere) != placementAsked { + return false + } + // The dial restored the connection's schema; a failed restore leaves the default. + let placement = adoptedPlacement(of: newSession, includingDatabase: !dialedHere) + let moved = placement != catalog.placement + guard enter(.adopt(newSession, by: lander)) else { return false } + commit(load, at: placement, onFailure: moved ? .empty : .refuse) + if dialedHere { + appState.bumpSessionRevision(connectionId) + } + await refreshTransaction() + return true + } + + // A session dialed or moved elsewhere sits on that database and schema, so the lists follow it before any write + // can name a table from the old list. + private func adoptedPlacement(of taken: ConnectionSession, includingDatabase: Bool) -> Placement { + let moved = movedPlacement(of: taken, includingDatabase: includingDatabase) + return Placement(database: moved.database ?? activeDatabase, schema: moved.schema ?? activeSchema) + } + + private func movedPlacement( + of taken: ConnectionSession, + from placement: Placement? = nil, + includingDatabase: Bool + ) -> (database: String?, schema: String?) { + let base = placement ?? catalog.placement + var database: String? + if includingDatabase { + let candidate = sessionDatabase(of: taken) + if supportsDatabaseSwitching, !candidate.isEmpty, candidate != base.database { database = candidate } + } + var schema: String? + if supportsSchemas, let current = taken.driver.currentSchema, current != base.schema { schema = current } + return (database, schema) + } + + private func sessionDatabase(of taken: ConnectionSession) -> String { + guard let live = appState.connectionManager.session(for: connectionId), live.driver === taken.driver else { + return taken.activeDatabase + } + return live.activeDatabase + } + + private static func answersPing(_ driver: any DatabaseDriver) async -> Bool { + do { + _ = try await driver.ping() + return true + } catch { + return false + } + } + + // MARK: - Following + + private var canFollow: Bool { + (session != nil || isLost) && switchesInFlight == 0 && !state.isReconnecting && !state.isOpening + } + + // The manager holds a session this model did not land, or moved its own: the lists follow it there. A follow + // asked while the model is busy runs once it is free, so a later move is never missed. + func follow(from scene: ConnectionCoordinator) async { + followRequester = scene + guard canFollow, !isFollowing else { + followRequested = true + return + } + isFollowing = true + defer { isFollowing = false } + repeat { + followRequested = false + await followOnce(from: followLead(or: scene)) + } while followRequested && canFollow + } + + // The window that asked hears the follow's outcome only while it still shows the live session; a retired one cannot + // answer its questions, and a pinned one's alert would describe a session it does not show. + private func followLead(or scene: ConnectionCoordinator) -> ConnectionCoordinator { + guard let followRequester, followRequester.showsLiveSession else { return scene } + return followRequester + } + + // What a busy model held back: a follow, and a catalog reload that would have read a placement in motion. + func runHeldBackWork(from scene: ConnectionCoordinator) async { + if followRequested, canFollow, !isFollowing { + await follow(from: followLead(or: scene)) + } + if catalogReloadHeldBack, switchesInFlight == 0 { + catalogReloadHeldBack = false + await reloadChangedCatalog() + } + } + + private func followOnce(from scene: ConnectionCoordinator) async { + if isLost { + guard appState.connectionManager.session(for: connectionId) != nil else { return } + await restoreSession(after: .whileIdle, from: scene) + return + } + await appState.sessionRedialer.settled(connectionId, consumer: scene, prompter: scene.prompts) + guard canFollow else { + followRequested = true + return + } + guard let held = session, let live = appState.connectionManager.session(for: connectionId) else { return } + if live.driver !== held.driver { + guard await takeSession(live, dialedHere: false, by: nil, yieldingToBusy: true) else { + followRequested = true + return + } + reportDroppedTransaction(pendingLoss, to: scene) + await loadSchemas(restoring: activeSchema, from: scene) + return + } + let moved = movedPlacement(of: live, includingDatabase: true) + guard moved.database != nil || moved.schema != nil else { return } + let request = beginPlacementRequest() + let target = Placement(database: moved.database ?? activeDatabase, schema: moved.schema ?? activeSchema) + let load = await loadTables(in: supportsSchemas ? target.schema : nil, on: live) + if let refusal = load.failure { + Self.logger.warning("Failed to load tables where the session moved: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + } + // Another move that landed while the tables loaded made them the wrong placement's. + guard canFollow, session?.driver === live.driver, + movedPlacement(of: live, includingDatabase: true) == moved else { + followRequested = true + return + } + enter(.adopt(live, by: nil)) + commit(load, at: target, onFailure: .empty, for: request) + // The next open restores where the session went, not a schema a Roll Back or a commit can still take back. + if let schema = moved.schema, activeSchema == schema, + appState.sessionTransactions.hold(for: connectionId, on: live.driver) == nil { + UserDefaults.standard.set(schema, forKey: ConnectionDefaultsKey.lastSchema.name(for: connectionId)) + } + // SQL Server lists another database's schemas. + if moved.database != nil { + await loadSchemas(restoring: activeSchema, from: scene) + } + await refreshTransaction() + } + + // MARK: - Database / Schema Switching + + func beginSwitch() { + switchesInFlight += 1 + } + + func endSwitch() { + switchesInFlight -= 1 + } + + // Runs with a `USE` in flight on the shared session, in any window. + private var databaseMoves = 0 + + var isDatabaseMoving: Bool { databaseMoves > 0 } + + func beginDatabaseMove() { + databaseMoves += 1 + } + + func endDatabaseMove() { + databaseMoves = max(0, databaseMoves - 1) + } + + private func beginPlacementRequest() -> Int { + placementEpoch += 1 + return placementEpoch + } + + func switchDatabase( + to name: String, + from scene: ConnectionCoordinator, + inside open: ConnectionAttempt? + ) async -> ConnectionFailure? { + let failure: ConnectionFailure? + if redialsToSwitchDatabase { + await reconnectWithDatabase(name, from: scene, inside: open) + failure = nil + } else { + failure = await useDatabase(name) + } + await refreshTransaction() + appState.bumpSessionRevision(connectionId) + return failure + } + + // SQL Server lists another database's schemas; a redial already read them on its new session. It runs once the + // switch ends, so restoring a schema there is not refused as a switch in flight. + func loadSchemasAfterSwitch(to name: String, from scene: ConnectionCoordinator) async { + guard !redialsToSwitchDatabase, activeDatabase == name else { return } + await loadSchemas(restoring: activeSchema, from: scene) + } + + private func useDatabase(_ name: String) async -> ConnectionFailure? { + let request = beginPlacementRequest() + do { + try await appState.connectionManager.switchDatabase(connectionId, to: name) + } catch { + return ConnectionFailure(error: error, title: ConnectionCoordinator.switchDatabaseFailedTitle, redialRepeats: false) + } + if let freshSession = appState.connectionManager.session(for: connectionId) { + enter(.adopt(freshSession, by: nil)) + } + UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastDB.name(for: connectionId)) + guard let session else { return nil } + let load = await loadTables(in: nil, on: session) + // The session is on the new database either way, so a refused list empties until a reload can read it. + commit(load, at: Placement(database: name, schema: activeSchema), onFailure: .empty, for: request) + guard let refusal = load.failure else { return nil } + return ConnectionFailure(error: refusal, title: ConnectionCoordinator.refreshTablesFailedTitle, redialRepeats: true) + } + + // Inside its open the switch is the open's step; otherwise it is an attempt of its own, which no older step outlives. + private func beginDatabaseSwitch(from scene: ConnectionCoordinator, inside open: ConnectionAttempt?) -> ConnectionAttempt? { + if let open { + return enter(.teardown(open)) ? open : nil + } + let attempt = ConnectionAttempt(.switchDatabase) + attempt.join(scene) + return enter(.switchDatabase(attempt)) ? attempt : nil + } + + // Every window reads no session while a PostgreSQL switch runs; a switch asked then runs after it. + var isSwitchingDatabase: Bool { + guard case .reconnecting(let attempt, _) = state else { return false } + return attempt.purpose == .switchDatabase + } + + // A dial, a session check or another window's switch in flight lands first: the teardown would cancel the dial, + // and the check's lost-transaction notice or the other switch would never land. The newest switch then wins. + // `open` is the open this switch is a step of: it is the one attempt the switch never waits for. + private func reconnectWithDatabase( + _ database: String, + from scene: ConnectionCoordinator, + inside open: ConnectionAttempt? + ) async { + while true { + await appState.sessionRedialer.settled(connectionId) + guard let running = state.attempt, running !== open, let task = running.task else { break } + _ = await task.value + if state.attempt === running { break } + } + // A window closed during that wait must not tear down the session the other windows now use. + guard scene.isAttached, activeDatabase != database, + let switching = beginDatabaseSwitch(from: scene, inside: open) else { return } + guard switching.purpose == .switchDatabase else { + await runDatabaseSwitch(to: database, as: switching, from: scene) + return + } + let task = Task { + await runDatabaseSwitch(to: database, as: switching, from: scene) + return false + } + switching.task = task + _ = await task.value + } + + private func runDatabaseSwitch( + to database: String, + as switching: ConnectionAttempt, + from scene: ConnectionCoordinator + ) async { + let request = beginPlacementRequest() + let previous = dialTarget + await appState.connectionManager.disconnect(connectionId) + + var newConnection = record + newConnection.database = database + + do { + let newSession = try await appState.sessionRedialer.redial( + newConnection, + schema: nil, + prompter: scene.prompts, + joining: false, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(switching) == true } + ).session + guard isCurrent(switching) else { return } + let load = await loadTables(in: nil, on: newSession) + if let refusal = load.failure { throw refusal } + guard isCurrent(switching) else { return } + enter(.adopt(newSession, by: switching)) + commit(load, at: Placement(database: database, schema: activeSchema), onFailure: .keep, for: request) + UserDefaults.standard.set(database, forKey: ConnectionDefaultsKey.lastDB.name(for: connectionId)) + await loadSchemas(restoring: activeSchema, from: scene) + } catch { + guard isCurrent(switching) else { return } + Self.logger.error("Failed to switch to database \(database, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + // Back to the database the picker still shows, not the saved one, so the picker stays true. + do { + let fallback = try await appState.sessionRedialer.redial( + previous, + schema: supportsSchemas ? activeSchema : nil, + prompter: scene.prompts, + joining: false, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(switching) == true } + ) + guard isCurrent(switching), + await takeSession(fallback.session, dialedHere: fallback.dialedHere, by: switching) else { return } + scene.presentFailure( + title: ConnectionCoordinator.switchDatabaseFailedTitle, + message: classify(error, operation: "switchDatabase").message + ) + } catch { + guard isCurrent(switching) else { return } + enter(.lose(classify(error, operation: "switchDatabase"), switching)) + } + } + } + + func switchSchema(to name: String, on session: ConnectionSession) async -> ConnectionFailure? { + let request = beginPlacementRequest() + do { + try await session.driver.switchSchema(to: name) + } catch { + return ConnectionFailure(error: error, title: ConnectionCoordinator.switchSchemaFailedTitle, redialRepeats: false) + } + UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastSchema.name(for: connectionId)) + appState.bumpSessionRevision(connectionId) + let load = await loadTables(in: name, on: session) + // The session is on the new schema either way, so a refused list empties until a reload can read it. + commit(load, at: Placement(database: activeDatabase, schema: name), onFailure: .empty, for: request) + guard let refusal = load.failure else { return nil } + return ConnectionFailure(error: refusal, title: ConnectionCoordinator.refreshTablesFailedTitle, redialRepeats: true) + } + + // MARK: - Tables + + func refreshTables() async -> ConnectionFailure? { + guard let session else { return nil } + let load = await loadTables(in: supportsSchemas ? activeSchema : nil, on: session) + commit(load, onFailure: .keep) + guard let refusal = load.failure else { return nil } + Self.logger.warning("Failed to refresh tables: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + return ConnectionFailure(error: refusal, title: ConnectionCoordinator.refreshTablesFailedTitle, redialRepeats: true) + } + + func refreshTransaction() async { + await appState.sessionTransactions.refresh(connectionId) + } + + // A Roll Back from any window can undo DDL; every window asks, and one reload serves them all. + func reloadAfterRollBack(from scene: ConnectionCoordinator) async { + guard !isReloadingAfterRollBack else { + rollBackReloadRequested = true + return + } + isReloadingAfterRollBack = true + defer { isReloadingAfterRollBack = false } + repeat { + rollBackReloadRequested = false + await reloadAfterRollBackOnce(from: scene) + } while rollBackReloadRequested + } + + // A failed read keeps the list it has. + private func reloadAfterRollBackOnce(from scene: ConnectionCoordinator) async { + guard !catalog.tablesRefused else { + await reloadMetadataIfReleased(from: scene) + return + } + // PostgreSQL undoes a `SET search_path` the transaction made, the picker's own among them, so the lists follow + // the session back. + if supportsSchemas, let reporting = session?.driver as? any SessionSchemaReporting { + await reporting.settleCurrentSchema() + if reporting === session?.driver, reporting.currentSchema.map({ $0 != activeSchema }) == true { + await follow(from: scene) + } + } + guard let session else { return } + let load = await loadTables(in: supportsSchemas ? activeSchema : nil, on: session) + commit(load, onFailure: .keep) + if let refusal = load.failure { + Self.logger.warning("Failed to reload tables after a Roll Back: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + } + // With autocommit off, a MySQL read opens a transaction. + await refreshTransaction() + } + + func reloadMetadataIfReleased(from scene: ConnectionCoordinator) async { + guard catalog.tablesRefused, let session, + appState.sessionTransactions.hold(for: connectionId, on: session.driver) == nil else { return } + let load = await loadTables(in: nil, on: session) + if let refusal = load.failure { + Self.logger.warning("Tables are still refused: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + return + } + guard commit(load, onFailure: .keep, clearingRefusal: true) else { return } + await loadDatabases(restoring: activeDatabase, from: scene) + await loadSchemas(restoring: activeSchema, from: scene) + } + + private enum OnFailure { + case keep + // The list stays, and reloads in full once nothing holds the session. + case refuse + case empty + } + + private struct CatalogLoad { + let driver: ObjectIdentifier + let from: Placement + let epoch: Int + var tables: Result<[TableInfo], any Error>? + var schemas: [String]? + var databases: [String]? + + var failure: (any Error)? { + guard case .failure(let error)? = tables else { return nil } + return error + } + } + + private func loadTables(in schema: String?, on session: ConnectionSession) async -> CatalogLoad { + let from = catalog.placement + let epoch = placementEpoch + let driver = ObjectIdentifier(session.driver) + do { + let tables = try await session.driver.fetchTables(schema: schema) + return CatalogLoad(driver: driver, from: from, epoch: epoch, tables: .success(tables)) + } catch { + return CatalogLoad(driver: driver, from: from, epoch: epoch, tables: .failure(error)) + } + } + + // The one place a read lands. Placement and lists change in one step, and only on the session the read used: a + // placement request only while it is the newest, any other read only while nothing moved since it started. + @discardableResult + private func commit( + _ load: CatalogLoad, + at placement: Placement? = nil, + onFailure: OnFailure, + clearingRefusal: Bool = false, + for request: Int? = nil + ) -> Bool { + guard let session, ObjectIdentifier(session.driver) == load.driver else { return false } + if let request { + guard request == placementEpoch else { return false } + } else { + guard load.epoch == placementEpoch, catalog.placement == load.from else { return false } + } + var next = catalog + next.placement = placement ?? catalog.placement + switch load.tables { + case .success(let tables)?: + next.tables = tables + if clearingRefusal { next.tablesRefused = false } + case .failure?: + switch onFailure { + case .keep: + break + case .refuse: + next.tablesRefused = true + case .empty: + next.tables = [] + next.tablesRefused = true + } + case nil: + break + } + if let schemas = load.schemas { next.schemas = schemas } + if let databases = load.databases { next.databases = databases } + if next != catalog { catalog = next } + return true + } + + // MARK: - Catalog Changes + + // `reloadedHere`: the caller already read the session, the one place a transaction's own DDL shows. + func noteCatalogChanged(_ kinds: SQLCatalogObjectKinds, reloadedHere: Bool) { + let board = appState.catalogChanges + let caughtUp = board.generation(for: connectionId) == seenCatalogGeneration + board.post(kinds, for: connectionId) + if reloadedHere, caughtUp { + seenCatalogGeneration = board.generation(for: connectionId) + } + } + + // A statement a window ran may have created or dropped what the lists show; failed reads keep the lists. + // One reload per connection: a later post waits for it, so an older answer never lands over a newer one. + func reloadChangedCatalog() async { + guard !isReloadingCatalog else { + catalogReloadRequested = true + return + } + isReloadingCatalog = true + defer { isReloadingCatalog = false } + repeat { + catalogReloadRequested = false + await reloadChangedCatalogOnce() + } while catalogReloadRequested + } + + private func reloadChangedCatalogOnce() async { + let board = appState.catalogChanges + let generation = board.generation(for: connectionId) + guard generation != seenCatalogGeneration else { return } + // A placement in motion would make this read the wrong one's, and a PostgreSQL switch leaves no session to + // read; the switch's end runs it again. + guard switchesInFlight == 0, !isSwitchingDatabase else { + catalogReloadHeldBack = true + return + } + guard let session else { return } + var wanted = board.kinds(for: connectionId, since: seenCatalogGeneration) + if !supportsSchemas { wanted.remove(.schemas) } + if !supportsDatabaseSwitching { wanted.remove(.databases) } + let kinds = wanted + let database = supportsDatabaseSwitching ? activeDatabase : "" + let schema = supportsSchemas ? activeSchema : nil + let readFrom = catalog.placement + let epoch = placementEpoch + let monitor = appState.sessionTransactions + let matchesSession = catalogMatchesSession(on: session.driver) + + var snapshot: CatalogSnapshot? + // On the session a read would replace ROW_COUNT() and the warnings, so it is only where the DDL is visible, or + // where a USE or a role moved the session off what the catalog connection sees, as Refresh reads it there too. + if matchesSession { + do { + snapshot = try await appState.connectionManager.readCatalog(for: connectionId, database: database) { + try await Self.readCatalog(kinds, schema: schema, from: $0) + } + } catch { + Self.logger.warning("Reloading the catalog after a change failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + seenCatalogGeneration = generation + return + } + } + if snapshot == nil { + // A server that has a catalog connection but could not open one keeps its lists: a read on the session would + // replace what the user's last statement left there. Only its own uncommitted DDL sends the read there. + if matchesSession, appState.connectionManager.opensCatalogConnections(for: connectionId) { + seenCatalogGeneration = generation + return + } + // A stream on the session would end under this read, so it waits for the session to go idle. + guard !monitor.isBusy(connectionId) else { return } + board.beginSessionRead(for: connectionId) + snapshot = try? await monitor.track(connectionId, reading: true) { + try await Self.readCatalog(kinds, schema: schema, from: session.driver) + } + board.endSessionRead(for: connectionId) + } + seenCatalogGeneration = generation + guard let snapshot else { return } + var load = CatalogLoad(driver: ObjectIdentifier(session.driver), from: readFrom, epoch: epoch) + if let tables = snapshot.tables { load.tables = .success(tables) } + load.schemas = snapshot.schemas + load.databases = snapshot.databases + commit(load, onFailure: .keep) + } + + private struct CatalogSnapshot: Sendable { + var tables: [TableInfo]? + var schemas: [String]? + var databases: [String]? + } + + // The table list is the read that matters: its failure fails the reload, so a dead connection is dropped. + nonisolated private static func readCatalog( + _ kinds: SQLCatalogObjectKinds, + schema: String?, + from driver: any DatabaseDriver + ) async throws -> CatalogSnapshot { + var snapshot = CatalogSnapshot() + if kinds.contains(.tables) { + snapshot.tables = try await driver.fetchTables(schema: schema) + } + if kinds.contains(.schemas) { + snapshot.schemas = try? await driver.fetchSchemas() + } + if kinds.contains(.databases) { + snapshot.databases = try? await driver.fetchDatabases() + } + return snapshot + } + + private static let transactionalDDLTypes: Set = [.postgresql, .redshift, .mssql] + + // PostgreSQL, Redshift and SQL Server keep DDL inside the transaction that ran it, where only the session sees it; + // a session that cannot say whether it holds one counts as holding one. + func catalogSeesSessionDDL(on driver: (any DatabaseDriver)?) -> Bool { + guard Self.transactionalDDLTypes.contains(record.type) else { return true } + return appState.sessionTransactions.isKnownIdle(connectionId, on: driver) + } + + // Completion reads beside the session only while both see the same tables: same DDL, same database, no table + // only a procedure on the session made. + func catalogMatchesSession(on driver: (any DatabaseDriver)?) -> Bool { + guard catalogSeesSessionDDL(on: driver) else { return false } + return !appState.catalogChanges.sessionLeftCatalog( + for: connectionId, driver: driver.map { ObjectIdentifier($0) } + ) + } + + // MARK: - Lists + + // `wanted` is where the connection was; the session may have opened elsewhere, and the asking window switches back, + // as a step of `open` when an open asks. + @discardableResult + private func loadDatabases( + restoring wanted: String, + from scene: ConnectionCoordinator?, + inside open: ConnectionAttempt? = nil + ) async -> [String]? { + guard let session, supportsDatabaseSwitching else { return nil } + do { + let listed = try await session.driver.fetchDatabases() + catalog.databases = listed + let sessionDB = appState.connectionManager.session(for: connectionId)?.activeDatabase ?? record.database + if !wanted.isEmpty, listed.contains(wanted) { + catalog.placement.database = sessionDB + if wanted != sessionDB { + await scene?.switchDatabase(to: wanted, inside: open) + } + } else { + catalog.placement.database = sessionDB + // A database the connection names by its id is listed by its name, which the driver reports. + if !listed.contains(activeDatabase), let reported = session.driver.currentDatabase, + listed.contains(reported) { + await scene?.switchDatabase(to: reported, inside: open) + } + } + return listed + } catch { + Self.logger.warning("Failed to load databases: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + return nil + } + } + + // `known` is a list the dial already read on this session. + private func loadSchemas(restoring wanted: String, from scene: ConnectionCoordinator?, listed known: [String]? = nil) async { + guard let session, supportsSchemas else { return } + do { + let listed: [String] + if let known { + listed = known + } else { + listed = try await session.driver.fetchSchemas() + } + catalog.schemas = listed + let currentSchema = session.driver.currentSchema ?? "public" + if listed.contains(wanted), wanted != currentSchema { + catalog.placement.schema = currentSchema + // A switch in flight lands its own schema, which the restore never moves back. + if switchesInFlight == 0 { + await restoreSchema(wanted, on: session, reportingTo: scene) + } + } else if !listed.contains(activeSchema) { + catalog.placement.schema = currentSchema + } + } catch { + Self.logger.warning("Failed to load schemas: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + + // On the model's own session, as a switch every window waits for: a window that asked may be pinned on another + // session. Its failure is that window's to hear only while it shows the live session. + private func restoreSchema(_ name: String, on session: ConnectionSession, reportingTo scene: ConnectionCoordinator?) async { + beginSwitch() + let failure = await switchSchema(to: name, on: session) + endSwitch() + if let failure, let scene, scene.showsLiveSession { + await scene.report(failure, operation: "switchSchema") + } + // What the restore held back goes now, as it does after a window's own switch. + if let lead = scene ?? followRequester { + await runHeldBackWork(from: lead) + } else if catalogReloadHeldBack, switchesInFlight == 0 { + catalogReloadHeldBack = false + await reloadChangedCatalog() + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionState.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionState.swift new file mode 100644 index 0000000000..f614df3246 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionState.swift @@ -0,0 +1,260 @@ +import Foundation +import TableProDatabase +import TableProModels + +// One connection's session lifecycle. Each async step carries the attempt it started under and lands only while the +// state still holds that attempt, so a dial or a load that returns after Cancel, or after a newer attempt, changes +// nothing. +enum ConnectionSessionState { + case notConnected + // `provisional` is the session an open reattached or dialed, held while the lists load. + case connecting(ConnectionAttempt, provisional: ConnectionSession?) + case connected(ConnectionSession) + // A session check pinging the session, before it knows whether to redial. + case checking(ConnectionAttempt, over: Resting) + // A redial, or a PostgreSQL database switch, which leaves no session until the new one lands. + case reconnecting(ConnectionAttempt, over: Resting) + case failed(AppError) + case lost(AppError) + + enum Resting { + case connected(ConnectionSession) + case lost(AppError) + + var phase: ConnectionCoordinator.ConnectionPhase { + switch self { + case .connected: return .connected + case .lost(let error): return .lost(error) + } + } + + var settled: ConnectionSessionState { + switch self { + case .connected(let session): return .connected(session) + case .lost(let error): return .lost(error) + } + } + } + + enum Event { + case open(ConnectionAttempt) + case check(ConnectionAttempt) + case dial(ConnectionAttempt) + case switchDatabase(ConnectionAttempt) + // A database switch inside an open drops the session the open had. + case teardown(ConnectionAttempt) + // With no attempt: a session another window moved or replaced, or a USE moved, taken in place. + case adopt(ConnectionSession, by: ConnectionAttempt?) + case settle(ConnectionAttempt) + case opened(ConnectionAttempt) + case fail(AppError, ConnectionAttempt) + case lose(AppError, ConnectionAttempt) + // The last window waiting left: an open goes back to the loss or failure it started from, so a window that + // still shows it keeps its Reconnect, and a check or a switch goes back to the session it had. + case abandon(ConnectionAttempt) + // The record changed how it dials, so nothing of the old session is kept. + case cancel + } + + // Nil when the state refuses the event: a step of an attempt it no longer holds, or one it cannot take. + func applying(_ event: Event) -> ConnectionSessionState? { + switch event { + case .open(let attempt): + if isOpening { return nil } + return .connecting(attempt, provisional: nil) + case .check(let attempt): + // A failed redial left nothing to ping, so a check over a loss dials again. + guard let resting else { return nil } + return .checking(attempt, over: resting) + case .dial(let attempt): + guard case .checking(let current, let over) = self, current === attempt else { return nil } + return .reconnecting(attempt, over: over) + case .switchDatabase(let attempt): + // A check in flight finishes first, or its lost-transaction notice would never be given. + guard let resting else { return nil } + return .reconnecting(attempt, over: resting) + case .teardown(let attempt): + guard case .connecting(let current, _) = self, current === attempt else { return nil } + return .connecting(attempt, provisional: nil) + case .adopt(let session, let lander): + guard let lander else { return replacingSession(with: session) } + guard self.attempt === lander else { return nil } + if isOpening { return .connecting(lander, provisional: session) } + return .connected(session) + case .settle(let attempt): + guard self.attempt === attempt else { return nil } + switch self { + case .checking(_, let over), .reconnecting(_, let over): return over.settled + case .notConnected, .connecting, .connected, .failed, .lost: return nil + } + case .opened(let attempt): + guard case .connecting(let current, let provisional?) = self, current === attempt else { return nil } + return .connected(provisional) + case .fail(let error, let attempt): + guard case .connecting(let current, _) = self, current === attempt else { return nil } + return .failed(error) + case .lose(let error, let attempt): + guard self.attempt === attempt else { return nil } + if case .checking = self { return nil } + return .lost(error) + case .abandon(let attempt): + guard self.attempt === attempt else { return nil } + return atRest + case .cancel: + return .notConnected + } + } + + // What a window shows once nothing is in flight: the loss or failure an open started from, or the session a + // check or a switch started over. + var atRest: ConnectionSessionState { + switch self { + case .connecting(let attempt, _): + switch attempt.origin { + case .failed(let error)?: return .failed(error) + case .lost(let error)?: return .lost(error) + case nil: return .notConnected + } + case .checking(_, let over), .reconnecting(_, let over): + return over.settled + case .notConnected, .connected, .failed, .lost: + return self + } + } + + // What an open started here goes back to when every window waiting on it leaves. + var openOrigin: ConnectionAttempt.Origin? { + switch self { + case .failed(let error): return .failed(error) + case .lost(let error): return .lost(error) + case .notConnected, .connecting, .connected, .checking, .reconnecting: return nil + } + } + + var attempt: ConnectionAttempt? { + switch self { + case .connecting(let attempt, _), .checking(let attempt, _), .reconnecting(let attempt, _): return attempt + case .notConnected, .connected, .failed, .lost: return nil + } + } + + // The running session check, which a later trigger joins. + var check: ConnectionAttempt? { + guard let attempt, attempt.purpose == .redial else { return nil } + return attempt + } + + var session: ConnectionSession? { + switch self { + case .connecting(_, let provisional): + return provisional + case .connected(let session), .checking(_, .connected(let session)): + return session + case .reconnecting(let attempt, .connected(let session)): + return attempt.purpose == .switchDatabase ? nil : session + case .notConnected, .checking(_, .lost), .reconnecting(_, .lost), .failed, .lost: + return nil + } + } + + var phase: ConnectionCoordinator.ConnectionPhase { + switch self { + case .notConnected: return .notConnected + case .connecting: return .connecting + case .connected: return .connected + case .checking(_, let over), .reconnecting(_, let over): return over.phase + case .failed(let error): return .failed(error) + case .lost(let error): return .lost(error) + } + } + + // A database switch names its target instead. + var isReconnecting: Bool { + guard case .reconnecting(let attempt, _) = self else { return false } + return attempt.purpose == .redial + } + + var isOpening: Bool { + if case .connecting = self { return true } + return false + } + + private var resting: Resting? { + switch self { + case .connected(let session): return .connected(session) + case .lost(let error): return .lost(error) + case .notConnected, .connecting, .checking, .reconnecting, .failed: return nil + } + } + + private func replacingSession(with session: ConnectionSession) -> ConnectionSessionState? { + switch self { + case .connecting(let attempt, _): + return .connecting(attempt, provisional: session) + case .connected: + return .connected(session) + case .checking(let attempt, .connected): + return .checking(attempt, over: .connected(session)) + case .reconnecting(let attempt, .connected) where attempt.purpose == .redial: + return .reconnecting(attempt, over: .connected(session)) + case .notConnected, .checking(_, .lost), .reconnecting, .failed, .lost: + return nil + } + } +} + +// One open, session check or PostgreSQL database switch. Its identity is what a late step is checked against. +final class ConnectionAttempt { + nonisolated enum Purpose { + case open + case redial + case switchDatabase + } + + enum Origin { + case failed(AppError) + case lost(AppError) + } + + let purpose: Purpose + // For an open: the loss or failure it started from. + let origin: Origin? + // The windows that asked for it or wait on it, first asker first; the last one to leave abandons it. + private(set) var consumers: [ObjectIdentifier] = [] + // A statement failed while this redial ran, so it may have ended the transaction itself. + var ambiguous = false + var task: Task? + + init(_ purpose: Purpose, origin: Origin? = nil) { + self.purpose = purpose + self.origin = origin + } + + func join(_ consumer: AnyObject) { + let id = ObjectIdentifier(consumer) + guard !consumers.contains(id) else { return } + consumers.append(id) + } + + // True when that consumer was the last one. + func leave(_ consumer: AnyObject) -> Bool { + guard let index = consumers.firstIndex(of: ObjectIdentifier(consumer)) else { return false } + consumers.remove(at: index) + return consumers.isEmpty + } +} + +// What a window's lists show and where they were read. +nonisolated struct ConnectionCatalog: Equatable { + nonisolated struct Placement: Hashable { + var database: String + var schema: String + } + + var placement = Placement(database: "", schema: "public") + var databases: [String] = [] + var schemas: [String] = [] + var tables: [TableInfo] = [] + // A table read was refused; the lists reload in full once nothing holds the session. + var tablesRefused = false +} diff --git a/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift b/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift index e7fcb97b34..2826f06557 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift @@ -5,18 +5,25 @@ import TableProLogRedaction import TableProModels import TableProSSHTransport -// Each scene has its own coordinator on the one shared session, so two windows coming back at once -// would each dial, and the later attempt cancels the earlier one's. +// One dial per connection at a time: a dial that would start a second one joins the first, or waits for it when it +// needs its own target, since the later attempt cancels the earlier one's. @MainActor final class SessionRedialer { struct Outcome { let session: ConnectionSession let dialedHere: Bool + // What the dial read to check the schema it was asked to restore; nil when it read nothing. + let schemas: [String]? + } + + private struct Landed: Sendable { + let session: ConnectionSession + let schemas: [String]? } private struct Dial { let id: UUID - let task: Task + let task: Task let prompter: SharedDialPrompter var consumers: Set } @@ -31,9 +38,12 @@ final class SessionRedialer { } // The schema goes back inside the dial, so a window that joins or adopts it never reads the default schema. + // `onlyIfListed` is for a schema remembered from an earlier session, which may since be dropped or belong to + // another database: PostgreSQL and SQL Server would move to it anyway and list no tables. func redial( _ connection: DatabaseConnection, schema: String?, + onlyIfListed: Bool = false, prompter: ConnectionPromptQueue?, joining: Bool = true, consumer: AnyObject? = nil, @@ -44,7 +54,8 @@ final class SessionRedialer { if joining { if let consumerId { dials[connection.id]?.consumers.insert(consumerId) } inFlight.prompter.add(consumerId, prompter) - return Outcome(session: try await inFlight.task.value, dialedHere: false) + let landed = try await inFlight.task.value + return Outcome(session: landed.session, dialedHere: false, schemas: landed.schemas) } // A database switch needs its own target, and a retry after a sign-in its new credentials. _ = try? await inFlight.task.value @@ -61,11 +72,12 @@ final class SessionRedialer { if dials[connection.id]?.id == dialId { dials[connection.id] = nil } } let session = try await manager.connect(connection, prompter: shared) - await Self.restore(schema, on: session.driver) - return session + let schemas = await Self.restore(schema, onlyIfListed: onlyIfListed, on: session.driver) + return Landed(session: session, schemas: schemas) } dials[connection.id] = Dial(id: dialId, task: task, prompter: shared, consumers: consumerId.map { [$0] } ?? []) - return Outcome(session: try await task.value, dialedHere: true) + let landed = try await task.value + return Outcome(session: landed.session, dialedHere: true, schemas: landed.schemas) } // A window that goes away stops waiting; the dial is cancelled only when no other window still waits on it, so @@ -97,13 +109,20 @@ final class SessionRedialer { } } - private static func restore(_ schema: String?, on driver: any DatabaseDriver) async { - guard let schema, let current = driver.currentSchema, current != schema else { return } + private static func restore(_ schema: String?, onlyIfListed: Bool, on driver: any DatabaseDriver) async -> [String]? { + guard let schema, let current = driver.currentSchema, current != schema else { return nil } + var listed: [String]? do { + if onlyIfListed { + let names = try await driver.fetchSchemas() + listed = names + guard names.contains(schema) else { return names } + } try await driver.switchSchema(to: schema) } catch { logger.warning("Failed to restore schema \(schema, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") } + return listed } } diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift index c9f143bcce..0cd74fc0b7 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift @@ -3,6 +3,25 @@ import Foundation import TableProModels extension DuckDBActor { + // A streamed BLOB is its bytes, as on every other engine. `decodeCell` keeps the base64 text `execute` returns. + static func streamedCell( + vector: duckdb_vector, + row: idx_t, + column: DuckDBStreamColumn, + options: StreamOptions, + ref: CellRef? + ) -> Cell { + guard !column.castToText, column.type == DUCKDB_TYPE_BLOB else { + return decodeCell(vector: vector, row: row, column: column, options: options, ref: ref) + } + if let validity = duckdb_vector_get_validity(vector), !duckdb_validity_row_is_valid(validity, row) { + return .null + } + guard let data = duckdb_vector_get_data(vector) else { return .null } + let value = data.assumingMemoryBound(to: duckdb_string_t.self)[Int(row)] + return .binary(byteCount: Int(duckdb_string_t_length(value)), ref: ref) + } + static func decodeCell( vector: duckdb_vector, row: idx_t, @@ -83,11 +102,7 @@ extension DuckDBActor { let length = Int(duckdb_string_t_length(value)) guard length > 0, let pointer = duckdb_string_t_data(&value) else { return .text("") } let text = String(bytes: UnsafeRawBufferPointer(start: pointer, count: length), encoding: .utf8) ?? "" - if length > options.textTruncationBytes { - let prefix = String(text.prefix(options.textTruncationBytes)) - return .truncatedText(prefix: prefix, totalBytes: length, ref: ref) - } - return .text(text) + return Cell.from(text: text, options: options, ref: ref) } private static func decodeBlob(_ data: UnsafeMutableRawPointer, row: idx_t) -> Cell { diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift index 79056a3882..c5e7f64c31 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift @@ -50,7 +50,12 @@ nonisolated extension DuckDBDriver { break } let cells = legacyRow.enumerated().map { index, value -> Cell in - Cell.from(legacyValue: value, columnTypeName: index < columns.count ? columns[index].typeName : nil, options: options) + let typeName = index < columns.count ? columns[index].typeName : nil + // `execute` carries a BLOB as base64, so the cell counts the bytes it decodes to. + if let value, typeName == DuckDBActor.typeName(for: DUCKDB_TYPE_BLOB) { + return .binary(byteCount: Self.base64ByteCount(value), ref: nil) + } + return Cell.from(legacyValue: value, columnTypeName: typeName, options: options) } continuation.yield(.row(Row(cells: cells))) emitted += 1 @@ -60,6 +65,12 @@ nonisolated extension DuckDBDriver { } } + private static func base64ByteCount(_ text: String) -> Int { + let utf8 = text.utf8 + let padding = utf8.reversed().prefix(2).prefix { $0 == UInt8(ascii: "=") }.count + return utf8.count / 4 * 3 - padding + } + func executeStreaming(query: String, options: StreamOptions) -> AsyncThrowingStream { let actor = self.actor let interruptHandle = actor.interruptHandle @@ -213,7 +224,7 @@ extension DuckDBActor { cells.append(.null) continue } - cells.append(Self.decodeCell( + cells.append(Self.streamedCell( vector: vector, row: row, column: columns[index], diff --git a/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift index 11186e7582..e00048133e 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift @@ -20,7 +20,7 @@ nonisolated private extension MSSQLRawResult { } } -nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { +nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "MSSQLDriver") nonisolated(unsafe) private var conn: FreeTDSConnection @@ -33,8 +33,18 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { var supportsSchemas: Bool { true } var supportsTransactions: Bool { true } var sendsBatchesWhole: Bool { true } + var reportsDatabaseWithoutStatement: Bool { false } nonisolated(unsafe) private(set) var currentSchema: String? = "dbo" + // A connection that names no database opens in the login's default one, which only the server knows. + private let sessionDatabaseName = OSAllocatedUnfairLock(initialState: nil) + // Where the app last put the session (the open, a switch), which a `USE` the user runs never moves. Catalog reads + // run there, so a list read while such a run is in flight still lists the picker's database. + private let appDatabaseName = OSAllocatedUnfairLock(initialState: nil) + + var currentDatabase: String? { + sessionDatabaseName.withLock { $0 } + } nonisolated(unsafe) private(set) var serverVersion: String? // This form collects no CA for FreeTDS, so a verify mode from the Mac or an import cannot be honoured here. @@ -106,6 +116,9 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } } + let opened = try? await sessionDatabase() + appDatabaseName.withLock { $0 = opened } + if let serverSchema = try? await runQuery(MSSQLSchemaQueries.currentSchema).rows.first?.first ?? nil, !serverSchema.isEmpty { currentSchema = serverSchema @@ -199,7 +212,6 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { let cells = zip(headerColumns, row).map { columnInfo, rawCell in cell( from: rawCell, - columnTypeName: columnInfo.typeName, options: options, ref: rowKey?.ref(column: columnInfo.name, primaryKey: primaryKey) ) @@ -234,12 +246,12 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } // A binary value reads back through `execute` as text, not its bytes, so it offers no full value to load. - private func cell(from raw: MSSQLRawCell, columnTypeName: String, options: StreamOptions, ref: CellRef?) -> Cell { + private func cell(from raw: MSSQLRawCell, options: StreamOptions, ref: CellRef?) -> Cell { switch raw { case .null: return .null case .string(let value): - return Cell.from(legacyValue: value, columnTypeName: columnTypeName, options: options, ref: ref) + return Cell.from(text: value, options: options, ref: ref) case .bytes(let data): return .binary(byteCount: data.count, ref: nil) } @@ -282,10 +294,12 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } catch let error as MSSQLCoreError { throw mapToConnectionError(error) } + sessionDatabaseName.withLock { $0 = name } + appDatabaseName.withLock { $0 = name } } func fetchSchemas() async throws -> [String] { - let result = try await runQuery(MSSQLSchemaQueries.schemas) + let result = try await runQuery(catalogRead(MSSQLSchemaQueries.schemas, in: nil)) return result.rows.compactMap { $0.first ?? nil } } @@ -293,12 +307,38 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { currentSchema = name } + func sessionDatabase() async throws -> String? { + let name = try await runQuery(MSSQLPrincipalQueries.currentDatabase).rows.first?.first ?? nil + sessionDatabaseName.withLock { $0 = name } + return name + } + // MARK: - Table Metadata private var effectiveSchema: String { currentSchema ?? "dbo" } + // A system procedure called through a database name runs in that database and leaves the session where it is. + static func catalogStatement(_ statement: String, in database: String?) -> String { + guard let database, !database.isEmpty else { return statement } + return MSSQLPrincipalStatements.inDatabase(database, statement) + } + + // The database a window names wins; a list, or a read that names none, goes where the app last put the session. + static func catalogTarget(named database: String?, appDatabase: String?) -> String? { + database.flatMap { $0.isEmpty ? nil : $0 } ?? appDatabase + } + + private func catalogRead(_ statement: String, in database: String?) -> String { + Self.catalogStatement( + statement, + in: Self.catalogTarget(named: database, appDatabase: appDatabaseName.withLock { $0 }) + ) + } + func fetchTables(schema: String?) async throws -> [TableInfo] { - let result = try await runQuery(MSSQLSchemaQueries.tables(schema: schema ?? effectiveSchema)) + let result = try await runQuery( + catalogRead(MSSQLSchemaQueries.tables(schema: schema ?? effectiveSchema), in: nil) + ) return result.rows.compactMap { row in MSSQLSchemaQueries.parseTableRow(row).map { TableInfo(name: $0.name, type: $0.isView ? .view : .table) @@ -307,7 +347,13 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } func fetchColumns(table: String, schema: String?) async throws -> [ColumnInfo] { - let result = try await runQuery(MSSQLSchemaQueries.columns(schema: schema ?? effectiveSchema, table: table)) + try await fetchColumns(table: table, schema: schema, database: nil) + } + + func fetchColumns(table: String, schema: String?, database: String?) async throws -> [ColumnInfo] { + let result = try await runQuery( + catalogRead(MSSQLSchemaQueries.columns(schema: schema ?? effectiveSchema, table: table), in: database) + ) return result.rows.enumerated().compactMap { idx, row in MSSQLSchemaQueries.parseColumnRow(row).map { parsed in ColumnInfo( @@ -327,7 +373,13 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } func fetchIndexes(table: String, schema: String?) async throws -> [IndexInfo] { - let result = try await runQuery(MSSQLSchemaQueries.indexes(schema: schema ?? effectiveSchema, table: table)) + try await fetchIndexes(table: table, schema: schema, database: nil) + } + + func fetchIndexes(table: String, schema: String?, database: String?) async throws -> [IndexInfo] { + let result = try await runQuery( + catalogRead(MSSQLSchemaQueries.indexes(schema: schema ?? effectiveSchema, table: table), in: database) + ) var byName: [String: (unique: Bool, primary: Bool, cols: [String])] = [:] for row in result.rows { guard let parsed = MSSQLSchemaQueries.parseIndexRow(row) else { continue } @@ -342,7 +394,13 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] { - let result = try await runQuery(MSSQLSchemaQueries.foreignKeys(schema: schema ?? effectiveSchema, table: table)) + try await fetchForeignKeys(table: table, schema: schema, database: nil) + } + + func fetchForeignKeys(table: String, schema: String?, database: String?) async throws -> [ForeignKeyInfo] { + let result = try await runQuery( + catalogRead(MSSQLSchemaQueries.foreignKeys(schema: schema ?? effectiveSchema, table: table), in: database) + ) return result.rows.compactMap { row in MSSQLSchemaQueries.parseForeignKeyRow(row).map { ForeignKeyInfo(name: $0.constraintName, column: $0.columnName, diff --git a/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift index 3170d07c7d..68fb65dfca 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift @@ -10,7 +10,7 @@ import TableProPluginKit import TableProSQLGrammar import TableProSQLQuoting -nonisolated final class MySQLDriver: DatabaseDriver, @unchecked Sendable { +nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @unchecked Sendable { private let actor = MySQLActor() private let host: String private let port: Int @@ -32,6 +32,13 @@ nonisolated final class MySQLDriver: DatabaseDriver, @unchecked Sendable { var supportsSchemas: Bool { false } var currentSchema: String? { nil } var supportsTransactions: Bool { true } + var reportsDatabaseWithoutStatement: Bool { false } + var enforcesReadOnlyMode: Bool { Self.runsReadOnlyTransactions(for: databaseType, banner: serverVersion) } + + func entersReadOnlyMode() async -> Bool { + guard enforcesReadOnlyMode else { return false } + return await actor.takesReadOnlyTransaction() + } private let settledFacts = OSAllocatedUnfairLock(initialState: nil) @@ -132,6 +139,10 @@ nonisolated final class MySQLDriver: DatabaseDriver, @unchecked Sendable { ) } + static func runsReadOnlyTransactions(for databaseType: DatabaseType, banner: String?) -> Bool { + MySQLReadOnlyTransaction.isAvailable(banner: banner, flavor: serverFlavor(for: databaseType, banner: banner)) + } + func connect() async throws { try await LocalNetworkPermission.shared.ensureAccess(for: host) try await actor.connect( @@ -207,11 +218,14 @@ nonisolated final class MySQLDriver: DatabaseDriver, @unchecked Sendable { let statement = killer.reserve() let databaseType = self.databaseType let session = settledLexicalFacts + let inReadOnlyTransaction = options.enforcesReadOnly && enforcesReadOnlyMode return AsyncThrowingStream { continuation in let task = Task { var openStream: MySQLStreamID? do { - switch try await actor.beginStream(query: query, statement: statement) { + switch try await actor.beginStream( + query: query, statement: statement, inReadOnlyTransaction: inReadOnlyTransaction + ) { case .noResult(let affectedRows): if affectedRows != 0 { continuation.yield(.rowsAffected(affectedRows)) @@ -376,6 +390,11 @@ nonisolated final class MySQLDriver: DatabaseDriver, @unchecked Sendable { throw MySQLError.unsupported("MySQL does not support schemas") } + // NULL once no database is selected, as after a DROP DATABASE of the one the session was in. + func sessionDatabase() async throws -> String? { + try await actor.execute("SELECT DATABASE()").rows.first?.first ?? nil + } + func fetchSchemas() async throws -> [String] { [] } func beginTransaction() async throws { @@ -432,6 +451,13 @@ nonisolated final class MySQLDriver: DatabaseDriver, @unchecked Sendable { } } +// The sign-in went through and the database was refused as a USE there is (scripts/probes/check-mysql-dial-database-refusal.sh). +nonisolated enum MySQLDatabaseRefusal { + static func isRefusal(code: UInt32) -> Bool { + code == 1_044 || code == 1_049 + } +} + nonisolated enum MySQLTokenSignIn { // A token session refuses a server without TLS before the token is sent; this names that refusal. static func connectFailure( @@ -652,6 +678,11 @@ nonisolated enum MySQLSessionTransaction { guard serverStatus & UInt32(SERVER_STATUS_IN_TRANS) != 0 else { return .idle } return serverStatus & UInt32(SERVER_STATUS_AUTOCOMMIT) != 0 ? .explicitTransaction : .implicitTransaction } + + // A Read-Only statement's own transaction opened on an idle session and ends with its stream. + static func state(infoResult: my_bool, serverStatus: UInt32, holdsReadOnlyTransaction: Bool) -> DriverTransactionState { + holdsReadOnlyTransaction ? .idle : state(infoResult: infoResult, serverStatus: serverStatus) + } } // MARK: - MySQL Actor (thread-safe C API access) @@ -729,8 +760,13 @@ private actor MySQLActor { ) { throw tokenFailure } - throw MariaDBSSLClassifier.classifySSLError(code: failure.code, message: failure.message) - ?? MySQLError.connectionFailed(failure.message) + if let tlsFailure = MariaDBSSLClassifier.classifySSLError(code: failure.code, message: failure.message) { + throw tlsFailure + } + if MySQLDatabaseRefusal.isRefusal(code: failure.code) { + throw MySQLError.databaseRefused(failure.message) + } + throw MySQLError.connectionFailed(failure.message) } guard MariaDBCharacterSet.establishSession(on: handle, encoding: encoding) else { @@ -859,8 +895,11 @@ private actor MySQLActor { showTextIsBinary && mysqlStatementListsServerMetadata(query) } + // Closing the session ends a read-only transaction on the server. func close() { killer.detach() + holdsReadOnlyTransaction = false + footprint.reset() guard let mysql else { return } if let result = streamingResult { // An open result points at the handle `mysql_close` frees, so it goes first; shutting the socket @@ -917,7 +956,9 @@ private actor MySQLActor { guard let mysql else { return .unknown } var serverStatus: UInt32 = 0 let infoResult = mariadb_get_info(mysql, MARIADB_CONNECTION_SERVER_STATUS, &serverStatus) - return MySQLSessionTransaction.state(infoResult: infoResult, serverStatus: serverStatus) + return MySQLSessionTransaction.state( + infoResult: infoResult, serverStatus: serverStatus, holdsReadOnlyTransaction: holdsReadOnlyTransaction + ) } /// `NO_BACKSLASH_ESCAPES` from the status flags of the last reply turns the backslash off in both string quotes. @@ -948,7 +989,10 @@ private actor MySQLActor { let start = Date() + let features = lexicalFeatures(of: mysql) + footprint.observe(query, lexicalFeatures: features) guard mysql_real_query(mysql, query, UInt(query.utf8.count)) == 0 else { + footprint.observeFailure(of: query, lexicalFeatures: features) throw MySQLError.queryFailed(message(from: mysql)) } @@ -1014,10 +1058,22 @@ private actor MySQLActor { private var streamingColumns: [ColumnInfo] = [] private var streamingDecoding = MySQLResultColumns() private var streamingStatement: SentStatement? - - func beginStream(query: String, statement: UInt64) throws -> MySQLBeginStreamResult { + // Set from START TRANSACTION READ ONLY until the ROLLBACK after the statement's last result. + private var holdsReadOnlyTransaction = false + // The status flags never show a LOCK TABLES, which a START TRANSACTION releases, so the statements say it. + private var footprint = MySQLSessionFootprint() + + // With `inReadOnlyTransaction` the state check, the transaction and the statement go out in this one actor call, + // so nothing else on the session runs between them. The session is opened without CLIENT_MULTI_STATEMENTS, so no + // `COMMIT; DELETE ...` can follow the read. + func beginStream( + query: String, + statement: UInt64, + inReadOnlyTransaction: Bool = false + ) throws -> MySQLBeginStreamResult { guard let mysql else { throw MySQLError.notConnected } endLeftoverStream(of: mysql) + if inReadOnlyTransaction { try beginReadOnlyTransaction() } streamGeneration &+= 1 let stream = MySQLStreamID(generation: streamGeneration) let sent = send(statement, on: mysql) @@ -1025,23 +1081,30 @@ private actor MySQLActor { // flight and kills it. guard !Task.isCancelled else { finish(sent, on: mysql) + try? endReadOnlyTransaction() throw CancellationError() } + let features = lexicalFeatures(of: mysql) + footprint.observe(query, lexicalFeatures: features) guard mysql_real_query(mysql, query, UInt(query.utf8.count)) == 0 else { + footprint.observeFailure(of: query, lexicalFeatures: features) + let failure = queryFailure(of: mysql) finish(sent, on: mysql) - throw MySQLError.queryFailed(message(from: mysql)) + try? endReadOnlyTransaction() + throw failure } guard let result = mysql_use_result(mysql) else { - defer { finish(sent, on: mysql) } - if mysql_field_count(mysql) != 0 { - throw MySQLError.queryFailed(message(from: mysql)) + return try settle(sent, on: mysql) { + if mysql_field_count(mysql) != 0 { + throw queryFailure(of: mysql) + } + let raw = mysql_affected_rows(mysql) + let affected = raw == .max ? 0 : Int(clamping: raw) + try consumeTrailingResults(of: mysql) + return .noResult(affectedRows: affected) } - let raw = mysql_affected_rows(mysql) - let affected = raw == .max ? 0 : Int(clamping: raw) - try consumeTrailingResults(of: mysql) - return .noResult(affectedRows: affected) } streamingResult = result @@ -1094,12 +1157,7 @@ private actor MySQLActor { case .bytes(let data): return .binary(byteCount: data.count, ref: ref) case .text(let text): - return Cell.from( - legacyValue: text, - columnTypeName: column.typeName, - options: options, - ref: ref - ) + return Cell.from(text: text, options: options, ref: ref) @unknown default: return .null } @@ -1118,11 +1176,7 @@ private actor MySQLActor { if let sent = streamingStatement { killer.rowCapReached(sent.id) } - do { - try releaseStream(of: mysql) - } catch { - guard flavor.isInterruptedByKill(errno: mysql_errno(mysql), message: message(from: mysql)) else { throw error } - } + try releaseStream(of: mysql, atRowCap: true) } /// The producer of a superseded stream only learns that its rows were cut off, so an error in the rest of @@ -1137,13 +1191,76 @@ private actor MySQLActor { } } - private func releaseStream(of mysql: UnsafeMutablePointer) throws { + // At the row cap the kill's 1317 is the cut the reader was already told of. + private func releaseStream(of mysql: UnsafeMutablePointer, atRowCap: Bool = false) throws { guard let result = streamingResult else { return } let sent = streamingStatement clearStream() - defer { if let sent { finish(sent, on: mysql) } } - try drain(result, of: mysql) - try consumeTrailingResults(of: mysql) + try settle(sent, on: mysql) { + do { + try drain(result, of: mysql) + try consumeTrailingResults(of: mysql) + } catch where atRowCap && flavor.isInterruptedByKill(errno: mysql_errno(mysql), message: message(from: mysql)) {} + } + } + + // The kill bookkeeping reads the statement's error code, so the ROLLBACK, which resets it, comes after. + private func settle( + _ sent: SentStatement?, + on mysql: UnsafeMutablePointer, + _ statement: () throws -> Value + ) throws -> Value { + let outcome = Result(catching: statement) + if let sent { finish(sent, on: mysql) } + switch outcome { + case .success(let value): + try endReadOnlyTransaction() + return value + case .failure(let error): + try? endReadOnlyTransaction() + throw error + } + } + + // A START TRANSACTION would commit the session's open transaction and release its LOCK TABLES. A `SET autocommit + // = 0` session is always in one, so the app does not ask for the read-only one there. + private func beginReadOnlyTransaction() throws { + guard takesReadOnlyTransaction() else { + throw ReadOnlyModeRefusal.sessionNotIdle(message: MySQLReadOnlyTransaction.notIdleMessage(.safeMode)) + } + do { + _ = try execute(MySQLReadOnlyTransaction.begin) + } catch { + _ = try? execute(MySQLReadOnlyTransaction.end) + throw error + } + holdsReadOnlyTransaction = true + } + + func takesReadOnlyTransaction() -> Bool { + mysql != nil && transactionState() == .idle && !footprint.hasLockedTables + } + + private func lexicalFeatures(of mysql: UnsafeMutablePointer) -> SQLLexicalFeatures { + var serverStatus: UInt32 = 0 + let known = mariadb_get_info(mysql, MARIADB_CONNECTION_SERVER_STATUS, &serverStatus) == 0 + let noBackslashEscapes = known ? serverStatus & UInt32(SERVER_STATUS_NO_BACKSLASH_ESCAPES) != 0 : nil + return MySQLLexicalFeatures.features(for: flavor, noBackslashEscapes: noBackslashEscapes) + } + + private func endReadOnlyTransaction() throws { + guard holdsReadOnlyTransaction else { return } + holdsReadOnlyTransaction = false + _ = try execute(MySQLReadOnlyTransaction.end) + } + + // Taken before the ROLLBACK, which resets the code. + private func queryFailure(of mysql: UnsafeMutablePointer) -> any Error { + let text = message(from: mysql) + guard holdsReadOnlyTransaction, mysql_errno(mysql) == MySQLReadOnlyTransaction.refusalCode else { + return MySQLError.queryFailed(text) + } + return ReadOnlyModeRefusal.refused(message: MySQLReadOnlyTransaction.refusalMessage(.safeMode), serverMessage: text) } private func clearStream() { @@ -1195,12 +1312,12 @@ private actor MySQLActor { private func consumeTrailingResults(of mysql: UnsafeMutablePointer) throws { while mysql_more_results(mysql) != 0 { let status = mysql_next_result(mysql) - if status > 0 { throw MySQLError.queryFailed(message(from: mysql)) } + if status > 0 { throw queryFailure(of: mysql) } if status < 0 { return } if let result = mysql_use_result(mysql) { try drain(result, of: mysql) } else if mysql_field_count(mysql) != 0 { - throw MySQLError.queryFailed(message(from: mysql)) + throw queryFailure(of: mysql) } } } @@ -1215,10 +1332,10 @@ private actor MySQLActor { /// `mysql_fetch_row` returns nil both at the end and on an error, and a command refused while the result /// was open leaves its own code on the handle, so only a code the fetch set means the rows ended in an error. - private func fetchFailure(of mysql: UnsafeMutablePointer, errorBefore: UInt32) -> MySQLError? { + private func fetchFailure(of mysql: UnsafeMutablePointer, errorBefore: UInt32) -> (any Error)? { let code = mysql_errno(mysql) guard code != 0, code != errorBefore else { return nil } - return .queryFailed(message(from: mysql)) + return queryFailure(of: mysql) } private func makeCellRef(column: String, row: MYSQL_ROW, options: StreamOptions, columns: [ColumnInfo]) -> CellRef? { @@ -1257,13 +1374,15 @@ nonisolated private struct RawMySQLResult: Sendable { nonisolated enum MySQLError: Error, LocalizedError { case connectionFailed(String) + // The database the handshake named was refused; it reads as any failed connect. + case databaseRefused(String) case notConnected case queryFailed(String) case unsupported(String) var errorDescription: String? { switch self { - case .connectionFailed(let msg): return "MySQL connection failed: \(msg)" + case .connectionFailed(let msg), .databaseRefused(let msg): return "MySQL connection failed: \(msg)" case .notConnected: return "Not connected to MySQL database" case .queryFailed(let msg): return "MySQL query failed: \(msg)" case .unsupported(let msg): return msg diff --git a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift index 8a7bc1df36..8e7d52ce96 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift @@ -129,7 +129,10 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { let coreStream = AsyncThrowingStream { coreContinuation in let coreTask = Task { do { - try await core.streamQuery(query, continuation: coreContinuation) + // The result buffer shows rows as they arrive, so none waits on a batch to fill. + try await core.streamQuery( + query, batching: OracleRowBatcher(rowsPerBatch: 1), continuation: coreContinuation + ) } catch { coreContinuation.finish(throwing: error) } @@ -167,7 +170,6 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { let cells = zip(headerColumns, row).map { columnInfo, rawCell in cell( from: rawCell, - columnTypeName: columnInfo.typeName, options: options, ref: rowKey?.ref(column: columnInfo.name, primaryKey: primaryKey) ) @@ -195,12 +197,12 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { } // A binary value reads back through `execute` as text, not its bytes, so it offers no full value to load. - private func cell(from raw: OracleRawCell, columnTypeName: String, options: StreamOptions, ref: CellRef?) -> Cell { + private func cell(from raw: OracleRawCell, options: StreamOptions, ref: CellRef?) -> Cell { switch raw { case .null: return .null case .string(let value): - return Cell.from(legacyValue: value, columnTypeName: columnTypeName, options: options, ref: ref) + return Cell.from(text: value, options: options, ref: ref) case .bytes(let data): return .binary(byteCount: data.count, ref: nil) } diff --git a/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift index 0130bd43be..61c8b7b681 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift @@ -225,8 +225,7 @@ nonisolated class PluginBackedDriver: DatabaseDriver, @unchecked Sendable { case .bytes(let data): return .binary(byteCount: data.count, ref: ref) case .text(let text): - let typeName = index < columns.count ? columns[index].typeName : nil - return Cell.from(legacyValue: text, columnTypeName: typeName, options: options, ref: ref) + return Cell.from(text: text, options: options, ref: ref) } } continuation.yield(.row(Row(cells: cells, locator: locator))) diff --git a/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift index 0389ae815c..ab6c89d52a 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift @@ -79,8 +79,8 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, columns: [String], values: [(column: String, value: String?)] ) throws -> [PluginRowWrite] { - // The plugin reads this text as "leave the column out", so storing it as a value cannot be expressed. - if values.contains(where: { $0.value == Self.defaultSentinel }) { + // A plugin that does not read the default mark reads this text as "leave the column out", so it cannot store it. + if !plugin.capabilities.contains(.columnDefaultMarks), values.contains(where: { $0.value == Self.defaultSentinel }) { throw DatabaseError(message: String(format: String( localized: "The text %@ cannot be inserted: this database reads it as the column's default." ), Self.defaultSentinel)) @@ -90,7 +90,9 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, guard let value = given[column] else { return .text(Self.defaultSentinel) } return PluginCellValue(optionalText: value) } - let rowChange = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + var rowChange = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let defaulted = Set(columns.indices.filter { given[columns[$0]] == nil }) + rowChange.defaultColumns = defaulted.isEmpty ? nil : defaulted return try generate( table: table, schema: schema, columns: columns, changes: [rowChange], inserted: [0: row], context: PluginRowWriteContext() @@ -158,7 +160,7 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, DatabaseError(message: String(localized: "This database cannot write this change from iPhone or iPad.")) } - // The Mac's grid marks a cell left at the column default with this text, and the drivers leave such a column out. + // What a column left at its default holds, for a plugin built before `PluginRowChange.defaultColumns`. static let defaultSentinel = "__DEFAULT__" private func generate( diff --git a/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift index f36df9f778..8408576e5d 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift @@ -9,7 +9,7 @@ import TableProQuery import TableProSQLGrammar import TableProSQLQuoting -nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { +nonisolated final class PostgreSQLDriver: DatabaseDriver, SessionSchemaReporting, @unchecked Sendable { static let logger = Logger(subsystem: "com.TablePro", category: "PostgreSQLDriver") private let actor = PostgreSQLActor() @@ -24,6 +24,8 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { var supportsSchemas: Bool { true } var supportsTransactions: Bool { true } + // As on the Mac, where the PostgreSQL plugin driver declares the mode and Redshift's does not. + var enforcesReadOnlyMode: Bool { databaseType == .postgresql } private let settledFacts = OSAllocatedUnfairLock(initialState: nil) @@ -38,8 +40,12 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { return SQLQuoting.stringLiteralBody(value, escapes: escapes) } - // Set once during connect()/switchSchema() before the driver is shared — safe for concurrent reads - nonisolated(unsafe) private(set) var currentSchema: String? = "public" + // A run's end writes it while a window reads it, so it sits behind a lock. + private let sessionSchema = OSAllocatedUnfairLock(initialState: "public") + + var currentSchema: String? { + sessionSchema.withLock { $0 } + } nonisolated(unsafe) private(set) var serverVersion: String? nonisolated(unsafe) private(set) var serverVersionNumber: Int32 = 0 nonisolated(unsafe) private(set) var catalogPresence = PostgreSQLDriver.presence(probedRows: nil) @@ -94,7 +100,7 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { serverVersionNumber = await actor.serverVersionNumber() catalogPresence = await probeCatalogPresence() columnReadSupport = PostgreSQLColumnReadSupport() - await adoptServerSchema() + await settleCurrentSchema() await settleLexicalFacts() } @@ -117,10 +123,11 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { } } - private func adoptServerSchema() async { - guard let schema = try? await actor.execute("SELECT pg_catalog.current_schema()").rows.first?.first ?? nil, - !schema.isEmpty else { return } - currentSchema = schema + // A typed `SET search_path` moves it, and a transaction's end takes back a `SET` made inside it, the picker's own + // among them. Inside a transaction block the schema waits for the block to end. + func settleCurrentSchema() async { + guard let schema = await actor.currentSchemaOutsideBlock(), !schema.isEmpty else { return } + sessionSchema.withLock { $0 = schema } } func disconnect() async throws { @@ -181,10 +188,13 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { let cancelHandle = actor.cancelHandle let id = cancelHandle.reserveQuery() let readsOnly = !SQLWriteClassifier.isWriteQuery(query, databaseType: databaseType) + let inReadOnlyTransaction = options.enforcesReadOnly && enforcesReadOnlyMode return AsyncThrowingStream { continuation in let task = Task { do { - switch try await actor.beginStream(query: query, id: id, readsOnly: readsOnly) { + switch try await actor.beginStream( + query: query, id: id, readsOnly: readsOnly, inReadOnlyTransaction: inReadOnlyTransaction + ) { case .commandOk(let affectedRows, let statusMessage): if affectedRows != 0 { continuation.yield(.rowsAffected(affectedRows)) @@ -279,7 +289,7 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, @unchecked Sendable { func switchSchema(to name: String) async throws { _ = try await actor.execute("SET search_path TO \(SQLQuoting.identifier(name))") - currentSchema = name + sessionSchema.withLock { $0 = name } } func fetchSchemas() async throws -> [String] { @@ -390,7 +400,7 @@ private actor PostgreSQLActor { ) guard let connection = PQconnectStart(connStr) else { - throw PostgreSQLError.connectionFailed(String(localized: "Could not start a connection.")) + throw PostgreSQLError.unreachable(String(localized: "Could not start a connection.")) } var adopted = false @@ -429,14 +439,14 @@ private actor PostgreSQLActor { throw PostgreSQLError.connectionFailed(Self.message(from: connection)) } guard Date() < deadline else { - throw PostgreSQLError.connectionFailed(String(localized: "Connection timed out.")) + throw PostgreSQLError.unreachable(String(localized: "Connection timed out.")) } let events = status == PGRES_POLLING_READING ? Int16(POLLIN) : Int16(POLLOUT) var descriptor = pollfd(fd: socket, events: events, revents: 0) let ready = poll(&descriptor, 1, Self.pollSliceMilliseconds) guard ready >= 0 else { - throw PostgreSQLError.connectionFailed(String(localized: "Connection failed while waiting on the socket.")) + throw PostgreSQLError.unreachable(String(localized: "Connection failed while waiting on the socket.")) } guard ready > 0 else { continue } @@ -475,6 +485,12 @@ private actor PostgreSQLActor { ) } + // A read inside a block would take a REPEATABLE READ transaction's snapshot, and a later SET TRANSACTION would fail. + func currentSchemaOutsideBlock() -> String? { + guard let conn, !PostgreSQLSessionTransaction.isInsideBlock(PQtransactionStatus(conn)) else { return nil } + return (try? execute("SELECT pg_catalog.current_schema()"))?.rows.first?.first ?? nil + } + func ping() throws { guard let conn else { throw PostgreSQLError.notConnected } guard PostgreSQLSessionTransaction.pingSendsStatement(in: PQtransactionStatus(conn)) else { @@ -632,18 +648,33 @@ private actor PostgreSQLActor { /// of a stream that a later query replaced can neither read nor end the later one. private var stream: PostgreSQLStreamState? - func beginStream(query: String, id: UInt64, readsOnly: Bool) throws -> PGBeginStreamResult { + // With `inReadOnlyTransaction` the state check, the transaction and the statement go out in this one actor call, so + // nothing else on the session runs between them. + func beginStream( + query: String, + id: UInt64, + readsOnly: Bool, + inReadOnlyTransaction: Bool = false + ) throws -> PGBeginStreamResult { endLeftoverStream() guard let conn else { throw PostgreSQLError.notConnected } let cancelsOutput = cancelsAbandonedOutput(conn) + let statusAtSend = PQtransactionStatus(conn) + if inReadOnlyTransaction { try beginReadOnlyTransaction(from: statusAtSend) } let state = PostgreSQLStreamState( id: id, cancelsAbandonedOutput: cancelsOutput, cancelsUnreadRows: cancelsOutput && readsOnly, - transactionStatusAtSend: PQtransactionStatus(conn) + transactionStatusAtSend: statusAtSend, + runsReadOnly: inReadOnlyTransaction ) - try send(query, id: id, on: conn) + do { + try send(query, id: id, on: conn, singleStatement: inReadOnlyTransaction) + } catch { + _ = endReadOnlyTransaction(of: state) + throw error + } guard PQsetSingleRowMode(conn) == 1 else { _ = drain(state) throw PostgreSQLError.queryFailed("Failed to enter single-row streaming mode") @@ -661,14 +692,14 @@ private actor PostgreSQLActor { case PGRES_TUPLES_OK: let columns = parseColumns(firstResult) PQclear(firstResult) - if let error = drain(state).error { throw error } + if let error = drainFailure(state) { throw error } stream = state return .tuples(columns) case PGRES_COMMAND_OK: let affected = Int(String(cString: PQcmdTuples(firstResult))) ?? 0 let commandTag = String(cString: PQcmdStatus(firstResult)) PQclear(firstResult) - if let error = drain(state).error { throw error } + if let error = drainFailure(state) { throw error } return .commandOk( affectedRows: affected, statusMessage: PostgreSQLSessionTransaction.statusMessage( @@ -679,18 +710,55 @@ private actor PostgreSQLActor { ) default: state.pendingResult = firstResult - if let error = drain(state).error { throw error } + if let error = drainFailure(state) { throw error } return .commandOk(affectedRows: 0, statusMessage: nil) } } + // Inside an open transaction the wrap would make the user's own transaction read-only. + private func beginReadOnlyTransaction(from status: PGTransactionStatusType) throws { + guard status == PQTRANS_IDLE else { + throw ReadOnlyModeRefusal.sessionNotIdle(message: LibPQReadOnlyTransaction.notIdleMessage(.safeMode)) + } + do { + _ = try execute(LibPQReadOnlyTransaction.begin) + } catch { + _ = rollBackReadOnlyTransaction() + throw error + } + } + + // Once per stream, after its last result. Nil when the ROLLBACK went through or there was none to send. + private func endReadOnlyTransaction(of state: PostgreSQLStreamState) -> String? { + guard state.holdsReadOnlyTransaction else { return nil } + state.holdsReadOnlyTransaction = false + return rollBackReadOnlyTransaction() + } + + // Registered with no query id, so Stop cannot cancel it and leave the read-only transaction open. A connection the + // drain dropped has nothing to roll back. + private func rollBackReadOnlyTransaction() -> String? { + guard let conn else { return nil } + cancelHandle.waitForPendingCancels() + guard let result = PQexec(conn, LibPQReadOnlyTransaction.end) else { return Self.message(from: conn) } + defer { PQclear(result) } + guard PQresultStatus(result) == PGRES_COMMAND_OK else { return String(cString: PQresultErrorMessage(result)) } + return nil + } + + private func drainFailure(_ state: PostgreSQLStreamState) -> (any Error)? { + drain(state).error(inReadOnlyTransaction: state.runsReadOnly) + } + /// A cancel still on its way could reach the server after this query and stop it instead, so the /// query waits for it. A consumer that leaves as the query goes out finds nothing in flight to /// cancel, so its cancel is sent here once the query is registered. - private func send(_ query: String, id: UInt64, on conn: OpaquePointer) throws { + private func send(_ query: String, id: UInt64, on conn: OpaquePointer, singleStatement: Bool) throws { cancelHandle.waitForPendingCancels() try Task.checkCancellation() - guard PQsendQuery(conn, query) == 1 else { + // The extended protocol takes one statement, so no `COMMIT; DELETE ...` can follow a read-only read. + let sent = singleStatement ? PQsendQueryParams(conn, query, 0, nil, nil, nil, nil, 0) : PQsendQuery(conn, query) + guard sent == 1 else { throw PostgreSQLError.queryFailed(String(cString: PQerrorMessage(conn))) } cancelHandle.querySent(id) @@ -715,7 +783,7 @@ private actor PostgreSQLActor { } guard PQresultStatus(result) == PGRES_SINGLE_TUPLE else { state.pendingResult = result - if let error = drain(state).error { throw error } + if let error = drainFailure(state) { throw error } return nil } defer { PQclear(result) } @@ -727,7 +795,7 @@ private actor PostgreSQLActor { guard let state = stream, state.id == id else { return } stream = nil let end = drain(state) - guard end != .cancelled, let error = end.error else { return } + guard end != .cancelled, let error = end.error(inReadOnlyTransaction: state.runsReadOnly) else { return } throw error } @@ -775,22 +843,36 @@ private actor PostgreSQLActor { _ = drain(leftover) } + // The connection closes next, which ends a read-only transaction on the server. private func discardStream() { guard let state = stream else { return } stream = nil if let readAhead = state.pendingResult { PQclear(readAhead) } state.pendingResult = nil + state.holdsReadOnlyTransaction = false markDrained(state) } private func markDrained(_ state: PostgreSQLStreamState) { state.isDrained = true cancelHandle.queryFinished(state.id) + if let failure = endReadOnlyTransaction(of: state) { + Self.logger.warning("The read-only transaction did not roll back: \(failure, privacy: .private)") + } } - /// Reads every result the stream's query still has in flight, ending any COPY among them. + // The ROLLBACK goes out once the results are read and the query is no longer in flight. Its failure is the stream's + // when the statement itself ended cleanly. private func drain(_ state: PostgreSQLStreamState) -> PostgreSQLStreamEnd { guard !state.isDrained else { return .completed } + let end = drainResults(state) + let rollbackFailure = endReadOnlyTransaction(of: state) + guard end == .completed, let rollbackFailure else { return end } + return .failed(rollbackFailure) + } + + // Reads every result the stream's query still has in flight, ending any COPY among them. + private func drainResults(_ state: PostgreSQLStreamState) -> PostgreSQLStreamEnd { state.isDrained = true defer { cancelHandle.queryFinished(state.id) } var readAhead = state.pendingResult @@ -817,9 +899,15 @@ private actor PostgreSQLActor { private static func pendingResult(of result: OpaquePointer) -> PostgreSQLPendingResult { if let copy = LibPQCopyState.copy(of: result) { return .copy(copy) } guard PostgreSQLStreamDrain.isFailure(PQresultStatus(result)) else { return .completed } - return .failed(String(cString: PQresultErrorMessage(result))) + return .failed( + String(cString: PQresultErrorMessage(result)), + sqlState: PQresultErrorField(result, sqlStateField).map { String(cString: $0) } + ) } + // PG_DIAG_SQLSTATE, a character macro Swift does not import. + private static let sqlStateField = Int32(UInt8(ascii: "C")) + private func cells(of result: OpaquePointer, options: StreamOptions, columns: [ColumnInfo]) -> [Cell] { var cells: [Cell] = [] cells.reserveCapacity(columns.count) @@ -913,20 +1001,27 @@ nonisolated private final class PostgreSQLStreamState { /// `UPDATE ... RETURNING` past the row cap has to finish, or its write is undone. let cancelsUnreadRows: Bool let transactionStatusAtSend: PGTransactionStatusType + // The statement ran in Read-Only's read-only transaction, so a refusal state is Read-Only's refusal. + let runsReadOnly: Bool /// A result read before its turn: the first row, or what the row-cap check took off the wire. var pendingResult: OpaquePointer? var isDrained = false + // Until the ROLLBACK goes out. + var holdsReadOnlyTransaction: Bool init( id: UInt64, cancelsAbandonedOutput: Bool, cancelsUnreadRows: Bool, - transactionStatusAtSend: PGTransactionStatusType + transactionStatusAtSend: PGTransactionStatusType, + runsReadOnly: Bool = false ) { self.id = id self.cancelsAbandonedOutput = cancelsAbandonedOutput self.cancelsUnreadRows = cancelsUnreadRows self.transactionStatusAtSend = transactionStatusAtSend + self.runsReadOnly = runsReadOnly + holdsReadOnlyTransaction = runsReadOnly } } @@ -1074,7 +1169,7 @@ nonisolated private final class PostgreSQLCancelHandle: @unchecked Sendable { /// A result read while a stream ends, reduced to what decides how the stream ends. nonisolated enum PostgreSQLPendingResult: Equatable, Sendable { case copy(LibPQCopy) - case failed(String) + case failed(String, sqlState: String? = nil) case completed } @@ -1082,9 +1177,14 @@ nonisolated enum PostgreSQLStreamEnd: Equatable, Sendable { case completed case cancelled case abandonedCopy(LibPQCopyDirection) - case failed(String) + case failed(String, sqlState: String? = nil) var error: (any Error)? { + error(inReadOnlyTransaction: false) + } + + // In Read-Only's read-only transaction a refusal state is the engine holding Read-Only, not a failed query. + func error(inReadOnlyTransaction: Bool) -> (any Error)? { switch self { case .completed: return nil @@ -1092,8 +1192,13 @@ nonisolated enum PostgreSQLStreamEnd: Equatable, Sendable { return CancellationError() case .abandonedCopy(let direction): return PostgreSQLError.unsupported(direction.mobileUnsupportedMessage) - case .failed(let message): - return PostgreSQLError.queryFailed(message) + case .failed(let message, let sqlState): + guard inReadOnlyTransaction, LibPQReadOnlyTransaction.isRefusal(sqlState: sqlState) else { + return PostgreSQLError.queryFailed(message) + } + return ReadOnlyModeRefusal.refused( + message: LibPQReadOnlyTransaction.refusalMessage(.safeMode), serverMessage: message + ) } } } @@ -1113,14 +1218,14 @@ nonisolated enum PostgreSQLStreamDrain { endCopy: (LibPQCopy) -> Void, afterCancel: @autoclosure () -> Bool ) -> (end: PostgreSQLStreamEnd, outcome: LibPQDrainOutcome) { - var firstFailure: String? + var firstFailure: (message: String, sqlState: String?)? let outcome = LibPQPendingResultDrain.drain( nextResult: { switch nextResult() { case .copy(let copy): return .copy(copy) - case .failed(let message): - if firstFailure == nil { firstFailure = message } + case .failed(let message, let sqlState): + if firstFailure == nil { firstFailure = (message, sqlState) } return .completed case .completed: return .completed @@ -1134,7 +1239,7 @@ nonisolated enum PostgreSQLStreamDrain { if let copy = outcome.abandonedCopy { end = .abandonedCopy(copy.direction) } else if let firstFailure { - end = afterCancel() ? .cancelled : .failed(firstFailure) + end = afterCancel() ? .cancelled : .failed(firstFailure.message, sqlState: firstFailure.sqlState) } else { end = .completed } @@ -1160,7 +1265,7 @@ nonisolated private func pgOidToTypeName(_ oid: UInt32) -> String { case 700: return "real" case 701: return "double precision" case 869: return "inet" - // PostgreSQL OID constants — separators would obscure the wire-protocol values + // PostgreSQL OID constants: separators would obscure the wire-protocol values // swiftlint:disable number_separator case 1042: return "char" case 1043: return "varchar" @@ -1189,13 +1294,15 @@ nonisolated private struct RawPGResult: Sendable { nonisolated enum PostgreSQLError: Error, LocalizedError { case connectionFailed(String) + // The connect never reached the server: it timed out or the socket failed on this device. + case unreachable(String) case notConnected case queryFailed(String) case unsupported(String) var errorDescription: String? { switch self { - case .connectionFailed(let msg): return "PostgreSQL connection failed: \(msg)" + case .connectionFailed(let msg), .unreachable(let msg): return "PostgreSQL connection failed: \(msg)" case .notConnected: return "Not connected to PostgreSQL database" case .queryFailed(let msg): return "PostgreSQL query failed: \(msg)" case .unsupported(let msg): return msg diff --git a/apple/TableProMobile/TableProMobile/Drivers/ReadOnlyModeRefusal.swift b/apple/TableProMobile/TableProMobile/Drivers/ReadOnlyModeRefusal.swift new file mode 100644 index 0000000000..78047408b4 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Drivers/ReadOnlyModeRefusal.swift @@ -0,0 +1,18 @@ +import Foundation + +// Safe Mode Read-Only's refusal from a driver that runs the statement in the engine's read-only transaction. The message +// is the one the engine's Mac plugin shows, from the file both apps compile. +nonisolated enum ReadOnlyModeRefusal: LocalizedError, Equatable { + case sessionNotIdle(message: String) + case refused(message: String, serverMessage: String) + + var errorDescription: String? { + switch self { + case .sessionNotIdle(let message): + return message + case .refused(let message, let serverMessage): + let server = serverMessage.trimmingCharacters(in: .whitespacesAndNewlines) + return server.isEmpty ? message : "\(message)\n\(server)" + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift index 980ad0e87e..c411752fdd 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift @@ -542,12 +542,8 @@ private actor SQLiteActor { default: if let str = Self.text(of: stmt, column: columnIndex) { let ref = makeCellRef(column: columns[i].name, columns: columns, statement: stmt, options: options) - cells.append(Cell.from( - legacyValue: str, - columnTypeName: columns[i].typeName, - options: options, - ref: ref - )) + // The storage class, not the declared type: a BLOB column can hold text. + cells.append(Cell.from(text: str, options: options, ref: ref)) } else { cells.append(.null) } diff --git a/apple/TableProMobile/TableProMobile/Drivers/SessionPlacementReporting.swift b/apple/TableProMobile/TableProMobile/Drivers/SessionPlacementReporting.swift new file mode 100644 index 0000000000..1737a8207e --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Drivers/SessionPlacementReporting.swift @@ -0,0 +1,24 @@ +import Foundation +import TableProDatabase + +// A statement the user runs can move the session to another database: a MySQL or SQL Server `USE`, a MongoDB `use`. +// The app's unqualified names then resolve there, so the picker follows what the session says. +nonisolated protocol SessionDatabaseReporting: DatabaseDriver { + // True when asking costs no statement on the session, which would replace what the user's last one left there. + var reportsDatabaseWithoutStatement: Bool { get } + func sessionDatabase() async throws -> String? +} + +// A statement the user runs, or a rollback that undoes a `SET`, can move the session's schema. +nonisolated protocol SessionSchemaReporting: DatabaseDriver { + // Reads the schema from the session into `currentSchema`; a session that cannot answer keeps what it had. + func settleCurrentSchema() async +} + +nonisolated extension PluginBackedDriver: SessionDatabaseReporting { + var reportsDatabaseWithoutStatement: Bool { true } + + func sessionDatabase() async throws -> String? { + try await settledCurrentDatabase() + } +} diff --git a/apple/TableProMobile/TableProMobile/Helpers/AppError+IAM.swift b/apple/TableProMobile/TableProMobile/Helpers/AppError+IAM.swift index 5a2a5a3cea..744f505dd1 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/AppError+IAM.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/AppError+IAM.swift @@ -84,6 +84,22 @@ nonisolated extension ErrorClassifier { recovery: String(localized: "Enter the RDS Endpoint in this connection's Authentication section."), opensEditor: true ) + case .endpointLookupFailed: + return failure( + error, + .network, + title: String(localized: "DNS Lookup Failed"), + recovery: String(localized: "Check the network connection and try again."), + opensEditor: false + ) + case .rdsEndpointNotDialed: + return failure( + error, + .config, + title: String(localized: "RDS Endpoint Does Not Match"), + recovery: String(localized: "Clear the RDS Endpoint in this connection's Authentication section."), + opensEditor: true + ) case .rdsEndpointInvalid: return failure( error, diff --git a/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift b/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift index 46638ebd8c..f92dfca54b 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift @@ -32,8 +32,20 @@ nonisolated enum ClipboardExporter { } nonisolated struct TooLarge: LocalizedError, Equatable { + enum Source: Equatable { + case page + case queryResult + } + + var source: Source = .page + var errorDescription: String? { - String(localized: "The full values on this page need more memory than this device has free. Choose a smaller page size and try again.") + switch source { + case .page: + String(localized: "The full values on this page need more memory than this device has free. Choose a smaller page size and try again.") + case .queryResult: + String(localized: "The full values in this result need more memory than this device has free. Run a query that returns fewer rows and try again.") + } } } @@ -171,21 +183,56 @@ nonisolated enum ClipboardExporter { for (i, col) in columns.enumerated() { let value = i < row.count ? row[i] : nil let key = " " + JSONText.stringLiteral(col.name) - if let value { - if isJsonNumber(value) { - pairs.append("\(key): \(value)") - } else if value == "true" || value == "false" { - pairs.append("\(key): \(value)") - } else { - pairs.append("\(key): \(JSONText.stringLiteral(value))") - } - } else { - pairs.append("\(key): null") - } + pairs.append("\(key): \(value.map { jsonValue($0, typeName: col.typeName) } ?? "null")") } return "{\n" + pairs.joined(separator: ",\n") + "\n}" } + // The column's type decides, as in the Mac export: text that reads like a number or `true` stays text. A column with + // no type, as SQLite reports for an expression, is read by its value. + private static func jsonValue(_ value: String, typeName: String) -> String { + if typeName.trimmingCharacters(in: .whitespaces).isEmpty { + let bare = isJsonNumber(value) || value == "true" || value == "false" + return bare ? value : JSONText.stringLiteral(value) + } + if isBooleanType(typeName), let flag = booleanLiteral(value) { + return flag + } + if isNumberType(typeName), isJsonNumber(value) { + return value + } + return JSONText.stringLiteral(value) + } + + // PostgreSQL's text format sends `t` or `f`, and SQLite keeps a boolean as 1 or 0. + private static func booleanLiteral(_ value: String) -> String? { + switch value.lowercased() { + case "true", "t", "1": "true" + case "false", "f", "0": "false" + default: nil + } + } + + // SQL Server documents `bit` as an integer type that holds 1, 0 or NULL. + private static func isNumberType(_ typeName: String) -> Bool { + ColumnInfo.isNumericType(typeName: typeName) || unwrappedTypeName(typeName) == "bit" + } + + private static func isBooleanType(_ typeName: String) -> Bool { + let base = unwrappedTypeName(typeName) + return base == "bool" || base == "boolean" + } + + // ClickHouse wraps the type, as in `Nullable(Bool)`. + private static func unwrappedTypeName(_ typeName: String) -> String { + var base = typeName.lowercased().trimmingCharacters(in: .whitespaces) + while base.hasSuffix(")"), let open = base.firstIndex(of: "("), + ["nullable", "lowcardinality"].contains(String(base[.. String { var lines: [String] = [] if includeHeader { diff --git a/apple/TableProMobile/TableProMobile/Helpers/IndexedRow.swift b/apple/TableProMobile/TableProMobile/Helpers/IndexedRow.swift index c90c8ed5e9..eb06023008 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/IndexedRow.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/IndexedRow.swift @@ -1,15 +1,6 @@ import Foundation -/// Identifiable wrapper used by iOS lists that need both the row payload and -/// its position index. Iterating over `[IndexedRow]` instead of -/// `rows.indices` keeps SwiftUI's `ForEach` diff stable when the underlying -/// row collection shrinks mid-render. This is the pattern that prevents the -/// `Array._checkSubscript` crashes seen in release 1.0 (build 11). struct IndexedRow: Identifiable { let id: Int let values: Element - - static func wrap(_ rows: [Element]) -> [IndexedRow] { - rows.enumerated().map { IndexedRow(id: $0.offset, values: $0.element) } - } } diff --git a/apple/TableProMobile/TableProMobile/Helpers/QueryHistoryStorage.swift b/apple/TableProMobile/TableProMobile/Helpers/QueryHistoryStorage.swift index 61b791ab28..6fab3334ef 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/QueryHistoryStorage.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/QueryHistoryStorage.swift @@ -1,4 +1,5 @@ import Foundation +import os nonisolated struct QueryHistoryItem: Identifiable, Codable, Hashable { let id: UUID @@ -35,10 +36,29 @@ nonisolated struct QueryHistoryItem: Identifiable, Codable, Hashable { } } -nonisolated struct QueryHistoryStorage { +// One per library. The file is read once; every later read comes from the copy in memory, which takes a change only +// once the file holds it. The lock orders a save running off the main actor with a delete or clear on it. +nonisolated final class QueryHistoryStorage: Sendable { private static let maxEntries = 200 + // The whole list is encoded on every save, so its text is held to this many bytes; the newest entry always stays, + // and no entry is ever cut, since running part of a script from history would run part of it. + static let maxTextBytes = 2 * 1_024 * 1_024 + private static let logger = Logger(subsystem: "com.TablePro", category: "QueryHistoryStorage") private let directory: URL + private let state = OSAllocatedUnfairLock(initialState: State()) + + private nonisolated struct State { + // Nil until the file has been read. + var items: [QueryHistoryItem]? + // Counts the writes, so a list read before a later write is never shown over it. + var generation = 0 + } + + nonisolated struct Listed: Sendable { + let items: [QueryHistoryItem] + let generation: Int + } init(directory: URL) { self.directory = directory @@ -50,43 +70,142 @@ nonisolated struct QueryHistoryStorage { @discardableResult func save(_ item: QueryHistoryItem) -> Bool { - var items = loadAll() - if let last = items.last, - last.query == item.query, - last.connectionId == item.connectionId, - last.wasSuccessful == item.wasSuccessful { - return false + record(item) != nil + } + + // The connection's list once the item is saved, read from memory; nil when nothing was saved. + func record(_ item: QueryHistoryItem) -> Listed? { + state.withLock { state -> Listed? in + guard var items = cached(&state) else { return nil } + if let last = items.last, + last.query == item.query, + last.connectionId == item.connectionId, + last.wasSuccessful == item.wasSuccessful { + return nil + } + items.append(item) + items = Self.withinBudget(items) + guard commit(items, to: &state) else { return nil } + return Listed(items: items.filter { $0.connectionId == item.connectionId }, generation: state.generation) } - items.append(item) - if items.count > Self.maxEntries { - items.removeFirst(items.count - Self.maxEntries) + } + + func listed(for connectionId: UUID) -> Listed { + state.withLock { state in + let items = cached(&state) ?? [] + return Listed(items: items.filter { $0.connectionId == connectionId }, generation: state.generation) } - return writeAll(items) } func loadAll() -> [QueryHistoryItem] { - guard let data = try? Data(contentsOf: fileURL), - let items = try? JSONDecoder().decode([QueryHistoryItem].self, from: data) else { - return [] - } - return items + state.withLock { cached(&$0) } ?? [] } func load(for connectionId: UUID) -> [QueryHistoryItem] { loadAll().filter { $0.connectionId == connectionId } } + var generation: Int { + state.withLock { $0.generation } + } + func delete(_ id: UUID) { - var items = loadAll() - items.removeAll { $0.id == id } - writeAll(items) + state.withLock { state in + guard var items = cached(&state) else { return } + items.removeAll { $0.id == id } + commit(items, to: &state) + } + } + + // The set-aside copy holds every connection's history: only a clear asked for after the footer named it removes it. + func clearAll(for connectionIds: Set, removingSetAsideCopies: Bool = false) { + state.withLock { state in + guard let items = cached(&state) else { return } + let kept = items.filter { !connectionIds.contains($0.connectionId) } + if kept.count != items.count { + guard commit(kept, to: &state) else { return } + } + if removingSetAsideCopies { + removeSetAsideCopies() + } + } + } + + static func withinBudget(_ items: [QueryHistoryItem]) -> [QueryHistoryItem] { + var kept: [QueryHistoryItem] = [] + var bytes = 0 + for item in items.reversed() { + let size = item.query.utf8.count + (item.errorMessage?.utf8.count ?? 0) + guard kept.isEmpty || (kept.count < maxEntries && bytes + size <= maxTextBytes) else { break } + kept.append(item) + bytes += size + } + return kept.reversed() } - func clearAll(for connectionIds: Set) { - let items = loadAll() - let kept = items.filter { !connectionIds.contains($0.connectionId) } - guard kept.count != items.count else { return } - writeAll(kept) + // nil while the file cannot be read (a locked device), so no write replaces it with a partial list. + private func cached(_ state: inout State) -> [QueryHistoryItem]? { + if state.items == nil { state.items = readAll() } + return state.items + } + + @discardableResult + private func commit(_ items: [QueryHistoryItem], to state: inout State) -> Bool { + guard writeAll(items) else { return false } + state.items = items + state.generation += 1 + return true + } + + var hasSetAsideCopies: Bool { + !setAsideCopies().isEmpty + } + + private func setAsideCopies() -> [URL] { + let names = (try? FileManager.default.contentsOfDirectory(atPath: directory.path(percentEncoded: false))) ?? [] + return names + .filter { $0.hasPrefix("query-history.damaged-") } + .map { directory.appendingPathComponent($0) } + } + + private func removeSetAsideCopies() { + for copy in setAsideCopies() { + do { + try FileManager.default.removeItem(at: copy) + } catch { + Self.logger.error("A set-aside query history copy could not be deleted: \(error.localizedDescription, privacy: .private)") + } + } + } + + private func readAll() -> [QueryHistoryItem]? { + guard FileManager.default.fileExists(atPath: fileURL.path(percentEncoded: false)) else { return [] } + guard let data = try? Data(contentsOf: fileURL) else { return nil } + if let items = try? JSONDecoder().decode([QueryHistoryItem].self, from: data) { + return items + } + return setAsideUndecodableFile() ? [] : nil + } + + private func setAsideUndecodableFile() -> Bool { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.dateFormat = "yyyyMMdd-HHmmss" + let stamp = formatter.string(from: Date()) + var attempt = 0 + var destination = directory.appendingPathComponent("query-history.damaged-\(stamp).json") + while FileManager.default.fileExists(atPath: destination.path(percentEncoded: false)) { + attempt += 1 + destination = directory.appendingPathComponent("query-history.damaged-\(stamp)-\(attempt).json") + } + do { + try FileManager.default.moveItem(at: fileURL, to: destination) + Self.logger.fault("Query history did not decode and was set aside") + return true + } catch { + Self.logger.error("Query history did not decode and could not be set aside: \(error.localizedDescription, privacy: .private)") + return false + } } @discardableResult diff --git a/apple/TableProMobile/TableProMobile/Helpers/ReadOnlyFunctionGuard.swift b/apple/TableProMobile/TableProMobile/Helpers/ReadOnlyFunctionGuard.swift new file mode 100644 index 0000000000..5fa3102e03 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Helpers/ReadOnlyFunctionGuard.swift @@ -0,0 +1,60 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProSQLGrammar +import TableProSQLVocabulary + +// In the engine's read-only mode the server refuses what a function writes inside the transaction. Without it, at +// Read-Only a call off the built-in list, on an engine whose functions can write, cannot be told from a write and is +// refused like one. +nonisolated enum ReadOnlyFunctionGuard { + static func refuses( + _ sql: String, + databaseType: DatabaseType, + level: SafeModeLevel, + enforcesReadOnlyMode: Bool + ) -> Bool { + let typeId = databaseType.rawValue + guard level.blocksWrites, !enforcesReadOnlyMode, + SQLReadOnlyFunctionCalls.queryFunctionsCanWrite.contains(typeId) else { return false } + return SQLReadOnlyFunctionCalls.callsUnlistedFunction( + sql, + databaseTypeId: typeId, + readings: SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil), + typeNames: SQLEngineVocabulary.curated(forDatabaseTypeId: typeId)?.dataTypes ?? [] + ) + } + + // A write never reaches the driver at Read-Only, so only a read asks the session. One holding a transaction or a + // table lock runs as it is, and MySQL with autocommit off always holds a transaction. + static func runsInReadOnlyMode(writes: Bool, level: SafeModeLevel, driver: any DatabaseDriver) async -> Bool { + guard !writes, level.blocksWrites else { return false } + return await driver.entersReadOnlyMode() + } + + static let callableCatalog = SQLCallableCatalogCache() + + // Oracle calls a function named without parentheses once the name is not a column in scope, so the session's own + // catalog says which names call. A catalog that cannot be read leaves the name unexplained, and it counts as a call. + static func refusesBareName( + _ sql: String, + databaseType: DatabaseType, + level: SafeModeLevel, + connectionId: UUID, + schema: String?, + read: @escaping SQLCallableCatalogCache.Reader + ) async -> Bool { + let typeId = databaseType.rawValue + guard level.blocksWrites, SQLBareNameCalls.engines.contains(typeId) else { return false } + let reading = SQLBareNameCalls.read( + sql, + databaseTypeId: typeId, + readings: SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) + ) + guard !reading.namesNothing else { return false } + let calls = try? await callableCatalog.callsFunction( + reading, connectionId: connectionId, schema: schema, read: read + ) + return calls ?? true + } +} diff --git a/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift b/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift index 51e30f4422..2ec25c6f66 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift @@ -7,6 +7,7 @@ nonisolated enum RowInsertPlanner { static func statements( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, columns: [ColumnInfo], @@ -21,6 +22,7 @@ nonisolated enum RowInsertPlanner { SQLBuilder.buildInsert( table: table, schema: schema, + database: database, type: type, driver: driver, columns: values.map(\.column), @@ -32,6 +34,7 @@ nonisolated enum RowInsertPlanner { static func batch( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, columns: [ColumnInfo], @@ -41,7 +44,7 @@ nonisolated enum RowInsertPlanner { ) throws -> RowWriteBatch? { guard let writer = driver as? any DriverRowWriting else { let sql = try statements( - table: table, schema: schema, type: type, driver: driver, columns: columns, rows: rows, + table: table, schema: schema, database: database, type: type, driver: driver, columns: columns, rows: rows, allowAllDefaults: allowAllDefaults, dropsEmptyPrimaryKey: dropsEmptyPrimaryKey ) return sql.isEmpty ? nil : .sql(sql, driver) @@ -80,13 +83,15 @@ nonisolated enum RowInserter { table: String, type: DatabaseType, schema: String?, + database: String? = nil, qualifier: String?, rows: [PayloadRow] ) async throws -> Int { - let columns = try await driver.fetchColumns(table: table, schema: schema) + let columns = try await driver.fetchColumns(table: table, schema: schema, database: database) guard let batch = try RowInsertPlanner.batch( table: table, schema: qualifier, + database: database, type: type, driver: driver, columns: columns, diff --git a/apple/TableProMobile/TableProMobile/Helpers/RowWriteBatch.swift b/apple/TableProMobile/TableProMobile/Helpers/RowWriteBatch.swift index 18174895e4..493980c306 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/RowWriteBatch.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/RowWriteBatch.swift @@ -32,6 +32,7 @@ nonisolated enum RowWriteBatch: Sendable { static func update( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, columns: [String], @@ -49,8 +50,8 @@ nonisolated enum RowWriteBatch: Sendable { } guard let keys = locator.keyValues else { return nil } let sql = SQLBuilder.buildUpdate( - table: table, schema: schema, type: type, driver: driver, changes: changes, primaryKeys: keys, - columns: columnDetails + table: table, schema: schema, database: database, type: type, driver: driver, changes: changes, + primaryKeys: keys, columns: columnDetails ) return .sql([sql], driver) } @@ -58,6 +59,7 @@ nonisolated enum RowWriteBatch: Sendable { static func delete( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, columns: [String], @@ -73,7 +75,8 @@ nonisolated enum RowWriteBatch: Sendable { } guard let keys = locator.keyValues, !keys.isEmpty else { return nil } let sql = SQLBuilder.buildDelete( - table: table, schema: schema, type: type, driver: driver, primaryKeys: keys, columns: columnDetails + table: table, schema: schema, database: database, type: type, driver: driver, primaryKeys: keys, + columns: columnDetails ) return .sql([sql], driver) } @@ -95,6 +98,7 @@ nonisolated struct PlannedRowWrite: Sendable { static func update( table: String, schema: String?, + database: String? = nil, type: DatabaseType, columns: [String], columnDetails: [ColumnInfo], @@ -103,7 +107,7 @@ nonisolated struct PlannedRowWrite: Sendable { ) -> PlannedRowWrite { PlannedRowWrite { driver in try RowWriteBatch.update( - table: table, schema: schema, type: type, driver: driver, columns: columns, + table: table, schema: schema, database: database, type: type, driver: driver, columns: columns, columnDetails: columnDetails, locator: locator, changes: changes ) } @@ -112,13 +116,14 @@ nonisolated struct PlannedRowWrite: Sendable { static func insert( table: String, schema: String?, + database: String? = nil, type: DatabaseType, columns: [ColumnInfo], values: [String: PayloadValue] ) -> PlannedRowWrite { PlannedRowWrite { driver in try RowInsertPlanner.batch( - table: table, schema: schema, type: type, driver: driver, columns: columns, + table: table, schema: schema, database: database, type: type, driver: driver, columns: columns, rows: [PayloadRow(values: values)], allowAllDefaults: true, dropsEmptyPrimaryKey: false ) } diff --git a/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift b/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift index e9f7dca291..4fbe342211 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift @@ -46,13 +46,15 @@ nonisolated enum SQLBuilder { } } - static func buildCount(table: String, schema: String?, type: DatabaseType) -> String { - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + static func buildCount(table: String, schema: String?, database: String? = nil, type: DatabaseType) -> String { + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) return "SELECT COUNT(*) FROM \(quoted)" } - static func buildSelect(table: String, schema: String?, type: DatabaseType, limit: Int, offset: Int) -> String { - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + static func buildSelect( + table: String, schema: String?, database: String? = nil, type: DatabaseType, limit: Int, offset: Int + ) -> String { + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let pagination = paginationClause(orderBy: "", limit: limit, offset: offset, for: type) return "SELECT * FROM \(quoted) \(pagination)" } @@ -62,12 +64,13 @@ nonisolated enum SQLBuilder { static func buildSingleRowLookup( table: String, schema: String?, + database: String? = nil, key: [(column: String, value: String)], columns: [ColumnInfo], type: DatabaseType, driver: any DatabaseDriver ) -> String { - let quotedTable = qualifiedIdentifier(table: table, schema: schema, for: type) + let quotedTable = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let predicate = key.isEmpty ? "1 = 0" : keyPredicate(key, columns: columns, type: type, driver: driver, exactText: false) let pagination = paginationClause(orderBy: "", limit: 1, offset: 0, for: type) @@ -135,24 +138,26 @@ nonisolated enum SQLBuilder { static func buildKeyMatchCount( table: String, schema: String?, + database: String? = nil, key: [(column: String, value: String)], columns: [ColumnInfo], type: DatabaseType, driver: any DatabaseDriver ) -> String { - let quotedTable = qualifiedIdentifier(table: table, schema: schema, for: type) + let quotedTable = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) return "SELECT COUNT(*) FROM \(quotedTable) WHERE \(keyPredicate(key, columns: columns, type: type, driver: driver))" } static func buildDelete( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, primaryKeys: [(column: String, value: String)], columns: [ColumnInfo] ) -> String { - let quotedTable = qualifiedIdentifier(table: table, schema: schema, for: type) + let quotedTable = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let predicate = keyPredicate(primaryKeys, columns: columns, type: type, driver: driver, uniqueIn: quotedTable) return "DELETE FROM \(quotedTable) WHERE \(predicate)" } @@ -160,13 +165,14 @@ nonisolated enum SQLBuilder { static func buildUpdate( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, changes: [(column: String, value: String?)], primaryKeys: [(column: String, value: String)], columns: [ColumnInfo] ) -> String { - let quotedTable = qualifiedIdentifier(table: table, schema: schema, for: type) + let quotedTable = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let assignments = changes.map { col, val in let qcol = quoteIdentifier(col, for: type) if let val { return "\(qcol) = \(stringLiteral(val, for: type, driver: driver))" } @@ -179,12 +185,13 @@ nonisolated enum SQLBuilder { static func buildInsert( table: String, schema: String?, + database: String? = nil, type: DatabaseType, driver: any DatabaseDriver, columns: [String], values: [String?] ) -> String? { - let qualifiedTable = qualifiedIdentifier(table: table, schema: schema, for: type) + let qualifiedTable = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) guard !columns.isEmpty else { return buildAllDefaultsInsert(qualifiedTable: qualifiedTable, for: type) } @@ -206,8 +213,8 @@ nonisolated enum SQLBuilder { } // SQLite has no TRUNCATE statement; a DELETE with no WHERE is its truncate. - static func truncateStatement(table: String, schema: String?, type: DatabaseType) -> String { - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + static func truncateStatement(table: String, schema: String?, database: String? = nil, type: DatabaseType) -> String { + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) return type.isSQLiteFamily ? "DELETE FROM \(quoted)" : "TRUNCATE TABLE \(quoted)" } @@ -215,25 +222,33 @@ nonisolated enum SQLBuilder { buildAllDefaultsInsert(qualifiedTable: "t", for: type) != nil } - static func qualifiedIdentifier(table: String, schema: String?, for type: DatabaseType) -> String { + // SQL Server resolves `[schema].[table]` in the session's database, which a `USE` run in the Query tab moves, so + // it names the database too. MySQL's database already sits in `schema`; other engines have no third part. + static func qualifiedIdentifier( + table: String, schema: String?, database: String? = nil, for type: DatabaseType + ) -> String { let quotedTable = quoteIdentifier(table, for: type) - guard let schema, !schema.isEmpty else { return quotedTable } - return "\(quoteIdentifier(schema, for: type)).\(quotedTable)" + let quotedSchema = schema.flatMap { $0.isEmpty ? nil : quoteIdentifier($0, for: type) } + if type == .mssql, let database, !database.isEmpty { + return "\(quoteIdentifier(database, for: type)).\(quotedSchema ?? "").\(quotedTable)" + } + guard let quotedSchema else { return quotedTable } + return "\(quotedSchema).\(quotedTable)" } static func buildSelect( - table: String, schema: String?, type: DatabaseType, + table: String, schema: String?, database: String? = nil, type: DatabaseType, sortState: SortState, limit: Int, offset: Int ) -> String { - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let orderBy = buildOrderByClause(sortState, for: type) let pagination = paginationClause(orderBy: orderBy, limit: limit, offset: offset, for: type) return "SELECT * FROM \(quoted) \(pagination)" } static func buildFilteredSelect( - table: String, schema: String?, type: DatabaseType, + table: String, schema: String?, database: String? = nil, type: DatabaseType, filters: [TableFilter], logicMode: FilterLogicMode, columns: [ColumnInfo] = [], session: SQLSessionLexicalFacts? = nil, @@ -242,7 +257,7 @@ nonisolated enum SQLBuilder { let dialect = dialectDescriptor(for: type) let generator = filterGenerator(dialect: dialect, type: type, session: session) let whereClause = generator.generateWhereClause(from: filters, logicMode: logicMode, columns: columns) - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let pagination = paginationClause(orderBy: "", limit: limit, offset: offset, for: type) var sql = "SELECT * FROM \(quoted)" if !whereClause.isEmpty { sql += " \(whereClause)" } @@ -251,7 +266,7 @@ nonisolated enum SQLBuilder { } static func buildFilteredSelect( - table: String, schema: String?, type: DatabaseType, + table: String, schema: String?, database: String? = nil, type: DatabaseType, filters: [TableFilter], logicMode: FilterLogicMode, columns: [ColumnInfo] = [], session: SQLSessionLexicalFacts? = nil, @@ -262,7 +277,7 @@ nonisolated enum SQLBuilder { let generator = filterGenerator(dialect: dialect, type: type, session: session) let whereClause = generator.generateWhereClause(from: filters, logicMode: logicMode, columns: columns) let orderBy = buildOrderByClause(sortState, for: type) - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let pagination = paginationClause(orderBy: orderBy, limit: limit, offset: offset, for: type) var sql = "SELECT * FROM \(quoted)" if !whereClause.isEmpty { sql += " \(whereClause)" } @@ -271,7 +286,7 @@ nonisolated enum SQLBuilder { } static func buildFilteredCount( - table: String, schema: String?, type: DatabaseType, + table: String, schema: String?, database: String? = nil, type: DatabaseType, filters: [TableFilter], logicMode: FilterLogicMode, columns: [ColumnInfo] = [], session: SQLSessionLexicalFacts? = nil @@ -279,7 +294,7 @@ nonisolated enum SQLBuilder { let dialect = dialectDescriptor(for: type) let generator = filterGenerator(dialect: dialect, type: type, session: session) let whereClause = generator.generateWhereClause(from: filters, logicMode: logicMode, columns: columns) - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) if whereClause.isEmpty { return "SELECT COUNT(*) FROM \(quoted)" } @@ -289,7 +304,7 @@ nonisolated enum SQLBuilder { // MARK: - Search static func buildSearchSelect( - table: String, schema: String?, type: DatabaseType, + table: String, schema: String?, database: String? = nil, type: DatabaseType, searchText: String, searchColumns: [ColumnInfo], filters: [TableFilter] = [], logicMode: FilterLogicMode = .and, columns: [ColumnInfo] = [], @@ -297,7 +312,7 @@ nonisolated enum SQLBuilder { sortState: SortState = SortState(), limit: Int, offset: Int ) -> String { - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let whereClause = buildSearchWhereClause( searchText: searchText, searchColumns: searchColumns, filters: filters, logicMode: logicMode, columns: columns, type: type, session: session @@ -311,13 +326,13 @@ nonisolated enum SQLBuilder { } static func buildSearchCount( - table: String, schema: String?, type: DatabaseType, + table: String, schema: String?, database: String? = nil, type: DatabaseType, searchText: String, searchColumns: [ColumnInfo], filters: [TableFilter] = [], logicMode: FilterLogicMode = .and, columns: [ColumnInfo] = [], session: SQLSessionLexicalFacts? = nil ) -> String { - let quoted = qualifiedIdentifier(table: table, schema: schema, for: type) + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) let whereClause = buildSearchWhereClause( searchText: searchText, searchColumns: searchColumns, filters: filters, logicMode: logicMode, columns: columns, type: type, session: session diff --git a/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift b/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift index 5f5c1d92c6..fb746b4caa 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift @@ -11,6 +11,9 @@ nonisolated enum SessionLocalStatements { // A `USE`, a role or a search path moved the session off what the catalog connection sees, or a procedure or // dynamic SQL may have made a table only the session sees. var leavesCatalog = false + // A `USE` ran, the one statement that moves a MySQL or SQL Server session to another database: a procedure or + // dynamic SQL cannot, and SQL Server puts the database back when its `EXEC` or procedure ends. + var usesDatabase = false } private static let identifier = #"(`(?:[^`]|``)+`|"(?:[^"]|"")+"|\[[^\]]+\]|[\w$#]+)"# @@ -66,8 +69,10 @@ nonisolated enum SessionLocalStatements { effect.temporaryTables.insert(name) } let range = NSRange(location: 0, length: (code as NSString).length) + let uses = usePattern?.firstMatch(in: code, range: range) != nil + if uses { effect.usesDatabase = true } // A MySQL executable comment can hide any statement, a temporary table among them, from the patterns. - if usePattern?.firstMatch(in: code, range: range) != nil + if uses || contextPattern?.firstMatch(in: code, range: range) != nil || SQLLeadingComments.conditionalCommentOpeners.contains(where: { (code as NSString).range(of: $0, options: .literal).location != NSNotFound diff --git a/apple/TableProMobile/TableProMobile/Info.plist b/apple/TableProMobile/TableProMobile/Info.plist index 8428493002..88d57f4c64 100644 --- a/apple/TableProMobile/TableProMobile/Info.plist +++ b/apple/TableProMobile/TableProMobile/Info.plist @@ -57,6 +57,7 @@ UIBackgroundModes fetch + remote-notification BGTaskSchedulerPermittedIdentifiers diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index d76dcd4161..d4c8c46296 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -1784,6 +1784,9 @@ }, "A cursor has no .%@() modifier." : { + }, + "A damaged history file from every connection was kept on this device. Clear All History deletes it." : { + }, "A database name must be shorter than %ld bytes." : { @@ -1924,6 +1927,9 @@ }, "A number in %@ is larger than a 64-bit integer. Write it as a decimal, such as {\"$numberDecimal\": \"12345678901234567890\"}." : { + }, + "A query that switches the database is running on this connection. Try again once it ends." : { + }, "A renamed column's foreign key is rewritten, so its MATCH and DEFERRABLE clauses are not carried over." : { "extractionState" : "manual", @@ -4828,6 +4834,9 @@ }, "Check the host name, or set SSL Mode to Verify CA to skip the name check." : { + }, + "Check the network connection and try again." : { + }, "Check the server's TLS settings and this connection's SSL Mode." : { @@ -5426,6 +5435,9 @@ } } } + }, + "Clear the RDS Endpoint in this connection's Authentication section." : { + }, "ClickHouse ended the result partway through a row, so the rows that arrived are incomplete." : { @@ -6618,6 +6630,10 @@ } } }, + "COPY FROM STDIN takes its rows from the lines after the line its ; ends, so no rows were sent." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Copy Key" : { }, @@ -7208,6 +7224,10 @@ } } }, + "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Could not run the credential_process command for profile \"%@\": %@" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -7484,6 +7504,9 @@ "Crown" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "DNS Lookup Failed" : { + }, "Data & Analytics" : { "extractionState" : "manual", @@ -7722,6 +7745,9 @@ } } } + }, + "Database Switch Running" : { + }, "Database Type" : { "localizations" : { @@ -10884,6 +10910,9 @@ } } } + }, + "Go to First Page" : { + }, "Go to Page" : { "localizations" : { @@ -12783,6 +12812,9 @@ }, "Its uncommitted changes are discarded." : { + }, + "Join Condition" : { + }, "JSON" : { "extractionState" : "manual", @@ -15404,6 +15436,9 @@ } } } + }, + "No Rows on This Page" : { + }, "No Tables" : { "localizations" : { @@ -18178,6 +18213,9 @@ } } } + }, + "Query Blocked" : { + }, "Query Error" : { "localizations" : { @@ -18459,6 +18497,9 @@ }, "RDS Endpoint (optional)" : { + }, + "RDS Endpoint Does Not Match" : { + }, "RDS Endpoint Invalid" : { @@ -19961,6 +20002,12 @@ } } } + }, + "Safe Mode is set to Read-Only for this connection, and it has a transaction open. Roll it back, then try again." : { + + }, + "Safe Mode is set to Read-Only for this connection, and the database refused the statement." : { + }, "Same Key as SSH Server" : { @@ -20713,6 +20760,9 @@ "Services" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Session in Another Database" : { + }, "Session Token" : { "extractionState" : "manual", @@ -21134,6 +21184,9 @@ } } } + }, + "Shown in part. Run a query that returns fewer rows or columns to see it in full." : { + }, "SID" : { "localizations" : { @@ -21441,6 +21494,9 @@ }, "Some documents may already have been changed when this failed. MongoDB does not undo them, so check the data before you run it again." : { + }, + "Some long values are shown in part to stay within memory." : { + }, "Sort" : { "localizations" : { @@ -22912,6 +22968,9 @@ }, "This pipeline is not something MongoDB can read: %@" : { + }, + "This query calls a function that can change data, which Read-Only does not run. Switch Safe Mode to %@ to run it." : { + }, "This row has no _id, so TablePro cannot tell which document to change." : { @@ -23245,6 +23304,10 @@ }, "TablePro closed %@ when it went to the background, which rolled back its open transaction. It has been reopened; run the statements again, and commit before you leave the app." : { + }, + "TablePro could not look up \"%@\" in DNS to find the AWS endpoint it points to. Check the network and try again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "TablePro could not read that file." : { "localizations" : { @@ -23464,6 +23527,9 @@ "TablePro on iPhone and iPad can't connect to %1$@ with the SSL mode %2$@." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "TablePro is reading or writing rows on this connection with settings of its own." : { + }, "TablePro on iPhone and iPad can't open %@ connections." : { "extractionState" : "manual", @@ -24422,6 +24488,9 @@ } } } + }, + "The connection moved to another database or schema. Choose the row again." : { + }, "The connection moved to another database or schema. Choose the table again." : { @@ -24919,6 +24988,9 @@ "The FreeTDS configuration could not be written: %@" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The full values in this result need more memory than this device has free. Run a query that returns fewer rows and try again." : { + }, "The full values on this page need more memory than this device has free. Choose a smaller page size and try again." : { @@ -25706,6 +25778,26 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The query moved the session to “%@” and it could not go back to “%@”. To save the unsaved changes, run USE %@ in the Query tab of the window that holds them." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The query moved the session to “%1$@” and it could not go back to “%2$@”. To save the unsaved changes, run USE %3$@ in the Query tab of the window that holds them." + } + } + } + }, + "The query moved the session to “%@” and it could not go back to “%@”. Unsaved changes still save to “%@”." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The query moved the session to “%1$@” and it could not go back to “%2$@”. Unsaved changes still save to “%3$@”." + } + } + } + }, "The query names its own output format, %@. Remove its FORMAT clause to read it as rows." : { }, @@ -25752,6 +25844,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The referential action of foreign key %@ is not one SQL Server has: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The report contains:" : { "localizations" : { "ko" : { @@ -26298,6 +26394,16 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The session is in “%@”, not “%@”, so this change cannot be saved now." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The session is in “%1$@”, not “%2$@”, so this change cannot be saved now." + } + } + } + }, "The shell drops a field named __proto__ from a new document. Remove the field, then set it on the saved document." : { }, @@ -26316,6 +26422,10 @@ }, "The socket path must start with /." : { + }, + "The RDS Endpoint \"%1$@\" is not the database at \"%2$@\", where the connection goes. Clear the RDS Endpoint unless you connect through a port forward." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The SQLite extensions for this connection could not be loaded." : { "extractionState" : "manual", @@ -26420,6 +26530,10 @@ }, "The statement before it stays applied." : { + }, + "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The table \"%@\" and all its data will be permanently deleted." : { "localizations" : { @@ -26742,6 +26856,13 @@ } } }, + "This connection called a stored routine, so TablePro cannot tell what the session is holding." : { + + }, + "This connection cannot load the rows of COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This connection goes through an SSH jump host, which TablePro on iPhone and iPad cannot dial. Open it in TablePro on the Mac." : { "localizations" : { "ko" : { @@ -26825,6 +26946,33 @@ } } } + }, + "This connection has a transaction open. Commit or roll it back in TablePro, then try again." : { + + }, + "This connection has an open transaction. Commit or roll it back first." : { + + }, + "This connection has open HANDLER cursors, which reconnecting would close." : { + + }, + "This connection has prepared statements, which reconnecting would discard." : { + + }, + "This connection has session settings changed, which reconnecting would reset." : { + + }, + "This connection has session variables set, which reconnecting would clear." : { + + }, + "This connection has temporary tables, which reconnecting would delete." : { + + }, + "This connection holds advisory locks, which reconnecting would release." : { + + }, + "This connection holds table locks, which reconnecting would release." : { + }, "This connection is in read-only mode. Write queries are not allowed." : { "localizations" : { @@ -26853,6 +27001,9 @@ } } } + }, + "This connection is read only for external clients, and the database refused the statement." : { + }, "This connection no longer exists. It may have been removed from another device." : { "localizations" : { @@ -26903,6 +27054,9 @@ }, "This connection signs in with the gcloud credentials on the Mac." : { + }, + "This connection switched database with USE, which reconnecting would undo." : { + }, "This connection uses a socket on a Mac and cannot be opened on iPhone or iPad." : { "extractionState" : "manual", @@ -27154,6 +27308,10 @@ } } }, + "This database has no column defaults, so a cell cannot be left to one. Enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This database has no Create Table form" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -27226,6 +27384,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This driver cannot send rows to COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This file also holds saved queries, which import only on a Mac." : { "localizations" : { "ko" : { @@ -27540,6 +27702,9 @@ "This Oracle server requires Native Network Encryption but did not offer it on this connection. Ask the DBA to enable it, or use TLS instead by setting an SSL mode." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "This page is past the last row." : { + }, "This query will modify data. Are you sure you want to continue?" : { "localizations" : { @@ -27889,6 +28054,9 @@ } } } + }, + "This window has unsaved changes on this connection, so the session stayed on its database. Save or discard them before switching database." : { + }, "Ticket" : { "extractionState" : "manual", @@ -29424,6 +29592,9 @@ }, "View as JSON" : { + }, + "View Full Value" : { + }, "Views" : { "localizations" : { diff --git a/apple/TableProMobile/TableProMobile/Models/ForeignKeyReference.swift b/apple/TableProMobile/TableProMobile/Models/ForeignKeyReference.swift index b65c0acfb5..ce9e1ce7a9 100644 --- a/apple/TableProMobile/TableProMobile/Models/ForeignKeyReference.swift +++ b/apple/TableProMobile/TableProMobile/Models/ForeignKeyReference.swift @@ -9,20 +9,32 @@ nonisolated struct ForeignKeyReference: Hashable, Identifiable, Sendable { let table: String let schema: String? + // Where the key was read. A SQL Server foreign key never crosses databases. + let database: String? let parts: [Part] var id: Self { self } + init(table: String, schema: String?, database: String? = nil, parts: [Part]) { + self.table = table + self.schema = schema + self.database = database + self.parts = parts + } + static func make( forColumn column: String, foreignKeys: [ForeignKeyInfo], columns: [ColumnInfo], cells: [Cell], - sourceSchema: String? + sourceSchema: String?, + sourceDatabase: String? = nil ) -> Self? { guard let target = columnIndex(named: column, in: columns) else { return nil } for key in keys(in: foreignKeys) where key.contains(where: { columnIndex(named: $0.column, in: columns) == target }) { - if let reference = resolve(key, columns: columns, cells: cells, sourceSchema: sourceSchema) { + if let reference = resolve( + key, columns: columns, cells: cells, sourceSchema: sourceSchema, sourceDatabase: sourceDatabase + ) { return reference } } @@ -33,11 +45,14 @@ nonisolated struct ForeignKeyReference: Hashable, Identifiable, Sendable { foreignKeys: [ForeignKeyInfo], columns: [ColumnInfo], cells: [Cell], - sourceSchema: String? + sourceSchema: String?, + sourceDatabase: String? = nil ) -> [(name: String, column: String, reference: Self)] { keys(in: foreignKeys).compactMap { key in guard let first = key.first, - let reference = resolve(key, columns: columns, cells: cells, sourceSchema: sourceSchema) + let reference = resolve( + key, columns: columns, cells: cells, sourceSchema: sourceSchema, sourceDatabase: sourceDatabase + ) else { return nil } return (name: first.name, column: key.map(\.column).joined(separator: ", "), reference: reference) } @@ -85,7 +100,8 @@ nonisolated struct ForeignKeyReference: Hashable, Identifiable, Sendable { _ key: [ForeignKeyInfo], columns: [ColumnInfo], cells: [Cell], - sourceSchema: String? + sourceSchema: String?, + sourceDatabase: String? ) -> Self? { guard let first = key.first else { return nil } var parts: [Part] = [] @@ -95,7 +111,10 @@ nonisolated struct ForeignKeyReference: Hashable, Identifiable, Sendable { else { return nil } parts.append(Part(column: pair.referencedColumn, value: value)) } - return Self(table: first.referencedTable, schema: first.referencedSchema ?? sourceSchema, parts: parts) + return Self( + table: first.referencedTable, schema: first.referencedSchema ?? sourceSchema, database: sourceDatabase, + parts: parts + ) } private static func columnIndex(named name: String, in columns: [ColumnInfo]) -> Int? { diff --git a/apple/TableProMobile/TableProMobile/Models/RowWindow.swift b/apple/TableProMobile/TableProMobile/Models/RowWindow.swift index 5180c3c4bc..715d89a375 100644 --- a/apple/TableProMobile/TableProMobile/Models/RowWindow.swift +++ b/apple/TableProMobile/TableProMobile/Models/RowWindow.swift @@ -5,8 +5,16 @@ struct RowWindow: Sendable { private(set) var rows: [Row] private(set) var firstAbsoluteIndex: Int private(set) var totalAppended: Int + private(set) var clears = 0 let capacity: Int + // Changes whenever the first rows may have, so a view can tell without reading them. + struct HeadIdentity: Equatable, Sendable { + let clears: Int + let firstAbsoluteIndex: Int + let count: Int + } + init(capacity: Int = 200) { self.rows = [] self.firstAbsoluteIndex = 0 @@ -37,6 +45,11 @@ struct RowWindow: Sendable { rows = [] firstAbsoluteIndex = 0 totalAppended = 0 + clears += 1 + } + + func headIdentity(count: Int) -> HeadIdentity { + HeadIdentity(clears: clears, firstAbsoluteIndex: firstAbsoluteIndex, count: min(rows.count, count)) } var isEmpty: Bool { diff --git a/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift b/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift index 8a262c15cc..534cc6dab5 100644 --- a/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift +++ b/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift @@ -5,8 +5,8 @@ import TableProModels @Observable final class StreamingResultBuffer { private(set) var columns: [ColumnInfo] = [] + // The one copy of the rows: a list derives each row's display text when it draws it. private(set) var window: RowWindow - private(set) var legacyRows: [[String?]] = [] private(set) var rowsAffected: Int? private(set) var statusMessage: String? private(set) var truncation: TruncationReason? @@ -21,8 +21,8 @@ final class StreamingResultBuffer { self.window = RowWindow(capacity: capacity) } - var count: Int { legacyRows.count } - var isEmpty: Bool { legacyRows.isEmpty } + var count: Int { window.count } + var isEmpty: Bool { window.isEmpty } func apply(_ element: StreamElement) { switch element { @@ -51,12 +51,7 @@ final class StreamingResultBuffer { flushTask?.cancel() flushTask = nil guard !pendingRows.isEmpty else { return } - let legacyBatch = pendingRows.map(\.legacyValues) window.append(contentsOf: pendingRows) - legacyRows.append(contentsOf: legacyBatch) - if legacyRows.count > window.count { - legacyRows.removeFirst(legacyRows.count - window.count) - } pendingRows.removeAll(keepingCapacity: true) } @@ -67,8 +62,6 @@ final class StreamingResultBuffer { func shrink(to maxCount: Int) { window.shrink(to: maxCount) - guard legacyRows.count > maxCount else { return } - legacyRows.removeFirst(legacyRows.count - maxCount) } func reset() { @@ -76,7 +69,6 @@ final class StreamingResultBuffer { flushTask = nil columns = [] window.clear() - legacyRows.removeAll(keepingCapacity: true) rowsAffected = nil statusMessage = nil truncation = nil diff --git a/apple/TableProMobile/TableProMobile/Platform/ConnectionLibraryPublisher.swift b/apple/TableProMobile/TableProMobile/Platform/ConnectionLibraryPublisher.swift index 43de7c3d9f..b4c561c7cf 100644 --- a/apple/TableProMobile/TableProMobile/Platform/ConnectionLibraryPublisher.swift +++ b/apple/TableProMobile/TableProMobile/Platform/ConnectionLibraryPublisher.swift @@ -5,39 +5,53 @@ import TableProLogRedaction import TableProModels import WidgetKit +// The index, the widget file and the shortcut parameters all outlive the process, so each is +// rewritten only when what it holds differs from the library, not at every launch. @MainActor final class ConnectionLibraryPublisher { private static let logger = Logger(subsystem: "com.TablePro", category: "LibraryPublisher") + static let publishedShortcutsKey = "com.TablePro.publishedShortcutProjection" private let searchIndex: any ConnectionSearchIndexing + private let readWidgetItems: () -> [WidgetConnectionItem] private let writeWidgetItems: ([WidgetConnectionItem]) -> Void private let refreshShortcutParameters: () -> Void + private let defaults: UserDefaults + private let build: String - private var shortcutProjection: [SearchableConnection]? private var indexed: [SearchableConnection]? private var queued: [SearchableConnection]? private var replacing: Task? init( searchIndex: any ConnectionSearchIndexing, + readWidgetItems: @escaping () -> [WidgetConnectionItem], writeWidgetItems: @escaping ([WidgetConnectionItem]) -> Void, - refreshShortcutParameters: @escaping () -> Void + refreshShortcutParameters: @escaping () -> Void, + defaults: UserDefaults, + build: String = "" ) { self.searchIndex = searchIndex + self.readWidgetItems = readWidgetItems self.writeWidgetItems = writeWidgetItems self.refreshShortcutParameters = refreshShortcutParameters + self.defaults = defaults + self.build = build } - static func live() -> ConnectionLibraryPublisher { + static func live(defaults: UserDefaults = .standard) -> ConnectionLibraryPublisher { ConnectionLibraryPublisher( searchIndex: SpotlightConnectionIndex(), + readWidgetItems: { SharedConnectionStore.read() }, writeWidgetItems: { items in SharedConnectionStore.write(items) WidgetCenter.shared.reloadAllTimelines() }, refreshShortcutParameters: { TableProShortcuts.updateAppShortcutParameters() - } + }, + defaults: defaults, + build: Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "" ) } @@ -64,11 +78,15 @@ final class ConnectionLibraryPublisher { } func publish(_ connections: [DatabaseConnection]) { - writeWidgetItems(Self.widgetItems(for: connections)) + let items = Self.widgetItems(for: connections) + if items != readWidgetItems() { + writeWidgetItems(items) + } let searchable = Self.searchableConnections(for: connections) - if searchable != shortcutProjection { - shortcutProjection = searchable + let digest = SearchableConnection.digest(of: searchable, build: build) + if digest != defaults.data(forKey: Self.publishedShortcutsKey) { refreshShortcutParameters() + defaults.set(digest, forKey: Self.publishedShortcutsKey) } queued = searchable guard replacing == nil else { return } @@ -83,8 +101,15 @@ final class ConnectionLibraryPublisher { while let next = queued { queued = nil guard next != indexed else { continue } + let state = SearchableConnection.digest(of: next, build: build) + // A new process learns what the index holds from the state the last replace stored. A state that + // cannot be read is treated as different, so the index is written rather than left as it is. + if indexed == nil, (try? await searchIndex.storedState()) == state { + indexed = next + continue + } do { - try await searchIndex.replaceConnections(with: next) + try await searchIndex.replaceConnections(with: next, state: state) indexed = next } catch { indexed = nil diff --git a/apple/TableProMobile/TableProMobile/Platform/ConnectionSecretsCopier.swift b/apple/TableProMobile/TableProMobile/Platform/ConnectionSecretsCopier.swift index 727fe85322..84b0ccc2d3 100644 --- a/apple/TableProMobile/TableProMobile/Platform/ConnectionSecretsCopier.swift +++ b/apple/TableProMobile/TableProMobile/Platform/ConnectionSecretsCopier.swift @@ -11,15 +11,18 @@ nonisolated struct ConnectionSecrets: Sendable { let secureStore: any SecureStore let certificateStore: any CertificateMaterialStoring let bookmarkStore: FileBookmarkStore + let materializer: CertificateMaterializer init( secureStore: any SecureStore, certificateStore: any CertificateMaterialStoring = CertificateMaterialStore(), - bookmarkStore: FileBookmarkStore = FileBookmarkStore() + bookmarkStore: FileBookmarkStore = FileBookmarkStore(), + materializer: CertificateMaterializer = CertificateMaterializer() ) { self.secureStore = secureStore self.certificateStore = certificateStore self.bookmarkStore = bookmarkStore + self.materializer = materializer } @discardableResult @@ -74,6 +77,8 @@ nonisolated struct ConnectionSecrets: Sendable { try? secureStore.delete(forKey: prefix + connectionId.uuidString) } certificateStore.deleteAll(for: connectionId) + // A connect wrote the decrypted client key to a file the drivers read. + materializer.release(for: connectionId) bookmarkStore.delete(for: connectionId) } } diff --git a/apple/TableProMobile/TableProMobile/Platform/IOSDriverFactory.swift b/apple/TableProMobile/TableProMobile/Platform/IOSDriverFactory.swift index 4227583c61..1365e95be7 100644 --- a/apple/TableProMobile/TableProMobile/Platform/IOSDriverFactory.swift +++ b/apple/TableProMobile/TableProMobile/Platform/IOSDriverFactory.swift @@ -8,6 +8,8 @@ import TableProRedisCore nonisolated final class IOSDriverFactory: DriverFactory { private static let logger = Logger(subsystem: "com.TablePro", category: "IOSDriverFactory") + // A lookup still waiting after this has failed; each hop would otherwise wait out dnssd's 30-second timeout. + private static let endpointLookupSeconds = 10 private let bookmarkStore: FileBookmarkStore private let materializer: CertificateMaterializer @@ -18,6 +20,7 @@ nonisolated final class IOSDriverFactory: DriverFactory { private let apiSessionConfiguration: @Sendable () -> URLSessionConfiguration // Times how long a MySQL kill may reuse the token its session signed in with. private let clock: any Clock + private let cnameResolver: any CNAMEResolving init( bookmarkStore: FileBookmarkStore = FileBookmarkStore(), @@ -26,7 +29,8 @@ nonisolated final class IOSDriverFactory: DriverFactory { mint: IAMPasswordMint = IAMPasswordMint(), secureStore: any SecureStore = KeychainSecureStore(), apiSessionConfiguration: @escaping @Sendable () -> URLSessionConfiguration = { .default }, - clock: any Clock = ContinuousClock() + clock: any Clock = ContinuousClock(), + cnameResolver: any CNAMEResolving = SystemCNAMEResolver() ) { self.bookmarkStore = bookmarkStore self.materializer = materializer @@ -35,6 +39,7 @@ nonisolated final class IOSDriverFactory: DriverFactory { self.secureStore = secureStore self.apiSessionConfiguration = apiSessionConfiguration self.clock = clock + self.cnameResolver = cnameResolver } private func sqliteSource(for connection: DatabaseConnection) throws -> LocalDatabaseFileSource { @@ -106,6 +111,32 @@ nonisolated final class IOSDriverFactory: DriverFactory { return resolved } + // RDS takes a token only for the endpoint AWS issued, so a CNAME of the user's own is dialled as that endpoint + // too: signing it while dialling the CNAME would hand the token to whoever answers the CNAME's next lookup. Not + // behind SSH, where the server resolves the name and its PermitOpen matches it as typed. + func resolvingEndpointAlias(for connection: DatabaseConnection) async throws -> DatabaseConnection { + guard connection.iamSignIn?.isAWS == true, + IAMSignIn.engine(for: connection.type)?.awsService == .rds, + !dialsSSHTunnel(for: connection) + else { return connection } + let lookup = await AWSEndpointHost.rdsEndpoint( + behind: connection.host, + resolver: cnameResolver, + within: .seconds(connection.connectTimeoutSeconds ?? Self.endpointLookupSeconds) + ) + try Task.checkCancellation() + switch lookup { + case .notFound: + return connection + case .failed: + throw AWSAuthError.endpointLookupFailed(host: connection.host) + case .found(let endpoint): + var resolved = connection + resolved.host = endpoint + return resolved + } + } + func resolvePassword( for connection: DatabaseConnection, stored: String?, diff --git a/apple/TableProMobile/TableProMobile/Platform/KeychainSecureStore.swift b/apple/TableProMobile/TableProMobile/Platform/KeychainSecureStore.swift index 4ae63bc82c..fc2890837d 100644 --- a/apple/TableProMobile/TableProMobile/Platform/KeychainSecureStore.swift +++ b/apple/TableProMobile/TableProMobile/Platform/KeychainSecureStore.swift @@ -14,6 +14,8 @@ nonisolated protocol KeychainItemStore: Sendable { func update(_ query: [String: Any], attributes: [String: Any]) -> OSStatus // The first item the query matches, with its data and modification date. func copyItem(_ query: [String: Any]) -> KeychainCopiedItem + // Every item's account the query matches; none when the read fails. + func accounts(_ query: [String: Any]) -> [String] func delete(_ query: [String: Any]) -> OSStatus } @@ -41,6 +43,16 @@ nonisolated struct SystemKeychainItemStore: KeychainItemStore { ) } + func accounts(_ query: [String: Any]) -> [String] { + var query = query + query[kSecReturnAttributes as String] = true + query[kSecMatchLimit as String] = kSecMatchLimitAll + var result: AnyObject? + guard SecItemCopyMatching(query as CFDictionary, &result) == errSecSuccess, + let items = result as? [[String: Any]] else { return [] } + return items.compactMap { $0[kSecAttrAccount as String] as? String } + } + func delete(_ query: [String: Any]) -> OSStatus { SecItemDelete(query as CFDictionary) } @@ -184,38 +196,38 @@ nonisolated final class KeychainSecureStore: SecureStore { } // Device-local items only: deleting a synced one here would remove it from the Mac that wrote it. - static func cleanOrphanedCredentials(validConnectionIds: Set) { - guard !validConnectionIds.isEmpty else { return } - + // The Keychain calls block, so they run off the main actor. The library is read once the listing + // returns, and a save adds its connection before its secrets, so a connection saved meanwhile keeps them. + @concurrent + static func cleanOrphanedCredentials( + items: any KeychainItemStore = SystemKeychainItemStore(), + keeping connectionIds: @MainActor @Sendable () -> Set? + ) async { let accessGroup = resolveAccessGroup() - let query: [String: Any] = [ + let accounts = items.accounts(applying(accessGroup: accessGroup, to: [ kSecClass as String: kSecClassGenericPassword, kSecAttrService as String: serviceName, - kSecReturnAttributes as String: true, - kSecMatchLimit as String: kSecMatchLimitAll, kSecAttrSynchronizable as String: false, kSecUseDataProtectionKeychain as String: true, - ] - var result: AnyObject? - guard SecItemCopyMatching(applying(accessGroup: accessGroup, to: query) as CFDictionary, &result) == errSecSuccess, - let items = result as? [[String: Any]] else { return } - - let accounts = items.compactMap { $0[kSecAttrAccount as String] as? String } - for account in ConnectionSecretKind.orphanedAccounts(accounts, keeping: validConnectionIds) { - deleteDeviceLocal(forKey: account, accessGroup: accessGroup) + ])) + guard !accounts.isEmpty, let kept = await connectionIds() else { return } + for account in ConnectionSecretKind.orphanedAccounts(accounts, keeping: kept) { + let status = items.delete(deviceLocalQuery(forKey: account, accessGroup: accessGroup)) + if status != errSecSuccess, status != errSecItemNotFound { + logger.error("Removing a deleted connection's secret failed: \(status)") + } } } // Unlike delete(forKey:), never matches a synced item of the same name. - private static func deleteDeviceLocal(forKey key: String, accessGroup: String?) { - let query: [String: Any] = [ + private static func deviceLocalQuery(forKey key: String, accessGroup: String?) -> [String: Any] { + applying(accessGroup: accessGroup, to: [ kSecClass as String: kSecClassGenericPassword, kSecAttrService as String: serviceName, kSecAttrAccount as String: key, kSecAttrSynchronizable as String: false, kSecUseDataProtectionKeychain as String: true, - ] - SecItemDelete(applying(accessGroup: accessGroup, to: query) as CFDictionary) + ]) } } diff --git a/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift b/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift index 8b58f12408..6ffa538a61 100644 --- a/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift +++ b/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift @@ -56,4 +56,18 @@ final class MemoryPressureMonitor { guard available > 0 else { return true } return available > requiredBytes } + + nonisolated func textBudgetBytes() -> Int { + Self.textBudgetBytes(availableBytes: availableMemoryBytes()) + } + + // A sixth of the headroom past the margin: the kept text, the four copies `exportFits` wants room for, and a + // share for the rest of the app. The call answers 0 outside an app, where only the ceiling bounds it. + nonisolated static func textBudgetBytes(availableBytes: Int) -> Int { + guard availableBytes > 0 else { return textBudgetCeilingBytes } + return min(textBudgetCeilingBytes, max(0, availableBytes - textBudgetMarginBytes) / 6) + } + + nonisolated private static let textBudgetCeilingBytes = 64 * 1_024 * 1_024 + nonisolated private static let textBudgetMarginBytes = 64 * 1_024 * 1_024 } diff --git a/apple/TableProMobile/TableProMobile/Platform/SceneLifecycle.swift b/apple/TableProMobile/TableProMobile/Platform/SceneLifecycle.swift index 7f65197c10..daa639a3c8 100644 --- a/apple/TableProMobile/TableProMobile/Platform/SceneLifecycle.swift +++ b/apple/TableProMobile/TableProMobile/Platform/SceneLifecycle.swift @@ -1,9 +1,57 @@ import Combine import Observation +import os import SwiftUI +import TableProSyncTransport import UIKit final class TableProAppDelegate: NSObject, UIApplicationDelegate { + // Set in the app's init, which runs before the system hands this delegate a launch or a notification. + static weak var appState: AppState? + + private static let logger = Logger(subsystem: "com.TablePro", category: "Sync") + + // iOS ends an app that has not answered a notification in 30 seconds; a run still going finishes when the app next runs. + private static let notificationBudget: Duration = .seconds(25) + + func application( + _ application: UIApplication, + didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil + ) -> Bool { + Self.appState?.syncCoordinator.registerForPushesIfEnabled() + return true + } + + func application(_ application: UIApplication, didFailToRegisterForRemoteNotificationsWithError error: any Error) { + Self.logger.warning("Push registration failed: \(error.localizedDescription, privacy: .private)") + } + + func application( + _ application: UIApplication, + didReceiveRemoteNotification userInfo: [AnyHashable: Any] + ) async -> UIBackgroundFetchResult { + guard SyncPushSubscription.announcesChange(userInfo), let appState = Self.appState else { return .noData } + let run = Task { await appState.syncFromBackground(.remoteChange) } + guard let ran = await Self.value(of: run, within: Self.notificationBudget) else { return .failed } + guard ran else { return .noData } + return appState.syncCoordinator.status.error == nil ? .newData : .failed + } + + private static func value(of task: Task, within budget: Duration) async -> Value? { + await withCheckedContinuation { continuation in + let answer = FirstAnswer(continuation) + let timer = Task { + try? await Task.sleep(for: budget) + answer.give(nil) + } + Task { + let value = await task.value + timer.cancel() + answer.give(value) + } + } + } + func application( _ application: UIApplication, configurationForConnecting connectingSceneSession: UISceneSession, @@ -15,6 +63,19 @@ final class TableProAppDelegate: NSObject, UIApplicationDelegate { } } +private final class FirstAnswer { + private var continuation: CheckedContinuation? + + init(_ continuation: CheckedContinuation) { + self.continuation = continuation + } + + func give(_ value: Value?) { + continuation?.resume(returning: value) + continuation = nil + } +} + enum LockCoverMode: Equatable { case locked case obscured diff --git a/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift b/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift index b74ed25335..dc2d6ee94f 100644 --- a/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift +++ b/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift @@ -44,6 +44,7 @@ final class SessionTransactionMonitor { private var rollBackCounts: [UUID: Int] = [:] @ObservationIgnored private var refreshGenerations: [UUID: Int] = [:] @ObservationIgnored private var workGenerations: [UUID: Int] = [:] + @ObservationIgnored private var statementWaiters: [UUID: [UUID: CheckedContinuation]] = [:] @ObservationIgnored private let liveDriver: (UUID) -> (any DatabaseDriver)? init(liveDriver: @escaping (UUID) -> (any DatabaseDriver)?) { @@ -88,6 +89,26 @@ final class SessionTransactionMonitor { statementCounts[connectionId, default: 0] > 0 } + // A run that moves the session's database lets a write already on its way finish where it was aimed. Stop ends the + // wait, as it ends a run queued behind another. + func statementsFinished(for connectionId: UUID) async { + guard isRunningStatements(connectionId) else { return } + let waiter = UUID() + await withTaskCancellationHandler { + await withCheckedContinuation { continuation in + guard !Task.isCancelled else { + continuation.resume() + return + } + statementWaiters[connectionId, default: [:]][waiter] = continuation + } + } onCancel: { + Task { @MainActor [weak self] in + self?.statementWaiters[connectionId]?.removeValue(forKey: waiter)?.resume() + } + } + } + // `reading` work keeps Commit and Roll Back off the session while it streams, and stales a dialog opened before it. func track(_ connectionId: UUID, reading: Bool = false, _ work: () async throws -> T) async rethrows -> T { workCounts[connectionId, default: 0] += 1 @@ -207,6 +228,10 @@ final class SessionTransactionMonitor { guard !reading else { return } let statements = statementCounts[connectionId, default: 1] - 1 statementCounts[connectionId] = statements > 0 ? statements : nil + guard statements <= 0 else { return } + for waiter in (statementWaiters.removeValue(forKey: connectionId) ?? [:]).values { + waiter.resume() + } } } diff --git a/apple/TableProMobile/TableProMobile/Platform/SpotlightConnectionIndex.swift b/apple/TableProMobile/TableProMobile/Platform/SpotlightConnectionIndex.swift index 0197142618..a2c7fe6b6a 100644 --- a/apple/TableProMobile/TableProMobile/Platform/SpotlightConnectionIndex.swift +++ b/apple/TableProMobile/TableProMobile/Platform/SpotlightConnectionIndex.swift @@ -1,4 +1,5 @@ import CoreSpotlight +import CryptoKit import Foundation import TableProModels @@ -19,21 +20,66 @@ nonisolated struct SearchableConnection: Equatable, Sendable { summary = [connection.type.mobileDisplayName, ConnectionDetailFormatter.detail(for: connection)] .joined(separator: ", ") } + + // The build is part of it, so an update that changes how an item is written publishes again once. + static func digest(of connections: [SearchableConnection], build: String) -> Data { + var bytes = Data() + for field in [build] + connections.flatMap({ [$0.id.uuidString, $0.title, $0.summary] }) { + let utf8 = Data(field.utf8) + withUnsafeBytes(of: UInt32(utf8.count).bigEndian) { bytes.append(contentsOf: $0) } + bytes.append(utf8) + } + return Data(SHA256.hash(data: bytes)) + } } nonisolated protocol ConnectionSearchIndexing: Sendable { - func replaceConnections(with connections: [SearchableConnection]) async throws + // The state the last finished replace stored, nil when none did. + func storedState() async throws -> Data? + func replaceConnections(with connections: [SearchableConnection], state: Data) async throws } nonisolated struct SpotlightConnectionIndex: ConnectionSearchIndexing { static let domainIdentifier = "com.TablePro.connections" + // CoreSpotlight keeps a client state only for a named index; the default one supports no batch. + static let indexName = "com.TablePro.connections" + + func storedState() async throws -> Data? { + guard CSSearchableIndex.isIndexingAvailable() else { return nil } + return try await Self.lastClientState(of: CSSearchableIndex(name: Self.indexName)) + } - func replaceConnections(with connections: [SearchableConnection]) async throws { + // Connections are all the app indexes. Deleting everything also resets the stored state, so an index a failed + // replace emptied never passes for the projection it held before. + func replaceConnections(with connections: [SearchableConnection], state: Data) async throws { guard CSSearchableIndex.isIndexingAvailable() else { return } - let index = CSSearchableIndex.default() - try await index.deleteSearchableItems(withDomainIdentifiers: [Self.domainIdentifier]) - guard !connections.isEmpty else { return } - try await index.indexSearchableItems(connections.map(Self.searchableItem(for:))) + let index = CSSearchableIndex(name: Self.indexName) + if (try? await Self.lastClientState(of: index)) == nil { + // Builds before the named index wrote through the default one. + try await CSSearchableIndex.default().deleteSearchableItems(withDomainIdentifiers: [Self.domainIdentifier]) + } + try await index.deleteAllSearchableItems() + index.beginBatch() + if !connections.isEmpty { + // Journaled with the batch, so ending the batch is the one call awaited. + index.indexSearchableItems(connections.map(Self.searchableItem(for:)), completionHandler: nil) + } + try await index.endBatch(withClientState: state) + } + + // The async form returns a non-optional Data, though the index answers nil before any batch stored a state. + private static func lastClientState(of index: CSSearchableIndex) async throws -> Data? { + let state: Data? = try await withCheckedThrowingContinuation { continuation in + index.fetchLastClientState { state, error in + if let error { + continuation.resume(throwing: error) + } else { + continuation.resume(returning: state) + } + } + } + guard let state, !state.isEmpty else { return nil } + return state } static func searchableItem(for connection: SearchableConnection) -> CSSearchableItem { diff --git a/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift b/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift index afacbf7bce..3d609ecd99 100644 --- a/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift +++ b/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift @@ -3,18 +3,17 @@ import Foundation import os import TableProSSHTransport -/// One authenticated libssh2 session forwarding a local port, built by `SSHTunnelFactory`. -/// -/// A class rather than an actor, the shape `LibSSH2Tunnel` already has on macOS. Every libssh2 -/// call is serialized on `sessionQueue`, the accept loop has a serial queue of its own and the -/// relays share a concurrent one, so nothing here blocks a thread of Swift's cooperative pool. -/// An actor gives mutual exclusion but never thread transfer, so a tunnel parked in `poll` held -/// a cooperative thread for its whole life: measured, 48 tunnels ran 12 at a time in 4,005ms -/// where queues ran all 48 in 1,003ms. `Task.detached` measured the same 12, because it is the -/// same pool. -/// -/// Its socket, session and listening socket are `let`, established before construction, so there -/// is no mutable state to protect and `close()` is synchronous. +// One authenticated libssh2 session forwarding a local port, built by `SSHTunnelFactory`. +// +// A class rather than an actor, the shape `LibSSH2Tunnel` already has on macOS. Every libssh2 call +// runs on the session's pump, the acceptor has a serial queue of its own and channel opens +// share a concurrent one, so nothing here blocks a thread of Swift's cooperative pool. An actor gives mutual +// exclusion but never thread transfer, so a tunnel parked in `poll` held a cooperative thread for its +// whole life: measured, 48 tunnels ran 12 at a time in 4,005ms where queues ran all 48 in 1,003ms. +// `Task.detached` measured the same 12, because it is the same pool. +// +// Its socket, session and listening socket are `let`, established before construction, so `close()` +// is synchronous. nonisolated final class SSHTunnel: @unchecked Sendable { let localPort: Int @@ -25,30 +24,30 @@ nonisolated final class SSHTunnel: @unchecked Sendable { private let listenFD: Int32 private let jumpHops: [SSHJumpHop] - /// Serial: every libssh2 call on this session. libssh2 is not thread-safe per session. - private let sessionQueue: DispatchQueue + // Every libssh2 call on the session, and the relays of its channels. libssh2 is not thread-safe per session. + private let pump: SSHSessionPump - /// Concurrent: relay loops, which poll, send and recv but make no libssh2 call directly. + // Concurrent: channel opens, which wait on the transport between attempts. private let relayQueue: DispatchQueue - /// Serial: the accept loop, which polls the listening socket and nothing else. + // Serial: the acceptor's handler, which accepts and makes no libssh2 call. private let acceptQueue: DispatchQueue private let aliveLatch = TeardownLatch() private let deathNotice = SSHTunnelDeathNotice() - /// The relays still running, so teardown frees the session only once none of them can touch - /// it. A group rather than a collection of tasks: a relay leaves it by finishing, which is the - /// one thing a `[Task]` pruned on `isCancelled` never noticed, so a tunnel that had served - /// clients carried every one of them until it closed. It is also all a task was ever worth - /// here, since the relay runs on `relayQueue` outside the task's cancellation scope and stops - /// on `aliveLatch` rather than on `Task.isCancelled`. + // Each accepted client from accept until its channel is freed or given up, so teardown frees the + // session only once none of them can touch it. private let clientRelays = DispatchGroup() - private var forwardingTask: Task? + private var acceptor: SSHListenerAcceptor? private var keepAliveTask: Task? private static let relayBufferSize = 32_768 + nonisolated private struct ChannelHandle: @unchecked Sendable { + let pointer: OpaquePointer + } + /// Bounds a forwarding channel open for a client that has already been accepted. libssh2 /// retries EAGAIN forever on its own, so without this a stuck open outlives the database /// driver's connect timeout and the client waits on a socket nothing will ever write to. @@ -56,12 +55,6 @@ nonisolated final class SSHTunnel: @unchecked Sendable { /// before the driver reports its own timeout, which names no cause. static let channelOpenDeadlineSeconds: TimeInterval = 6 - /// How long the accept loop waits per poll before rechecking the latch. Small enough that - /// noticing a client the kernel already accepted costs a slim part of the margin above. - static let acceptPollTimeoutMs: Int32 = 200 - - private static let keepAliveIntervalSeconds = 10 - init( session: OpaquePointer, socketFD: Int32, @@ -74,7 +67,7 @@ nonisolated final class SSHTunnel: @unchecked Sendable { self.socketFD = socketFD self.listenFD = listenFD self.localPort = localPort - self.sessionQueue = sessionQueue + self.pump = SSHSessionPump(transportFD: socketFD, queue: sessionQueue) self.jumpHops = jumpHops let label = UUID().uuidString self.relayQueue = DispatchQueue(label: "com.TablePro.ssh.relay.\(label)", qos: .utility, attributes: .concurrent) @@ -88,58 +81,48 @@ nonisolated final class SSHTunnel: @unchecked Sendable { // MARK: - Forwarding func startForwarding(destination: SSHForwardDestination) { - sessionQueue.sync { libssh2_session_set_blocking(session, 0) } + pump.perform { libssh2_session_set_blocking(session, 0) } - forwardingTask = Task.detached { [weak self] in - guard let self else { return } - - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.acceptQueue.async { [weak self] in - defer { continuation.resume() } - guard let self else { return } - - let target = destination.logDescription - Self.logger.info("Forwarding started on port \(self.localPort) -> \(target)") - - while self.isRunning { - guard let client = self.acceptClient() else { continue } - self.spawnClient( - clientFD: client.fd, - acceptedAt: client.acceptedAt, - destination: destination - ) - } - - Self.logger.info("Forwarding loop ended for port \(self.localPort)") - } + // A handler already running when the latch is claimed may still spawn one relay; teardown + // waits for the cancel handler before the relay group, so it waits for that relay too. + let acceptor = SSHListenerAcceptor(listenFD: listenFD, queue: acceptQueue) { [weak self] client in + guard let self, self.isRunning else { + Darwin.close(client.descriptor) + return } + self.spawnClient( + clientFD: client.descriptor, + acceptedAt: client.acceptedAt, + destination: destination + ) } + self.acceptor = acceptor + acceptor.start() + + let target = destination.logDescription + Self.logger.info("Forwarding started on port \(self.localPort) -> \(target)") } // MARK: - Keep-Alive func startKeepAlive() { - sessionQueue.sync { libssh2_keepalive_config(session, 1, 30) } + pump.perform { LibSSH2KeepAlive.configure(session: session) } keepAliveTask = Task.detached { [weak self] in guard let self else { return } while !Task.isCancelled && self.isRunning { - let failed = await withCheckedContinuation { (continuation: CheckedContinuation) in - self.sessionQueue.async { - var secondsToNext: Int32 = 0 - let rc = libssh2_keepalive_send(self.session, &secondsToNext) - continuation.resume(returning: sshKeepAliveDidFail(rc)) - } + let outcome = await self.pump.performAsync { + LibSSH2KeepAlive.send(session: self.session, transportFD: self.socketFD) } - if failed { + if outcome.failed { Self.logger.warning("Keep-alive failed, marking tunnel dead") self.markDead() break } - try? await Task.sleep(for: .seconds(Self.keepAliveIntervalSeconds)) + try? await Task.sleep(for: .seconds(outcome.secondsToNext)) } } } @@ -162,17 +145,18 @@ nonisolated final class SSHTunnel: @unchecked Sendable { deathNotice.died() } - // shutdown wakes every poll on the transport but keeps the number, so no other socket can take it mid-poll. - // Darwin's shutdown does not wake a poll on a listening socket, so the accept loop ends on the latch instead. - // The relays are awaited after the accept loop, the only thing that starts one, so the group can only empty. - // The hops go last, in the Mac's order. + // shutdown wakes every open or free waiting on the transport but keeps the number, so no other socket can take it mid-poll. + // The listening socket closes only after the acceptor's cancel handler ran, once no accept handler is running. + // The clients are awaited after that, since the accept handler is the only thing that starts one, and the pump + // last, so no source of it can fire on the descriptors closed below. The hops go last, in the Mac's order. private func performTeardown() { - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() + pump.stop() shutdown(socketFD, SHUT_RDWR) - let sessionQueue = self.sessionQueue + let pump = self.pump let relayQueue = self.relayQueue let clientRelays = self.clientRelays let session = self.session @@ -180,17 +164,18 @@ nonisolated final class SSHTunnel: @unchecked Sendable { let listenFD = self.listenFD let jumpHops = self.jumpHops let localPort = self.localPort - let forwardingTask = self.forwardingTask + let acceptor = self.acceptor let keepAliveTask = self.keepAliveTask Task.detached { - await forwardingTask?.value + await acceptor?.waitForCancellation() await keepAliveTask?.value await withCheckedContinuation { (continuation: CheckedContinuation) in clientRelays.notify(queue: relayQueue) { continuation.resume() } } + await pump.waitUntilStopped() // Free before close: a closed number can go to the next tunnel's socket, and this session's DISCONNECT would land in it. - sessionQueue.sync { + pump.perform { libssh2_session_set_blocking(session, 1) tablepro_libssh2_session_disconnect(session, "Closing tunnel") libssh2_session_free(session) @@ -205,38 +190,15 @@ nonisolated final class SSHTunnel: @unchecked Sendable { // MARK: - Private - /// The accept timestamp is taken here, not once the open reaches `openAndRelay`, because the - /// client's own connect timeout is already running by then and the scheduling hops in between - /// would push the deadline past it. - private func acceptClient() -> (fd: Int32, acceptedAt: Date)? { - var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) - let pollResult = poll(&pollFD, 1, Self.acceptPollTimeoutMs) - - guard pollResult > 0, pollFD.revents & Int16(POLLIN) != 0 else { return nil } - - var clientAddr = sockaddr_in() - var addrLen = socklen_t(MemoryLayout.size) - - let clientFD = withUnsafeMutablePointer(to: &clientAddr) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - accept(listenFD, $0, &addrLen) - } - } - - guard clientFD >= 0 else { return nil } - return (clientFD, Date()) - } - - /// Opens the channel and relays one accepted client, off the accept loop so a slow open - /// cannot delay the next accept. The loop runs on `relayQueue` (concurrent); individual - /// libssh2 calls are dispatched to `sessionQueue` (serial) for thread safety. + // On `relayQueue`, off the accept handler, so a slow open cannot delay the next accept. The client + // stays in `clientRelays` until its relay ended and its channel was freed. private func spawnClient(clientFD: Int32, acceptedAt: Date, destination: SSHForwardDestination) { let clientRelays = self.clientRelays clientRelays.enter() relayQueue.async { [weak self] in - defer { clientRelays.leave() } guard let self else { Darwin.close(clientFD) + clientRelays.leave() return } self.openAndRelay(clientFD: clientFD, acceptedAt: acceptedAt, destination: destination) @@ -244,12 +206,15 @@ nonisolated final class SSHTunnel: @unchecked Sendable { } private func openAndRelay(clientFD: Int32, acceptedAt: Date, destination: SSHForwardDestination) { - let pump = SSHForwardChannelOpenPump( + let turn = SSHSendTurn(pump: pump) + defer { turn.end() } + let openPump = SSHForwardChannelOpenPump( opener: LibSSH2ForwardChannelOpener( session: session, destination: destination, originPort: localPort, - sessionQueue: sessionQueue + pump: pump, + turn: turn ), isActive: { [weak self] in self?.isRunning ?? false }, deadline: acceptedAt.addingTimeInterval(Self.channelOpenDeadlineSeconds), @@ -260,11 +225,13 @@ nonisolated final class SSHTunnel: @unchecked Sendable { } ) - let outcome = pump.run() + let outcome = openPump.run() logChannelOpenOutcome(outcome, destination: destination) handleChannelOpenOutcome(outcome, clientFD: clientFD) { channel in - runRelay(clientFD: clientFD, channel: channel, destination: destination) + relayChannel(clientFD: clientFD, channel: channel, destination: destination) } + if case .opened = outcome { return } + clientRelays.leave() } private func logChannelOpenOutcome(_ outcome: ChannelOpenOutcome, destination: SSHForwardDestination) { @@ -285,32 +252,29 @@ nonisolated final class SSHTunnel: @unchecked Sendable { } } - private func runRelay(clientFD: Int32, channel: OpaquePointer, destination: SSHForwardDestination) { - let relay = SSHChannelRelay( - localFD: clientFD, - transportFD: socketFD, - channelIO: LibSSH2ChannelIO(channel: channel, session: session, sessionQueue: sessionQueue), - bufferSize: Self.relayBufferSize, - isActive: { [weak self] in self?.isRunning ?? false } - ) - + private func relayChannel(clientFD: Int32, channel: OpaquePointer, destination: SSHForwardDestination) { let startedAt = Date() - let termination = relay.run() - let elapsed = Date().timeIntervalSince(startedAt) - let target = destination.logDescription - Self.logger.debug( - "Relay to \(target) ended as \(String(describing: termination)) after \(elapsed, format: .fixed(precision: 1))s" - ) - - Darwin.close(clientFD) - guard isRunning else { return } + let clientRelays = self.clientRelays + let handle = ChannelHandle(pointer: channel) + pump.relay( + localFD: clientFD, + channel: LibSSH2ChannelIO(channel: channel, session: session), + bufferSize: Self.relayBufferSize + ) { [weak self] termination in + defer { clientRelays.leave() } + let elapsed = Date().timeIntervalSince(startedAt) + Self.logger.debug( + "Relay to \(target) ended as \(String(describing: termination)) after \(elapsed, format: .fixed(precision: 1))s" + ) + guard let self, self.isRunning else { return } - freeChannel(channel) + self.freeChannel(handle.pointer) - if termination == .transportHangup { - Self.logger.info("SSH transport hung up, marking tunnel dead on port \(self.localPort)") - markDead() + if termination == .transportHangup { + Self.logger.info("SSH transport hung up, marking tunnel dead on port \(self.localPort)") + self.markDead() + } } } @@ -318,7 +282,7 @@ nonisolated final class SSHTunnel: @unchecked Sendable { let outcome = LibSSH2ChannelFree.run( channel: channel, session: session, - sessionQueue: sessionQueue, + pump: pump, transportFD: socketFD, isActive: { [weak self] in self?.isRunning ?? false } ) diff --git a/apple/TableProMobile/TableProMobile/Security/CertificateMaterializer.swift b/apple/TableProMobile/TableProMobile/Security/CertificateMaterializer.swift index 88aee704b9..d724150c54 100644 --- a/apple/TableProMobile/TableProMobile/Security/CertificateMaterializer.swift +++ b/apple/TableProMobile/TableProMobile/Security/CertificateMaterializer.swift @@ -17,15 +17,26 @@ nonisolated final class CertificateMaterializer: @unchecked Sendable { private let store: any CertificateMaterialStoring private let fileManager: FileManager + private let root: URL? - init(store: any CertificateMaterialStoring = CertificateMaterialStore(), fileManager: FileManager = .default) { + // `root` is the folder that holds one folder per connection; nil is Application Support's. + init( + store: any CertificateMaterialStoring = CertificateMaterialStore(), + fileManager: FileManager = .default, + root: URL? = nil + ) { self.store = store self.fileManager = fileManager + self.root = root } + // A role the store no longer holds loses its file, so a certificate removed in the form leaves no key behind. func materialize(for connectionId: UUID) throws -> MaterializedCertificates { let paths = try CertificateRole.allCases.reduce(into: [CertificateRole: String]()) { result, role in - guard let pem = store.pem(role: role, for: connectionId) else { return } + guard let pem = store.pem(role: role, for: connectionId) else { + release(role, for: connectionId) + return + } result[role] = try write(pem, role: role, for: connectionId) } @@ -39,7 +50,31 @@ nonisolated final class CertificateMaterializer: @unchecked Sendable { func release(for connectionId: UUID) { let directory = directory(for: connectionId) guard fileManager.fileExists(atPath: directory.path) else { return } - try? fileManager.removeItem(at: directory) + do { + try fileManager.removeItem(at: directory) + } catch { + Self.logger.error("Client certificate files were not removed: \(error.localizedDescription, privacy: .private)") + } + } + + // Deletes before this release left the files of connections that are gone. Only a loaded library may ask. + func releaseAll(except kept: Set) { + guard let root = try? rootDirectory(), + let names = try? fileManager.contentsOfDirectory(atPath: root.path) else { return } + for name in names { + guard let connectionId = UUID(uuidString: name), !kept.contains(connectionId) else { continue } + release(for: connectionId) + } + } + + func release(_ role: CertificateRole, for connectionId: UUID) { + let file = directory(for: connectionId).appendingPathComponent("\(role.rawValue).pem") + guard fileManager.fileExists(atPath: file.path) else { return } + do { + try fileManager.removeItem(at: file) + } catch { + Self.logger.error("A stale client certificate file was not removed: \(error.localizedDescription, privacy: .private)") + } } private func write(_ pem: String, role: CertificateRole, for connectionId: UUID) throws -> String { @@ -72,6 +107,7 @@ nonisolated final class CertificateMaterializer: @unchecked Sendable { } private func rootDirectory() throws -> URL { + if let root { return root } let support = try fileManager.url( for: .applicationSupportDirectory, in: .userDomainMask, diff --git a/apple/TableProMobile/TableProMobile/Sync/IOSSyncCoordinator.swift b/apple/TableProMobile/TableProMobile/Sync/IOSSyncCoordinator.swift index 9420079f9f..d9c831c2ef 100644 --- a/apple/TableProMobile/TableProMobile/Sync/IOSSyncCoordinator.swift +++ b/apple/TableProMobile/TableProMobile/Sync/IOSSyncCoordinator.swift @@ -76,6 +76,16 @@ final class IOSSyncCoordinator { @ObservationIgnored private var editGenerations: [EditKey: Int] = [:] @ObservationIgnored private var accountChangeObservation: AnyCancellable? + // A scene activation asks only for what other devices sent, so one download per interval answers every + // activation in it. The same rule as the Mac's SyncCoordinator. + nonisolated static let activationInterval: Duration = .seconds(60) + @ObservationIgnored private let activationClock: any Clock + @ObservationIgnored private var activationCooldown: Task? + @ObservationIgnored private var activationHeld = false + @ObservationIgnored private var activationAwaitsRun = false + @ObservationIgnored private var downloadIsAheadInRun = false + @ObservationIgnored private let pushChannel: SyncPushChannel + // Each returns whether the merged list was saved; false holds the change token for a replay. @ObservationIgnored var onConnectionsChanged: (([DatabaseConnection]) -> Bool)? @ObservationIgnored var onGroupsChanged: (([ConnectionGroup]) -> Bool)? @@ -99,13 +109,17 @@ final class IOSSyncCoordinator { makeTransport: @escaping () -> any IOSSyncTransport = { CloudKitSyncEngine() }, isEnabled: @escaping () -> Bool = { AppPreferences.isCloudSyncEnabled }, notificationCenter: NotificationCenter = .default, - networkMonitor: SyncNetworkMonitor? = SyncNetworkMonitor() + networkMonitor: SyncNetworkMonitor? = SyncNetworkMonitor(), + activationClock: any Clock = ContinuousClock(), + pushRegistrar: SyncPushRegistrar = .live ) { + self.pushChannel = SyncPushChannel(registrar: pushRegistrar, metadata: metadata) self.metadata = metadata self.recordCache = recordCache self.makeTransport = makeTransport self.isEnabled = isEnabled self.networkMonitor = networkMonitor + self.activationClock = activationClock self.status = isEnabled() ? metadata.zoneState.initialStatus : .disabled(.userDisabled) self.lastSyncDate = metadata.lastSyncDate accountChangeObservation = notificationCenter.publisher(for: .CKAccountChanged) @@ -124,6 +138,7 @@ final class IOSSyncCoordinator { deinit { debounceTask?.cancel() retryTask?.cancel() + activationCooldown?.cancel() } private func currentTransport() -> any IOSSyncTransport { @@ -148,12 +163,20 @@ final class IOSSyncCoordinator { // MARK: - Enable / Disable + // At launch, from the app delegate: nothing registers while sync is off. + func registerForPushesIfEnabled() { + guard isEnabled() else { return } + pushChannel.register() + } + func setEnabled(_ enabled: Bool) { resetRetryState() guard enabled else { debounceTask?.cancel() debounceTask = nil pendingTriggers = [] + endActivationHold() + pushChannel.unregister() metadata.lastSyncDate = nil lastSyncDate = nil decide(.disabled(.userDisabled)) @@ -164,6 +187,8 @@ final class IOSSyncCoordinator { prepareFullUpload() } decide(.idle) + pushChannel.forgetSubscription() + registerForPushesIfEnabled() Task { await sync(.userRequest) } } @@ -181,6 +206,7 @@ final class IOSSyncCoordinator { } private func prepareFullUpload() { + pushChannel.forgetSubscription() metadata.saveToken(nil) metadata.zoneState = .unknown dropRecordCache() @@ -199,7 +225,9 @@ final class IOSSyncCoordinator { return } if let runningSync { - if !pendingTriggers.contains(trigger) { + // The download still ahead in this run brings what a notice announced. + let coveredByRun = trigger == .remoteChange && downloadIsAheadInRun + if !coveredByRun, !pendingTriggers.contains(trigger) { pendingTriggers.append(trigger) } await runningSync.value @@ -210,6 +238,54 @@ final class IOSSyncCoordinator { await task.value } + // The scene's entry point. A background refresh and every other trigger call `sync(_:)` and are never held. + func syncOnSceneActivation() async { + if let runningSync { + // A run whose download has not started brings what the activation asks for. + guard !downloadIsAheadInRun else { return } + activationAwaitsRun = true + await runningSync.value + return + } + if activationCooldown != nil { + activationHeld = true + return + } + await sync(.activation) + } + + var isHoldingActivations: Bool { + activationCooldown != nil + } + + // After a download, activations wait for the interval to end and then run once, so a change made on + // another device still arrives. + private func startActivationCooldown() { + activationCooldown?.cancel() + activationHeld = false + let clock = activationClock + activationCooldown = Task { [weak self] in + do { + try await clock.sleep(for: Self.activationInterval) + } catch { + return + } + // A newer download may have replaced this hold after the sleep ended. + guard !Task.isCancelled, let self else { return } + activationCooldown = nil + guard activationHeld else { return } + activationHeld = false + await syncOnSceneActivation() + } + } + + private func endActivationHold() { + activationCooldown?.cancel() + activationCooldown = nil + activationHeld = false + activationAwaitsRun = false + } + private func runThenPending(_ trigger: SyncTrigger) async { var next: SyncTrigger? = trigger while let current = next { @@ -258,6 +334,16 @@ final class IOSSyncCoordinator { /// The trigger that may do the most among those that arrived during the last run. private func takePendingTrigger() -> SyncTrigger? { + if activationAwaitsRun { + activationAwaitsRun = false + // The download began before the activation and may have missed what it asks for, so it runs once the + // interval ends rather than now. + if activationCooldown != nil { + activationHeld = true + } else if !pendingTriggers.contains(.activation) { + pendingTriggers.append(.activation) + } + } guard !pendingTriggers.isEmpty else { return nil } let error = status.error /// The network coming back mid-run matters only if the run ended unable to reach iCloud. @@ -281,6 +367,8 @@ final class IOSSyncCoordinator { /// Nil when the status moved on while the run was in flight. private func perform(_ requested: SyncAdmission, generation: Int) async -> RunResult? { + downloadIsAheadInRun = true + defer { downloadIsAheadInRun = false } let transport = currentTransport() var admission = requested switch await confirmAccount(using: transport, generation: generation) { @@ -294,6 +382,7 @@ final class IOSSyncCoordinator { admission = .full } } + await pushChannel.saveSubscriptionIfNeeded { try await transport.saveSubscription() } guard let result = await pullThenPush(admission, generation: generation, using: transport) else { return nil } let zone = metadata.zoneState @@ -303,6 +392,7 @@ final class IOSSyncCoordinator { if nextZone == .removed, zone != .removed { dropRecordCache() metadata.saveToken(nil) + pushChannel.forgetSubscription() } metadata.zoneState = nextZone return result @@ -359,6 +449,7 @@ final class IOSSyncCoordinator { } let pulled: PullResult + downloadIsAheadInRun = false do { pulled = try await fetchChanges(using: transport) } catch { @@ -387,6 +478,7 @@ final class IOSSyncCoordinator { guard applied != .notSaved else { return RunResult(admission: admission, failure: .pullNotSaved) } + startActivationCooldown() // Committed before the upload, so a failed upload does not make every later run download everything again. if applied == .saved, let newToken = pulled.newToken { metadata.saveToken(newToken) diff --git a/apple/TableProMobile/TableProMobile/Sync/IOSSyncTransport.swift b/apple/TableProMobile/TableProMobile/Sync/IOSSyncTransport.swift index 13e86f88b2..a07f844dcd 100644 --- a/apple/TableProMobile/TableProMobile/Sync/IOSSyncTransport.swift +++ b/apple/TableProMobile/TableProMobile/Sync/IOSSyncTransport.swift @@ -7,6 +7,7 @@ nonisolated protocol IOSSyncTransport: Sendable { func accountStatus() async throws -> CKAccountStatus func currentAccountId() async throws -> String func ensureZoneExists() async throws + func saveSubscription() async throws func pull(since token: CKServerChangeToken?) async throws -> PullResult func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome func fetch(recordIDs: [CKRecord.ID]) async throws -> [CKRecord.ID: CKRecord] diff --git a/apple/TableProMobile/TableProMobile/Sync/SyncPushRegistrar+UIKit.swift b/apple/TableProMobile/TableProMobile/Sync/SyncPushRegistrar+UIKit.swift new file mode 100644 index 0000000000..17be11ea42 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Sync/SyncPushRegistrar+UIKit.swift @@ -0,0 +1,9 @@ +import TableProSyncTransport +import UIKit + +extension SyncPushRegistrar { + static let live = SyncPushRegistrar( + register: { UIApplication.shared.registerForRemoteNotifications() }, + unregister: { UIApplication.shared.unregisterForRemoteNotifications() } + ) +} diff --git a/apple/TableProMobile/TableProMobile/TableProMobileApp.swift b/apple/TableProMobile/TableProMobile/TableProMobileApp.swift index d4d054dadf..f7a93cad27 100644 --- a/apple/TableProMobile/TableProMobile/TableProMobileApp.swift +++ b/apple/TableProMobile/TableProMobile/TableProMobileApp.swift @@ -21,6 +21,7 @@ struct TableProMobileApp: App { ConnectionListTips.configure() let appState = AppState() _appState = State(initialValue: appState) + TableProAppDelegate.appState = appState // At launch, so a connection opened before the scene first turns active still sends its first-use event. if !TestRuntime.isActive { Self.startTelemetry(appState: appState) @@ -54,7 +55,7 @@ struct TableProMobileApp: App { if appState.onboarding.isCloudSyncEnabled && appState.loadStatus == .ready { syncTask?.cancel() syncTask = Task { - await appState.syncCoordinator.sync(.activation) + await appState.syncCoordinator.syncOnSceneActivation() } } Telemetry.shared.sceneBecameActive() @@ -106,15 +107,7 @@ struct TableProMobileApp: App { @Sendable private func runBackgroundSync() async { scheduleBackgroundSync() - guard AppPreferences.isCloudSyncEnabled else { return } - await MainActor.run { appState.retryLoadIfFailed() } - let status = await MainActor.run { appState.loadStatus } - guard status == .ready else { - Self.backgroundLogger.warning("Background sync skipped: persistence load not ready (likely device locked)") - return - } - Self.backgroundLogger.info("Background sync starting") - await appState.syncCoordinator.sync(.activation) + guard await appState.syncFromBackground(.activation) else { return } Self.backgroundLogger.info("Background sync completed") } } diff --git a/apple/TableProMobile/TableProMobile/TableProMobileRelease.entitlements b/apple/TableProMobile/TableProMobile/TableProMobileRelease.entitlements index 3ae2bfe544..169be5c6a7 100644 --- a/apple/TableProMobile/TableProMobile/TableProMobileRelease.entitlements +++ b/apple/TableProMobile/TableProMobile/TableProMobileRelease.entitlements @@ -2,6 +2,8 @@ + aps-environment + development com.apple.developer.icloud-container-identifiers iCloud.com.TablePro diff --git a/apple/TableProMobile/TableProMobile/ViewModels/ConnectionFormViewModel+Certificates.swift b/apple/TableProMobile/TableProMobile/ViewModels/ConnectionFormViewModel+Certificates.swift index 2b4a3991b0..c3397a0e25 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/ConnectionFormViewModel+Certificates.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/ConnectionFormViewModel+Certificates.swift @@ -84,6 +84,8 @@ extension ConnectionFormViewModel { func persistCertificates(for connectionId: UUID) -> Bool { for role in removedCertificates where pendingCertificates[role] == nil { certificateStore.delete(role: role, for: connectionId) + // A connect wrote it to a file the drivers read, and no connect may follow to clear it. + CertificateMaterializer(store: certificateStore).release(role, for: connectionId) storedCertificateRoles.remove(role) } removedCertificates.removeAll() diff --git a/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift index 924322f039..cc770621ae 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift @@ -18,13 +18,13 @@ final class DataBrowserViewModel { private static let logger = Logger(subsystem: "com.TablePro", category: "DataBrowserViewModel") private let buffer: StreamingResultBuffer + private let textBudgetBytes: @MainActor () -> Int private(set) var totalRows: Int? private(set) var phase: Phase = .idle private(set) var executionTime: TimeInterval = 0 var columns: [ColumnInfo] { buffer.columns } var window: RowWindow { buffer.window } - var legacyRows: [[String?]] { buffer.legacyRows } var rowsAffected: Int? { buffer.rowsAffected } var statusMessage: String? { buffer.statusMessage } @@ -44,8 +44,10 @@ final class DataBrowserViewModel { @ObservationIgnored private var session: ConnectionSession? @ObservationIgnored private var table: TableInfo? @ObservationIgnored private var databaseType: DatabaseType = .mysql - // The schema the rows on screen came from. Every statement reads it, so an edit cannot target another schema. + // The schema and database the rows on screen came from. Every statement reads them, so an edit cannot target + // another schema or database. private(set) var schema: String? + private(set) var database: String? @ObservationIgnored private var host: String = "" @ObservationIgnored private var trackRead: SessionReadTracker? @ObservationIgnored private var pageEndedOnItsOwn = false @@ -55,7 +57,11 @@ final class DataBrowserViewModel { @ObservationIgnored private var keyReadTask: Task? @ObservationIgnored private var loadGeneration = 0 @ObservationIgnored private var hasSeededFilters = false - @ObservationIgnored private var singleRowMatch = SingleRowMatch.inactive + @ObservationIgnored private var foreignKeysRead = false + // Bumped when what the count counts changes; a page turn, a sort or a page size change leaves it. + @ObservationIgnored private var countEpoch = 0 + @ObservationIgnored private var countingEpoch: Int? + private var singleRowMatch = SingleRowMatch.inactive private enum SingleRowMatch { case inactive @@ -63,8 +69,12 @@ final class DataBrowserViewModel { case found } - init(windowCapacity: Int = 1_000) { + init( + windowCapacity: Int = 1_000, + textBudgetBytes: @escaping @MainActor () -> Int = { MemoryPressureMonitor.shared.textBudgetBytes() } + ) { self.buffer = StreamingResultBuffer(capacity: windowCapacity) + self.textBudgetBytes = textBudgetBytes self.pagination = PaginationState(pageSize: AppPreferences.defaultPageSize, currentPage: 0) } @@ -82,17 +92,20 @@ final class DataBrowserViewModel { private var rowIdentity: RowIdentity { RowLocator.identity(of: session?.driver) } var showsPaginationBar: Bool { - !legacyRows.isEmpty || hasActiveSearch || hasActiveFilters || isPageLoading + !buffer.isEmpty || hasActiveSearch || hasActiveFilters || isPageLoading || pagination.currentPage > 0 } var canGoToPreviousPage: Bool { pagination.currentPage > 0 && !isLoading } - var canGoToNextPage: Bool { pagination.hasNextPage && !isLoading } + var canGoToNextPage: Bool { pagination.hasNextPage && !isLoading && !isPastTheEnd } + + // An empty page after the first: Next or Go to Page went past the last row before the count said where it was. + var isPastTheEnd: Bool { buffer.isEmpty && pagination.currentPage > 0 } var paginationLabel: String { - guard !legacyRows.isEmpty else { return "" } + guard !buffer.isEmpty else { return "" } // A memory trim drops rows from the head of the page, so the range starts at the first row still held. let start = pagination.currentOffset + window.firstAbsoluteIndex + 1 - let end = start + legacyRows.count - 1 + let end = start + buffer.count - 1 guard let total = pagination.totalRows else { return "\(start)-\(end)" } return Self.pageRangeLabel(start: start, end: end, total: total) } @@ -112,14 +125,22 @@ final class DataBrowserViewModel { databaseType: DatabaseType, host: String, schema: String? = nil, + database: String? = nil, trackRead: SessionReadTracker? = nil ) { + // Another session, schema or database holds other rows, so neither the total nor a count still running applies. + let movesSource = session?.driver !== self.session?.driver || schema != self.schema || database != self.database + || table.name != self.table?.name self.session = session self.table = table self.databaseType = databaseType self.host = host self.schema = schema + self.database = database self.trackRead = trackRead + if movesSource { + resetTotal() + } } // Only the first call counts, so filters the user changed afterwards survive the view's task running again. @@ -131,6 +152,11 @@ final class DataBrowserViewModel { singleRowMatch = hasActiveFilters ? .awaitingFirstLoad : .inactive } + var hasSingleRowMatch: Bool { + if case .found = singleRowMatch { return true } + return false + } + func takeSingleRowMatch() -> Bool { guard case .found = singleRowMatch else { return false } singleRowMatch = .inactive @@ -155,7 +181,7 @@ final class DataBrowserViewModel { return } - if isInitial || legacyRows.isEmpty { + if isInitial || buffer.isEmpty { isLoading = true } else { isPageLoading = true @@ -189,13 +215,13 @@ final class DataBrowserViewModel { isLoading = false isPageLoading = false guard case .awaitingFirstLoad = singleRowMatch else { return } - singleRowMatch = loadError == nil && legacyRows.count == 1 ? .found : .inactive + singleRowMatch = loadError == nil && buffer.count == 1 ? .found : .inactive } private func loadRows(session: ConnectionSession, table: TableInfo, isInitial: Bool, generation: Int) async { do { if columnDetails.isEmpty || isInitial { - let fetched = try await session.driver.fetchColumns(table: table.name, schema: schema) + let fetched = try await session.driver.fetchColumns(table: table.name, schema: schema, database: database) guard isCurrent(generation) else { return } columnDetails = fetched } @@ -221,19 +247,26 @@ final class DataBrowserViewModel { return } settleTotalRowsFromShortPage() + // The page is on screen and takes taps; the key links and the pager's total fill in after it. + finishLoad() - if foreignKeys.isEmpty || isInitial { + if !foreignKeysRead || isInitial { do { - let fetched = try await session.driver.fetchForeignKeys(table: table.name, schema: schema) + let fetched = try await session.driver.fetchForeignKeys( + table: table.name, schema: schema, database: database + ) guard isCurrent(generation) else { return } foreignKeys = fetched + foreignKeysRead = true } catch { Self.logger.warning("Failed to fetch foreign keys: \(error.localizedDescription, privacy: .private)") } } - if isCurrent(generation), pagination.totalRows == nil { - await fetchTotalRows(session: session, table: table, generation: generation) + // A count already running for the same rows lands for this page too, so a page turn neither drops it nor + // starts another behind it. + if isCurrent(generation), pagination.totalRows == nil, countingEpoch != countEpoch { + await fetchTotalRows(session: session, table: table) } } catch { guard isCurrent(generation) else { return } @@ -259,7 +292,9 @@ final class DataBrowserViewModel { pagination.totalRows = page.totalCount let result = page.result - await loadPage(options: pageOptions(lazyContext: nil), startedAt: start, generation: generation) { options in + // A key's values have no key of their own to be read again by, so they stay whole within a budget. + let keyOptions = pageOptions(lazyContext: nil, textBudget: TextBudget(bytes: textBudgetBytes())) + await loadPage(options: keyOptions, startedAt: start, generation: generation) { options in QueryResultStreaming.stream(options: options) { result } } guard isCurrent(generation) else { return } @@ -278,18 +313,22 @@ final class DataBrowserViewModel { } } - // A page cut short by Stop, a closed screen or another statement says nothing about where the table ends. + // A page cut short by Stop, a closed screen or another statement says nothing about where the table ends, and an + // empty page after the first says only that it ends before it. private func settleTotalRowsFromShortPage() { - guard pageEndedOnItsOwn, legacyRows.count < pagination.pageSize, pagination.totalRows == nil else { return } - pagination.totalRows = pagination.currentOffset + legacyRows.count + guard pageEndedOnItsOwn, buffer.count < pagination.pageSize, pagination.totalRows == nil, !isPastTheEnd else { + return + } + pagination.totalRows = pagination.currentOffset + buffer.count } - private func pageOptions(lazyContext: LazyContext?) -> StreamOptions { + private func pageOptions(lazyContext: LazyContext?, textBudget: TextBudget? = nil) -> StreamOptions { StreamOptions( textTruncationBytes: 4_096, inlineBinary: false, maxRows: pagination.pageSize, - lazyContext: lazyContext + lazyContext: lazyContext, + textBudget: textBudget ) } @@ -297,7 +336,7 @@ final class DataBrowserViewModel { let activeSort = effectiveSortState() if hasActiveSearch { return SQLBuilder.buildSearchSelect( - table: table.name, schema: schema, type: databaseType, + table: table.name, schema: schema, database: database, type: databaseType, searchText: activeSearchText, searchColumns: searchableColumns(), filters: filters, logicMode: filterLogicMode, columns: columnDetails, session: session?.driver.settledLexicalFacts, @@ -307,7 +346,7 @@ final class DataBrowserViewModel { } if hasActiveFilters { return SQLBuilder.buildFilteredSelect( - table: table.name, schema: schema, type: databaseType, + table: table.name, schema: schema, database: database, type: databaseType, filters: filters, logicMode: filterLogicMode, columns: columnDetails, session: session?.driver.settledLexicalFacts, sortState: activeSort, @@ -316,26 +355,36 @@ final class DataBrowserViewModel { } if activeSort.isSorting { return SQLBuilder.buildSelect( - table: table.name, schema: schema, type: databaseType, + table: table.name, schema: schema, database: database, type: databaseType, sortState: activeSort, limit: pagination.pageSize, offset: pagination.currentOffset ) } return SQLBuilder.buildSelect( - table: table.name, schema: schema, type: databaseType, + table: table.name, schema: schema, database: database, type: databaseType, limit: pagination.pageSize, offset: pagination.currentOffset ) } - // SQL Server's `OFFSET FETCH` requires `ORDER BY`. Without a user sort, a key column keeps paging stable. + // The whole key orders the page, after the user's sort as its tiebreak. PostgreSQL and ClickHouse return an + // unpredictable subset for LIMIT and OFFSET without an order, so Next Page repeats and skips rows; a ClickHouse key + // need not be unique, so rows that share one can still trade places. SQL Server and Oracle `OFFSET FETCH` needs an + // ORDER BY, so a keyless table there sorts by its first column. Redshift keys are not enforced and its sort is a full + // sort, so it pages as it scans. private func effectiveSortState() -> SortState { - if sortState.isSorting { return sortState } - guard databaseType == .mssql || databaseType == .oracle else { return sortState } - let fallback = columnDetails.first(where: \.isPrimaryKey)?.name ?? columnDetails.first?.name - guard let fallback else { return sortState } - return SortState(columns: [SortColumn(name: fallback, ascending: true)]) + guard Self.keyOrderedPaging.contains(databaseType) else { return sortState } + var keyColumns = columnDetails.filter(\.isPrimaryKey).map(\.name) + if keyColumns.isEmpty, !sortState.isSorting, databaseType == .mssql || databaseType == .oracle, + let first = columnDetails.first { + keyColumns = [first.name] + } + let tiebreak = keyColumns.filter { key in !sortState.columns.contains { $0.name == key } } + guard !tiebreak.isEmpty else { return sortState } + return SortState(columns: sortState.columns + tiebreak.map { SortColumn(name: $0, ascending: true) }) } + private static let keyOrderedPaging: Set = [.postgresql, .clickhouse, .mssql, .oracle] + private func readRequest(table: TableInfo) -> TableReadRequest { TableReadRequest( table: table.name, @@ -349,40 +398,43 @@ final class DataBrowserViewModel { } private func searchableColumns() -> [ColumnInfo] { - columns.filter { col in - let upper = col.typeName.uppercased() - return !upper.contains("BLOB") && !upper.contains("BYTEA") && !upper.contains("BINARY") - && !upper.contains("VARBINARY") && !upper.contains("IMAGE") - } + columns.filter { !Cell.holdsBytes(typeName: $0.typeName) } } - private func fetchTotalRows(session: ConnectionSession, table: TableInfo, generation: Int) async { + private func fetchTotalRows(session: ConnectionSession, table: TableInfo) async { + let epoch = countEpoch + countingEpoch = epoch + defer { + if countingEpoch == epoch { countingEpoch = nil } + } do { if let builder = session.driver as? any DriverQueryBuilding { let total = try await builder.rowCount(readRequest(table: table)) - guard isCurrent(generation) else { return } + guard epoch == countEpoch else { return } pagination.totalRows = total return } let countQuery: String if hasActiveSearch { countQuery = SQLBuilder.buildSearchCount( - table: table.name, schema: schema, type: databaseType, + table: table.name, schema: schema, database: database, type: databaseType, searchText: activeSearchText, searchColumns: searchableColumns(), filters: filters, logicMode: filterLogicMode, columns: columnDetails, session: session.driver.settledLexicalFacts ) } else if hasActiveFilters { countQuery = SQLBuilder.buildFilteredCount( - table: table.name, schema: schema, type: databaseType, + table: table.name, schema: schema, database: database, type: databaseType, filters: filters, logicMode: filterLogicMode, columns: columnDetails, session: session.driver.settledLexicalFacts ) } else { - countQuery = SQLBuilder.buildCount(table: table.name, schema: schema, type: databaseType) + countQuery = SQLBuilder.buildCount( + table: table.name, schema: schema, database: database, type: databaseType + ) } let countResult = try await session.driver.execute(query: countQuery) - guard isCurrent(generation) else { return } + guard epoch == countEpoch else { return } if let firstRow = countResult.rows.first, let firstCol = firstRow.first { pagination.totalRows = Int(firstCol ?? "0") } @@ -416,10 +468,10 @@ final class DataBrowserViewModel { await navigatePage() } + // The page size and the order cannot change how many rows there are, so the total stays. func changePageSize(_ size: Int) async { pagination.pageSize = size pagination.currentPage = 0 - pagination.totalRows = nil await navigatePage() } @@ -431,7 +483,6 @@ final class DataBrowserViewModel { func applySort() async { pagination.currentPage = 0 - pagination.totalRows = nil await load() } @@ -440,7 +491,7 @@ final class DataBrowserViewModel { activeSearchText = trimmed guard !trimmed.isEmpty, !columns.isEmpty else { return } pagination.currentPage = 0 - pagination.totalRows = nil + resetTotal() searchTask?.cancel() let task = Task { [weak self] in guard let self else { return } @@ -453,7 +504,7 @@ final class DataBrowserViewModel { func clearSearch() async { activeSearchText = "" pagination.currentPage = 0 - pagination.totalRows = nil + resetTotal() searchTask?.cancel() let task = Task { [weak self] in guard let self else { return } @@ -465,17 +516,22 @@ final class DataBrowserViewModel { func applyFilters() async { pagination.currentPage = 0 - pagination.totalRows = nil + resetTotal() await load() } func clearFilters() async { filters.removeAll() pagination.currentPage = 0 - pagination.totalRows = nil + resetTotal() await load() } + private func resetTotal() { + pagination.totalRows = nil + countEpoch += 1 + } + // MARK: - Row Operations private static var rowNotDeleted: AppError { @@ -488,9 +544,34 @@ final class DataBrowserViewModel { ) } + // Where the rows on screen were read. Another window's switch re-attaches the browser to another schema or + // database, where the same key names another row. + struct RowSource: Equatable { + let driver: ObjectIdentifier? + let schema: String? + let database: String? + } + + var rowSource: RowSource { + RowSource(driver: session.map { ObjectIdentifier($0.driver) }, schema: schema, database: database) + } + + func refusesDelete(raisedOn source: RowSource) -> Bool { + guard source != rowSource else { return false } + operationError = AppError( + category: .query, + title: String(localized: "Row Not Deleted"), + message: String(localized: "The connection moved to another database or schema. Choose the row again."), + recovery: nil, + underlying: nil + ) + return true + } + func deleteRow(_ locator: RowLocator) async -> Bool { guard let session, let table else { return false } let schema = schema + let database = database if let keys = locator.keyValues, let shared = await SharedRowKey.error( table: table.name, schema: schema, key: keys, columns: columnDetails, type: databaseType, driver: session.driver @@ -510,6 +591,7 @@ final class DataBrowserViewModel { guard let batch = try RowWriteBatch.delete( table: table.name, schema: schema, + database: database, type: databaseType, driver: session.driver, columns: locator.columnNames(of: columns), @@ -522,8 +604,8 @@ final class DataBrowserViewModel { var isGone = deleted > 0 if !isGone, let keys = locator.keyValues, !keys.isEmpty { isGone = try await !rowExists( - table: table.name, schema: schema, primaryKey: keys, locator: locator.documentLocator, - driver: session.driver + table: table.name, schema: schema, database: database, primaryKey: keys, + locator: locator.documentLocator, driver: session.driver ) } else if !isGone, let writer { isGone = try await writer.matchingRowCount( @@ -574,6 +656,7 @@ final class DataBrowserViewModel { private func rowExists( table: String, schema: String?, + database: String?, primaryKey: [(column: String, value: String)], locator: String?, driver: any DatabaseDriver @@ -581,21 +664,28 @@ final class DataBrowserViewModel { let components = primaryKey.map { PrimaryKeyComponent(column: $0.column, value: $0.value) } let ref = CellRef(table: table, column: components[0].column, primaryKey: components, locator: locator) do { - _ = try await readValue(driver: driver, ref: ref, databaseType: databaseType, schema: schema, ofOneRow: false) + _ = try await readValue( + driver: driver, ref: ref, databaseType: databaseType, schema: schema, database: database, ofOneRow: false + ) return true } catch CellLookupError.rowNotFound { return false } } - // The caller passes the schema its rows came from: a schema switch while a read is in flight must not move it. - func loadFullValue(driver: DatabaseDriver, ref: CellRef, databaseType: DatabaseType, schema: String?) async throws -> String? { - try await readValue(driver: driver, ref: ref, databaseType: databaseType, schema: schema, ofOneRow: true) + // The caller passes the schema and database its rows came from: a switch while a read is in flight must not move it. + func loadFullValue( + driver: DatabaseDriver, ref: CellRef, databaseType: DatabaseType, schema: String?, database: String? = nil + ) async throws -> String? { + try await readValue( + driver: driver, ref: ref, databaseType: databaseType, schema: schema, database: database, ofOneRow: true + ) } // A full value is read only from a key that names one row; whether a deleted row is still there is not. private func readValue( - driver: DatabaseDriver, ref: CellRef, databaseType: DatabaseType, schema: String?, ofOneRow: Bool + driver: DatabaseDriver, ref: CellRef, databaseType: DatabaseType, schema: String?, database: String?, + ofOneRow: Bool ) async throws -> String? { if let builder = driver as? any DriverQueryBuilding { return try await builder.storedValue( @@ -606,7 +696,7 @@ final class DataBrowserViewModel { locator: ref.locator ) } - let table = SQLBuilder.qualifiedIdentifier(table: ref.table, schema: schema, for: databaseType) + let table = SQLBuilder.qualifiedIdentifier(table: ref.table, schema: schema, database: database, for: databaseType) let predicate = SQLBuilder.keyPredicate( ref.primaryKey.map { (column: $0.column, value: $0.value) }, columns: columnDetails, @@ -652,6 +742,7 @@ final class DataBrowserViewModel { throw ClipboardExporter.TooLarge() } let schema = schema + let database = database var complete: [[Cell]] = [] complete.reserveCapacity(rows.count) for cells in rows { @@ -659,7 +750,9 @@ final class DataBrowserViewModel { for (index, cell) in cells.enumerated() { guard case .truncatedText(_, _, let ref?) = cell else { continue } try Task.checkCancellation() - let value = try await loadFullValue(driver: driver, ref: ref, databaseType: databaseType, schema: schema) + let value = try await loadFullValue( + driver: driver, ref: ref, databaseType: databaseType, schema: schema, database: database + ) row[index] = value.map(Cell.text) ?? .null } complete.append(row) diff --git a/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift index 9d4ee99662..461c4b6539 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift @@ -81,7 +81,9 @@ final class ForeignKeyPreviewModel { // The key columns' types pick each literal, as they do for the filter Open Table runs. var keyColumns: [ColumnInfo] = [] do { - keyColumns = try await driver.fetchColumns(table: reference.table, schema: reference.schema) + keyColumns = try await driver.fetchColumns( + table: reference.table, schema: reference.schema, database: reference.database + ) } catch is CancellationError { return } catch { @@ -98,6 +100,7 @@ final class ForeignKeyPreviewModel { let query = SQLBuilder.buildSingleRowLookup( table: reference.table, schema: reference.schema, + database: reference.database, key: key, columns: keyColumns, type: databaseType, @@ -126,12 +129,17 @@ final class ForeignKeyPreviewModel { var foreignKeys: [ForeignKeyInfo] = [] do { - foreignKeys = try await driver.fetchForeignKeys(table: reference.table, schema: reference.schema) + foreignKeys = try await driver.fetchForeignKeys( + table: reference.table, schema: reference.schema, database: reference.database + ) } catch { Self.logger.warning("Foreign key preview could not read keys: \(error.localizedDescription, privacy: .private)") } - fields = Self.fields(columns: result.columns, row: row, foreignKeys: foreignKeys, schema: reference.schema) + fields = Self.fields( + columns: result.columns, row: row, foreignKeys: foreignKeys, schema: reference.schema, + database: reference.database + ) phase = .loaded } @@ -140,7 +148,8 @@ final class ForeignKeyPreviewModel { columns: [ColumnInfo], row: [String?], foreignKeys: [ForeignKeyInfo], - schema: String? + schema: String?, + database: String? ) -> [Field] { let options = StreamOptions(textTruncationBytes: .max) let cells = row.indices.map { index in @@ -160,7 +169,8 @@ final class ForeignKeyPreviewModel { foreignKeys: foreignKeys, columns: columns, cells: cells, - sourceSchema: schema + sourceSchema: schema, + sourceDatabase: database ) ) } diff --git a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift index 0b8a94ef79..c72841ce51 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift @@ -47,12 +47,14 @@ final class QueryEditorViewModel { private static let budgetCheckInterval = 500 private let buffer: StreamingResultBuffer + private let textBudgetBytes: @MainActor () -> Int private(set) var phase: Phase = .idle private(set) var executionTime: TimeInterval = 0 private(set) var requestCount = 0 private(set) var currentRequest = 0 private(set) var rowsAffectedTotal = 0 private(set) var interruption: ScriptInterruption? + private(set) var cutLongValues = false // Each request as it went to the server, split again after a statement changed how the session reads literals. @ObservationIgnored private(set) var sentStatements: [String] = [] @ObservationIgnored private(set) var ranTransactionRollBack = false @@ -61,14 +63,18 @@ final class QueryEditorViewModel { @ObservationIgnored private var runID: UInt64 = 0 @ObservationIgnored private var fetchTask: Task? @ObservationIgnored private var startedAt: Date? + @ObservationIgnored private var requestBudget: TextBudget? - init(windowCapacity: Int = QueryEditorViewModel.maxBufferedRows) { + init( + windowCapacity: Int = QueryEditorViewModel.maxBufferedRows, + textBudgetBytes: @escaping @MainActor () -> Int = { MemoryPressureMonitor.shared.textBudgetBytes() } + ) { self.buffer = StreamingResultBuffer(capacity: windowCapacity) + self.textBudgetBytes = textBudgetBytes } var columns: [ColumnInfo] { buffer.columns } var window: RowWindow { buffer.window } - var legacyRows: [[String?]] { buffer.legacyRows } var statusMessage: String? { buffer.statusMessage } var truncationReason: TruncationReason? { buffer.truncation } @@ -88,7 +94,7 @@ final class QueryEditorViewModel { var truncationMessage: String? { guard let truncationReason else { return nil } - let shown = buffer.legacyRows.count + let shown = buffer.count switch truncationReason { case .rowCap: return String(format: String(localized: "Showing the first %d rows. Add LIMIT to fetch more."), shown) @@ -101,6 +107,51 @@ final class QueryEditorViewModel { } } + var longValuesNotice: String? { + cutLongValues ? String(localized: "Some long values are shown in part to stay within memory.") : nil + } + + // Formats off the main actor: 10,000 rows of JSON take long enough to hold up the screen. + func exportResults( + as format: ExportFormat, + databaseType: DatabaseType, + driver: (any DatabaseDriver)?, + availableBytes: Int + ) async throws -> String { + try await Self.export( + rows: buffer.window.rows, columns: buffer.columns, format: format, + databaseType: databaseType, driver: driver, availableBytes: availableBytes + ) + } + + @concurrent + nonisolated private static func export( + rows: [Row], + columns: [ColumnInfo], + format: ExportFormat, + databaseType: DatabaseType, + driver: (any DatabaseDriver)?, + availableBytes: Int + ) async throws -> String { + guard resultsFitExport(rows: rows, availableBytes: availableBytes) else { + throw ClipboardExporter.TooLarge(source: .queryResult) + } + return try ClipboardExporter.exportRows( + columns: columns, rows: rows.map(\.cells), format: format, databaseType: databaseType, driver: driver + ) + } + + // The export text and the pasteboard's copy of it are held beside the values already on screen. + nonisolated static func resultsFitExport(rows: [Row], availableBytes: Int) -> Bool { + let textBytes = rows.reduce(0) { total, row in + row.cells.reduce(total) { sum, cell in + guard case .text(let value) = cell else { return sum } + return sum + value.utf8.count + } + } + return DataBrowserViewModel.exportFits(pendingBytes: textBytes, availableBytes: availableBytes) + } + var progressMessage: String { guard requestCount > 1 else { return String(localized: "Executing...") } switch unitKind { @@ -216,13 +267,19 @@ final class QueryEditorViewModel { return error.message } - func run(driver: DatabaseDriver, script: QueryScript, maxRows: Int = QueryEditorViewModel.maxBufferedRows) async { + func run( + driver: DatabaseDriver, + script: QueryScript, + maxRows: Int = QueryEditorViewModel.maxBufferedRows, + enforcesReadOnly: Bool = false + ) async { fetchTask?.cancel() let options = StreamOptions( textTruncationBytes: 4_096, inlineBinary: false, maxRows: maxRows, - lazyContext: nil + lazyContext: nil, + enforcesReadOnly: enforcesReadOnly ) runID &+= 1 let run = runID @@ -232,6 +289,8 @@ final class QueryEditorViewModel { executionTime = 0 rowsAffectedTotal = 0 interruption = nil + requestBudget = nil + cutLongValues = false ranTransactionRollBack = false unitKind = script.unitKind requestCount = script.requestCount @@ -259,6 +318,8 @@ final class QueryEditorViewModel { executionTime = 0 rowsAffectedTotal = 0 interruption = nil + requestBudget = nil + cutLongValues = false ranTransactionRollBack = false requestCount = 0 currentRequest = 0 @@ -351,6 +412,7 @@ final class QueryEditorViewModel { guard run == runID else { return } buffer.flush() finalizeTiming() + settleLongValues() resolvePhase() } @@ -370,8 +432,13 @@ final class QueryEditorViewModel { /// False when the request was cut short: by Stop, by memory pressure, or by another query that took over the /// connection, whose driver then ends this stream as cancelled. private func stream(_ sql: String, driver: DatabaseDriver, options: StreamOptions) async throws -> Bool { + // Only this request's rows stay on screen, so it gets a budget of its own, sized from the headroom now. + let budget = TextBudget(bytes: textBudgetBytes()) + requestBudget = budget + var requestOptions = options + requestOptions.textBudget = budget var sinceBudgetCheck = 0 - for try await element in driver.executeStreaming(query: sql, options: options) { + for try await element in driver.executeStreaming(query: sql, options: requestOptions) { if Task.isCancelled { return false } buffer.apply(element) guard case .row = element else { continue } @@ -393,6 +460,7 @@ final class QueryEditorViewModel { guard run == runID else { return } buffer.flush() finalizeTiming() + settleLongValues() let reason = buffer.truncation ?? .cancelled if requestCount > 1 { let cause: ScriptInterruption.Cause @@ -424,6 +492,7 @@ final class QueryEditorViewModel { guard run == runID else { return } buffer.flush() finalizeTiming() + settleLongValues() if requestCount > 1 { interruption = ScriptInterruption( cause: .failed(sql: sql), @@ -462,6 +531,10 @@ final class QueryEditorViewModel { } } + private func settleLongValues() { + cutLongValues = requestBudget?.hasRefused ?? false + } + private func finalizeTiming() { if let startedAt { executionTime = Date().timeIntervalSince(startedAt) diff --git a/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift index 224de52ca0..bdf4e73f61 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift @@ -11,13 +11,16 @@ final class RowDetailViewModel { let columns: [ColumnInfo] let columnDetails: [ColumnInfo] - let foreignKeys: [ForeignKeyInfo] + // Read when shown: the browser reads the keys after its rows appear, so a row opened first gets them too. + @ObservationIgnored private let readForeignKeys: () -> [ForeignKeyInfo] + var foreignKeys: [ForeignKeyInfo] { readForeignKeys() } let table: TableInfo? // Decides what the row is and how it is located; writes go to `session`, which follows a redial. private let openedSession: ConnectionSession? @ObservationIgnored private let liveSession: (() -> Result)? let databaseType: DatabaseType let schema: String? + let database: String? @ObservationIgnored private let readSafeModeLevel: () -> SafeModeLevel private(set) var rows: [Row] @@ -46,8 +49,9 @@ final class RowDetailViewModel { columnDetails: [ColumnInfo] = [], databaseType: DatabaseType = .sqlite, schema: String? = nil, + database: String? = nil, safeModeLevel: @escaping () -> SafeModeLevel = { .off }, - foreignKeys: [ForeignKeyInfo] = [], + foreignKeys: @escaping @autoclosure () -> [ForeignKeyInfo] = [], onSaved: (() -> Void)? = nil, loadFullValue: ((CellRef) async throws -> String?)? = nil, liveSession: (() -> Result)? = nil @@ -61,8 +65,9 @@ final class RowDetailViewModel { self.columnDetails = columnDetails self.databaseType = databaseType self.schema = schema + self.database = database self.readSafeModeLevel = safeModeLevel - self.foreignKeys = foreignKeys + self.readForeignKeys = foreignKeys self.onSaved = onSaved self.loadFullValueProvider = loadFullValue } @@ -156,6 +161,27 @@ final class RowDetailViewModel { } } + // Past this the row shows an excerpt, and the whole value opens on a screen of its own. + static let inlineByteLimit = Cell.previewByteLimit + + static func excerpt(of value: String) -> String { + Cell.preview(of: value, maxBytes: inlineByteLimit) + } + + func longValue(forRow rowIndex: Int, at index: Int) -> String? { + guard let value = completeValue(forRow: rowIndex, at: index), value.utf8.count > Self.inlineByteLimit else { + return nil + } + return value + } + + // Cut with no key to read it again by: a query result past its memory budget, or a table without a primary key. + func isShownInPart(forRow rowIndex: Int, at index: Int) -> Bool { + let rowCells = cells(at: rowIndex) + guard rowCells.indices.contains(index), case .truncatedText(_, _, .none) = rowCells[index] else { return false } + return true + } + // MariaDB's catalog names a JSON column longtext while its result column says JSON, so either type decides. func isJSON(at index: Int) -> Bool { guard index >= 0, index < columns.count else { return false } @@ -409,6 +435,7 @@ final class RowDetailViewModel { let write = PlannedRowWrite.update( table: table.name, schema: schema, + database: database, type: databaseType, columns: locator.columnNames(of: columns), columnDetails: columnDetails, diff --git a/apple/TableProMobile/TableProMobile/Views/Components/LiteralInputTraits.swift b/apple/TableProMobile/TableProMobile/Views/Components/LiteralInputTraits.swift index 7ebbf7c185..dcd3b61333 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/LiteralInputTraits.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/LiteralInputTraits.swift @@ -28,3 +28,16 @@ extension UITextView { writingToolsBehavior = .none } } + +// The bar keeps traits of its own beside its field's, so both are set. +extension UISearchBar { + func applyLiteralInputTraits() { + autocorrectionType = .no + autocapitalizationType = .none + spellCheckingType = .no + smartQuotesType = .no + smartDashesType = .no + smartInsertDeleteType = .no + searchTextField.applyLiteralInputTraits() + } +} diff --git a/apple/TableProMobile/TableProMobile/Views/Components/LiteralSearchField.swift b/apple/TableProMobile/TableProMobile/Views/Components/LiteralSearchField.swift new file mode 100644 index 0000000000..89923ad7a2 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Views/Components/LiteralSearchField.swift @@ -0,0 +1,92 @@ +import SwiftUI +import UIKit + +// A search the server runs with LIKE: SwiftUI's `.searchable` field cannot turn smart quotes off, and Smart Punctuation +// would send `O’Brien` for `O'Brien`. +struct LiteralSearchField: UIViewRepresentable { + private let prompt: String + @Binding private var text: String + @Binding private var isEditing: Bool + private let onSubmit: () -> Void + + init(_ prompt: LocalizedStringResource, text: Binding, isEditing: Binding, onSubmit: @escaping () -> Void) { + self.prompt = String(localized: prompt) + self._text = text + self._isEditing = isEditing + self.onSubmit = onSubmit + } + + static func configure(_ bar: UISearchBar) { + bar.searchBarStyle = .minimal + bar.returnKeyType = .search + bar.applyLiteralInputTraits() + } + + func makeUIView(context: Context) -> UISearchBar { + let bar = UISearchBar() + Self.configure(bar) + bar.delegate = context.coordinator + bar.placeholder = prompt + bar.text = text + return bar + } + + func updateUIView(_ bar: UISearchBar, context: Context) { + context.coordinator.field = self + // Text being composed is the input method's until it is committed. + if bar.searchTextField.markedTextRange == nil, bar.text != text { + bar.text = text + } + if bar.placeholder != prompt { + bar.placeholder = prompt + } + bar.isUserInteractionEnabled = context.environment.isEnabled + // Focusing sends the delegate's begin-editing call, which writes `isEditing`; not while SwiftUI is updating. + if isEditing, !bar.searchTextField.isFirstResponder { + DispatchQueue.main.async { bar.searchTextField.becomeFirstResponder() } + } + } + + func sizeThatFits(_ proposal: ProposedViewSize, uiView bar: UISearchBar, context: Context) -> CGSize? { + guard let width = proposal.width, width.isFinite else { return nil } + return CGSize(width: width, height: bar.intrinsicContentSize.height) + } + + func makeCoordinator() -> Coordinator { + Coordinator(field: self) + } + + final class Coordinator: NSObject, UISearchBarDelegate { + var field: LiteralSearchField + + init(field: LiteralSearchField) { + self.field = field + } + + func searchBar(_ searchBar: UISearchBar, textDidChange searchText: String) { + field.text = searchText + } + + func searchBarTextDidBeginEditing(_ searchBar: UISearchBar) { + searchBar.setShowsCancelButton(true, animated: true) + field.isEditing = true + } + + func searchBarTextDidEndEditing(_ searchBar: UISearchBar) { + searchBar.setShowsCancelButton(false, animated: true) + field.isEditing = false + } + + func searchBarSearchButtonClicked(_ searchBar: UISearchBar) { + field.onSubmit() + searchBar.resignFirstResponder() + } + + // As the system search does, Cancel clears the search and ends it. + func searchBarCancelButtonClicked(_ searchBar: UISearchBar) { + searchBar.text = "" + field.text = "" + searchBar.resignFirstResponder() + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift b/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift index 3e6b56d507..08b338c458 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift @@ -103,6 +103,7 @@ final class QuerySuggestionBar: UIView { case .alias: String(localized: "Alias") case .operator: String(localized: "Operator") case .favorite: String(localized: "Favorite") + case .joinCondition: String(localized: "Join Condition") } } } diff --git a/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift b/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift index 90f1c2adea..ce04f75535 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift @@ -88,11 +88,11 @@ nonisolated struct RowGridViewport: Equatable, Sendable { let leadingInset: CGFloat } -struct RowGrid: View { +struct RowGrid: View { let columns: [ColumnInfo] - let rows: [[String?]] + let window: RowWindow let keyColumn: Int? - @ViewBuilder let row: (_ index: Int, _ line: RowGridLine) -> Row + @ViewBuilder let row: (_ index: Int, _ line: RowGridLine) -> Line @Environment(\.dynamicTypeSize) private var dynamicTypeSize @Environment(\.displayScale) private var displayScale @@ -103,9 +103,12 @@ struct RowGrid: View { @State private var computedLayout: (key: LayoutKey, layout: RowGridLayout)? @State private var viewport = RowGridViewport(width: 0, leadingInset: 0) + private var rows: [Row] { window.rows } + private var layoutKey: LayoutKey { LayoutKey( columnNames: columns.map(\.name), + sampleIdentity: window.headIdentity(count: RowGridLayout.sampledRowCount), sample: rows.prefix(RowGridLayout.sampledRowCount), keyColumn: keyColumn, dynamicTypeSize: dynamicTypeSize, @@ -125,7 +128,8 @@ struct RowGrid: View { } var body: some View { - let layout = currentLayout.limitingKeyColumn(toViewportWidth: viewport.width) + let key = layoutKey + let layout = currentLayout(for: key).limitingKeyColumn(toViewportWidth: viewport.width) ScrollView([.horizontal, .vertical]) { grid(layout, width: max(layout.totalWidth, viewport.width)) } @@ -140,14 +144,13 @@ struct RowGrid: View { viewport = newViewport } // Reads only the key: the closure's other captures belong to the previous body. - .onChange(of: layoutKey, initial: true) { _, key in + .onChange(of: key, initial: true) { _, key in computedLayout = (key, Self.makeLayout(key)) } } // Until onChange stores the layout for these inputs, it is measured here, so no pass draws an empty grid. - private var currentLayout: RowGridLayout { - let key = layoutKey + private func currentLayout(for key: LayoutKey) -> RowGridLayout { if let computedLayout, computedLayout.key == key { return computedLayout.layout } return Self.makeLayout(key) } @@ -156,7 +159,7 @@ struct RowGrid: View { let measure = RowGridTextMeasure(dynamicTypeSize: key.dynamicTypeSize) return RowGridLayout.make( columnNames: key.columnNames, - rows: key.sample, + rows: key.sample.map { $0.displayValues() }, keyColumn: key.keyColumn, metrics: key.metrics, measure: measure.width(of:isHeader:) @@ -167,9 +170,10 @@ struct RowGrid: View { let style = style return LazyVStack(alignment: .leading, spacing: 0, pinnedViews: [.sectionHeaders]) { Section { - ForEach(IndexedRow.wrap(rows)) { item in - row(item.id, RowGridLine( - columns: columns, values: item.values, layout: layout, width: width, style: style + // Indices of the same array the closure reads, so a result that shrank cannot put one out of range. + ForEach(rows.indices, id: \.self) { index in + row(index, RowGridLine( + columns: columns, values: rows[index].displayValues(), layout: layout, width: width, style: style )) } } header: { @@ -202,12 +206,20 @@ struct RowGrid: View { .accessibilityAddTraits(.isHeader) } + // The sample rows are compared by where they sit in the window, so a body pass reads none of their text. private struct LayoutKey: Equatable { let columnNames: [String] - let sample: ArraySlice<[String?]> + let sampleIdentity: RowWindow.HeadIdentity + let sample: ArraySlice let keyColumn: Int? let dynamicTypeSize: DynamicTypeSize let metrics: RowGridMetrics + + static func == (lhs: LayoutKey, rhs: LayoutKey) -> Bool { + lhs.columnNames == rhs.columnNames && lhs.sampleIdentity == rhs.sampleIdentity + && lhs.keyColumn == rhs.keyColumn && lhs.dynamicTypeSize == rhs.dynamicTypeSize + && lhs.metrics == rhs.metrics + } } } diff --git a/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift b/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift index 6b87d7f96f..2fa227aa8a 100644 --- a/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift +++ b/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift @@ -86,9 +86,13 @@ struct ConnectedView: View { guard !Task.isCancelled else { return } switch resolved.phase { case .connected: - resolved.loadHistory() + await resolved.loadHistory() hapticSuccess.toggle() - case .failed, .lost: + case .lost: + // A lost open comes back to the tabs through Reconnect, which never runs this task. + await resolved.loadHistory() + hapticError.toggle() + case .failed: hapticError.toggle() case .notConnected, .connecting: break @@ -123,24 +127,32 @@ struct ConnectedView: View { case .notConnected: statusScreen { notConnectedView } case .failed(let error): - statusScreen { unavailableView(error, title: nil, systemImage: nil) } + statusScreen { unavailableView(error, title: nil, systemImage: nil) { retries += 1 } } case .lost(let error): let presentation = ConnectionStatusPresentation(coordinator.displayStatus) statusScreen { - unavailableView(error, title: presentation.title, systemImage: presentation.symbol?.systemName) + // The toolbar's Reconnect: a redial where the window was, not a fresh open of the saved database. + unavailableView(error, title: presentation.title, systemImage: presentation.symbol?.systemName) { + await coordinator.reconnect() + } } case .tabs: connectedContent(coordinator) } } - private func unavailableView(_ error: AppError, title: String?, systemImage: String?) -> some View { + private func unavailableView( + _ error: AppError, + title: String?, + systemImage: String?, + retry: @escaping () async -> Void + ) -> some View { ErrorView( error: error, title: title, systemImage: systemImage, retryTitle: String(localized: "Reconnect"), - onRetry: error.canRetry ? { retries += 1 } : nil, + onRetry: error.canRetry ? retry : nil, onEditConnection: connection.isSample ? nil : { presenter.presentConnectionEditor(for: connection.id) } ) } diff --git a/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift b/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift index 80a1d76327..a73e67880b 100644 --- a/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift +++ b/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift @@ -22,7 +22,7 @@ struct DataBrowserView: View { // Not persisted. This search runs on the server and is applied on submit, so a value restored // into the field would name a filter the rows on screen were never fetched under. @State private var searchText = "" - @FocusState private var searchFocused: Bool + @State private var searchFocused = false @State private var showInsertSheet = false @State private var showFilterSheet = false @State private var showShareSheet = false @@ -31,7 +31,7 @@ struct DataBrowserView: View { @State private var showMatchedRow = false @State private var showGoToPage = false @State private var goToPageInput = "" - @State private var deleteTarget: RowLocator? + @State private var deleteTarget: PendingRowDelete? @State private var fkPreviewItem: FKPreviewItem? @State private var shareText = "" @State private var isCompletingCells = false @@ -40,7 +40,7 @@ struct DataBrowserView: View { @AppStorage(AppPreferences.rowLayoutKey) private var preferredLayout: RowLayout? private var activeSchema: String? { - coordinator.supportsSchemas ? coordinator.activeSchema : nil + coordinator.tableQualifier } private var browserSchema: String? { @@ -48,6 +48,12 @@ struct DataBrowserView: View { return activeSchema } + // A SQL Server foreign key never leaves its database, so a referenced table is read where the key was. + private var browserDatabase: String? { + if let reference { return reference.database } + return coordinator.tableDatabase + } + private var displaySchema: String? { reference?.displaySchema(current: activeSchema) } @@ -70,7 +76,7 @@ struct DataBrowserView: View { } private var columns: [ColumnInfo] { viewModel.columns } - private var rows: [[String?]] { viewModel.legacyRows } + private var rows: [Row] { viewModel.window.rows } private var pageReads: DataBrowserViewModel.SessionReadTracker { let viewModel = viewModel @@ -124,23 +130,27 @@ struct DataBrowserView: View { .task { viewModel.attach( session: session, table: table, databaseType: connection.type, - host: connection.host, schema: browserSchema, trackRead: pageReads + host: connection.host, schema: browserSchema, database: browserDatabase, trackRead: pageReads ) if let filters = reference?.filters() { viewModel.seedFilters(filters) } await viewModel.load(isInitial: true) - guard reference != nil, !Task.isCancelled else { return } - if viewModel.takeSingleRowMatch() { - showMatchedRow = true - } } - // A redial or another window's database switch hands this table a new session; a referenced table keeps its schema. - .onChange(of: BrowseSource(session: session, schema: browserSchema)) { _, source in + // Pushed when the one row lands, not after the key read and the count that follow it. + .onChange(of: viewModel.hasSingleRowMatch) { _, found in + guard found, reference != nil, viewModel.takeSingleRowMatch() else { return } + showMatchedRow = true + } + // A redial or another window's switch hands this table a new session; a referenced table keeps its schema and database. + .onChange(of: BrowseSource(session: session, schema: browserSchema, database: browserDatabase)) { _, source in + // The confirmation names a row of the table this browser showed before. + showDeleteConfirmation = false + deleteTarget = nil guard source.driver != nil else { return } viewModel.attach( session: session, table: table, databaseType: connection.type, - host: connection.host, schema: source.schema, trackRead: pageReads + host: connection.host, schema: source.schema, database: source.database, trackRead: pageReads ) Task { await viewModel.load(isInitial: true) } } @@ -214,7 +224,10 @@ struct DataBrowserView: View { .sensoryFeedback(.success, trigger: hapticSuccess) .sensoryFeedback(.error, trigger: hapticError) .navigationDestination(isPresented: $showStructure) { - StructureView(table: table, session: session, databaseType: connection.type, schema: viewModel.schema) + StructureView( + table: table, session: session, databaseType: connection.type, schema: viewModel.schema, + database: viewModel.database + ) .transactionBar(for: coordinator) } .navigationDestination(isPresented: $showMatchedRow) { @@ -249,11 +262,12 @@ struct DataBrowserView: View { content .tableTitle(table.name, schema: displaySchema) .navigationBarTitleDisplayMode(.inline) - .searchable(text: $searchText, prompt: "Search all columns") - .searchFocused($searchFocused) - .textInputAutocapitalization(.never) - .onSubmit(of: .search) { - Task { await viewModel.applySearch(searchText) } + .safeAreaInset(edge: .top, spacing: 0) { + LiteralSearchField("Search all columns", text: $searchText, isEditing: $searchFocused) { + Task { await viewModel.applySearch(searchText) } + } + .padding(.horizontal, 8) + .background(.bar) } .onChange(of: searchText) { oldValue, newValue in if newValue.isEmpty, !oldValue.isEmpty, viewModel.hasActiveSearch { @@ -279,6 +293,17 @@ struct DataBrowserView: View { } else if rows.isEmpty, viewModel.isPageLoading { ProgressView() .frame(maxWidth: .infinity, maxHeight: .infinity) + } else if viewModel.isPastTheEnd { + ContentUnavailableView { + Label("No Rows on This Page", systemImage: "tray") + } description: { + Text("This page is past the last row.") + } actions: { + Button("Go to First Page") { + Task { await viewModel.goToPage(1) } + } + .buttonStyle(.borderedProminent) + } } else if rows.isEmpty, viewModel.hasActiveSearch { ContentUnavailableView.search(text: viewModel.activeSearchText) } else if rows.isEmpty { @@ -312,10 +337,10 @@ struct DataBrowserView: View { } private var rowList: some View { - let indexed = IndexedRow.wrap(rows) + let rows = rows return List { - ForEach(indexed) { item in - rowLink(index: item.id, row: item.values) + ForEach(rows.indices, id: \.self) { index in + rowLink(index: index, row: rows[index].displayValues()) } } .listStyle(.plain) @@ -325,7 +350,7 @@ struct DataBrowserView: View { let primaryKeys = primaryKeyNames return RowGrid( columns: columns, - rows: rows, + window: viewModel.window, keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: primaryKeys) ) { index, line in NavigationLink { @@ -341,7 +366,7 @@ struct DataBrowserView: View { RowCard( columns: columns, columnDetails: viewModel.columnDetails, - row: rows.indices.contains(index) ? rows[index] : [], + row: rows.indices.contains(index) ? rows[index].displayValues() : [], previewFieldCount: DuoLayoutResolver.regularPreviewFieldCount ) .padding() @@ -366,7 +391,8 @@ struct DataBrowserView: View { } private func rowDetail(index: Int) -> some View { - RowDetailView( + let viewModel = viewModel + return RowDetailView( columns: columns, rows: viewModel.window.rows, initialIndex: index, @@ -375,15 +401,17 @@ struct DataBrowserView: View { columnDetails: viewModel.columnDetails, databaseType: connection.type, schema: viewModel.schema, + database: viewModel.database, safeModeLevel: { [coordinator] in coordinator.connection.safeModeLevel }, foreignKeys: viewModel.foreignKeys, onSaved: { Task { await viewModel.load() } }, - loadFullValue: { [schema = viewModel.schema] ref in + loadFullValue: { [schema = viewModel.schema, database = viewModel.database] ref in guard let session else { return nil } // With autocommit off, a MySQL read opens a transaction; a failed read refreshes too, and never redials. do { let value = try await viewModel.loadFullValue( - driver: session.driver, ref: ref, databaseType: connection.type, schema: schema + driver: session.driver, ref: ref, databaseType: connection.type, schema: schema, + database: database ) await coordinator.refreshTransaction() return value @@ -435,7 +463,7 @@ struct DataBrowserView: View { } private func confirmDelete(_ locator: RowLocator) { - deleteTarget = locator + deleteTarget = PendingRowDelete(locator: locator, source: viewModel.rowSource) showDeleteConfirmation = true } @@ -446,12 +474,8 @@ struct DataBrowserView: View { Menu("Share Row") { ForEach(ExportFormat.offered(for: connection.type)) { format in Button(format.rawValue) { - exportCompleted([cells]) { exportColumns, completed in - shareText = try ClipboardExporter.exportRow( - columns: exportColumns, cells: completed[0], - format: format, tableName: table.name, - databaseType: connection.type, driver: session?.driver - ) + exportCompleted([cells], asOneRow: true, format: format) { text in + shareText = text showShareSheet = true } } @@ -461,12 +485,7 @@ struct DataBrowserView: View { Menu("Copy Row") { ForEach(ExportFormat.offered(for: connection.type)) { format in Button(format.rawValue) { - exportCompleted([cells]) { exportColumns, completed in - let text = try ClipboardExporter.exportRow( - columns: exportColumns, cells: completed[0], - format: format, tableName: table.name, - databaseType: connection.type, driver: session?.driver - ) + exportCompleted([cells], asOneRow: true, format: format) { text in ClipboardExporter.copyToClipboard(text) } } @@ -476,7 +495,7 @@ struct DataBrowserView: View { } let references = ForeignKeyReference.all( foreignKeys: viewModel.foreignKeys, columns: columns, - cells: cells, sourceSchema: viewModel.schema + cells: cells, sourceSchema: viewModel.schema, sourceDatabase: viewModel.database ) if !references.isEmpty { Divider() @@ -512,12 +531,7 @@ struct DataBrowserView: View { Section("Export") { ForEach(ExportFormat.offered(for: connection.type)) { format in Button { - exportCompleted(viewModel.window.rows.map(\.cells)) { exportColumns, completed in - let text = try ClipboardExporter.exportRows( - columns: exportColumns, rows: completed, - format: format, tableName: table.name, - databaseType: connection.type, driver: session?.driver - ) + exportCompleted(viewModel.window.rows.map(\.cells), asOneRow: false, format: format) { text in ClipboardExporter.copyToClipboard(text) } } label: { @@ -632,6 +646,7 @@ struct DataBrowserView: View { session: session, databaseType: connection.type, schema: viewModel.schema, + database: viewModel.database, safeModeLevel: { [coordinator] in coordinator.connection.safeModeLevel }, onInserted: { Task { await viewModel.load() } } ) @@ -640,11 +655,15 @@ struct DataBrowserView: View { // Rows that cannot all be completed skip the reads: the exporter refuses them anyway. private func exportCompleted( _ pageRows: [[Cell]], - export: @escaping @MainActor ([ColumnInfo], [[Cell]]) throws -> Void + asOneRow: Bool, + format: ExportFormat, + deliver: @escaping (String) -> Void ) { guard !isCompletingCells else { return } let exportColumns = columns let driver = session?.driver + let tableName = table.name + let databaseType = connection.type isCompletingCells = true Task { defer { isCompletingCells = false } @@ -653,7 +672,10 @@ struct DataBrowserView: View { if let driver, pageRows.allSatisfy(DataBrowserViewModel.canComplete) { completed = try await viewModel.completeRows(pageRows, driver: driver) } - try export(exportColumns, completed) + deliver(try await Self.exportText( + completed, columns: exportColumns, asOneRow: asOneRow, format: format, + tableName: tableName, databaseType: databaseType, driver: driver + )) } catch let error where error is ClipboardExporter.IncompleteValues || error is ClipboardExporter.TooLarge || error is CellLookupError { viewModel.operationError = AppError( @@ -674,7 +696,35 @@ struct DataBrowserView: View { } } - private func performDelete(_ locator: RowLocator) async { + // Off the main actor, so a page of 500 rows formats without holding up the screen. + @concurrent + nonisolated private static func exportText( + _ rows: [[Cell]], + columns: [ColumnInfo], + asOneRow: Bool, + format: ExportFormat, + tableName: String, + databaseType: DatabaseType, + driver: (any DatabaseDriver)? + ) async throws -> String { + if asOneRow, let cells = rows.first { + return try ClipboardExporter.exportRow( + columns: columns, cells: cells, format: format, tableName: tableName, + databaseType: databaseType, driver: driver + ) + } + return try ClipboardExporter.exportRows( + columns: columns, rows: rows, format: format, tableName: tableName, + databaseType: databaseType, driver: driver + ) + } + + private func performDelete(_ pending: PendingRowDelete) async { + guard !viewModel.refusesDelete(raisedOn: pending.source) else { + hapticError.toggle() + return + } + let locator = pending.locator let success = await coordinator.track(failed: { !$0 }) { await viewModel.deleteRow(locator) } if success { hapticSuccess.toggle() @@ -684,12 +734,19 @@ struct DataBrowserView: View { } } +private struct PendingRowDelete { + let locator: RowLocator + let source: DataBrowserViewModel.RowSource +} + private struct BrowseSource: Equatable { let driver: ObjectIdentifier? let schema: String? + let database: String? - init(session: ConnectionSession?, schema: String?) { + init(session: ConnectionSession?, schema: String?, database: String?) { driver = session.map { ObjectIdentifier($0.driver) } self.schema = schema + self.database = database } } diff --git a/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift b/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift index b6fc1ae4f3..26860ecb69 100644 --- a/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift +++ b/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift @@ -33,8 +33,7 @@ private struct ForeignKeyPreviewModifier: ViewModifier { @State private var openedReference: ForeignKeyReference? private var currentSchema: String? { - guard let coordinator, coordinator.supportsSchemas else { return nil } - return coordinator.activeSchema + coordinator?.tableQualifier } func body(content: Content) -> some View { diff --git a/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift b/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift index cea001a54a..bd92195011 100644 --- a/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift +++ b/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift @@ -8,6 +8,7 @@ struct InsertRowView: View { let session: ConnectionSession? let databaseType: DatabaseType let schema: String? + let database: String? let safeModeLevel: () -> SafeModeLevel var onInserted: (() -> Void)? @@ -36,7 +37,10 @@ struct InsertRowView: View { } private var insertWrite: PlannedRowWrite { - PlannedRowWrite.insert(table: table.name, schema: schema, type: databaseType, columns: columnDetails, values: fields) + PlannedRowWrite.insert( + table: table.name, schema: schema, database: database, type: databaseType, columns: columnDetails, + values: fields + ) } var body: some View { diff --git a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift index b2b420f9ac..c18385660b 100644 --- a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift +++ b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift @@ -25,7 +25,7 @@ struct QueryEditorView: View { @State private var executeTask: Task? private var resultRowCount: Int { - viewModel.legacyRows.count + viewModel.window.count } @State private var draft = QueryDraftSaver() @State private var deferredAdds: [String] = [] @@ -40,11 +40,13 @@ struct QueryEditorView: View { @State private var executionStartTime: Date? @State private var showWriteConfirmation = false @State private var showWriteBlockedAlert = false + @State private var showFunctionBlockedAlert = false @State private var pendingWriteQuery = "" @State private var showClearConfirmation = false @State private var showShareSheet = false @State private var shareText = "" - @State private var resultsExportError: ClipboardExporter.IncompleteValues? + @State private var resultsExportError: (any Error)? + @State private var isExportingResults = false @State private var hapticSuccess = false @State private var hapticError = false @State private var containerSize: CGSize = .zero @@ -112,6 +114,28 @@ struct QueryEditorView: View { } } + private var catalogForeignKeyReader: QueryForeignKeyReader { + let manager = appState.connectionManager + let connectionId = connectionId + let database = coordinator.supportsDatabaseSwitching ? coordinator.activeDatabase : "" + let activeSchema = activeSchema + let coordinator = coordinator + return { table, schema in + guard await coordinator.catalogMatchesSession else { return nil } + let namespace = schema ?? activeSchema + let keys = try await manager.readCatalog(for: connectionId, database: database) { driver in + try await driver.fetchForeignKeys(table: table, schema: namespace) + } + // MySQL, which has no schemas, names a key's database there. A key into another namespace would pair with + // a same-named table of this one. + let home = namespace.flatMap { $0.isEmpty ? nil : $0 } ?? database + return keys?.filter { key in + guard let target = key.referencedSchema, !target.isEmpty else { return true } + return target.caseInsensitiveCompare(home) == .orderedSame + } + } + } + private var catalogTableReader: QuerySchemaTableReader { let manager = appState.connectionManager let connectionId = connectionId @@ -199,7 +223,8 @@ struct QueryEditorView: View { readColumns: sessionColumnReader, readCatalogColumns: catalogColumnReader, readSchemaTables: sessionTableReader, - readCatalogSchemaTables: catalogTableReader + readCatalogSchemaTables: catalogTableReader, + readCatalogForeignKeys: catalogForeignKeyReader ) } .alert(String(localized: "Write Query Blocked"), isPresented: $showWriteBlockedAlert) { @@ -207,9 +232,14 @@ struct QueryEditorView: View { } message: { Text("This connection is in read-only mode. Write queries are not allowed.") } + .alert(String(localized: "Query Blocked"), isPresented: $showFunctionBlockedAlert) { + Button("OK", role: .cancel) {} + } message: { + Text("This query calls a function that can change data, which Read-Only does not run. Switch Safe Mode to \(SafeModeLevel.confirmWrites.displayName) to run it.") + } .confirmationDialog(String(localized: "Execute Write Query?"), isPresented: $showWriteConfirmation, titleVisibility: .visible) { Button(String(localized: "Execute"), role: .destructive) { - executeTask = Task { await executeQueryDirect(pendingWriteQuery) } + executeTask = Task { await executeQueryDirect(pendingWriteQuery, enforcesReadOnly: false) } } } message: { Text("This query will modify data. Are you sure you want to continue?") @@ -380,7 +410,7 @@ struct QueryEditorView: View { } private var canRun: Bool { - !isExecuting && !query.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + !isExecuting && !coordinator.isDatabaseMoving && !query.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } private var runAtCursorTitle: String { @@ -413,9 +443,9 @@ struct QueryEditorView: View { } } .frame(maxWidth: .infinity, maxHeight: .infinity) - } else if viewModel.columns.isEmpty || viewModel.legacyRows.isEmpty && !runCompleted { + } else if viewModel.columns.isEmpty || viewModel.window.isEmpty && !runCompleted { commandOutcomeView - } else if viewModel.legacyRows.isEmpty { + } else if viewModel.window.isEmpty { VStack(spacing: 0) { resultNotices(includesTruncation: false) ContentUnavailableView( @@ -521,6 +551,10 @@ struct QueryEditorView: View { noticeBanner(message, systemImage: "exclamationmark.triangle.fill", tint: .orange) Divider() } + if includesTruncation, let notice = viewModel.longValuesNotice { + noticeBanner(notice, systemImage: "info.circle.fill", tint: .secondary) + Divider() + } if let summary = viewModel.scriptSummary { noticeBanner(summary, systemImage: "checkmark.circle.fill", tint: .green) Divider() @@ -555,14 +589,15 @@ struct QueryEditorView: View { private var resultGrid: some View { let columns = viewModel.columns - let rows = viewModel.legacyRows + let window = viewModel.window + let rows = window.rows return RowGrid( columns: columns, - rows: rows, + window: window, keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: []) ) { index, line in NavigationLink { - RowDetailView(columns: columns, rows: viewModel.window.rows, initialIndex: index) + resultRowDetail(columns: columns, initialIndex: index) } label: { line } @@ -570,7 +605,7 @@ struct QueryEditorView: View { .contextMenu { resultRowContextMenu(columns: columns, rowIndex: index) } preview: { - resultRowCard(columns: columns, row: rows.indices.contains(index) ? rows[index] : []) + resultRowCard(columns: columns, row: rows.indices.contains(index) ? rows[index].displayValues() : []) .padding() .frame(maxWidth: 360, alignment: .leading) } @@ -578,28 +613,34 @@ struct QueryEditorView: View { } private var resultList: some View { - let indexed = IndexedRow.wrap(viewModel.legacyRows) + let columns = viewModel.columns + let rows = viewModel.window.rows return List { - ForEach(indexed) { item in - let rowIndex = item.id - let row = item.values + ForEach(rows.indices, id: \.self) { rowIndex in NavigationLink { - RowDetailView( - columns: viewModel.columns, - rows: viewModel.window.rows, - initialIndex: rowIndex - ) + resultRowDetail(columns: columns, initialIndex: rowIndex) } label: { - resultRowCard(columns: viewModel.columns, row: row) + resultRowCard(columns: columns, row: rows[rowIndex].displayValues()) } .contextMenu { - resultRowContextMenu(columns: viewModel.columns, rowIndex: rowIndex) + resultRowContextMenu(columns: columns, rowIndex: rowIndex) } } } .listStyle(.plain) } + // No table, so nothing to edit; the engine's type and session decide how Copy Row quotes and escapes. + private func resultRowDetail(columns: [ColumnInfo], initialIndex: Int) -> some View { + RowDetailView( + columns: columns, + rows: viewModel.window.rows, + initialIndex: initialIndex, + session: session, + databaseType: databaseType + ) + } + private func resultRowCard(columns: [ColumnInfo], row: [String?]) -> some View { let budget = DuoLayoutResolver.previewFieldCount(for: duoWidthClass) let preview = Array(zip(columns, row).prefix(budget)) @@ -700,42 +741,31 @@ struct QueryEditorView: View { } } - if !viewModel.legacyRows.isEmpty { + if !viewModel.window.isEmpty { RowLayoutPicker() Section("Share Results") { ForEach(ExportFormat.offered(for: databaseType)) { format in Button { - do throws(ClipboardExporter.IncompleteValues) { - shareText = try ClipboardExporter.exportRows( - columns: viewModel.columns, rows: viewModel.window.rows.map(\.cells), - format: format, - databaseType: databaseType, driver: coordinator.session?.driver - ) + exportResults(as: format) { text in + shareText = text showShareSheet = true - } catch { - resultsExportError = error } } label: { Label(format.rawValue, systemImage: "square.and.arrow.up") } + .disabled(isExportingResults) } } Section("Copy Results") { ForEach(ExportFormat.offered(for: databaseType)) { format in Button { - do throws(ClipboardExporter.IncompleteValues) { - let text = try ClipboardExporter.exportRows( - columns: viewModel.columns, rows: viewModel.window.rows.map(\.cells), - format: format, - databaseType: databaseType, driver: coordinator.session?.driver - ) + exportResults(as: format) { text in ClipboardExporter.copyToClipboard(text) - } catch { - resultsExportError = error } } label: { Label(format.rawValue, systemImage: "doc.on.clipboard") } + .disabled(isExportingResults) } } } @@ -766,6 +796,25 @@ struct QueryEditorView: View { } } + private func exportResults(as format: ExportFormat, deliver: @escaping (String) -> Void) { + guard !isExportingResults else { return } + isExportingResults = true + let viewModel = viewModel + let databaseType = databaseType + let driver = coordinator.session?.driver + let availableBytes = MemoryPressureMonitor.shared.availableMemoryBytes() + Task { + defer { isExportingResults = false } + do { + deliver(try await viewModel.exportResults( + as: format, databaseType: databaseType, driver: driver, availableBytes: availableBytes + )) + } catch { + resultsExportError = error + } + } + } + // MARK: - Execution private func runAtCursor(dismissingKeyboard: Bool) { @@ -799,7 +848,11 @@ struct QueryEditorView: View { return } - if SQLWriteClassifier.isWriteQuery(trimmed, databaseType: databaseType) { + let writes = SQLWriteClassifier.isWriteQuery(trimmed, databaseType: databaseType) + let readOnlyMode = await ReadOnlyFunctionGuard.runsInReadOnlyMode( + writes: writes, level: safeModeLevel, driver: session.driver + ) + if writes { switch safeModeLevel.writePermission { case .blocked: showWriteBlockedAlert = true @@ -811,16 +864,46 @@ struct QueryEditorView: View { case .proceed: break } + } else if ReadOnlyFunctionGuard.refuses( + trimmed, databaseType: databaseType, level: safeModeLevel, enforcesReadOnlyMode: readOnlyMode + ) { + showFunctionBlockedAlert = true + return + } else if await refusesBareFunctionName(trimmed, on: session) { + showFunctionBlockedAlert = true + return } - await executeQueryDirect(trimmed) + await executeQueryDirect(trimmed, enforcesReadOnly: readOnlyMode) + } + + // Read on the session itself, whose current schema the query resolves its names in. + private func refusesBareFunctionName(_ sql: String, on session: ConnectionSession) async -> Bool { + let transactions = appState.sessionTransactions + let connectionId = connectionId + let driver = session.driver + return await ReadOnlyFunctionGuard.refusesBareName( + sql, + databaseType: databaseType, + level: safeModeLevel, + connectionId: connectionId, + schema: activeSchema + ) { query in + try await transactions.track(connectionId, reading: true) { try await driver.execute(query: query).rows } + } } // Claimed before the first await, so a second Run while the session reports how it reads literals starts nothing. - private func executeQueryDirect(_ trimmed: String) async { + private func executeQueryDirect(_ trimmed: String, enforcesReadOnly: Bool) async { guard let session, !isExecuting else { return } + // A run with a `USE` in any window holds the session until the picker follows it. + guard !coordinator.isDatabaseMoving else { + appError = ConnectionCoordinator.databaseMovingRefusal + return + } isExecuting = true + let movesDatabase = coordinator.beginDatabaseMove(ifRunning: trimmed) let startedAt = Date() executionStartTime = startedAt appError = nil @@ -829,6 +912,7 @@ struct QueryEditorView: View { await completion.finishColumnReads() let script = await plan(trimmed, on: session) guard !script.isEmpty else { + if movesDatabase { coordinator.endDatabaseMove() } isExecuting = false completion.reopenColumnReads() executionStartTime = nil @@ -843,12 +927,15 @@ struct QueryEditorView: View { startedAt: startedAt ) + // A write another window started before this run lands where it was aimed before the run can move the session. + if movesDatabase { await appState.sessionTransactions.statementsFinished(for: connectionId) } let holdsCatalog = SessionLocalStatements.changesDatabases(trimmed, databaseType: databaseType) if holdsCatalog { await appState.connectionManager.holdCatalogConnections(for: connectionId) } // Waited on before the Stop check, and nothing suspends from here until `track` marks the session busy, which // keeps a new list reload off it. await appState.catalogChanges.sessionReadsFinished(for: connectionId) guard !Task.isCancelled else { + if movesDatabase { coordinator.endDatabaseMove() } if holdsCatalog { appState.connectionManager.releaseCatalogConnections(for: connectionId) } recordHistory(query: trimmed, outcome: .stopped, errorMessage: QueryExecutionOutcome.stopped.historyMessage) isExecuting = false @@ -858,21 +945,38 @@ struct QueryEditorView: View { return } + // Another window may have begun a run with a `USE` while this one planned; nothing suspends from here to `track`. + guard movesDatabase || !coordinator.isDatabaseMoving else { + if holdsCatalog { appState.connectionManager.releaseCatalogConnections(for: connectionId) } + appError = ConnectionCoordinator.databaseMovingRefusal + isExecuting = false + completion.reopenColumnReads() + executionStartTime = nil + await appState.queryActivities.end(token: token, outcome: QueryExecutionOutcome.failed.activityOutcome) + return + } + let progressUpdater = startActivityProgressUpdater(token: token) await coordinator.track( failed: { _ in QueryExecutionOutcome(phase: viewModel.phase) == .failed }, rolledBack: { _ in viewModel.ranTransactionRollBack } ) { - await viewModel.run(driver: session.driver, script: script) + await viewModel.run(driver: session.driver, script: script, enforcesReadOnly: enforcesReadOnly) } progressUpdater.cancel() // Released before the change posts, so the reload it starts can open a catalog connection again. if holdsCatalog { appState.connectionManager.releaseCatalogConnections(for: connectionId) } - postCatalogChanges(on: session) - // Waits for a request Stop ended to return, which the run's end must not. - Task { await coordinator.followSessionDatabase() } + let move = postCatalogChanges(on: session) + let follow = Task { + await coordinator.followSessionPlacement(after: move) + if movesDatabase { coordinator.endDatabaseMove() } + } + // The follow waits for a statement Stop ended to return, which the run's end must not; Run stays off until the + // move lands. A plugin driver's read waits for its stopped request too. + let stopped = QueryExecutionOutcome(phase: viewModel.phase) == .stopped + if !stopped, !(session.driver is PluginBackedDriver) { await follow.value } let phase = viewModel.phase let outcome = QueryExecutionOutcome(phase: phase) @@ -912,17 +1016,18 @@ struct QueryEditorView: View { // Every statement sent counts as run, a failed one's too: a missed reload leaves the lists naming what is gone. // They are read as sent, since a run splits the rest of its text again after a statement that changes how the // session reads literals. - private func postCatalogChanges(on session: ConnectionSession) { + private func postCatalogChanges(on session: ConnectionSession) -> ConnectionCoordinator.SessionMove { let statements = viewModel.sentStatements // A key or a collection a write made or removed is on the lists too; only a proven read leaves them as they are. guard databaseType.speaksSQL else { if statements.contains(where: { SQLWriteClassifier.isWriteQuery($0, databaseType: databaseType) }) { coordinator.noteCatalogChanged([.tables, .databases]) } - return + return ConnectionCoordinator.SessionMove() } + let sessionEffect = SessionLocalStatements.effect(of: statements, databaseType: databaseType) appState.catalogChanges.noteSessionEffect( - SessionLocalStatements.effect(of: statements, databaseType: databaseType), + sessionEffect, for: connectionId, driver: ObjectIdentifier(session.driver) ) @@ -934,6 +1039,12 @@ struct QueryEditorView: View { // A commit shows the transaction's DDL to the catalog connection, a rollback takes it back from the session. if effect.endsTransaction { kinds.formUnion([.tables, .schemas, .databases]) } coordinator.noteCatalogChanged(kinds) + // A transaction's end takes back a `SET LOCAL` or a `SET` inside it, a function may set the search path, and a + // typed ROLLBACK of a held transaction also reaches the Roll Back reload. + return ConnectionCoordinator.SessionMove( + database: sessionEffect.usesDatabase, + schema: sessionEffect.leavesCatalog || effect.endsTransaction + ) } private func plan(_ text: String, on session: ConnectionSession) async -> QueryScript { @@ -959,14 +1070,13 @@ struct QueryEditorView: View { } private func recordHistory(query: String, outcome: QueryExecutionOutcome, errorMessage: String?) { - coordinator.addHistoryItem( - QueryHistoryItem( - query: query, - connectionId: connectionId, - wasSuccessful: outcome == .completed, - errorMessage: errorMessage - ) + let item = QueryHistoryItem( + query: query, + connectionId: connectionId, + wasSuccessful: outcome == .completed, + errorMessage: errorMessage ) + Task { await coordinator.addHistoryItem(item) } } // MARK: - Live Activity @@ -977,7 +1087,7 @@ struct QueryEditorView: View { while !Task.isCancelled { try? await Task.sleep(for: .seconds(1)) if Task.isCancelled { return } - guard let count = viewModel?.legacyRows.count else { return } + guard let count = viewModel?.window.count else { return } await controller.update(token: token, rowsStreamed: count) } } diff --git a/apple/TableProMobile/TableProMobile/Views/QueryHistoryView.swift b/apple/TableProMobile/TableProMobile/Views/QueryHistoryView.swift index cf189a40a3..a253415db8 100644 --- a/apple/TableProMobile/TableProMobile/Views/QueryHistoryView.swift +++ b/apple/TableProMobile/TableProMobile/Views/QueryHistoryView.swift @@ -50,11 +50,15 @@ struct QueryHistoryView: View { } } - if !coordinator.queryHistory.isEmpty { + if !coordinator.queryHistory.isEmpty || coordinator.keepsDamagedHistoryCopy { Section { Button(String(localized: "Clear All History"), role: .destructive) { showClearConfirmation = true } + } footer: { + if coordinator.keepsDamagedHistoryCopy { + Text("A damaged history file from every connection was kept on this device. Clear All History deletes it.") + } } } } @@ -65,7 +69,7 @@ struct QueryHistoryView: View { } } .overlay { - if coordinator.queryHistory.isEmpty { + if coordinator.queryHistory.isEmpty, !coordinator.keepsDamagedHistoryCopy { ContentUnavailableView( "No History", systemImage: "clock", diff --git a/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift b/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift index e0572c3948..335954bee4 100644 --- a/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift +++ b/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift @@ -15,6 +15,7 @@ struct RowDetailView: View { @State private var showSaveConfirmation = false @State private var jsonEditorField: IndexedRow? @State private var jsonViewerRoute: JSONValueViewer.Route? + @State private var fullValueRoute: FullValueRoute? private let coordinator: ConnectionCoordinator? init( @@ -26,8 +27,9 @@ struct RowDetailView: View { columnDetails: [ColumnInfo] = [], databaseType: DatabaseType = .sqlite, schema: String? = nil, + database: String? = nil, safeModeLevel: @escaping () -> SafeModeLevel = { .off }, - foreignKeys: [ForeignKeyInfo] = [], + foreignKeys: @escaping @autoclosure () -> [ForeignKeyInfo] = [], onSaved: (() -> Void)? = nil, loadFullValue: ((CellRef) async throws -> String?)? = nil, coordinator: ConnectionCoordinator? = nil @@ -42,8 +44,9 @@ struct RowDetailView: View { columnDetails: columnDetails, databaseType: databaseType, schema: schema, + database: database, safeModeLevel: safeModeLevel, - foreignKeys: foreignKeys, + foreignKeys: foreignKeys(), onSaved: onSaved, loadFullValue: loadFullValue, liveSession: coordinator.map { coordinator in { coordinator.writableSession } } @@ -57,9 +60,9 @@ struct RowDetailView: View { rowContent(at: viewModel.currentIndex) } else { TabView(selection: $viewModel.currentIndex) { - ForEach(IndexedRow.wrap(viewModel.rows)) { item in - rowContent(at: item.id) - .tag(item.id) + ForEach(viewModel.rows.indices, id: \.self) { index in + rowContent(at: index) + .tag(index) } } .tabViewStyle(.page(indexDisplayMode: .never)) @@ -89,6 +92,12 @@ struct RowDetailView: View { .navigationDestination(item: $jsonViewerRoute) { route in JSONValueViewer(route: route) } + .navigationDestination(item: $fullValueRoute) { route in + FullValueScreen( + title: viewModel.columns.indices.contains(route.column) ? viewModel.columns[route.column].name : "", + text: viewModel.completeValue(forRow: route.row, at: route.column) ?? "" + ) + } .toolbar { rowDetailToolbar } // An unsaved edit keeps this screen up over a lost session, so the state and Reconnect show here. .connectionStatusToolbar(for: connectionCoordinator) @@ -238,7 +247,11 @@ struct RowDetailView: View { } } else { let jsonText = viewModel.jsonText(forRow: rowIndex, at: index) - fieldContent(value: value, jsonPreview: viewModel.isJSON(at: index) ? jsonText : nil) + let longValue = viewModel.longValue(forRow: rowIndex, at: index) + fieldContent( + value: longValue.map { RowDetailViewModel.excerpt(of: $0) } ?? value, + jsonPreview: viewModel.isJSON(at: index) ? jsonText : nil + ) .contextMenu { if let value = viewModel.completeValue(forRow: rowIndex, at: index) { Button { @@ -259,6 +272,17 @@ struct RowDetailView: View { } label: { Label("View as JSON", systemImage: "curlybraces") } + } else if longValue != nil { + Button { + fullValueRoute = FullValueRoute(row: rowIndex, column: index) + } label: { + Label("View Full Value", systemImage: "doc.plaintext") + } + } + if viewModel.isShownInPart(forRow: rowIndex, at: index) { + Text("Shown in part. Run a query that returns fewer rows or columns to see it in full.") + .font(.footnote) + .foregroundStyle(.secondary) } if index < cells.count, cells[index].isLoadable, !viewModel.hasOverride(forRow: rowIndex, cellIndex: index) { @@ -267,7 +291,8 @@ struct RowDetailView: View { if let fk = viewModel.foreignKeys.first(where: { $0.column == column.name }), let reference = ForeignKeyReference.make( forColumn: column.name, foreignKeys: viewModel.foreignKeys, - columns: viewModel.columns, cells: cells, sourceSchema: viewModel.schema + columns: viewModel.columns, cells: cells, sourceSchema: viewModel.schema, + sourceDatabase: viewModel.database ) { Button { fkPreviewItem = FKPreviewItem(reference: reference) @@ -424,4 +449,29 @@ struct RowDetailView: View { hapticError.toggle() } } + + // Names the cell rather than carrying its text, so the route stays cheap to compare. + private struct FullValueRoute: Hashable { + let row: Int + let column: Int + } +} + +private struct FullValueScreen: View { + let title: String + let text: String + + var body: some View { + JSONTextView(readOnlyText: text) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .navigationTitle(title) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button("Copy", systemImage: "doc.on.doc") { + ClipboardExporter.copyToClipboard(text) + } + } + } + } } diff --git a/apple/TableProMobile/TableProMobile/Views/StructureView.swift b/apple/TableProMobile/TableProMobile/Views/StructureView.swift index d1ff0db75d..7deac729de 100644 --- a/apple/TableProMobile/TableProMobile/Views/StructureView.swift +++ b/apple/TableProMobile/TableProMobile/Views/StructureView.swift @@ -9,6 +9,7 @@ struct StructureView: View { let session: ConnectionSession? let databaseType: DatabaseType let schema: String? + let database: String? private static let logger = Logger(subsystem: "com.TablePro", category: "StructureView") @@ -242,9 +243,11 @@ struct StructureView: View { appError = nil do { - async let fetchedColumns = session.driver.fetchColumns(table: table.name, schema: schema) - async let fetchedIndexes = session.driver.fetchIndexes(table: table.name, schema: schema) - async let fetchedForeignKeys = session.driver.fetchForeignKeys(table: table.name, schema: schema) + async let fetchedColumns = session.driver.fetchColumns(table: table.name, schema: schema, database: database) + async let fetchedIndexes = session.driver.fetchIndexes(table: table.name, schema: schema, database: database) + async let fetchedForeignKeys = session.driver.fetchForeignKeys( + table: table.name, schema: schema, database: database + ) self.columns = try await fetchedColumns self.indexes = try await fetchedIndexes diff --git a/apple/TableProMobile/TableProMobile/Views/TableListView.swift b/apple/TableProMobile/TableProMobile/Views/TableListView.swift index 21c06dbf9e..bd66c57ec8 100644 --- a/apple/TableProMobile/TableProMobile/Views/TableListView.swift +++ b/apple/TableProMobile/TableProMobile/Views/TableListView.swift @@ -162,7 +162,8 @@ struct TableListView: View { guard let driver = driver(for: action) else { return } do { let statement = SQLBuilder.truncateStatement( - table: action.table.name, schema: action.placement.schema, type: connection.type + table: action.table.name, schema: action.placement.schema, + database: action.placement.database, type: connection.type ) _ = try await coordinator.track { try await driver.executeWrite([statement]) } await coordinator.refreshTables() @@ -192,7 +193,8 @@ struct TableListView: View { guard let driver = driver(for: action) else { return } do { let quoted = SQLBuilder.qualifiedIdentifier( - table: action.table.name, schema: action.placement.schema, for: connection.type + table: action.table.name, schema: action.placement.schema, + database: action.placement.database, for: connection.type ) _ = try await coordinator.track { try await driver.executeWrite(["DROP TABLE \(quoted)"]) } await coordinator.refreshTables() diff --git a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift index f5322a5f2e..ac042a81c5 100644 --- a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift @@ -242,7 +242,7 @@ struct QueryCompletionCatalogTests { #expect(labels(first) == ["invoices", "Quarterly Totals"]) #expect(first.map(\.insertText) == ["invoices", "\"Quarterly Totals\""]) #expect(labels(again) == labels(first)) - #expect(await reads.count == 1) + #expect(await reads.readCount == 1) } @Test("another schema's tables come from the catalog connection first, from the session only while reads are open") @@ -266,7 +266,7 @@ struct QueryCompletionCatalogTests { #expect(labels(sales) == ["from_catalog"]) #expect(closed.isEmpty) #expect(labels(open) == ["log"]) - #expect(await sessionReads.count == 1) + #expect(await sessionReads.readCount == 1) } @Test("closing session reads waits for a schema read already sent on the session") @@ -304,7 +304,7 @@ struct QueryCompletionCatalogTests { #expect(items.map(\.insertText) == ["public", "sales", "\"Sales\""]) #expect(labels(lower) == ["lower_orders"]) #expect(labels(upper) == ["upper_orders"]) - #expect(await reads.count == 2) + #expect(await reads.readCount == 2) } @Test("a schema that differs from the active one only in case lists its own tables") @@ -331,7 +331,7 @@ struct QueryCompletionCatalogTests { #expect(await catalog.isKnownSchema("ARCHIVE")) #expect(labels(await catalog.tableCompletionItems(inSchema: "archive")) == ["orders_2019"]) - #expect(await reads.count == 1) + #expect(await reads.readCount == 1) } @Test("an enum column offers its values: the driver's labels, or the ENUM type MySQL and DuckDB spell out") @@ -400,6 +400,35 @@ struct QueryCompletionCatalogTests { #expect(await source.reads == ["orders", "customers"]) } + @Test("cached column suggestions are built once, and again only after a read, a table list or a namespace list") + func cachedColumnsAreBuiltOnce() async { + let source = CatalogColumnSource(["orders": [column("id")], "customers": [column("name")]]) + let catalog = catalog(source, tables: [table("orders"), table("customers")]) + _ = await catalog.columnCompletionItems(for: "orders", schema: nil) + + let first = await catalog.allColumnsFromCachedTables() + let second = await catalog.allColumnsFromCachedTables() + #expect(labels(second) == labels(first)) + #expect(await catalog.tableColumnItemsBuildCount == 1) + + _ = await catalog.columnCompletionItems(for: "customers", schema: nil) + let afterRead = await catalog.allColumnsFromCachedTables() + #expect(labels(afterRead) == ["name", "id"]) + #expect(await catalog.tableColumnItemsBuildCount == 2) + + let listedBefore = await catalog.tableCompletionItems() + #expect(labels(listedBefore) == ["orders", "customers"]) + await catalog.replaceTables([table("orders"), table("customers"), table("refunds")]) + _ = await catalog.allColumnsFromCachedTables() + #expect(await catalog.tableColumnItemsBuildCount == 3) + let listedAfter = await catalog.tableCompletionItems() + #expect(labels(listedAfter) == ["orders", "customers", "refunds"]) + + await catalog.replaceNamespaces(schemas: ["main"], databases: []) + _ = await catalog.allColumnsFromCachedTables() + #expect(await catalog.tableColumnItemsBuildCount == 4) + } + @Test("the catalog connection reads first and is not closed by a run; the session reads only without one") func catalogConnectionFirst() async { let catalogSource = CatalogColumnSource(["orders": [column("id")]]) @@ -508,7 +537,7 @@ private actor SchemaTableReads { private var answersNothing = false private var heldReads: [CheckedContinuation] = [] private var readWaiters: [CheckedContinuation] = [] - private(set) var count = 0 + private(set) var readCount = 0 private(set) var finished = 0 init(_ tables: [String: [TableInfo]], held: Bool = false) { @@ -525,7 +554,7 @@ private actor SchemaTableReads { } func waitForRead() async { - guard count == 0 else { return } + guard readCount == 0 else { return } await withCheckedContinuation { readWaiters.append($0) } } @@ -537,7 +566,7 @@ private actor SchemaTableReads { private func read(_ schema: String) async -> [TableInfo]? { guard !answersNothing else { return nil } - count += 1 + readCount += 1 readWaiters.forEach { $0.resume() } readWaiters = [] if isHeld { @@ -613,3 +642,83 @@ private actor CatalogColumnSource { return columns[table] ?? [] } } + +extension QueryCompletionCatalogTests { + private func foreignKey(_ column: String, to table: String, _ referenced: String, schema: String? = nil) -> ForeignKeyInfo { + ForeignKeyInfo( + name: "\(column)_fk", column: column, referencedTable: table, referencedColumn: referenced, referencedSchema: schema + ) + } + + @Test("a table's foreign keys are read once on the catalog connection, and again once marked stale") + func foreignKeysReadOncePerGeneration() async { + let keys = ForeignKeySource(["orders": [foreignKey("customer_id", to: "customers", "id")]]) + let catalog = QueryCompletionCatalog( + tables: [table("orders"), table("customers")], databaseType: .postgresql, schema: "public", + readColumns: CatalogColumnSource([:]).reader, readCatalogForeignKeys: keys.reader + ) + + let first = await catalog.foreignKeys(of: "orders", schema: nil) + _ = await catalog.foreignKeys(of: "orders", schema: nil) + await catalog.markColumnsStale() + _ = await catalog.foreignKeys(of: "orders", schema: nil) + + #expect(first == [SQLCompletionForeignKey( + name: "customer_id_fk", columns: ["customer_id"], referencedTable: "customers", referencedColumns: ["id"] + )]) + #expect(await keys.reads == ["orders", "orders"]) + } + + @Test("a key into another schema is left out, and the session is never asked for keys") + func foreignKeysStayInTheSchema() async { + let keys = ForeignKeySource(["orders": [ + foreignKey("customer_id", to: "customers", "id", schema: "public"), + foreignKey("auditor_id", to: "auditors", "id", schema: "audit"), + ]]) + let session = CatalogColumnSource([:]) + let catalog = QueryCompletionCatalog( + tables: [table("orders")], databaseType: .postgresql, schema: "public", + readColumns: session.reader, readCatalogForeignKeys: keys.reader + ) + + let read = await catalog.foreignKeys(of: "orders", schema: nil) + + #expect(read.map { $0.referencedTable } == ["customers"]) + #expect(await session.reads.isEmpty) + } + + @Test("without a catalog connection no keys are offered") + func noForeignKeysWithoutCatalogReader() async { + let catalog = catalog(CatalogColumnSource([:]), tables: [table("orders")], type: .postgresql, schema: "public") + #expect(await catalog.foreignKeys(of: "orders", schema: nil).isEmpty) + } + + @Test("a name is written with the engine's quotes, and a statement name only where it was quoted") + func writtenNames() { + let postgres = catalog(CatalogColumnSource([:]), tables: [], type: .postgresql, schema: "public") + #expect(postgres.writtenText(of: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("CustomerId")])) + == "\"O\".\"CustomerId\"") + #expect(postgres.writtenText(of: SQLCompletionIdentifier([.statement("Ord", quoted: false), .catalog("id")])) + == "Ord.id") + let mysql = catalog(CatalogColumnSource([:]), tables: [], type: .mysql) + #expect(mysql.writtenText(of: SQLCompletionIdentifier([.statement("o", quoted: false), .catalog("order")])) == "o.`order`") + } +} + +private actor ForeignKeySource { + private let keys: [String: [ForeignKeyInfo]] + private(set) var reads: [String] = [] + + init(_ keys: [String: [ForeignKeyInfo]]) { + self.keys = keys + } + + nonisolated var reader: QueryForeignKeyReader { + { table, _ in await self.read(table) } + } + + private func read(_ table: String) -> [ForeignKeyInfo]? { + reads.append(table) + return keys[table] ?? [] + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift index 7d2745486f..3f68ab827e 100644 --- a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift @@ -621,3 +621,49 @@ extension QueryCompletionControllerTests { #expect(await !suggest(controller, "/*\n" + notes + "it's done */\nSELECT * FROM or").isEmpty) } } + +extension QueryCompletionControllerTests { + @Test("right after a join's ON the foreign key's condition leads and goes in quoted for the engine") + func joinConditionFromForeignKey() async throws { + let source = ControllerColumnSource([ + "orders": [ColumnInfo(name: "Id", typeName: "integer"), ColumnInfo(name: "CustomerId", typeName: "integer")], + "customers": [ColumnInfo(name: "Id", typeName: "integer")], + ]) + let keys: QueryForeignKeyReader = { table, _ in + guard table == "orders" else { return [] } + return [ForeignKeyInfo(name: "orders_customer_fk", column: "CustomerId", referencedTable: "customers", referencedColumn: "Id")] + } + let controller = QueryCompletionController() + controller.update( + scope: scope(.postgresql, tables: ["orders", "customers"], database: "shop", schema: "public"), + readColumns: source.reader, + readCatalogForeignKeys: keys + ) + let text = "SELECT * FROM orders o JOIN customers c ON " + + let suggestions = await suggest(controller, text) + #expect(suggestions.first?.label == "o.CustomerId = c.Id") + #expect(suggestions.first?.kind == .joinCondition) + let edit = try accept("o.CustomerId = c.Id", in: text, with: controller) + #expect(applying(edit, to: text) == text + "o.\"CustomerId\" = c.\"Id\" ") + } + + // Two copies of `o.Id` were offered, and the first went in bare, which PostgreSQL folds to `o.id`. + @Test("a qualified column after a join's ON is offered once and keeps its quotes") + func qualifiedColumnOnceAfterOn() async throws { + let source = ControllerColumnSource([ + "orders": [ColumnInfo(name: "Id", typeName: "integer")], + "customers": [ColumnInfo(name: "Id", typeName: "integer")], + ]) + let controller = controller( + scope(.postgresql, tables: ["orders", "customers"], database: "shop", schema: "public"), + source: source + ) + let text = "SELECT * FROM orders o JOIN customers c ON " + + let suggestions = await suggest(controller, text) + #expect(suggestions.filter { $0.label == "o.Id" }.count == 1) + let edit = try accept("o.Id", in: text, with: controller) + #expect(applying(edit, to: text) == text + "o.\"Id\"") + } +} diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift index 3011a5fb98..7e6d2ecb4f 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift @@ -112,6 +112,29 @@ struct ConnectionCoordinatorBackgroundTests { ) } + // iPad: the transaction ran in one window, and the user comes back to the app in another. + @Test("A transaction lost in the background is reported once, in the window that reopens the file") + func reopenInAnotherWindowReportsOnce() async throws { + let left = try await connectedCoordinator(on: fileDriver(.explicitTransaction)) + #expect(left.transactionHold == .open) + + await releaseForTheBackground() + let reopened = fileDriver(.idle) + factory.queue(reopened) + let right = ConnectionCoordinator(connection: connection, appState: appState) + await right.connect() + await left.followLiveSession() + + #expect(right.session?.driver === reopened) + #expect(left.session?.driver === reopened) + #expect( + right.failureAlertMessage + == ConnectionCoordinator.lostTransactionMessage(.inBackground, connectionName: "Ledger") + ) + #expect(!left.showFailureAlert) + #expect(monitor.rollBackCount(for: connection.id) == 1) + } + @Test("A reopen that fails keeps the loss and reports it when a later reopen lands") func failedReopenReportsLater() async throws { let coordinator = try await connectedCoordinator(on: fileDriver(.explicitTransaction)) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift index b8843230d4..1ba67a44a7 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift @@ -241,7 +241,7 @@ struct ConnectionCoordinatorCatalogTests { let stopped = Task { @MainActor in withUnsafeCurrentTask { $0?.cancel() } - await coordinator.followSessionDatabase() + await coordinator.followSessionPlacement(after: .init()) } await stopped.value @@ -267,7 +267,7 @@ struct ConnectionCoordinatorCatalogTests { let stopped = Task { try await session.fetchTables(schema: nil) } for await _ in entered { break } - let follow = Task { await coordinator.followSessionDatabase() } + let follow = Task { await coordinator.followSessionPlacement(after: .init()) } release.yield() _ = try await stopped.value await follow.value @@ -285,7 +285,7 @@ struct ConnectionCoordinatorCatalogTests { appState.setEditorHold(true, by: otherWindow, on: connection.id) session.used = "archive" - await coordinator.followSessionDatabase() + await coordinator.followSessionPlacement(after: .init()) #expect(coordinator.activeDatabase == "app_prod") #expect(session.used == "app_prod") @@ -322,6 +322,192 @@ struct ConnectionCoordinatorCatalogTests { #expect(session.switchedDatabases.isEmpty) } + @Test("A USE run in the Query tab moves the picker, the lists and the redial to the session's database") + func useMovesThePickerToTheSessionsDatabase() async { + let session = driver(tables: ["orders"]) + session.scriptedDatabases = ["app_prod", "app_staging"] + let coordinator = await connected(session) + session.scriptedSessionDatabase = "app_staging" + session.scriptedTables = [TableInfo(name: "orders"), TableInfo(name: "staging_only")] + #expect(coordinator.tableQualifier == "app_prod") + + await coordinator.followSessionPlacement(after: .init(database: true)) + + #expect(coordinator.activeDatabase == "app_staging") + #expect(names(coordinator) == ["orders", "staging_only"]) + #expect(appState.connectionManager.session(for: connection.id)?.activeDatabase == "app_staging") + #expect(coordinator.tablePlacement.database == "app_staging") + #expect(coordinator.tableQualifier == "app_staging") + #expect(!coordinator.showFailureAlert) + } + + @Test("A run that cannot have moved the session sends it no statement") + func runWithoutUseLeavesTheSessionAlone() async { + let session = driver(tables: ["orders"]) + let coordinator = await connected(session) + session.scriptedSessionDatabase = "app_staging" + + await coordinator.followSessionPlacement(after: .init()) + + #expect(session.sessionDatabaseReads == 0) + #expect(coordinator.activeDatabase == "app_prod") + } + + @Test("While this window holds an unsaved edit, a USE is undone so the edit saves where it was made") + func ownHeldEditUndoesTheUse() async { + let session = driver(tables: ["orders"]) + let coordinator = await connected(session) + coordinator.isHeldByEditor = true + session.scriptedSessionDatabase = "app_staging" + + await coordinator.followSessionPlacement(after: .init(database: true)) + + #expect(coordinator.activeDatabase == "app_prod") + #expect(session.switchedDatabases.last == "app_prod") + #expect(coordinator.failureAlertMessage == ConnectionCoordinator.editorHeldHereMessage) + coordinator.isHeldByEditor = false + } + + @Test("When the session cannot go back under another window's unsaved edit, the lists follow it and the edit keeps its database") + func failedMoveBackFollowsTheSession() async { + let session = driver(tables: ["orders"]) + let coordinator = await connected(session) + let editing = ConnectionCoordinator(connection: connection, appState: appState) + editing.isHeldByEditor = true + session.scriptedSessionDatabase = "app_staging" + session.refusedDatabases = ["app_prod"] + + await coordinator.followSessionPlacement(after: .init(database: true)) + + #expect(coordinator.activeDatabase == "app_staging") + #expect(coordinator.showFailureAlert) + #expect(editing.activeDatabase == "app_prod") + #expect(editing.tableQualifier == "app_prod") + editing.isHeldByEditor = false + } + + @Test("A run with a USE holds Run in every window, and the Tables tab keeps naming the database it shows", arguments: [ + DatabaseType.mssql, .mysql + ]) + func useRunHoldsOnlyRun(type: DatabaseType) async { + var record = connection + record.type = type + let session = driver(tables: ["orders"]) + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: record, appState: appState) + await coordinator.connect() + let otherWindow = ConnectionCoordinator(connection: record, appState: appState) + await otherWindow.connect() + + #expect(!coordinator.beginDatabaseMove(ifRunning: "SELECT * FROM orders")) + #expect(!otherWindow.isDatabaseMoving) + + #expect(coordinator.beginDatabaseMove(ifRunning: "USE archive\nDELETE FROM orders")) + session.scriptedCurrentDatabase = "archive" + #expect(otherWindow.isDatabaseMoving) + #expect(otherWindow.tableDatabase == "app_prod") + #expect(otherWindow.tablePlacement.database == "app_prod") + + coordinator.endDatabaseMove() + #expect(!otherWindow.isDatabaseMoving) + } + + @Test("A SQL Server window names the database it shows while the session sits in another, a pinned window too") + func sqlServerWindowNamesTheDatabaseItShows() async { + var mssql = connection + mssql.type = .mssql + let session = driver(tables: ["orders"]) + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: mssql, appState: appState) + await coordinator.connect() + let editing = ConnectionCoordinator(connection: mssql, appState: appState) + editing.isHeldByEditor = true + session.scriptedSessionDatabase = "app_staging" + session.refusedDatabases = ["app_prod"] + + await coordinator.followSessionPlacement(after: .init(database: true)) + + #expect(coordinator.tableDatabase == "app_staging") + #expect(editing.tableDatabase == "app_prod") + #expect(editing.tablePlacement.database == "app_prod") + #expect(coordinator.failureAlertMessage == String( + format: String(localized: "The query moved the session to “%@” and it could not go back to “%@”. Unsaved changes still save to “%@”."), + "app_staging", "app_prod", "app_prod" + )) + editing.isHeldByEditor = false + } + + @Test("A SQL Server connection naming no database names the one it opened in, even when the database list fails") + func failedDatabaseListNamesTheOpenedDatabase() async { + var mssql = connection + mssql.type = .mssql + mssql.database = "" + let session = driver(tables: ["orders"]) + session.scriptedCurrentDatabase = "app_default" + session.fetchDatabasesError = MockDatabaseDriver.MockError.scripted + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: mssql, appState: appState) + await coordinator.connect() + + #expect(session.fetchDatabasesCalls > 0) + #expect(coordinator.activeDatabase == "app_default") + #expect(coordinator.tableDatabase == "app_default") + } + + @Test("A COMMIT that ends a SET LOCAL search_path moves the schema picker back") + func commitMovesTheSchemaPickerBack() async { + let session = driver(tables: ["orders"]) + session.scriptedSchemas = ["archive", "public"] + var postgres = connection + postgres.type = .postgresql + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: postgres, appState: appState) + await coordinator.connect() + session.scriptedSessionSchema = "archive" + await coordinator.followSessionPlacement(after: .init(schema: true)) + #expect(coordinator.activeSchema == "archive") + + session.scriptedSessionSchema = "public" + await coordinator.followSessionPlacement(after: .init(schema: true)) + + #expect(coordinator.activeSchema == "public") + } + + @Test("A typed SET search_path moves the schema picker to where the session reads") + func setSearchPathMovesTheSchemaPicker() async { + let session = driver(tables: ["orders"]) + session.scriptedSchemas = ["archive", "public"] + var postgres = connection + postgres.type = .postgresql + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: postgres, appState: appState) + await coordinator.connect() + #expect(coordinator.activeSchema == "public") + session.scriptedSessionSchema = "archive" + + await coordinator.followSessionPlacement(after: .init(schema: true)) + + #expect(coordinator.activeSchema == "archive") + } + + @Test("A Roll Back that undoes the picker's schema switch moves the picker back") + func rollBackMovesTheSchemaPickerBack() async { + let session = driver(tables: ["orders"]) + session.scriptedSchemas = ["archive", "public"] + var postgres = connection + postgres.type = .postgresql + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: postgres, appState: appState) + await coordinator.connect() + await coordinator.switchSchema(to: "archive") + #expect(coordinator.activeSchema == "archive") + session.scriptedSessionSchema = "public" + + await coordinator.reloadAfterRollBack() + + #expect(coordinator.activeSchema == "public") + } + private func usedDatabaseDriver(_ plugin: StubPluginDriver) -> UsedDatabaseDriver { UsedDatabaseDriver( plugin: plugin, dialedHosts: [], readsOnly: { _ in true }, diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorHistoryTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorHistoryTests.swift index 3e20144874..a0599724f2 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorHistoryTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorHistoryTests.swift @@ -16,29 +16,29 @@ struct ConnectionCoordinatorHistoryTests { #expect(appState.addConnection(connection)) } - private func makeCoordinator() -> ConnectionCoordinator { + private func makeCoordinator() async -> ConnectionCoordinator { let coordinator = ConnectionCoordinator(connection: connection, appState: appState) - coordinator.loadHistory() + await coordinator.loadHistory() return coordinator } @Test("Running the same query twice in a row lists it once, as the store keeps it") - func repeatedQueryIsListedOnce() { - let coordinator = makeCoordinator() + func repeatedQueryIsListedOnce() async { + let coordinator = await makeCoordinator() - coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) - coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) #expect(coordinator.queryHistory.count == 1) #expect(coordinator.queryHistory.map(\.id) == appState.queryHistory.load(for: connection.id).map(\.id)) } @Test("The same query failing after it succeeded is listed twice") - func differentOutcomeIsListedAgain() { - let coordinator = makeCoordinator() + func differentOutcomeIsListedAgain() async { + let coordinator = await makeCoordinator() - coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) - coordinator.addHistoryItem( + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) + await coordinator.addHistoryItem( QueryHistoryItem(query: "SELECT 1", connectionId: connection.id, wasSuccessful: false, errorMessage: "gone") ) @@ -46,30 +46,57 @@ struct ConnectionCoordinatorHistoryTests { } @Test("A query past the store's cap drops the oldest entry from the list too") - func listFollowsTheStoreCap() { + func listFollowsTheStoreCap() async { for index in 0..<200 { appState.queryHistory.save(QueryHistoryItem(query: "SELECT \(index)", connectionId: connection.id)) } - let coordinator = makeCoordinator() + let coordinator = await makeCoordinator() #expect(coordinator.queryHistory.count == 200) - coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 200", connectionId: connection.id)) + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 200", connectionId: connection.id)) #expect(coordinator.queryHistory.count == 200) #expect(coordinator.queryHistory.first?.query == "SELECT 1") #expect(coordinator.queryHistory.last?.query == "SELECT 200") } + @Test("A delete lists what the store holds after it, a run another window saved meanwhile included") + func deleteListsRunsSavedMeanwhile() async throws { + let coordinator = await makeCoordinator() + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) + let first = try #require(coordinator.queryHistory.first?.id) + appState.queryHistory.save(QueryHistoryItem(query: "SELECT 2", connectionId: connection.id)) + + await coordinator.deleteHistoryItem(first).value + + #expect(coordinator.queryHistory.map(\.query) == ["SELECT 2"]) + #expect(appState.queryHistory.load(for: connection.id).map(\.query) == ["SELECT 2"]) + } + + @Test("Clearing empties the list and the store for this connection only") + func clearEmptiesThisConnection() async { + let other = UUID() + appState.queryHistory.save(QueryHistoryItem(query: "SELECT 0", connectionId: other)) + let coordinator = await makeCoordinator() + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) + + await coordinator.clearHistory().value + + #expect(coordinator.queryHistory.isEmpty) + #expect(appState.queryHistory.load(for: connection.id).isEmpty) + #expect(appState.queryHistory.load(for: other).map(\.query) == ["SELECT 0"]) + } + @Test("A history file that cannot be read or written leaves the listed history in place") - func unreadableStoreKeepsTheList() throws { + func unreadableStoreKeepsTheList() async throws { appState.queryHistory.save(QueryHistoryItem(query: "SELECT 1", connectionId: connection.id)) - let coordinator = makeCoordinator() + let coordinator = await makeCoordinator() let historyFile = fixture.libraryDirectory.appendingPathComponent("query-history.json") try FileManager.default.removeItem(at: historyFile) try FileManager.default.createDirectory(at: historyFile, withIntermediateDirectories: false) try Data("locked".utf8).write(to: historyFile.appendingPathComponent("entry")) - coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 2", connectionId: connection.id)) + await coordinator.addHistoryItem(QueryHistoryItem(query: "SELECT 2", connectionId: connection.id)) #expect(coordinator.queryHistory.map(\.query) == ["SELECT 1"]) } diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift index c1d210522b..cd77db9040 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift @@ -3,6 +3,7 @@ import TableProDatabase @testable import TableProMobile import TableProModels import TableProQuery +import TableProSQLGrammar import Testing @MainActor @@ -339,21 +340,22 @@ struct ConnectionCoordinatorReconnectTests { #expect(following.tablePlacement != chosen) } + // The connection reads the new schema's tables once, for both windows. @Test("A window following a schema switch keeps its old placement until the new schema's tables arrive") func followCommitsPlacementWithItsTables() async throws { let first = driver(tables: ["facts"]) first.currentSchema = "public" let (switching, following) = try await warehouseCoordinators(on: first) - await switching.switchSchema(to: "staging") let fetch = TestGate() first.scriptedTables = [TableInfo(name: "staging_facts")] first.beforeFetchTables = { await fetch.enter() } - let follow = Task { await following.followLiveSession() } + let switched = Task { await switching.switchSchema(to: "staging") } await fetch.waitUntilEntered() let duringFetch = (following.tablePlacement.schema, tableNames(following)) await fetch.open() - await follow.value + await switched.value + await following.followLiveSession() #expect(duringFetch.0 == "public") #expect(duringFetch.1 == ["facts"]) @@ -366,18 +368,18 @@ struct ConnectionCoordinatorReconnectTests { let first = driver(tables: ["facts"]) first.currentSchema = "public" let (switching, following) = try await warehouseCoordinators(on: first) - await switching.switchSchema(to: "staging") let fetch = TestGate() first.scriptedTables = [TableInfo(name: "staging_facts")] first.beforeFetchTables = { await fetch.enter() } - let follow = Task { await following.followLiveSession() } + let switched = Task { await switching.switchSchema(to: "staging") } await fetch.waitUntilEntered() following.isHeldByEditor = true first.beforeFetchTables = nil await fetch.open() - await follow.value + await switched.value + #expect(switching.tablePlacement.schema == "staging") #expect(following.tablePlacement.schema == "public") #expect(tableNames(following) == ["facts"]) @@ -387,31 +389,234 @@ struct ConnectionCoordinatorReconnectTests { #expect(tableNames(following) == ["staging_facts"]) } - @Test("A follow that another switch overtakes while its tables load lands on the newer placement") - func followOvertakenBySecondSwitchFollowsTheNewest() async throws { + @Test("A switch that another window's switch overtakes while its tables load never lands") + func overtakenSwitchNeverLands() async throws { let first = driver(tables: ["facts"]) first.currentSchema = "public" let (switching, following) = try await warehouseCoordinators(on: first) - await switching.switchSchema(to: "staging") let fetch = TestGate() first.scriptedTables = [TableInfo(name: "staging_facts")] first.beforeFetchTables = { await fetch.enter() } - let follow = Task { await following.followLiveSession() } + let overtaken = Task { await switching.switchSchema(to: "staging") } await fetch.waitUntilEntered() first.beforeFetchTables = nil - first.currentSchema = "archive" first.scriptedTables = [TableInfo(name: "archive_facts")] + await following.switchSchema(to: "archive") await fetch.open() - await follow.value - let afterOvertaken = following.tablePlacement.schema - await following.resumedFollow?.value + await overtaken.value - #expect(afterOvertaken == "public") + #expect(switching.tablePlacement.schema == "archive") #expect(following.tablePlacement.schema == "archive") #expect(tableNames(following) == ["archive_facts"]) } + @Test("A database switch asked during a check over an open transaction waits for it, so the loss is still reported") + func switchDuringACheckKeepsTheLossNotice() async throws { + let first = driver(tables: ["facts"], state: .explicitTransaction) + first.currentSchema = "public" + let (coordinator, _) = try await warehouseCoordinators(on: first) + #expect(coordinator.transactionHold == .open) + + first.pingError = MockDatabaseDriver.MockError.scripted + let redialed = driver(tables: ["facts"]) + redialed.currentSchema = "public" + let loading = TestGate() + redialed.beforeFetchTables = { await loading.enter() } + factory.queue(redialed) + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + factory.queue(staging) + + let check = Task { await coordinator.reconnectIfNeeded() } + await loading.waitUntilEntered() + let switched = Task { await coordinator.switchDatabase(to: "staging") } + await ObservedCondition.wait { coordinator.isSwitching } + redialed.beforeFetchTables = nil + await loading.open() + await check.value + await switched.value + + #expect( + coordinator.failureAlertMessage + == ConnectionCoordinator.lostTransactionMessage(.whileIdle, connectionName: "Warehouse") + ) + #expect(monitor.rollBackCount(for: coordinator.connection.id) == 1) + #expect(coordinator.activeDatabase == "staging") + #expect(coordinator.session?.driver === staging) + } + + @Test("A PostgreSQL switch asked while another window's runs lands after it, so the newest switch wins") + func overlappingPostgreSQLSwitchesKeepTheNewest() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + let (busy, other) = try await warehouseCoordinators(on: first) + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + let dialing = TestGate() + staging.beforeConnect = { await dialing.enter() } + factory.queue(staging) + let archive = driver(tables: ["archive_facts"]) + archive.currentSchema = "public" + factory.queue(archive) + + let busySwitch = Task { await busy.switchDatabase(to: "staging") } + await dialing.waitUntilEntered() + let otherSwitch = Task { await other.switchDatabase(to: "archive") } + await ObservedCondition.wait { other.isSwitching } + await dialing.open() + await busySwitch.value + await otherSwitch.value + + #expect(factory.dialed.suffix(2).map { $0.database } == ["staging", "archive"]) + #expect(busy.session?.driver === archive) + #expect(other.activeDatabase == "archive") + #expect(tableNames(busy) == ["archive_facts"]) + } + + @Test("A catalog reload held back while another window switches schema runs once the switch ends") + func catalogReloadHeldBackBySwitchRunsAfterIt() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + first.scriptedSchemas = ["public", "sales"] + let (posting, switching) = try await warehouseCoordinators(on: first) + let fetch = TestGate() + first.beforeFetchTables = { await fetch.enter() } + let switched = Task { await switching.switchSchema(to: "sales") } + await fetch.waitUntilEntered() + + first.scriptedSchemas = ["public", "sales", "reporting"] + posting.noteCatalogChanged(.schemas) + await posting.reloadChangedCatalog() + let whileSwitching = posting.schemas + await fetch.open() + await switched.value + + #expect(whileSwitching == ["public", "sales"]) + #expect(posting.schemas == ["public", "sales", "reporting"]) + } + + @Test("A follow asked while another window switches runs once the switch ends") + func followAskedDuringASwitchRunsAfterIt() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + let (switching, following) = try await warehouseCoordinators(on: first) + let fetch = TestGate() + first.beforeFetchTables = { await fetch.enter() } + let switched = Task { await switching.switchSchema(to: "staging") } + await fetch.waitUntilEntered() + + let replacement = driver(tables: ["archive_facts"]) + replacement.currentSchema = "public" + factory.queue(replacement) + var archive = following.connection + archive.database = "archive" + _ = try await appState.connectionManager.connect(archive) + await following.followLiveSession() + let whileSwitching = following.session?.driver + first.beforeFetchTables = nil + await fetch.open() + await switched.value + + #expect(whileSwitching === first) + #expect(following.session?.driver === replacement) + #expect(following.activeDatabase == "archive") + #expect(tableNames(following) == ["archive_facts"]) + } + + @Test("A follow held back for a window that has since pinned an edit reports to the window that freed the model") + func heldBackFollowSkipsAPinnedRequester() async throws { + let first = driver(tables: ["facts"], state: .explicitTransaction) + first.currentSchema = "public" + let (switching, editing) = try await warehouseCoordinators(on: first) + let fetch = TestGate() + first.beforeFetchTables = { await fetch.enter() } + let switched = Task { await switching.switchSchema(to: "staging") } + await fetch.waitUntilEntered() + + let replacement = driver(tables: ["archive_facts"]) + replacement.currentSchema = "public" + factory.queue(replacement) + var archive = editing.connection + archive.database = "archive" + _ = try await appState.connectionManager.connect(archive) + await ObservedCondition.wait { monitor.hold(for: archive.id) == nil } + await editing.followLiveSession() + editing.isHeldByEditor = true + first.beforeFetchTables = nil + await fetch.open() + await switched.value + + #expect(switching.session?.driver === replacement) + #expect( + switching.failureAlertMessage + == ConnectionCoordinator.lostTransactionMessage(.whileIdle, connectionName: "Warehouse") + ) + #expect(!editing.showFailureAlert) + #expect(editing.session?.driver === first) + } + + @Test("A database change posted during a PostgreSQL switch reloads the list once the switch lands") + func catalogPostDuringAPostgreSQLSwitchReloadsAfterIt() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + first.scriptedDatabases = ["analytics", "staging"] + let (posting, switching) = try await warehouseCoordinators(on: first) + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + staging.scriptedDatabases = ["analytics", "staging", "reporting"] + let dialing = TestGate() + staging.beforeConnect = { await dialing.enter() } + factory.queue(staging) + let switched = Task { await switching.switchDatabase(to: "staging") } + await dialing.waitUntilEntered() + + posting.noteCatalogChanged(.databases) + await posting.reloadChangedCatalog() + let whileSwitching = posting.databases + await dialing.open() + await switched.value + + #expect(whileSwitching == ["analytics", "staging"]) + #expect(posting.databases == ["analytics", "staging", "reporting"]) + } + + @Test("A table created while a SQL Server schema restore reads its list shows in both windows once the restore ends") + func catalogPostDuringASchemaRestoreReloadsAfterIt() async throws { + let first = driver(tables: ["orders"]) + first.currentSchema = "dbo" + first.scriptedSchemas = ["dbo", "sales"] + let ledger = DatabaseConnection(name: "Ledger", type: .mssql, host: "sql.example.com", port: 1_433, database: "app") + factory.queue(first) + _ = try await appState.connectionManager.connect(ledger) + let switching = ConnectionCoordinator(connection: ledger, appState: appState) + await switching.connect() + let creating = ConnectionCoordinator(connection: ledger, appState: appState) + await creating.connect() + await switching.switchSchema(to: "sales") + // The session lands on the new database's default schema, so the USE puts it back on sales. + first.currentSchema = "dbo" + let restoring = TestGate() + first.beforeFetchTables = { [weak first] in + if first?.currentSchema == "sales" { await restoring.enter() } + } + factory.opensCatalogs = true + factory.queue(driver(tables: ["orders", "invoices"])) + + let switched = Task { await switching.switchDatabase(to: "history") } + await restoring.waitUntilEntered() + creating.noteCatalogChanged(.tables) + await creating.reloadChangedCatalog() + let whileRestoring = tableNames(creating) + await restoring.open() + await switched.value + + #expect(whileRestoring == ["orders"]) + #expect(switching.activeSchema == "sales") + #expect(tableNames(switching) == ["orders", "invoices"]) + #expect(tableNames(creating) == ["orders", "invoices"]) + } + @Test("A window closed while its database switch waits on another window's dial leaves that dial's session alone") func closedWindowSwitchDoesNotTearDownTheSharedSession() async throws { let first = driver(tables: ["facts"]) @@ -463,6 +668,25 @@ struct ConnectionCoordinatorReconnectTests { #expect(following.schemas == ["dbo", "archive"]) } + // The list the switching window had belongs to the database it left. + @Test("A window that switches a SQL Server database lists the new database's schemas") + func switchReloadsSchemasForTheNewDatabase() async { + let first = driver(tables: ["orders"]) + first.currentSchema = "dbo" + first.scriptedSchemas = ["dbo", "sales"] + let ledger = DatabaseConnection(name: "Ledger", type: .mssql, host: "sql.example.com", port: 1_433, database: "app") + factory.queue(first) + let switching = ConnectionCoordinator(connection: ledger, appState: appState) + await switching.connect() + #expect(switching.schemas == ["dbo", "sales"]) + + first.scriptedSchemas = ["dbo", "archive"] + await switching.switchDatabase(to: "history") + + #expect(switching.activeDatabase == "history") + #expect(switching.schemas == ["dbo", "archive"]) + } + @Test("A PostgreSQL database switch in one window moves the other window onto the new session") func otherWindowFollowsAPostgreSQLSwitch() async throws { let first = driver(tables: ["facts"]) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift index b89b21270a..17b19e114f 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift @@ -181,6 +181,45 @@ struct ConnectionCoordinatorStatusTests { #expect(isConnected(coordinator)) } + // The lost screen's Reconnect calls this too, so neither reopens the saved database and switches back. + @Test("Reconnect on a lost PostgreSQL window redials the database and schema it was on, in one dial") + func reconnectRedialsWhereTheWindowWas() async { + let warehouse = DatabaseConnection( + name: "Warehouse", + type: .postgresql, + host: "pg.example.com", + port: 5_432, + database: "analytics" + ) + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + factory.queue(first) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + await coordinator.connect() + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + factory.queue(staging) + await coordinator.switchDatabase(to: "staging") + await coordinator.switchSchema(to: "sales") + staging.pingError = MockDatabaseDriver.MockError.scripted + factory.queue(refusingDriver()) + await coordinator.reconnectIfNeeded() + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) + let dialsBefore = factory.dialed.count + + let recovered = driver(tables: ["sales_facts"]) + recovered.currentSchema = "public" + factory.queue(recovered) + await coordinator.reconnect() + + #expect(factory.dialed.count == dialsBefore + 1) + #expect(factory.dialed.last?.database == "staging") + #expect(recovered.currentSchema == "sales") + #expect(coordinator.session?.driver === recovered) + #expect(coordinator.activeDatabase == "staging") + #expect(coordinator.activeSchema == "sales") + } + @Test("Tracked work that fails on a lost session dials again") func failedWorkOnALostSessionRedials() async throws { let coordinator = try await lostCoordinator(on: driver(tables: ["orders"])) @@ -208,6 +247,25 @@ struct ConnectionCoordinatorStatusTests { #expect(appState.sessionTransactions.rollBackCount(for: connection.id) == 1) } + @Test("Cancel in a window opening a lost connection leaves another window on its lost screen") + func cancelledOpenKeepsAnotherWindowLost() async throws { + let lost = try await lostCoordinator(on: driver(tables: ["orders"])) + let opening = ConnectionCoordinator(connection: connection, appState: appState) + let dialed = driver(tables: ["orders"]) + let connecting = TestGate() + dialed.beforeConnect = { await connecting.enter() } + factory.queue(dialed) + let open = Task { await opening.connect() } + await connecting.waitUntilEntered() + + opening.cancelConnect() + await connecting.open() + await open.value + + #expect(opening.displayStatus == .notConnected) + #expect(lost.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) + } + @Test("A window whose redial failed takes the session another window dialed") func lostWindowFollowsAnotherWindowsDial() async throws { let lost = try await lostCoordinator(on: driver(tables: ["orders"])) @@ -376,6 +434,13 @@ struct ConnectionCoordinatorStatusTests { #expect(ranUpdate(second)) } + // An open dials the remembered database first; this one refuses it as a dropped database. + private func goneStagingDriver() -> MockDatabaseDriver { + let gone = driver() + gone.connectError = MySQLError.databaseRefused("Unknown database 'app_staging'") + return gone + } + // A window on app_staging whose redial failed; `beforeLoss` runs while it is still connected. private func lostOnStaging( beforeLoss: (ConnectionCoordinator) -> Void = { _ in } @@ -392,8 +457,9 @@ struct ConnectionCoordinatorStatusTests { return coordinator } - // Another window opens the connection, which dials the saved app_prod. + // Another window opens the connection once app_staging is gone, so it lands on the saved app_prod. private func reconnectAnotherWindow() async -> MockDatabaseDriver { + factory.queue(goneStagingDriver()) let second = driver(tables: ["orders"]) factory.queue(second) await ConnectionCoordinator(connection: connection, appState: appState).connect() @@ -416,9 +482,11 @@ struct ConnectionCoordinatorStatusTests { let saved = await viewModel.saveChanges() #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) #expect(coordinator.activeDatabase == "app_staging") #expect(coordinator.tables.map { $0.name } == ["staging_orders"]) #expect(!saved) + #expect(viewModel.operationError?.title == ConnectionStatusPresentation(.lost(cause: nil)).title) #expect(!ranUpdate(second)) coordinator.isHeldByEditor = false @@ -430,23 +498,34 @@ struct ConnectionCoordinatorStatusTests { #expect(isConnected(coordinator)) } + // The other window's open reads the saved database's tables once, for both windows. @Test("An unsaved edit that starts while a lost window loads another database's tables keeps the window where it was") func editArrivingDuringTheFetchPinsTheWindow() async throws { let coordinator = try await lostOnStaging() - let second = await reconnectAnotherWindow() + let second = driver(tables: ["orders"]) let loading = TestGate() second.beforeFetchTables = { await loading.enter() } + factory.queue(goneStagingDriver()) + factory.queue(second) + let other = ConnectionCoordinator(connection: connection, appState: appState) - let check = Task { await coordinator.reconnectIfNeeded() } + let opening = Task { await other.connect() } await loading.waitUntilEntered() coordinator.isHeldByEditor = true second.beforeFetchTables = nil await loading.open() - await check.value + await opening.value + #expect(appState.connectionManager.session(for: connection.id)?.activeDatabase == "app_prod") #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) #expect(coordinator.activeDatabase == "app_staging") #expect(coordinator.tables.map { $0.name } == ["staging_orders"]) + guard case .failure(let refusal) = coordinator.writableSession else { + Issue.record("Expected the pinned window to refuse a write") + return + } + #expect(refusal.title == ConnectionStatusPresentation(.lost(cause: nil)).title) coordinator.isHeldByEditor = false await coordinator.resumedFollow?.value @@ -480,11 +559,13 @@ struct ConnectionCoordinatorStatusTests { nonisolated enum ScriptedFailure: LocalizedError { case unreachable case refused + case noSuchDatabase var errorDescription: String? { switch self { case .unreachable: "Lost connection to the server during the query" case .refused: "The server refused the request" + case .noSuchDatabase: "FATAL: the requested database does not exist" } } } diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift index 66a22a8243..f9db011a6b 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift @@ -3,6 +3,7 @@ import TableProConnectionLibrary import TableProDatabase @testable import TableProMobile import TableProModels +import TableProPluginKit import Testing @MainActor @@ -45,6 +46,43 @@ struct ConnectionCoordinatorSwitchTests { return refused } + private func missingDatabaseDriver() -> MockDatabaseDriver { + let refused = driver() + refused.connectError = ScriptedFailure.noSuchDatabase + return refused + } + + private func mySQLDriver(tables: [String]) -> MockDatabaseDriver { + let session = driver(tables: tables) + session.currentSchema = nil + session.scriptedDatabases = ["app_prod", "app_staging"] + return session + } + + private func mySQLFamily(_ type: DatabaseType) -> DatabaseConnection { + DatabaseConnection(name: "Prod", type: type, host: "db.example.com", port: 3_306, database: "app_prod") + } + + private func postgreSQLDriver(tables: [String]) -> MockDatabaseDriver { + let session = driver(tables: tables) + session.scriptedDatabases = ["analytics", "staging"] + session.scriptedSchemas = ["public", "sales"] + return session + } + + // Where the connection was when it last closed, which the first window restores. + private func remember(database: String, schema: String? = nil, for record: DatabaseConnection) { + UserDefaults.standard.set(database, forKey: ConnectionDefaultsKey.lastDB.name(for: record.id)) + if let schema { + UserDefaults.standard.set(schema, forKey: ConnectionDefaultsKey.lastSchema.name(for: record.id)) + } + } + + private func forget(_ record: DatabaseConnection) { + UserDefaults.standard.removeObject(forKey: ConnectionDefaultsKey.lastDB.name(for: record.id)) + UserDefaults.standard.removeObject(forKey: ConnectionDefaultsKey.lastSchema.name(for: record.id)) + } + private func attachedCoordinator(to record: DatabaseConnection, on first: MockDatabaseDriver) async throws -> ConnectionCoordinator { factory.queue(first) _ = try await appState.connectionManager.connect(record) @@ -155,6 +193,454 @@ struct ConnectionCoordinatorSwitchTests { #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) } + @Test("An open whose remembered and saved databases both refuse fails as a connect does, and the next connect dials the remembered one again") + func refusedRememberedAndSavedDatabasesFailTheOpen() async { + remember(database: "staging", for: warehouse) + defer { forget(warehouse) } + factory.queue(missingDatabaseDriver()) + factory.queue(refusingDriver()) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging", "analytics"]) + #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .failed(cause: ScriptedFailure.unreachable.localizedDescription)) + #expect(!coordinator.showFailureAlert) + + let recovered = driver(tables: ["staging_facts"]) + factory.queue(recovered) + await coordinator.connect() + + #expect(factory.dialed.count == 3) + #expect(factory.dialed.last?.database == "staging") + #expect(coordinator.session?.driver === recovered) + #expect(coordinator.activeDatabase == "staging") + #expect(coordinator.displayStatus == .connected) + } + + // MARK: - Open + + @Test("A PostgreSQL open dials the remembered database once, restores the schema inside the dial and reads each list once") + func openDialsTheRememberedPlacement() async { + remember(database: "staging", schema: "sales", for: warehouse) + defer { forget(warehouse) } + let session = postgreSQLDriver(tables: ["sales_facts"]) + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging"]) + #expect(session.switchedSchemas == ["sales"]) + #expect(session.fetchTablesSchemas == [nil]) + #expect(session.fetchSchemasCalls == 1) + #expect(session.fetchDatabasesCalls == 1) + #expect(session.transactionStateReads == 1) + #expect(coordinator.session?.driver === session) + #expect(coordinator.activeDatabase == "staging") + #expect(coordinator.activeSchema == "sales") + #expect(coordinator.schemas == ["public", "sales"]) + #expect(tableNames(coordinator) == ["sales_facts"]) + #expect(!coordinator.showFailureAlert) + } + + @Test( + "A MySQL-family open dials the remembered database once, with no USE, and reads each list once", + arguments: [DatabaseType.mysql, .mariadb, .tidb] + ) + func mySQLFamilyOpenDialsTheRememberedDatabase(_ type: DatabaseType) async { + let record = mySQLFamily(type) + remember(database: "app_staging", for: record) + defer { forget(record) } + let session = mySQLDriver(tables: ["staging_orders"]) + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: record, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_staging"]) + #expect(session.switchedDatabases.isEmpty) + #expect(session.fetchTablesSchemas == [nil]) + #expect(session.fetchDatabasesCalls == 1) + #expect(session.transactionStateReads == 1) + #expect(coordinator.session?.driver === session) + #expect(coordinator.activeDatabase == "app_staging") + #expect(coordinator.tableQualifier == "app_staging") + #expect(tableNames(coordinator) == ["staging_orders"]) + #expect(!coordinator.showFailureAlert) + } + + // A contained SQL Server user signs in against the database the login names; OceanBase is unmeasured. + @Test( + "A SQL Server or OceanBase open dials the saved database and moves to the remembered one with USE", + arguments: [DatabaseType.mssql, .oceanbase] + ) + func useEngineOpenDialsTheSavedDatabase(_ type: DatabaseType) async { + let record = DatabaseConnection(name: "Ledger", type: type, host: "db.example.com", port: 1_433, database: "app") + remember(database: "history", for: record) + defer { forget(record) } + let session = driver(tables: ["orders"]) + session.currentSchema = "dbo" + session.scriptedSchemas = ["dbo"] + session.scriptedDatabases = ["app", "history"] + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: record, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app"]) + #expect(session.switchedDatabases == ["history"]) + #expect(coordinator.activeDatabase == "history") + #expect(!coordinator.showFailureAlert) + } + + @Test("A MySQL database refused at sign-in falls back to the saved one, whose USE says why as an open always did") + func refusedMySQLDatabaseFallsBackToUse() async { + remember(database: "app_staging", for: connection) + defer { forget(connection) } + let refused = driver() + refused.connectError = MySQLError.databaseRefused("Access denied for user 'app'@'%' to database 'app_staging'") + let saved = mySQLDriver(tables: ["orders"]) + let useRefusal = MySQLError.queryFailed("Access denied for user 'app'@'%' to database 'app_staging'") + saved.switchDatabaseError = useRefusal + factory.queue(refused) + factory.queue(saved) + let coordinator = ConnectionCoordinator(connection: connection, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_staging", "app_prod"]) + #expect(coordinator.session?.driver === saved) + #expect(coordinator.displayStatus == .connected) + #expect(coordinator.activeDatabase == "app_prod") + #expect(tableNames(coordinator) == ["orders"]) + #expect(coordinator.failureAlertTitle == ConnectionCoordinator.switchDatabaseFailedTitle) + #expect(coordinator.failureAlertMessage == useRefusal.localizedDescription) + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) == "app_staging") + } + + @Test("A MySQL database the server no longer lists falls back quietly and is forgotten") + func droppedMySQLDatabaseIsForgotten() async { + remember(database: "app_archive", for: connection) + defer { forget(connection) } + let refused = driver() + refused.connectError = MySQLError.databaseRefused("Unknown database 'app_archive'") + let saved = mySQLDriver(tables: ["orders"]) + factory.queue(refused) + factory.queue(saved) + let coordinator = ConnectionCoordinator(connection: connection, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_archive", "app_prod"]) + #expect(coordinator.session?.driver === saved) + #expect(saved.switchedDatabases.isEmpty) + #expect(coordinator.activeDatabase == "app_prod") + #expect(!coordinator.showFailureAlert) + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) == nil) + } + + @Test("A MySQL sign-in refused on the remembered database fails the open after one dial, and the next open dials the saved one and moves with USE") + func refusedMySQLSignInDialsOnce() async { + remember(database: "app_staging", for: connection) + defer { forget(connection) } + let denied = driver() + let refusal = MySQLError.connectionFailed("Access denied for user 'app'@'10.0.0.2' (using password: YES)") + denied.connectError = refusal + let saved = mySQLDriver(tables: ["staging_orders"]) + factory.queue(denied) + factory.queue(saved) + let coordinator = ConnectionCoordinator(connection: connection, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_staging"]) + #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .failed(cause: refusal.localizedDescription)) + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_staging", "app_prod"]) + #expect(coordinator.session?.driver === saved) + #expect(saved.switchedDatabases == ["app_staging"]) + #expect(coordinator.activeDatabase == "app_staging") + #expect(coordinator.displayStatus == .connected) + } + + // Another server speaking the protocol may number a missing database its own way, as 1105. + @Test("A MySQL dial refused in a way it cannot pin on the database fails once, and the next open forgets one no longer listed") + func unrecognizedMySQLRefusalYieldsToTheSavedDatabase() async { + remember(database: "app_archive", for: connection) + defer { forget(connection) } + let refused = driver() + let refusal = MySQLError.connectionFailed("Unknown database 'app_archive'") + refused.connectError = refusal + factory.queue(refused) + let coordinator = ConnectionCoordinator(connection: connection, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_archive"]) + #expect(coordinator.displayStatus == .failed(cause: refusal.localizedDescription)) + + let saved = mySQLDriver(tables: ["orders"]) + factory.queue(saved) + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_archive", "app_prod"]) + #expect(coordinator.session?.driver === saved) + #expect(saved.switchedDatabases.isEmpty) + #expect(coordinator.activeDatabase == "app_prod") + #expect(!coordinator.showFailureAlert) + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) == nil) + } + + @Test("A MySQL database whose table list is refused falls back to the saved one and moves there with USE") + func refusedMySQLTableListFallsBackToUse() async { + remember(database: "app_staging", for: connection) + defer { forget(connection) } + let refusing = driver() + refusing.fetchTablesError = ScriptedFailure.refused + let saved = mySQLDriver(tables: ["staging_orders"]) + factory.queue(refusing) + factory.queue(saved) + let coordinator = ConnectionCoordinator(connection: connection, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["app_staging", "app_prod"]) + #expect(appState.connectionManager.session(for: connection.id)?.driver === saved) + #expect(saved.switchedDatabases == ["app_staging"]) + #expect(coordinator.activeDatabase == "app_staging") + #expect(tableNames(coordinator) == ["staging_orders"]) + #expect(!coordinator.showFailureAlert) + } + + @Test("A remembered database that refuses the open falls back to the saved one once and says why") + func refusedRememberedDatabaseFallsBackOnce() async { + remember(database: "staging", for: warehouse) + defer { forget(warehouse) } + let fallback = postgreSQLDriver(tables: ["facts"]) + factory.queue(missingDatabaseDriver()) + factory.queue(fallback) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging", "analytics"]) + #expect(coordinator.session?.driver === fallback) + #expect(coordinator.displayStatus == .connected) + #expect(coordinator.activeDatabase == "analytics") + #expect(tableNames(coordinator) == ["facts"]) + #expect(coordinator.failureAlertTitle == ConnectionCoordinator.switchDatabaseFailedTitle) + #expect(coordinator.failureAlertMessage == ScriptedFailure.noSuchDatabase.localizedDescription) + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastDB.name(for: warehouse.id)) == "staging") + } + + @Test("A remembered database that never answered fails the open once, and the next open starts from the saved one") + func hungRememberedDatabaseYieldsToTheSavedOne() async { + remember(database: "staging", for: warehouse) + defer { forget(warehouse) } + let hung = driver() + hung.connectError = PostgreSQLError.unreachable("Connection timed out.") + factory.queue(hung) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging"]) + #expect(coordinator.session == nil) + + let saved = driver(tables: ["facts"]) + factory.queue(saved) + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging", "analytics"]) + #expect(coordinator.session?.driver === saved) + #expect(coordinator.displayStatus == .connected) + } + + @Test("A remembered database the open cannot reach fails it after one dial, without dialing the saved one") + func unreachableRememberedDatabaseDialsOnce() async { + remember(database: "staging", for: warehouse) + defer { forget(warehouse) } + factory.queue(refusingDriver()) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging"]) + #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .failed(cause: ScriptedFailure.unreachable.localizedDescription)) + + let recovered = driver(tables: ["staging_facts"]) + factory.queue(recovered) + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging", "staging"]) + #expect(coordinator.session?.driver === recovered) + #expect(coordinator.displayStatus == .connected) + } + + @Test("A PostgreSQL refusal goes to the saved database unless it came from the network, the device or the sign-in") + func fallbackFollowsTheRefusal() { + #expect(ConnectionSessionModel.mayBlameDatabase(ScriptedFailure.noSuchDatabase)) + #expect(ConnectionSessionModel.mayBlameDatabase(ScriptedFailure.refused)) + #expect(ConnectionSessionModel.mayBlameDatabase(Refusal(message: "permission denied for database \"staging\""))) + #expect(ConnectionSessionModel.mayBlameDatabase(Refusal(message: "FATAL: la base de données « staging » n'existe pas"))) + #expect(!ConnectionSessionModel.mayBlameDatabase(ScriptedFailure.unreachable)) + #expect(!ConnectionSessionModel.mayBlameDatabase(URLError(.timedOut))) + #expect(!ConnectionSessionModel.mayBlameDatabase(POSIXError(.ETIMEDOUT))) + #expect(!ConnectionSessionModel.mayBlameDatabase(PostgreSQLError.unreachable("연결 시간이 초과되었습니다."))) + #expect(!ConnectionSessionModel.mayBlameDatabase(SSHTunnelError.hostKeyRejected("db.example.com"))) + #expect(!ConnectionSessionModel.mayBlameDatabase(CancellationError())) + #expect(!ConnectionSessionModel.mayBlameDatabase(Refusal(message: "password authentication failed for user \"app\""))) + #expect(!ConnectionSessionModel.mayBlameDatabase(Refusal( + message: "password authentication failed for user \"app\"\nno pg_hba.conf entry for host \"10.0.0.2\", user \"app\", database \"staging\", no encryption" + ))) + #expect(!ConnectionSessionModel.mayBlameDatabase(Refusal(message: "could not translate host name \"db.example\" to address"))) + } + + @Test("A remembered database whose table list is refused falls back to the saved one and says why") + func refusedRememberedTableListFallsBack() async { + remember(database: "staging", for: warehouse) + defer { forget(warehouse) } + let refusing = driver() + refusing.fetchTablesError = ScriptedFailure.refused + let fallback = postgreSQLDriver(tables: ["facts"]) + factory.queue(refusing) + factory.queue(fallback) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging", "analytics"]) + #expect(appState.connectionManager.session(for: warehouse.id)?.driver === fallback) + #expect(coordinator.session?.driver === fallback) + #expect(coordinator.activeDatabase == "analytics") + #expect(coordinator.failureAlertTitle == ConnectionCoordinator.switchDatabaseFailedTitle) + #expect(coordinator.failureAlertMessage == ScriptedFailure.refused.localizedDescription) + } + + @Test("A remembered database the server no longer lists falls back quietly and is forgotten") + func droppedRememberedDatabaseIsForgotten() async { + remember(database: "archive", for: warehouse) + defer { forget(warehouse) } + let fallback = postgreSQLDriver(tables: ["facts"]) + factory.queue(missingDatabaseDriver()) + factory.queue(fallback) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["archive", "analytics"]) + #expect(coordinator.session?.driver === fallback) + #expect(coordinator.activeDatabase == "analytics") + #expect(!coordinator.showFailureAlert) + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastDB.name(for: warehouse.id)) == nil) + } + + @Test("A SQL Server open that needs a sign-in dials the saved database once and offers it") + func signInOnTheRememberedDatabaseSkipsTheFallback() async throws { + let ledger = DatabaseConnection(name: "Ledger", type: .mssql, host: "sql.example.com", port: 1_433, database: "app") + remember(database: "history", for: ledger) + defer { forget(ledger) } + let expired = driver() + expired.connectError = EntraOAuthError.signInRequired + factory.queue(expired) + let coordinator = ConnectionCoordinator(connection: ledger, appState: appState) + coordinator.restorePersistedState() + + let open = Task { await coordinator.connect() } + await ObservedCondition.wait { coordinator.prompts.current != nil } + let offer = try #require(coordinator.prompts.current) + coordinator.prompts.answer(offer.id, accepted: false) + await open.value + + #expect(factory.dialed.map { $0.database } == ["app"]) + #expect(coordinator.session == nil) + #expect(!coordinator.showFailureAlert) + guard case .failed = coordinator.phase else { + Issue.record("Expected the declined sign-in to fail the open") + return + } + } + + // lastSchema is kept per connection, so it can name a schema of another database. + @Test("A remembered schema the database does not list is not restored") + func unlistedRememberedSchemaStaysUnrestored() async { + remember(database: "staging", schema: "etl", for: warehouse) + defer { forget(warehouse) } + let session = postgreSQLDriver(tables: ["staging_facts"]) + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["staging"]) + #expect(session.switchedSchemas.isEmpty) + #expect(session.fetchSchemasCalls == 1) + #expect(coordinator.activeSchema == "public") + #expect(tableNames(coordinator) == ["staging_facts"]) + } + + @Test("A PostgreSQL open with nothing remembered dials the saved database and switches nothing") + func openWithNothingRememberedDialsTheSavedDatabase() async { + let session = postgreSQLDriver(tables: ["facts"]) + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.map { $0.database } == ["analytics"]) + #expect(session.switchedSchemas.isEmpty) + #expect(session.fetchTablesSchemas == [nil]) + #expect(session.fetchSchemasCalls == 1) + #expect(session.fetchDatabasesCalls == 1) + #expect(session.transactionStateReads == 1) + #expect(coordinator.activeDatabase == "analytics") + #expect(coordinator.activeSchema == "public") + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastDB.name(for: warehouse.id)) == nil) + #expect(UserDefaults.standard.string(forKey: ConnectionDefaultsKey.lastSchema.name(for: warehouse.id)) == nil) + } + + // The placement's schema reads "public" until a session says otherwise. + @Test("A SQL Server open with nothing remembered stays on the server's default schema") + func sqlServerOpenKeepsItsDefaultSchema() async { + let ledger = DatabaseConnection(name: "Ledger", type: .mssql, host: "sql.example.com", port: 1_433, database: "app") + let session = driver(tables: ["orders"]) + session.currentSchema = "dbo" + session.scriptedSchemas = ["dbo", "sales"] + factory.queue(session) + let coordinator = ConnectionCoordinator(connection: ledger, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(session.switchedSchemas.isEmpty) + #expect(session.fetchSchemasCalls == 1) + #expect(coordinator.activeSchema == "dbo") + #expect(coordinator.schemas == ["dbo", "sales"]) + #expect(tableNames(coordinator) == ["orders"]) + } + @Test("A refresh that fails on a dead session redials and shows the new list without an alert") func refreshOnADeadSessionRedials() async throws { let first = driver(tables: ["orders"]) @@ -185,3 +671,9 @@ struct ConnectionCoordinatorSwitchTests { #expect(coordinator.failureAlertMessage == ScriptedFailure.refused.localizedDescription) } } + +private struct Refusal: LocalizedError { + let message: String + + var errorDescription: String? { message } +} diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionLibrary/ConnectionLibraryPublisherTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionLibrary/ConnectionLibraryPublisherTests.swift index f4072639ef..47a486aafd 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionLibrary/ConnectionLibraryPublisherTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionLibrary/ConnectionLibraryPublisherTests.swift @@ -7,20 +7,33 @@ import Testing @MainActor @Suite("Connection library publisher") struct ConnectionLibraryPublisherTests { + // Outlives a publisher, as the app group file and the defaults outlive a launch. private final class Recorder { var shortcutRefreshes = 0 + var widgetFile: [WidgetConnectionItem] = [] var widgetWrites: [[WidgetConnectionItem]] = [] } private let index = RecordingSearchIndex() private let recorder = Recorder() + private let defaults: UserDefaults - private func makePublisher() -> ConnectionLibraryPublisher { + init() throws { + defaults = try #require(UserDefaults(suiteName: "com.TablePro.tests.LibraryPublisher.\(UUID().uuidString)")) + } + + private func makePublisher(build: String = "1") -> ConnectionLibraryPublisher { let recorder = recorder return ConnectionLibraryPublisher( searchIndex: index, - writeWidgetItems: { recorder.widgetWrites.append($0) }, - refreshShortcutParameters: { recorder.shortcutRefreshes += 1 } + readWidgetItems: { recorder.widgetFile }, + writeWidgetItems: { items in + recorder.widgetFile = items + recorder.widgetWrites.append(items) + }, + refreshShortcutParameters: { recorder.shortcutRefreshes += 1 }, + defaults: defaults, + build: build ) } @@ -145,6 +158,101 @@ struct ConnectionLibraryPublisherTests { await publisher.settle() } + @Test("A launch over an unchanged library touches neither Spotlight, the widget nor Shortcuts") + func unchangedLibraryOnRelaunchPublishesNothing() async { + let library = [connection("A"), connection("B", sortOrder: 1)] + let first = makePublisher() + first.publish(library) + await first.settle() + #expect(index.replacements.count == 1) + #expect(recorder.widgetWrites.count == 1) + #expect(recorder.shortcutRefreshes == 1) + + let relaunched = makePublisher() + relaunched.publish(library) + await relaunched.settle() + + #expect(index.replacements.count == 1) + #expect(recorder.widgetWrites.count == 1) + #expect(recorder.shortcutRefreshes == 1) + #expect(index.stateReads == 2) + #expect(index.indexedIds == Set(library.map(\.id))) + } + + @Test("A library changed between launches is published again on the next one") + func changedLibraryOnRelaunchPublishes() async { + let kept = connection("A") + let first = makePublisher() + first.publish([kept]) + await first.settle() + + var renamed = kept + renamed.name = "Renamed" + let added = connection("B", sortOrder: 1) + let relaunched = makePublisher() + relaunched.publish([renamed, added]) + await relaunched.settle() + + #expect(index.replacements.count == 2) + #expect(index.indexedIds == [kept.id, added.id]) + #expect(index.contents.first { $0.id == kept.id }?.title == "Renamed") + #expect(recorder.widgetWrites.count == 2) + #expect(recorder.widgetFile.map(\.name) == ["Renamed", "B"]) + #expect(recorder.shortcutRefreshes == 2) + } + + @Test("A launch that cannot read the index's state writes the index rather than trust it") + func unreadableStateReindexes() async { + let library = [connection("A")] + let first = makePublisher() + first.publish(library) + await first.settle() + + index.failNextStateRead() + let relaunched = makePublisher() + relaunched.publish(library) + await relaunched.settle() + + #expect(index.replacements.count == 2) + #expect(index.indexedIds == [library[0].id]) + } + + @Test("A new build indexes and refreshes Shortcuts once, and leaves an unchanged widget file alone") + func newBuildPublishesOnce() async { + let library = [connection("A")] + let first = makePublisher(build: "1") + first.publish(library) + await first.settle() + + let updated = makePublisher(build: "2") + updated.publish(library) + await updated.settle() + let relaunched = makePublisher(build: "2") + relaunched.publish(library) + await relaunched.settle() + + #expect(index.replacements.count == 2) + #expect(recorder.shortcutRefreshes == 2) + #expect(recorder.widgetWrites.count == 1) + } + + @Test("A widget file written by an older build is rewritten once") + func olderWidgetFileIsRewritten() async { + let current = connection("A") + recorder.widgetFile = [ + WidgetConnectionItem(id: current.id, name: "A", type: current.type.rawValue, sortOrder: 0, glyph: nil) + ] + + for _ in 0..<2 { + let publisher = makePublisher() + publisher.publish([current]) + await publisher.settle() + } + + #expect(recorder.widgetWrites.count == 1) + #expect(recorder.widgetFile.first?.glyph != nil) + } + @Test("Widget items sort by order, then name, and a blank name shows the host") func widgetItemMapping() { let unnamed = DatabaseConnection(name: "", type: .mysql, host: "cache.local", sortOrder: 1) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionSessionStateTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionSessionStateTests.swift new file mode 100644 index 0000000000..a1539cdef1 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/ConnectionSessionStateTests.swift @@ -0,0 +1,203 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import Testing + +@MainActor +struct ConnectionSessionStateTests { + private let failure = AppError( + category: .network, + title: "Connection Failed", + message: "The server closed the connection.", + recovery: nil, + underlying: nil + ) + + private func session(on driver: MockDatabaseDriver) -> ConnectionSession { + ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "app") + } + + private func isConnected(_ state: ConnectionSessionState?, on driver: MockDatabaseDriver) -> Bool { + guard case .connected(let landed)? = state else { return false } + return landed.driver === driver + } + + private func isLost(_ state: ConnectionSessionState?) -> Bool { + guard case .lost? = state else { return false } + return true + } + + private func isNotConnected(_ state: ConnectionSessionState?) -> Bool { + guard case .notConnected? = state else { return false } + return true + } + + private func showsConnected(_ state: ConnectionSessionState?) -> Bool { + guard case .connected? = state?.phase else { return false } + return true + } + + private func showsLost(_ state: ConnectionSessionState?) -> Bool { + guard case .lost? = state?.phase else { return false } + return true + } + + @Test("An open lands only with a session, so a connected state always has one") + func openLandsOnlyWithASession() { + let open = ConnectionAttempt(.open) + let driver = MockDatabaseDriver() + let started = ConnectionSessionState.notConnected.applying(.open(open)) + + #expect(started?.isOpening == true) + #expect(started?.session == nil) + #expect(started?.applying(.opened(open)) == nil) + + let loading = started?.applying(.adopt(session(on: driver), by: open)) + #expect(loading?.isOpening == true) + #expect(loading?.session?.driver === driver) + #expect(isConnected(loading?.applying(.opened(open)), on: driver)) + } + + @Test("A database restore that fails both ways inside an open ends it lost, and the open can no longer land") + func failedRestoreEndsTheOpenLost() { + let open = ConnectionAttempt(.open) + let loading = ConnectionSessionState.connecting(open, provisional: session(on: MockDatabaseDriver())) + + let tornDown = loading.applying(.teardown(open)) + #expect(tornDown?.isOpening == true) + #expect(tornDown?.session == nil) + + let lost = tornDown?.applying(.lose(failure, open)) + #expect(isLost(lost)) + #expect(lost?.applying(.opened(open)) == nil) + #expect(lost?.applying(.adopt(session(on: MockDatabaseDriver()), by: open)) == nil) + } + + @Test("A step of an attempt the state no longer holds is refused") + func staleStepsAreRefused() { + let driver = MockDatabaseDriver() + let first = ConnectionAttempt(.open) + let abandoned = ConnectionSessionState.connecting(first, provisional: nil).applying(.abandon(first)) + + #expect(isNotConnected(abandoned)) + #expect(abandoned?.applying(.adopt(session(on: driver), by: first)) == nil) + #expect(abandoned?.applying(.fail(failure, first)) == nil) + #expect(ConnectionSessionState.connecting(first, provisional: nil).applying(.open(ConnectionAttempt(.open))) == nil) + + let check = ConnectionAttempt(.redial) + let checking = ConnectionSessionState.connected(session(on: driver)).applying(.check(check)) + let reopened = checking?.applying(.open(ConnectionAttempt(.open))) + #expect(reopened?.isOpening == true) + #expect(reopened?.applying(.dial(check)) == nil) + #expect(reopened?.applying(.lose(failure, check)) == nil) + #expect(reopened?.applying(.adopt(session(on: driver), by: check)) == nil) + } + + @Test("A check pings in place, and only its redial reads as reconnecting") + func checkReconnectsOnlyOnceItDials() { + let held = MockDatabaseDriver() + let check = ConnectionAttempt(.redial) + let checking = ConnectionSessionState.connected(session(on: held)).applying(.check(check)) + + #expect(checking?.check === check) + #expect(checking?.isReconnecting == false) + #expect(checking?.session?.driver === held) + #expect(showsConnected(checking)) + + let dialing = checking?.applying(.dial(check)) + #expect(dialing?.isReconnecting == true) + #expect(dialing?.session?.driver === held) + #expect(isLost(dialing?.applying(.lose(failure, check)))) + + let redialed = MockDatabaseDriver() + let retry = ConnectionAttempt(.redial) + let lostChecking = ConnectionSessionState.lost(failure).applying(.check(retry)) + #expect(showsLost(lostChecking)) + #expect(lostChecking?.session == nil) + let lostDialing = lostChecking?.applying(.dial(retry)) + #expect(lostDialing?.isReconnecting == true) + #expect(isConnected(lostDialing?.applying(.adopt(session(on: redialed), by: retry)), on: redialed)) + } + + @Test("A check starts only over a connected or a lost session") + func checkNeedsASessionOrALoss() { + let driver = MockDatabaseDriver() + let switching = ConnectionAttempt(.switchDatabase) + let states: [ConnectionSessionState] = [ + .notConnected, + .failed(failure), + .connecting(ConnectionAttempt(.open), provisional: session(on: driver)), + .reconnecting(switching, over: .connected(session(on: driver))) + ] + for state in states { + #expect(state.applying(.check(ConnectionAttempt(.redial))) == nil) + } + } + + @Test("A session another window moved lands under a running check without ending it") + func followLandsUnderACheck() { + let check = ConnectionAttempt(.redial) + let moved = MockDatabaseDriver() + let checking = ConnectionSessionState.checking(check, over: .connected(session(on: MockDatabaseDriver()))) + + let followed = checking.applying(.adopt(session(on: moved), by: nil)) + #expect(followed?.check === check) + #expect(followed?.session?.driver === moved) + #expect(isConnected(followed?.applying(.settle(check)), on: moved)) + } + + @Test("A PostgreSQL database switch shows no session until the new one lands, and is not a reconnect") + func databaseSwitchHasNoSessionUntilItLands() { + let switching = ConnectionAttempt(.switchDatabase) + let staging = MockDatabaseDriver() + let torn = ConnectionSessionState.connected(session(on: MockDatabaseDriver())).applying(.switchDatabase(switching)) + + #expect(torn?.session == nil) + #expect(torn?.isReconnecting == false) + #expect(showsConnected(torn)) + #expect(torn?.applying(.adopt(session(on: staging), by: nil)) == nil) + #expect(isConnected(torn?.applying(.adopt(session(on: staging), by: switching)), on: staging)) + #expect(isLost(torn?.applying(.lose(failure, switching)))) + } + + @Test("A database switch waits for a running check, and may start over a loss") + func databaseSwitchWaitsForACheck() { + let held = session(on: MockDatabaseDriver()) + let check = ConnectionAttempt(.redial) + let checking = ConnectionSessionState.checking(check, over: .connected(held)) + let redialing = ConnectionSessionState.reconnecting(check, over: .connected(held)) + + #expect(checking.applying(.switchDatabase(ConnectionAttempt(.switchDatabase))) == nil) + #expect(redialing.applying(.switchDatabase(ConnectionAttempt(.switchDatabase))) == nil) + + let overLoss = ConnectionSessionState.lost(failure).applying(.switchDatabase(ConnectionAttempt(.switchDatabase))) + #expect(overLoss?.session == nil) + #expect(showsLost(overLoss)) + #expect(overLoss?.isReconnecting == false) + } + + @Test("An open every window left goes back to the loss it started from, and a reset ends neutral") + func abandonedOpenKeepsItsLoss() { + let fromLoss = ConnectionAttempt(.open, origin: .lost(failure)) + let reopening = ConnectionSessionState.connecting(fromLoss, provisional: session(on: MockDatabaseDriver())) + let fresh = ConnectionAttempt(.open) + + #expect(isLost(reopening.applying(.abandon(fromLoss)))) + #expect(isNotConnected(ConnectionSessionState.connecting(fresh, provisional: nil).applying(.abandon(fresh)))) + #expect(reopening.applying(.abandon(fresh)) == nil) + #expect(isNotConnected(reopening.applying(.cancel))) + } + + @Test("At rest, an open shows the loss it started from and a switch the session it started over") + func restingStateSkipsWhatIsInFlight() { + let held = MockDatabaseDriver() + let reopening = ConnectionSessionState.connecting(ConnectionAttempt(.open, origin: .lost(failure)), provisional: nil) + let switching = ConnectionSessionState.reconnecting( + ConnectionAttempt(.switchDatabase), over: .connected(session(on: held)) + ) + + #expect(isLost(reopening.atRest)) + #expect(isConnected(switching.atRest, on: held)) + #expect(isNotConnected(ConnectionSessionState.connecting(ConnectionAttempt(.open), provisional: nil).atRest)) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/DataBrowserDriverBuiltTests.swift b/apple/TableProMobile/TableProMobileTests/DataBrowserDriverBuiltTests.swift index 578ada74cb..c21b319ac7 100644 --- a/apple/TableProMobile/TableProMobileTests/DataBrowserDriverBuiltTests.swift +++ b/apple/TableProMobile/TableProMobileTests/DataBrowserDriverBuiltTests.swift @@ -44,7 +44,55 @@ struct DataBrowserDriverBuiltTests { #expect(page.request.searchText == nil) #expect(driver.countRequests.count == 1) #expect(viewModel.pagination.totalRows == 420) - #expect(viewModel.legacyRows.count == viewModel.pagination.pageSize) + #expect(viewModel.window.shownValues.count == viewModel.pagination.pageSize) + } + + @Test("A page turn while the count runs keeps that count instead of starting another") + func pageTurnKeepsTheRunningCount() async { + let driver = MockQueryBuildingDriver() + driver.scriptedCount = 5_000 + let counting = TestGate() + driver.beforeCount = { await counting.enter() } + let viewModel = browser(driver) + driver.scriptedRows = fullPage(viewModel) + + let firstLoad = Task { await viewModel.load(isInitial: true) } + await counting.waitUntilEntered() + #expect(viewModel.isLoading == false) + await viewModel.goToNextPage() + #expect(viewModel.pagination.currentPage == 1) + #expect(driver.countRequests.count == 1) + + await counting.open() + await firstLoad.value + #expect(viewModel.pagination.totalRows == 5_000) + #expect(driver.countRequests.count == 1) + } + + @Test("A page past the last row keeps the pager and sets no total, and the count still lands") + func pagePastTheEndSetsNoTotal() async { + let driver = MockQueryBuildingDriver() + let counting = TestGate() + driver.beforeCount = { await counting.enter() } + let viewModel = browser(driver) + let pageSize = viewModel.pagination.pageSize + driver.scriptedCount = pageSize + driver.scriptedRows = fullPage(viewModel) + + let firstLoad = Task { await viewModel.load(isInitial: true) } + await counting.waitUntilEntered() + driver.scriptedRows = [] + await viewModel.goToNextPage() + + #expect(viewModel.isPastTheEnd) + #expect(viewModel.pagination.totalRows == nil) + #expect(viewModel.showsPaginationBar) + #expect(viewModel.canGoToPreviousPage) + #expect(!viewModel.canGoToNextPage) + + await counting.open() + await firstLoad.value + #expect(viewModel.pagination.totalRows == pageSize) } @Test("Search and the enabled filters reach the driver; binary columns are not searched") diff --git a/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift index 9de74f2a30..0cd439b7fc 100644 --- a/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift @@ -7,7 +7,6 @@ import TableProQuery import Testing @MainActor -@Suite("DataBrowserViewModel") struct DataBrowserViewModelTests { private func makeSession(driver: MockDatabaseDriver) -> ConnectionSession { ConnectionSession( @@ -54,7 +53,7 @@ struct DataBrowserViewModelTests { vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost") await vm.load(isInitial: true) - #expect(vm.legacyRows.count == 2) + #expect(vm.window.shownValues.count == 2) #expect(vm.columnDetails.count == 2) #expect(vm.hasPrimaryKeys == true) #expect(vm.loadError == nil) @@ -85,7 +84,7 @@ struct DataBrowserViewModelTests { #expect(log.tracked == 1) #expect(log.callsInside == driver.fetchColumnsCalls + driver.executedQueries.count) #expect(log.callsInside > 0) - #expect(vm.legacyRows.count == 1) + #expect(vm.window.shownValues.count == 1) } @Test("A page cut short is not taken as the end of the table") @@ -103,7 +102,7 @@ struct DataBrowserViewModelTests { vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost") await vm.load(isInitial: true) - #expect(vm.legacyRows.count == 1) + #expect(vm.window.shownValues.count == 1) #expect(vm.pagination.totalRows == 120) #expect(vm.canGoToNextPage) } @@ -150,7 +149,7 @@ struct DataBrowserViewModelTests { let vm = DataBrowserViewModel() vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost") await vm.load(isInitial: true) - #expect(vm.legacyRows.count == 1) + #expect(vm.window.shownValues.count == 1) #expect(vm.isLoading == false) #expect(vm.isPageLoading == false) @@ -162,7 +161,7 @@ struct DataBrowserViewModelTests { #expect(vm.isLoading == false) #expect(vm.isPageLoading == false) - #expect(vm.legacyRows.count == 2) + #expect(vm.window.shownValues.count == 2) } @Test("pagination prev/next clamps at boundaries") @@ -220,7 +219,7 @@ struct DataBrowserViewModelTests { vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "customers"), databaseType: .mysql, host: "localhost") await vm.load(isInitial: true) - #expect(vm.legacyRows.count == 1) + #expect(vm.window.shownValues.count == 1) #expect(vm.rowLocator(forRowAt: 0) == nil) } @@ -411,6 +410,67 @@ struct DataBrowserViewModelTests { #expect((userDriver.executedQueries.first ?? "").contains("\"code\" IN (5, '5')")) } + private func fullPage(_ vm: DataBrowserViewModel, from start: Int = 0) -> Result { + let rows = (start.. Result { + .success(QueryResult(columns: [], rows: [[String(total)]], rowsAffected: 0, executionTime: 0)) + } + + @Test("The rows show and take taps while the row count still runs") + func rowsShowBeforeTheCount() async { + let driver = MockDatabaseDriver() + driver.scriptedColumns = makeColumns() + let gate = ExecuteGate(holding: [2]) + driver.beforeExecute = { await gate.pass() } + let vm = DataBrowserViewModel() + driver.scriptedExecuteResults = [fullPage(vm), rowCount(5_000)] + vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost") + + let load = Task { await vm.load(isInitial: true) } + await gate.arrival(of: 2) + + #expect(vm.isLoading == false) + #expect(vm.isPageLoading == false) + #expect(vm.window.count == vm.pagination.pageSize) + #expect(driver.fetchForeignKeysCalls == 1) + #expect(vm.pagination.totalRows == nil) + #expect(vm.canGoToNextPage) + #expect(vm.paginationLabel == "1-\(vm.pagination.pageSize)") + + await gate.release(2) + await load.value + #expect(vm.pagination.totalRows == 5_000) + } + + @Test("A sort or a new page size keeps the total, and a page turn reads the keys of a table without any only once") + func reloadsReadNoCountOrKeysAgain() async { + let driver = MockDatabaseDriver() + driver.scriptedColumns = makeColumns() + let vm = DataBrowserViewModel() + driver.scriptedExecuteResults = [fullPage(vm), rowCount(5_000)] + vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost") + await vm.load(isInitial: true) + #expect(driver.executedQueries.count == 2) + + driver.scriptedExecuteResults = [fullPage(vm)] + vm.sortState.columns = [SortColumn(name: "name", ascending: false)] + await vm.applySort() + #expect(driver.executedQueries.count == 3) + #expect(vm.pagination.totalRows == 5_000) + + driver.scriptedExecuteResults = [fullPage(vm, from: vm.pagination.pageSize)] + await vm.goToNextPage() + #expect(driver.executedQueries.count == 4) + #expect(driver.fetchForeignKeysCalls == 1) + + await vm.changePageSize(50) + #expect(driver.executedQueries.count == 5) + #expect(vm.pagination.totalRows == 5_000) + } + @Test("A load overtaken during its row count leaves the newer load's total and loading state alone") func overtakenCountIsDropped() async { let driver = MockDatabaseDriver() @@ -440,7 +500,7 @@ struct DataBrowserViewModelTests { await gate.release(3) await filtered.value #expect(vm.pagination.totalRows == 1) - #expect(vm.legacyRows.count == 1) + #expect(vm.window.shownValues.count == 1) #expect(vm.isLoading == false) #expect(vm.isPageLoading == false) } @@ -469,6 +529,40 @@ struct DataBrowserViewModelTests { #expect(driver.executedQueries.contains(where: { $0.uppercased().hasPrefix("DELETE") })) } + @Test("A delete confirmed after another window moved the browser to another schema is refused") + func deleteRaisedInAnotherSchemaIsRefused() { + let driver = MockDatabaseDriver() + let session = makeSession(driver: driver) + let vm = DataBrowserViewModel() + vm.attach(session: session, table: TableInfo(name: "users"), databaseType: .mssql, host: "localhost", schema: "dbo") + let raised = vm.rowSource + #expect(!vm.refusesDelete(raisedOn: raised)) + #expect(vm.operationError == nil) + + vm.attach(session: session, table: TableInfo(name: "users"), databaseType: .mssql, host: "localhost", schema: "archive") + + #expect(vm.refusesDelete(raisedOn: raised)) + #expect(vm.operationError != nil) + #expect(driver.executedQueries.isEmpty) + } + + @Test("A delete confirmed after a redial handed the browser a new session is refused") + func deleteRaisedOnAnOlderSessionIsRefused() { + let vm = DataBrowserViewModel() + vm.attach( + session: makeSession(driver: MockDatabaseDriver()), table: TableInfo(name: "users"), + databaseType: .postgresql, host: "localhost", schema: "public" + ) + let raised = vm.rowSource + + vm.attach( + session: makeSession(driver: MockDatabaseDriver()), table: TableInfo(name: "users"), + databaseType: .postgresql, host: "localhost", schema: "public" + ) + + #expect(vm.refusesDelete(raisedOn: raised)) + } + @Test("A binary, NULL or ref-less cut key cell gives no key, and a cut one with a ref gives the whole key") func nonTextKeyGivesNoKey() async { let columns = [ @@ -525,7 +619,7 @@ struct DataBrowserViewModelTests { #expect(success == false) #expect(vm.operationError?.title == String(localized: "Row Not Deleted")) #expect(driver.executedQueries.contains { $0.hasPrefix("SELECT `id` FROM `users` WHERE `id` = 1") }) - #expect(vm.legacyRows == [["1", "Alice"]]) + #expect(vm.window.shownValues == [["1", "Alice"]]) } @Test("A delete whose count reads 0, as under SET NOCOUNT ON, succeeds once the row is gone") @@ -766,7 +860,7 @@ struct DataBrowserViewModelTests { } } - @Test("changePageSize resets currentPage and totalRows") + @Test("changePageSize returns to the first page") func changePageSizeResets() async { let driver = MockDatabaseDriver() driver.scriptedColumns = makeColumns() @@ -820,7 +914,7 @@ struct DataBrowserViewModelTests { driver.scriptedExecuteResults = [emptyResult()] await vm.applySearch("nobody") - #expect(vm.legacyRows.isEmpty) + #expect(vm.window.shownValues.isEmpty) #expect(vm.showsPaginationBar) } @@ -837,7 +931,7 @@ struct DataBrowserViewModelTests { driver.scriptedExecuteResults = [emptyResult()] await vm.applyFilters() - #expect(vm.legacyRows.isEmpty) + #expect(vm.window.shownValues.isEmpty) #expect(vm.showsPaginationBar) } @@ -872,6 +966,162 @@ struct DataBrowserViewModelTests { return QueryResult(columns: makeColumns(), rows: rows, rowsAffected: 0, executionTime: 0) } + private func firstPageQuery( + _ type: DatabaseType, + columns: [ColumnInfo], + sort: SortState = SortState() + ) async -> String? { + let driver = MockDatabaseDriver() + driver.scriptedColumns = columns + let vm = DataBrowserViewModel() + vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "events"), databaseType: type, host: "localhost") + vm.sortState = sort + await vm.load(isInitial: true) + // The page size is the user's default, so the expected text names it as `{limit}`. + return driver.executedQueries.first?.replacingOccurrences(of: "LIMIT \(vm.pagination.pageSize) ", with: "LIMIT {limit} ") + .replacingOccurrences(of: "NEXT \(vm.pagination.pageSize) ROWS", with: "NEXT {limit} ROWS") + } + + private var compositeKey: [ColumnInfo] { + [ + ColumnInfo(name: "tenant", typeName: "INT", isPrimaryKey: true, ordinalPosition: 0), + ColumnInfo(name: "id", typeName: "INT", isPrimaryKey: true, ordinalPosition: 1), + ColumnInfo(name: "status", typeName: "TEXT", ordinalPosition: 2) + ] + } + + @Test("A PostgreSQL page orders by every key column, so paging neither repeats nor skips rows") + func postgresPageOrdersByTheWholeKey() async { + let query = await firstPageQuery(.postgresql, columns: compositeKey) + #expect(query == #"SELECT * FROM "events" ORDER BY "tenant" ASC, "id" ASC LIMIT {limit} OFFSET 0"#) + } + + @Test("A ClickHouse page orders by its key columns") + func clickHousePageOrdersByTheKey() async { + let columns = [ + ColumnInfo(name: "day", typeName: "Date", isPrimaryKey: true, ordinalPosition: 0), + ColumnInfo(name: "hits", typeName: "UInt64", ordinalPosition: 1) + ] + let query = await firstPageQuery(.clickhouse, columns: columns) + #expect(query == "SELECT * FROM `events` ORDER BY `day` ASC LIMIT {limit} OFFSET 0") + } + + @Test("A PostgreSQL table with no key pages as before") + func keylessPostgresPageHasNoOrder() async { + let columns = [ColumnInfo(name: "note", typeName: "TEXT", ordinalPosition: 0)] + let query = await firstPageQuery(.postgresql, columns: columns) + #expect(query == #"SELECT * FROM "events" LIMIT {limit} OFFSET 0"#) + } + + @Test("The user's sort comes first and the key breaks its ties") + func userSortKeepsTheKeyAsTiebreak() async { + let sort = SortState(columns: [SortColumn(name: "status", ascending: false)]) + let query = await firstPageQuery(.postgresql, columns: compositeKey, sort: sort) + #expect(query == #"SELECT * FROM "events" ORDER BY "status" DESC, "tenant" ASC, "id" ASC LIMIT {limit} OFFSET 0"#) + } + + @Test("A key column the user sorts by is not repeated") + func userSortOnAKeyColumnIsNotRepeated() async { + let sort = SortState(columns: [SortColumn(name: "id", ascending: false)]) + let query = await firstPageQuery(.postgresql, columns: compositeKey, sort: sort) + #expect(query == #"SELECT * FROM "events" ORDER BY "id" DESC, "tenant" ASC LIMIT {limit} OFFSET 0"#) + } + + @Test("SQL Server orders by the whole key, and a keyless table by its first column") + func sqlServerOrdersByTheWholeKey() async { + let keyed = await firstPageQuery(.mssql, columns: compositeKey) + #expect(keyed == "SELECT * FROM [events] ORDER BY [tenant] ASC, [id] ASC OFFSET 0 ROWS FETCH NEXT {limit} ROWS ONLY") + let keyless = await firstPageQuery(.mssql, columns: [ColumnInfo(name: "note", typeName: "NVARCHAR", ordinalPosition: 0)]) + #expect(keyless == "SELECT * FROM [events] ORDER BY [note] ASC OFFSET 0 ROWS FETCH NEXT {limit} ROWS ONLY") + } + + @Test("A SQL Server page, its count and its column and key reads name the browser's database") + func sqlServerPageNamesTheDatabase() async { + let driver = MockDatabaseDriver() + driver.scriptedColumns = makeColumns() + let vm = DataBrowserViewModel() + // A full page leaves the total to the count. + driver.scriptedExecuteResults = [ + .success(numberedPage(from: 1, count: vm.pagination.pageSize)), + .success(QueryResult(columns: [], rows: [["500"]], rowsAffected: 0, executionTime: 0)) + ] + vm.attach( + session: makeSession(driver: driver), table: TableInfo(name: "orders"), databaseType: .mssql, + host: "localhost", schema: "dbo", database: "shop" + ) + await vm.load(isInitial: true) + + #expect(driver.executedQueries.first?.hasPrefix("SELECT * FROM [shop].[dbo].[orders] ORDER BY [id] ASC") == true) + #expect(driver.executedQueries.contains("SELECT COUNT(*) FROM [shop].[dbo].[orders]")) + #expect(driver.fetchColumnsDatabases == ["shop"]) + #expect(driver.fetchForeignKeysDatabases == ["shop"]) + #expect(vm.pagination.totalRows == 500) + } + + @Test("A SQL Server delete, and the read that finds its row gone, name the browser's database") + func sqlServerDeleteNamesTheDatabase() async { + let driver = MockDatabaseDriver() + driver.scriptedColumns = makeColumns() + driver.scriptedExecuteResults = [ + .success(QueryResult(columns: makeColumns(), rows: [["1", "Alice"]], rowsAffected: 0, executionTime: 0)) + ] + let vm = DataBrowserViewModel() + vm.attach( + session: makeSession(driver: driver), table: TableInfo(name: "orders"), databaseType: .mssql, + host: "localhost", schema: "dbo", database: "shop" + ) + await vm.load(isInitial: true) + // SET NOCOUNT ON reports 0, so the delete looks for the row afterwards. + driver.scriptedExecuteResults = [ + .success(QueryResult(columns: [], rows: [], rowsAffected: 0, executionTime: 0)), + .success(QueryResult(columns: [], rows: [], rowsAffected: 0, executionTime: 0)) + ] + + let deleted = await vm.deleteRow(RowLocator(row: ["1", "Alice"], keyValues: [(column: "id", value: "1")])) + + #expect(deleted) + #expect(driver.executedQueries.contains("DELETE FROM [shop].[dbo].[orders] WHERE [id] = 1")) + #expect(driver.executedQueries.contains("SELECT TOP 1 [id] FROM [shop].[dbo].[orders] WHERE [id] = 1")) + } + + @Test("A delete confirmed after the browser moved to another database is refused") + func deleteRaisedInAnotherDatabaseIsRefused() { + let session = makeSession(driver: MockDatabaseDriver()) + let vm = DataBrowserViewModel() + vm.attach( + session: session, table: TableInfo(name: "orders"), databaseType: .mssql, host: "localhost", + schema: "dbo", database: "shop" + ) + let raised = vm.rowSource + + vm.attach( + session: session, table: TableInfo(name: "orders"), databaseType: .mssql, host: "localhost", + schema: "dbo", database: "staging" + ) + + #expect(vm.refusesDelete(raisedOn: raised)) + } + + @Test("A SQL Server full value is read from the database its row came from") + func sqlServerFullValueNamesTheDatabase() async throws { + let driver = MockDatabaseDriver() + let vm = DataBrowserViewModel() + driver.scriptedExecuteResults = [.success(QueryResult(columns: [], rows: [["full"]], rowsAffected: 0, executionTime: 0))] + + let value = try await vm.loadFullValue( + driver: driver, ref: bodyRef("bio"), databaseType: .mssql, schema: "dbo", database: "shop" + ) + + #expect(value == "full") + #expect(driver.executedQueries.first?.hasPrefix("SELECT TOP 1 [bio] FROM [shop].[dbo].[users] WHERE") == true) + } + + @Test("MySQL and Redshift pages keep their scan order") + func scanOrderedEnginesAddNoOrder() async { + #expect(await firstPageQuery(.mysql, columns: compositeKey) == "SELECT * FROM `events` LIMIT {limit} OFFSET 0") + #expect(await firstPageQuery(.redshift, columns: compositeKey) == #"SELECT * FROM "events" LIMIT {limit} OFFSET 0"#) + } + @Test("A page trimmed under memory pressure labels the rows it kept, not the ones it dropped") func trimmedPageLabelsTheRowsItKept() async { let driver = MockDatabaseDriver() @@ -889,8 +1139,8 @@ struct DataBrowserViewModelTests { await vm.handlePressure(.warning) - #expect(vm.legacyRows.count == 100) - #expect(vm.legacyRows.first?.first == "901") + #expect(vm.window.shownValues.count == 100) + #expect(vm.window.shownValues.first?.first == "901") #expect(vm.paginationLabel == DataBrowserViewModel.pageRangeLabel(start: 901, end: 1_000, total: 3_503)) } @@ -909,7 +1159,7 @@ struct DataBrowserViewModelTests { driver.scriptedExecuteResults = [emptyResult()] await vm.applyFilters() - #expect(vm.legacyRows.isEmpty) + #expect(vm.window.shownValues.isEmpty) #expect(vm.memoryWarning == nil) } @@ -950,7 +1200,7 @@ struct DataBrowserViewModelTests { #expect(vm.pagination.totalRows == pageSize * 3) #expect(vm.columnDetails.map(\.name) == ["index", "element"]) #expect(vm.hasPrimaryKeys == false) - #expect(vm.legacyRows.first == [String(pageSize), "e\(pageSize)"]) + #expect(vm.window.shownValues.first == [String(pageSize), "e\(pageSize)"]) #expect(vm.loadError == nil) #expect(vm.isLoading == false) } @@ -977,7 +1227,7 @@ struct DataBrowserViewModelTests { await overtaken.value #expect(driver.pageRequests.count == 2) - #expect(vm.legacyRows.first == [String(pageSize), "e\(pageSize)"]) + #expect(vm.window.shownValues.first == [String(pageSize), "e\(pageSize)"]) #expect(vm.loadError == nil) #expect(vm.isLoading == false) } @@ -1017,10 +1267,41 @@ struct DataBrowserViewModelTests { await vm.load(isInitial: true) #expect(vm.loadError?.title == String(localized: "Key Not Found")) - #expect(vm.legacyRows.isEmpty) + #expect(vm.window.shownValues.isEmpty) #expect(vm.isLoading == false) #expect(driver.executedQueries.isEmpty) } + + @Test("a long value in a key's contents stays whole within the budget, and the grid shows a preview of it") + func longKeyValueStaysWhole() async { + let long = String(repeating: "v", count: 10_000) + let driver = MockKeyContentsDriver() + driver.scriptedPages = [ + .success(KeyContentsPage( + result: QueryResult( + columns: [ColumnInfo(name: "value", typeName: "string", ordinalPosition: 0)], + rows: [[long]], + rowsAffected: 0, + executionTime: 0 + ), + totalCount: 1 + )) + ] + let vm = DataBrowserViewModel(textBudgetBytes: { 1_000_000 }) + let session = ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "db0", tables: []) + vm.attach(session: session, table: TableInfo(name: "profile"), databaseType: .redis, host: "localhost") + + await vm.load(isInitial: true) + + guard case .text(let kept)? = vm.cells(at: 0).first else { + Issue.record("the value should be kept whole, got \(vm.cells(at: 0))") + return + } + #expect(kept == long) + #expect(DataBrowserViewModel.canComplete(vm.cells(at: 0))) + let shown = vm.window.shownValues.first?.first ?? nil + #expect(shown?.utf8.count == Cell.previewByteLimit) + } } private actor ExecuteGate { diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift index 4b674276ca..92f843a957 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift @@ -95,7 +95,15 @@ final class DuckDBDriverTests: XCTestCase { XCTAssertEqual(streamed.count, 1) XCTAssertEqual(materialized.rows.count, 1) - XCTAssertEqual(streamed[0], materialized.rows[0], "streamed output must match the materialized path exactly") + // A streamed BLOB is a binary cell, as on every other engine; `execute` still carries its base64. + let payload = 10 + XCTAssertEqual(streamed[0][payload], Cell.binary(byteCount: 3, ref: nil).displayString) + XCTAssertEqual(materialized.rows[0][payload], Data("abc".utf8).base64EncodedString()) + var streamedRest = streamed[0] + var materializedRest = materialized.rows[0] + streamedRest.remove(at: payload) + materializedRest.remove(at: payload) + XCTAssertEqual(streamedRest, materializedRest, "streamed output must match the materialized path for every other type") } func testStreamingMultiRowOrder() async throws { @@ -323,6 +331,30 @@ final class DuckDBDriverTests: XCTestCase { } } +struct DuckDBBlobStreamingTests { + @Test("A streamed BLOB is a binary cell of its byte count, with the ref every other column gets") + func blobStreamsAsBinary() async throws { + let driver = DuckDBDriver(source: .inMemory) + try await driver.connect() + _ = try await driver.execute(query: "CREATE TABLE files (id INTEGER PRIMARY KEY, payload BLOB)") + _ = try await driver.execute(query: "INSERT INTO files VALUES (1, 'abc'::BLOB)") + let options = StreamOptions(maxRows: 10, lazyContext: LazyContext(table: "files", primaryKeyColumns: ["id"])) + + var cells: [Cell] = [] + for try await element in driver.executeStreaming(query: "SELECT id, payload FROM files", options: options) { + if case .row(let row) = element { cells = row.cells } + } + try await driver.disconnect() + + guard cells.count == 2, case .binary(let byteCount, let ref) = cells[1] else { + Issue.record("the BLOB should stream as a binary cell, got \(cells)") + return + } + #expect(byteCount == 3) + #expect(ref?.primaryKey == [PrimaryKeyComponent(column: "id", value: "1")]) + } +} + struct DuckDBDriverTransactionStateTests { private func connectedDriver() async throws -> DuckDBDriver { let driver = DuckDBDriver(source: .inMemory) diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBSafeModeReadOnlyTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBSafeModeReadOnlyTests.swift new file mode 100644 index 0000000000..17535c2a15 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBSafeModeReadOnlyTests.swift @@ -0,0 +1,117 @@ +// +// DuckDBSafeModeReadOnlyTests.swift +// TableProMobileTests +// +// Plugins/DuckDBDriverPlugin/DuckDBReadOnlyStatement.swift, compiled into this target: the Mac plugin is a registry +// plugin with no test bundle of its own. +// + +import CDuckDB +import Foundation +import TableProPluginKit +import Testing + +struct DuckDBSafeModeReadOnlyTests { + private struct Refused: Error {} + + private final class Session { + private var database: duckdb_database? + private var connection: duckdb_connection? + + init() throws { + guard duckdb_open_ext(nil, &database, nil, nil) != DuckDBError, + duckdb_connect(database, &connection) != DuckDBError + else { throw Refused() } + try run("CREATE TABLE kept (a INTEGER)") + try run("INSERT INTO kept VALUES (1)") + } + + deinit { + duckdb_disconnect(&connection) + duckdb_close(&database) + } + + var handle: duckdb_connection { + get throws { + guard let connection else { throw Refused() } + return connection + } + } + + func run(_ sql: String) throws { + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + guard duckdb_query(try handle, sql, &result) != DuckDBError else { throw Refused() } + } + + // The CAST fails unless the count matches, so the check needs no value reading. + func holds(_ rows: Int) -> Bool { + (try? run("SELECT CAST(CASE WHEN count(*) = \(rows) THEN '1' ELSE 'x' END AS INTEGER) FROM kept")) != nil + } + } + + @Test("An external read names external clients, Read-Only from any origin names Safe Mode, an app read alone neither") + func reasonFollowsTheContext() { + var externalRead = PluginStatementContext() + externalRead.readOnly = true + externalRead.origin = .external + var externalSafeModeRead = externalRead + externalSafeModeRead.enforcesReadOnly = true + var externalWrite = PluginStatementContext() + externalWrite.origin = .external + externalWrite.enforcesReadOnly = true + var catalogRead = PluginStatementContext() + catalogRead.readOnly = true + catalogRead.readsCatalog = true + + #expect(DuckDBReadOnlyStatement.reason(for: externalRead) == .externalClients) + #expect(DuckDBReadOnlyStatement.reason(for: externalSafeModeRead) == .externalClients) + #expect(DuckDBReadOnlyStatement.reason(for: PluginStatementContext(enforcesReadOnly: true)) == .safeMode) + #expect(DuckDBReadOnlyStatement.reason(for: externalWrite) == .safeMode) + #expect(DuckDBReadOnlyStatement.reason(for: catalogRead) == nil) + #expect(DuckDBReadOnlyStatement.reason(for: PluginStatementContext()) == nil) + } + + @Test("At Read-Only a write is refused inside the read-only transaction, and the table keeps its rows") + func safeModeRefusesWrites() throws { + let session = try Session() + + for sql in ["INSERT INTO kept VALUES (2)", "UPDATE kept SET a = 5", "CREATE TABLE added (a INTEGER)"] { + #expect(throws: Refused.self, "\(sql)") { + try DuckDBReadOnlyStatement.run(on: try session.handle, session: .idle, reason: .safeMode) { + try session.run(sql) + } + } + } + let read = try DuckDBReadOnlyStatement.run(on: try session.handle, session: .idle, reason: .safeMode) { + session.holds(1) + } + + #expect(read) + try session.run("INSERT INTO kept VALUES (3)") + #expect(session.holds(2)) + } + + @Test( + "At Read-Only a session not known to be idle is refused before anything runs, naming Safe Mode", + arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction, .unknown] + ) + func safeModeRefusesUnlessIdle(state: PluginSessionTransactionState) throws { + let session = try Session() + try session.run("BEGIN TRANSACTION") + try session.run("INSERT INTO kept VALUES (500)") + + var ran = false + #expect(throws: DuckDBReadOnlyStatement.Refusal.safeModeTransactionOpen) { + try DuckDBReadOnlyStatement.run(on: try session.handle, session: state, reason: .safeMode) { ran = true } + } + + #expect(!ran) + #expect( + DuckDBReadOnlyStatement.Refusal.safeModeTransactionOpen.pluginErrorMessage + != DuckDBReadOnlyStatement.Refusal.transactionOpen.pluginErrorMessage + ) + try session.run("COMMIT") + #expect(session.holds(2)) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/FakeMySQLServer.swift b/apple/TableProMobile/TableProMobileTests/Drivers/FakeMySQLServer.swift index cc15f357aa..3cc708f3cf 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/FakeMySQLServer.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/FakeMySQLServer.swift @@ -13,10 +13,37 @@ import TableProTLSTestFixtures // statement does, or with `killedQueriesComplete` succeeds as one that finished before the kill landed. Unkilled, // or after `release`, it succeeds. A query starting with `streamsRowsStarting` returns `streamedRows` one-column rows, // cut by a kill for its session with 1317. A cleartext password passed to `refuse` is answered with 1045, as an expired -// token is. +// token is. With `refusesLogin` every login gets that error, as a refused sign-in or database does. A query in +// `failures` is answered with its error, after its rows when it has any. Replies carry SERVER_STATUS_IN_TRANS from a +// BEGIN or START TRANSACTION to the COMMIT or ROLLBACK that ends it. final class FakeMySQLServer: @unchecked Sendable { struct SetupFailure: Error {} + struct Failure: Sendable { + let code: UInt16 + let sqlState: String + let message: String + var rowsFirst: Int? + + // ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION. + static func readOnlyRefusal(rowsFirst: Int? = nil) -> Failure { + Failure( + code: 1_792, sqlState: "25006", message: "Cannot execute statement in a READ ONLY transaction.", + rowsFirst: rowsFirst + ) + } + } + + struct LoginRefusal { + let code: UInt16 + let sqlState: String + let message: String + + var packet: [UInt8] { + [0xFF, UInt8(code & 0xFF), UInt8(code >> 8)] + Array("#\(sqlState)".utf8) + Array(message.utf8) + } + } + struct Identity { let certificate: String let key: String @@ -66,7 +93,6 @@ final class FakeMySQLServer: @unchecked Sendable { private static let comQuit: UInt8 = 0x01 private static let comQuery: UInt8 = 0x03 - private static let okPacket: [UInt8] = [0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00] private static let sslRequestLength = 32 private static let switchToCleartext: [UInt8] = [0xFE] + Array("mysql_clear_password".utf8) + [0] private static let switchToSHA2: [UInt8] = [0xFE] + Array("caching_sha2_password".utf8) + [0] @@ -78,8 +104,6 @@ final class FakeMySQLServer: @unchecked Sendable { // ER_ACCESS_DENIED_ERROR (1045), SQLSTATE 28000. private static let accessDeniedPacket: [UInt8] = [0xFF, 0x15, 0x04] + Array("#28000".utf8) + Array("Access denied for user (using password: YES)".utf8) - // No warnings, SERVER_STATUS_AUTOCOMMIT. - private static let eofPacket: [UInt8] = [0xFE, 0x00, 0x00, 0x02, 0x00] // A utf8mb4 VARCHAR(255) named `n`: catalog, schema, table, org_table, name and org_name, then charset 45, length // 1020, type 0xFD, no flags or decimals. private static let textColumnDefinition: [UInt8] = ["def", "", "", "", "n", ""] @@ -96,6 +120,9 @@ final class FakeMySQLServer: @unchecked Sendable { private let killedQueriesComplete: Bool private let streamedPrefix: String? private let streamedRows: Int + private let serverVersion: String + private let failures: [String: Failure] + private let loginRefusal: LoginRefusal? private let fixtures: URL? private let kills = KillBoard() private let refusedPasswords = OSAllocatedUnfairLock>(initialState: []) @@ -122,7 +149,10 @@ final class FakeMySQLServer: @unchecked Sendable { holdSeconds: TimeInterval = 10, killedQueriesComplete: Bool = false, streamsRowsStarting streamedPrefix: String? = nil, - streamedRows: Int = 0 + streamedRows: Int = 0, + serverVersion: String = "8.0.36", + failures: [String: Failure] = [:], + refusesLogin loginRefusal: LoginRefusal? = nil ) throws { self.asksForCleartext = asksForCleartext self.asksForFullSHA2Auth = asksForFullSHA2Auth @@ -132,6 +162,9 @@ final class FakeMySQLServer: @unchecked Sendable { self.killedQueriesComplete = killedQueriesComplete self.streamedPrefix = streamedPrefix self.streamedRows = streamedRows + self.serverVersion = serverVersion + self.failures = failures + self.loginRefusal = loginRefusal if let identity { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("FakeMySQLServer-\(UUID().uuidString)", isDirectory: true) @@ -276,20 +309,32 @@ final class FakeMySQLServer: @unchecked Sendable { return } } - guard channel.send(Self.okPacket, sequence: sequence &+ 1) else { return } + if let loginRefusal { + _ = channel.send(loginRefusal.packet, sequence: sequence &+ 1) + return + } + guard channel.send(Self.okPacket(inTransaction: false), sequence: sequence &+ 1) else { return } session.loggedIn = true + var inTransaction = false while let command = channel.readPacket() { if command.payload.first == Self.comQuit { session.quit = true return } - var reply = Self.okPacket + var reply = Self.okPacket(inTransaction: inTransaction) if command.payload.first == Self.comQuery { let query = String(decoding: command.payload.dropFirst(), as: UTF8.self) session.queries.append(query) + inTransaction = Self.inTransaction(after: query, was: inTransaction) + reply = Self.okPacket(inTransaction: inTransaction) + if let failure = failures[query] { + guard fail(with: failure, on: channel, after: command.sequence, inTransaction: inTransaction) else { return } + continue + } if let streamedPrefix, query.hasPrefix(streamedPrefix) { - guard streamRows(on: channel, after: command.sequence, into: &session) else { return } + guard streamRows(on: channel, after: command.sequence, inTransaction: inTransaction, into: &session) + else { return } continue } if let killed = Self.killedSession(in: query) { @@ -305,13 +350,19 @@ final class FakeMySQLServer: @unchecked Sendable { // A text result set without CLIENT_DEPRECATE_EOF, which the handshake does not offer: the column count, the column, // an EOF, the rows and a closing EOF, or 1317 in place of the rest once a kill for the session arrives. - private func streamRows(on channel: ServerChannel, after sequence: UInt8, into session: inout Session) -> Bool { + private func streamRows( + on channel: ServerChannel, + after sequence: UInt8, + inTransaction: Bool, + into session: inout Session + ) -> Bool { var next = sequence func send(_ payload: [UInt8]) -> Bool { next &+= 1 return channel.send(payload, sequence: next) } - guard send([1]), send(Self.textColumnDefinition), send(Self.eofPacket) else { return false } + let eof = Self.eofPacket(inTransaction: inTransaction) + guard send([1]), send(Self.textColumnDefinition), send(eof) else { return false } for index in 0.. Bool { + var next = sequence + func send(_ payload: [UInt8]) -> Bool { + next &+= 1 + return channel.send(payload, sequence: next) + } + if let rows = failure.rowsFirst { + guard send([1]), send(Self.textColumnDefinition), send(Self.eofPacket(inTransaction: inTransaction)) else { + return false + } + for index in 0..> 8)] + Array("#\(failure.sqlState)".utf8) + return send(error + Array(failure.message.utf8)) + } + + private static func inTransaction(after query: String, was open: Bool) -> Bool { + let head = query.trimmingCharacters(in: .whitespaces).uppercased() + if head.hasPrefix("BEGIN") || head.hasPrefix("START TRANSACTION") { return true } + if head.hasPrefix("COMMIT") || head.hasPrefix("ROLLBACK") { return false } + return open + } + + // SERVER_STATUS_AUTOCOMMIT, with SERVER_STATUS_IN_TRANS inside a transaction. + private static func status(inTransaction: Bool) -> [UInt8] { + [inTransaction ? 0x03 : 0x02, 0x00] + } + + private static func okPacket(inTransaction: Bool) -> [UInt8] { + [0x00, 0x00, 0x00] + status(inTransaction: inTransaction) + [0x00, 0x00] + } + + // No warnings. + private static func eofPacket(inTransaction: Bool) -> [UInt8] { + [0xFE, 0x00, 0x00] + status(inTransaction: inTransaction) } private static func lengthEncoded(_ text: String) -> [UInt8] { @@ -341,7 +432,7 @@ final class FakeMySQLServer: @unchecked Sendable { capabilities |= Capability.ssl } var payload: [UInt8] = [10] - payload += Array("8.0.36".utf8) + [0] + payload += Array(serverVersion.utf8) + [0] payload += (0..<4).map { UInt8((connectionId >> ($0 * 8)) & 0xFF) } payload += Array("abcdefgh".utf8) + [0] payload += [UInt8(capabilities & 0xFF), UInt8((capabilities >> 8) & 0xFF)] diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift b/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift index 4f1d047357..f17ab72448 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift @@ -1,21 +1,36 @@ import Darwin import Foundation +import os // A loopback PostgreSQL server for one plaintext session: it refuses SSL and GSS encryption, accepts any startup without -// a password, and answers every simple query with an empty completion. +// a password, and answers a statement through the simple or the extended protocol with its scripted reply, or with an +// empty completion. It keeps the transaction status a BEGIN, START TRANSACTION, COMMIT or ROLLBACK leaves, and answers a +// cancel request on a second connection without acting on it. final class FakePostgreSQLServer: @unchecked Sendable { struct SetupFailure: Error {} + enum Reply: Sendable { + // One text column `n`, its values `row 0`, `row 1` and on. + case rows(Int) + // An ErrorResponse after `rowsFirst` rows. + case failure(sqlState: String, message: String, rowsFirst: Int = 0) + } + struct Session: Sendable { var startedUp = false + // Every statement in the order it arrived, as a simple Query or a Parse. var queries: [String] = [] + var extendedQueries: [String] = [] var terminated = false } private static let protocolVersion: UInt32 = 196_608 private static let sslRequest: UInt32 = 80_877_103 private static let gssEncryptionRequest: UInt32 = 80_877_104 + private static let cancelRequest: UInt32 = 80_877_102 private static let pollMilliseconds: Int32 = 10_000 + private static let cancelPollMilliseconds: Int32 = 100 + private static let textOid: UInt32 = 25 private static let parameters: [(String, String)] = [ ("server_version", "16.4"), ("server_encoding", "UTF8"), @@ -28,8 +43,11 @@ final class FakePostgreSQLServer: @unchecked Sendable { let port: Int private let listener: Int32 + private let replies: [String: Reply] + private let sessionEnded = OSAllocatedUnfairLock(initialState: false) - init() throws { + init(replies: [String: Reply] = [:]) throws { + self.replies = replies let socketFD = socket(AF_INET, SOCK_STREAM, 0) guard socketFD >= 0 else { throw SetupFailure() } var option: Int32 = 1 @@ -76,27 +94,106 @@ final class FakePostgreSQLServer: @unchecked Sendable { private func runSession() -> Session { var session = Session() - guard let client = acceptClient() else { return session } + guard let client = acceptClient(within: Self.pollMilliseconds) else { return session } defer { Darwin.close(client) } + // Stopped and waited for before the session returns, so it never polls a listener the test has closed. + let cancels = DispatchGroup() + cancels.enter() + DispatchQueue(label: "com.TablePro.tests.fake-postgresql.cancel").async { + self.answerCancels() + cancels.leave() + } + defer { + sessionEnded.withLock { $0 = true } + cancels.wait() + } guard startUp(client) else { return session } session.startedUp = true + var conversation = Conversation() while let type = receive(client, 1)?.first, let body = receiveBody(client) { - switch type { - case UInt8(ascii: "Q"): - session.queries.append(String(decoding: body.prefix { $0 != 0 }, as: UTF8.self)) - guard send(client, Self.message("C", Array("SELECT 0".utf8) + [0])), - send(client, Self.readyForQuery) else { return session } - case UInt8(ascii: "X"): + guard type != UInt8(ascii: "X") else { session.terminated = true return session - default: - return session } + guard let reply = conversation.answer(type, body, server: self, session: &session), + reply.isEmpty || send(client, reply) else { return session } } return session } + // libpq's PQcancel opens a connection of its own, sends the request and waits for the server to close it. + private func answerCancels() { + while !sessionEnded.withLock({ $0 }) { + var pollFD = pollfd(fd: listener, events: Int16(POLLIN), revents: 0) + let ready = poll(&pollFD, 1, Self.cancelPollMilliseconds) + guard ready >= 0, pollFD.revents & Int16(POLLNVAL) == 0 else { return } + guard ready > 0 else { continue } + let connection = accept(listener, nil, nil) + guard connection >= 0 else { continue } + _ = receiveBody(connection) + Darwin.close(connection) + } + } + + // What one session has said so far: the transaction status and, in the extended protocol, the statement parsed and + // whether an error has the rest skipped up to the Sync. + private struct Conversation { + var status = UInt8(ascii: "I") + var parsed = "" + var skippingToSync = false + + mutating func answer(_ type: UInt8, _ body: [UInt8], server: FakePostgreSQLServer, session: inout Session) -> [UInt8]? { + switch type { + case UInt8(ascii: "Q"): + let query = String(decoding: body.prefix { $0 != 0 }, as: UTF8.self) + session.queries.append(query) + return run(query, server: server, describes: true) + FakePostgreSQLServer.message("Z", [status]) + case UInt8(ascii: "P"): + let fields = body.split(separator: 0, maxSplits: 2, omittingEmptySubsequences: false) + parsed = fields.count > 1 ? String(decoding: fields[1], as: UTF8.self) : "" + session.queries.append(parsed) + session.extendedQueries.append(parsed) + return FakePostgreSQLServer.message("1", []) + case UInt8(ascii: "B"): + return skippingToSync ? [] : FakePostgreSQLServer.message("2", []) + case UInt8(ascii: "D"): + return skippingToSync ? [] : FakePostgreSQLServer.rowDescription + case UInt8(ascii: "E"): + guard !skippingToSync else { return [] } + if case .failure? = server.replies[parsed] { skippingToSync = true } + return run(parsed, server: server, describes: false) + case UInt8(ascii: "S"): + skippingToSync = false + return FakePostgreSQLServer.message("Z", [status]) + case UInt8(ascii: "H"): + return [] + default: + return nil + } + } + + private mutating func run(_ query: String, server: FakePostgreSQLServer, describes: Bool) -> [UInt8] { + let head = query.trimmingCharacters(in: .whitespaces).uppercased() + switch server.replies[query] { + case .rows(let count)?: + return (describes ? FakePostgreSQLServer.rowDescription : []) + FakePostgreSQLServer.dataRows(count) + + FakePostgreSQLServer.message("C", Array("SELECT \(count)".utf8) + [0]) + case .failure(let sqlState, let message, let rowsFirst)?: + if status == UInt8(ascii: "T") { status = UInt8(ascii: "E") } + let header = rowsFirst > 0 && describes ? FakePostgreSQLServer.rowDescription : [] + return header + FakePostgreSQLServer.dataRows(rowsFirst) + FakePostgreSQLServer.errorResponse(sqlState, message) + case nil: + if head.hasPrefix("BEGIN") || head.hasPrefix("START TRANSACTION") { + status = UInt8(ascii: "T") + } else if ["COMMIT", "ROLLBACK", "END", "ABORT"].contains(where: { head.hasPrefix($0) }) { + status = UInt8(ascii: "I") + } + return FakePostgreSQLServer.message("C", Array("SELECT 0".utf8) + [0]) + } + } + } + // The client may ask for SSL or GSS encryption first; each gets "N" and the real startup follows. private func startUp(_ client: Int32) -> Bool { while let packet = receiveBody(client), packet.count >= 4 { @@ -111,14 +208,33 @@ final class FakePostgreSQLServer: @unchecked Sendable { reply += Self.message("S", Array(name.utf8) + [0] + Array(value.utf8) + [0]) } reply += Self.message("K", Self.bytes(UInt32(4_242)) + Self.bytes(UInt32(7_777))) - reply += Self.readyForQuery + reply += Self.message("Z", [UInt8(ascii: "I")]) return send(client, reply) } return false } - private static var readyForQuery: [UInt8] { - message("Z", [UInt8(ascii: "I")]) + // A text column `n`, from no table. + private static var rowDescription: [UInt8] { + message( + "T", + bytes16(1) + Array("n".utf8) + [0] + bytes(UInt32(0)) + bytes16(0) + bytes(textOid) + bytes16(0xFFFF) + + bytes(UInt32.max) + bytes16(0) + ) + } + + private static func dataRows(_ count: Int) -> [UInt8] { + var rows: [UInt8] = [] + for index in 0.. [UInt8] { + let fields: [(Character, String)] = [("S", "ERROR"), ("V", "ERROR"), ("C", sqlState), ("M", text)] + return message("E", fields.flatMap { [$0.0.asciiValue ?? 0] + Array($0.1.utf8) + [0] } + [0]) } private static func message(_ type: Character, _ body: [UInt8]) -> [UInt8] { @@ -129,6 +245,10 @@ final class FakePostgreSQLServer: @unchecked Sendable { [UInt8((value >> 24) & 0xFF), UInt8((value >> 16) & 0xFF), UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] } + private static func bytes16(_ value: UInt16) -> [UInt8] { + [UInt8(value >> 8), UInt8(value & 0xFF)] + } + private static func uint32(_ bytes: C) -> UInt32 where C.Element == UInt8 { bytes.reduce(0) { $0 << 8 | UInt32($1) } } @@ -141,9 +261,9 @@ final class FakePostgreSQLServer: @unchecked Sendable { return length == 4 ? [] : receive(client, length - 4) } - private func acceptClient() -> Int32? { + private func acceptClient(within milliseconds: Int32) -> Int32? { var pollFD = pollfd(fd: listener, events: Int16(POLLIN), revents: 0) - guard poll(&pollFD, 1, Self.pollMilliseconds) > 0 else { return nil } + guard poll(&pollFD, 1, milliseconds) > 0 else { return nil } let client = accept(listener, nil, nil) guard client >= 0 else { return nil } var noSignal: Int32 = 1 diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MSSQLCatalogDatabaseTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MSSQLCatalogDatabaseTests.swift new file mode 100644 index 0000000000..02996a82ff --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MSSQLCatalogDatabaseTests.swift @@ -0,0 +1,47 @@ +@testable import TableProMobile +import TableProMSSQLCore +import Testing + +struct MSSQLCatalogDatabaseTests { + @Test("A catalog read for a database runs through that database's sp_executesql, its quotes doubled") + func runsInTheDatabase() { + let statement = "SELECT name FROM sys.tables WHERE name = N'o''brien'" + + #expect(MSSQLDriver.catalogStatement(statement, in: "shop") + == "EXEC [shop].sys.sp_executesql N'SELECT name FROM sys.tables WHERE name = N''o''''brien'''") + } + + @Test("A closing bracket in the database name is doubled") + func quotesTheDatabase() { + #expect(MSSQLDriver.catalogStatement("SELECT 1", in: "shop]; DROP TABLE x --") + == "EXEC [shop]]; DROP TABLE x --].sys.sp_executesql N'SELECT 1'") + } + + @Test("No database, or an empty one, leaves the statement as it is") + func noDatabaseLeavesTheStatement() { + #expect(MSSQLDriver.catalogStatement("SELECT 1", in: nil) == "SELECT 1") + #expect(MSSQLDriver.catalogStatement("SELECT 1", in: "") == "SELECT 1") + } + + @Test("The database a window names wins, then the one the app last put the session in, then none") + func catalogTargetOrder() { + #expect(MSSQLDriver.catalogTarget(named: "shop", appDatabase: "staging") == "shop") + #expect(MSSQLDriver.catalogTarget(named: nil, appDatabase: "staging") == "staging") + #expect(MSSQLDriver.catalogTarget(named: "", appDatabase: "staging") == "staging") + #expect(MSSQLDriver.catalogTarget(named: nil, appDatabase: nil) == nil) + } + + @Test("Column, index and foreign key reads run whole inside the database, so OBJECT_ID resolves there") + func tableReadsRunInTheDatabase() { + let reads = [ + MSSQLSchemaQueries.columns(schema: "dbo", table: "orders"), + MSSQLSchemaQueries.indexes(schema: "dbo", table: "orders"), + MSSQLSchemaQueries.foreignKeys(schema: "dbo", table: "orders") + ] + + for read in reads { + #expect(MSSQLDriver.catalogStatement(read, in: "shop") + == "EXEC [shop].sys.sp_executesql \(MSSQLStringLiteral.quoted(read))") + } + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift index 3d73e531ea..98201d824a 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift @@ -73,7 +73,7 @@ struct MongoDBOnIOSTests { offset: 100 ) - #expect(query == #"db.users.find({"$and": [{"name": "ann"}, {"$or": [{"name": {"$regex": "x", "$options": "i"}}, {"email": {"$regex": "x", "$options": "i"}}]}]}).sort({"name": 1}).skip(100).limit(50)"#) + #expect(query == #"db.users.find({"$and": [{"name": "ann"}, {"$or": [{"name": {"$regex": "x", "$options": "i"}}, {"email": {"$regex": "x", "$options": "i"}}]}]}).sort({"name": 1, "_id": 1}).skip(100).limit(50)"#) } @Test("From MongoDB 4.2, search matches a number, a date or an ObjectId by its text as well") diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDialRefusalTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDialRefusalTests.swift new file mode 100644 index 0000000000..59e9d7e251 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDialRefusalTests.swift @@ -0,0 +1,96 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct MySQLDialRefusalTests { + @Test( + "A database refused at sign-in is told apart, and reads as any failed connect", + arguments: [ + FakeMySQLServer.LoginRefusal(code: 1_049, sqlState: "42000", message: "Unknown database 'app_archive'"), + FakeMySQLServer.LoginRefusal( + code: 1_044, sqlState: "42000", message: "Access denied for user 'app'@'%' to database 'app_archive'" + ), + ] + ) + func refusedDatabase(_ refusal: FakeMySQLServer.LoginRefusal) async throws { + let error = try await connectError(answering: refusal) + + guard case MySQLError.databaseRefused(let message)? = error else { + Issue.record("Expected a refused database, got \(String(describing: error))") + return + } + #expect(message == refusal.message) + #expect(error?.localizedDescription == "MySQL connection failed: \(refusal.message)") + } + + // 1524 is a sign-in plugin the server has not loaded; 1105 is what another server may send for anything. + @Test( + "Any other refusal stays a failed connect, whatever database the dial names", + arguments: [ + FakeMySQLServer.LoginRefusal( + code: 1_045, sqlState: "28000", message: "Access denied for user 'app'@'10.0.0.2' (using password: YES)" + ), + FakeMySQLServer.LoginRefusal( + code: 1_698, sqlState: "28000", message: "Access denied for user 'app'@'10.0.0.2'" + ), + FakeMySQLServer.LoginRefusal( + code: 4_151, sqlState: "HY000", message: "Access denied, this account is locked" + ), + FakeMySQLServer.LoginRefusal(code: 1_040, sqlState: "08004", message: "Too many connections"), + FakeMySQLServer.LoginRefusal( + code: 1_524, sqlState: "HY000", message: "Plugin 'mysql_native_password' is not loaded" + ), + FakeMySQLServer.LoginRefusal(code: 1_105, sqlState: "HY000", message: "Unknown database 'app_archive'"), + ] + ) + func refusedSignIn(_ refusal: FakeMySQLServer.LoginRefusal) async throws { + let error = try await connectError(answering: refusal) + + guard case MySQLError.connectionFailed(let message)? = error else { + Issue.record("Expected a failed connect, got \(String(describing: error))") + return + } + #expect(message == refusal.message) + } + + // Preferred asks for TLS first, then signs in again without it, where the server refuses the database. + @Test("A database refused after Preferred falls back to plaintext is still told apart") + func refusedDatabaseAfterPlaintextRetry() async throws { + let refusal = FakeMySQLServer.LoginRefusal(code: 1_049, sqlState: "42000", message: "Unknown database 'app_archive'") + let error = try await connectError( + answering: refusal, + ssl: DriverSSLConfiguration(mode: .require, isPreferred: true), + sessions: 2 + ) + + guard case MySQLError.databaseRefused(let message)? = error else { + Issue.record("Expected a refused database, got \(String(describing: error))") + return + } + #expect(message == refusal.message) + } + + private func connectError( + answering refusal: FakeMySQLServer.LoginRefusal, + ssl: DriverSSLConfiguration = .disabled, + sessions: Int = 1 + ) async throws -> (any Error)? { + let server = try FakeMySQLServer(identity: nil, refusesLogin: refusal) + defer { server.close() } + async let served = server.serve(sessions: sessions) + let driver = MySQLDriver( + host: "127.0.0.1", port: server.port, user: "app", password: "app-password", database: "app_archive", + ssl: ssl + ) + var thrown: (any Error)? + do { + try await driver.connect() + } catch { + thrown = error + } + _ = await served + return thrown + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDriverTransactionTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDriverTransactionTests.swift index 3299ed8c72..a9f57b7381 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDriverTransactionTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLDriverTransactionTests.swift @@ -87,6 +87,20 @@ struct MySQLSessionTransactionTests { ) } + @Test("Read-Only's own transaction reads as idle while it runs, and the flags decide once it ends") + func readOnlyTransactionReadsAsIdle() { + #expect( + MySQLSessionTransaction.state( + infoResult: 0, serverStatus: inTransaction | autocommit, holdsReadOnlyTransaction: true + ) == .idle + ) + #expect( + MySQLSessionTransaction.state( + infoResult: 0, serverStatus: inTransaction | autocommit, holdsReadOnlyTransaction: false + ) == .explicitTransaction + ) + } + @Test("A transaction the server opened because autocommit is off is implicit") func autocommitOffTransactionIsImplicit() { #expect(MySQLSessionTransaction.state(infoResult: 0, serverStatus: inTransaction) == .implicitTransaction) diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MySQLReadOnlyRunTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLReadOnlyRunTests.swift new file mode 100644 index 0000000000..d86d63178a --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLReadOnlyRunTests.swift @@ -0,0 +1,238 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import TableProModels +import Testing + +// The fake server gives the first session thread id 1; a kill arrives on a second session. +@Suite(.timeLimit(.minutes(1))) +struct MySQLReadOnlyRunTests { + private static let scan = "SELECT body FROM events" + private static let purge = "SELECT purge_events()" + private static let readOnly = StreamOptions(maxRows: 100, enforcesReadOnly: true) + + private static func wrapped(_ statement: String) -> [String] { + [MySQLReadOnlyTransaction.begin, statement, MySQLReadOnlyTransaction.end, "SELECT 2"] + } + + @Test("A Read-Only statement runs inside START TRANSACTION READ ONLY, and ROLLBACK follows its last row") + func statementRunsInReadOnlyTransaction() async throws { + let server = try FakeMySQLServer(identity: nil, streamsRowsStarting: Self.scan, streamedRows: 3) + let run = try await Self.run(on: server) { driver in + #expect(driver.enforcesReadOnlyMode) + return try await Self.read(driver.executeStreaming(query: Self.scan, options: Self.readOnly)) + } + + #expect(run.read.rows == 3) + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped(Self.scan)) + } + + @Test("A session holding LOCK TABLES takes no read-only transaction, which would release the lock") + func tableLockKeepsTheReadOnlyTransactionOut() async throws { + let server = try FakeMySQLServer(identity: nil) + let run = try await Self.run(on: server) { driver in + let before = await driver.entersReadOnlyMode() + _ = try await driver.execute(query: "LOCK TABLES events READ") + let locked = await driver.entersReadOnlyMode() + _ = try await driver.execute(query: "UNLOCK TABLES") + return (before: before, locked: locked, unlocked: await driver.entersReadOnlyMode()) + } + + #expect(run.read.before) + #expect(!run.read.locked) + #expect(run.read.unlocked) + } + + @Test("A statement with no result set ends its read-only transaction too") + func noResultEndsTheTransaction() async throws { + let server = try FakeMySQLServer(identity: nil) + let run = try await Self.run(on: server) { driver in + try await Self.read(driver.executeStreaming(query: "DO 1", options: Self.readOnly)) + } + + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped("DO 1")) + } + + @Test("Without Read-Only, on TiDB and on a server older than READ ONLY, the statement runs with no transaction around it", arguments: [ + (DatabaseType.mysql, "8.0.36", false), (.tidb, "8.0.11-TiDB-v8.5.1", true), (.mysql, "5.6.4-log", true), + (.mariadb, "5.5.64-MariaDB", true), + ]) + func otherRunsAreNotWrapped(type: DatabaseType, banner: String, enforcesReadOnly: Bool) async throws { + let server = try FakeMySQLServer( + identity: nil, streamsRowsStarting: Self.scan, streamedRows: 3, serverVersion: banner + ) + let options = StreamOptions(maxRows: 100, enforcesReadOnly: enforcesReadOnly) + let run = try await Self.run(on: server, databaseType: type) { driver in + try await Self.read(driver.executeStreaming(query: Self.scan, options: options)) + } + + #expect(run.read.rows == 3) + #expect(Array(run.served.queries.suffix(2)) == [Self.scan, "SELECT 2"]) + #expect(!run.served.queries.contains(MySQLReadOnlyTransaction.begin)) + } + + @Test("A statement that fails stays a query error, and its read-only transaction still ends") + func failedStatementEndsTheTransaction() async throws { + let missing = FakeMySQLServer.Failure(code: 1_146, sqlState: "42S02", message: "Table 'app.events' doesn't exist") + let server = try FakeMySQLServer(identity: nil, failures: [Self.scan: missing]) + let run = try await Self.run(on: server) { driver in + await #expect(throws: MySQLError.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.scan, options: Self.readOnly)) + } + } + + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped(Self.scan)) + } + + @Test("1792 is Read-Only's refusal, whether it ends the statement or its rows", arguments: [nil, 2] as [Int?]) + func refusalCodeIsReadOnlysRefusal(rowsFirst: Int?) async throws { + let server = try FakeMySQLServer( + identity: nil, failures: [Self.purge: .readOnlyRefusal(rowsFirst: rowsFirst)] + ) + let run = try await Self.run(on: server) { driver in + await #expect(throws: ReadOnlyModeRefusal.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.purge, options: Self.readOnly)) + } + } + + guard case .refused(let message, let serverMessage)? = run.read else { + Issue.record("expected Read-Only's refusal, got \(String(describing: run.read))") + return + } + #expect(message == MySQLReadOnlyTransaction.refusalMessage(.safeMode)) + #expect(serverMessage.contains("Cannot execute statement in a READ ONLY transaction.")) + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped(Self.purge)) + } + + @Test("A session with a transaction open is refused before anything is sent") + func openTransactionIsRefused() async throws { + let server = try FakeMySQLServer(identity: nil, streamsRowsStarting: Self.scan, streamedRows: 3) + let run = try await Self.run(on: server, closingWith: "ROLLBACK") { driver in + _ = try await driver.execute(query: "BEGIN") + return await #expect(throws: ReadOnlyModeRefusal.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.scan, options: Self.readOnly)) + } + } + + #expect(run.read == .sessionNotIdle(message: MySQLReadOnlyTransaction.notIdleMessage(.safeMode))) + #expect(Array(run.served.queries.suffix(2)) == ["BEGIN", "ROLLBACK"]) + } + + @Test("A Read-Only read on a session holding LOCK TABLES is refused before the read-only transaction goes out") + func tableLockIsRefusedByTheDriver() async throws { + let server = try FakeMySQLServer(identity: nil, streamsRowsStarting: Self.scan, streamedRows: 3) + let run = try await Self.run(on: server, closingWith: "UNLOCK TABLES") { driver in + _ = try await driver.execute(query: "LOCK TABLES events READ") + return await #expect(throws: ReadOnlyModeRefusal.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.scan, options: Self.readOnly)) + } + } + + #expect(run.read == .sessionNotIdle(message: MySQLReadOnlyTransaction.notIdleMessage(.safeMode))) + #expect(Array(run.served.queries.suffix(2)) == ["LOCK TABLES events READ", "UNLOCK TABLES"]) + } + + @Test("The session reads as idle during and after a Read-Only read") + func streamReadsAsIdle() async throws { + let server = try FakeMySQLServer(identity: nil, streamsRowsStarting: Self.scan, streamedRows: 10_000) + let run = try await Self.run(on: server) { driver in + var midStream: DriverTransactionState? + let options = StreamOptions(maxRows: 100_000, enforcesReadOnly: true) + for try await element in driver.executeStreaming(query: Self.scan, options: options) { + guard case .row = element, midStream == nil else { continue } + midStream = await driver.sessionTransactionState() + } + return (midStream: midStream, after: await driver.sessionTransactionState()) + } + + #expect(run.read.midStream == .idle) + #expect(run.read.after == .idle) + } + + @Test("At the row cap the statement is killed and its read-only transaction ends") + func rowCapEndsTheTransaction() async throws { + let server = try FakeMySQLServer(identity: nil, streamsRowsStarting: Self.scan, streamedRows: 1_000_000) + defer { server.close() } + async let sessions = server.serveConcurrently(sessions: 2) + let driver = Self.driver(port: server.port) + try await driver.connect() + + let read = try await Self.read(driver.executeStreaming(query: Self.scan, options: Self.readOnly)) + _ = try await driver.execute(query: "SELECT 2") + try await driver.disconnect() + + let served = await sessions + #expect(read.rows == 100) + #expect(read.cappedAt == 100) + #expect(served[1].queries == ["KILL QUERY 1"]) + #expect(served[0].streamWasKilled) + #expect(Array(served[0].queries.suffix(4)) == Self.wrapped(Self.scan)) + } + + @Test("A query sent while a Read-Only read streams reaches the server after its ROLLBACK") + func replacedStreamEndsTheTransaction() async throws { + let rows = 50_000 + let server = try FakeMySQLServer(identity: nil, streamsRowsStarting: Self.scan, streamedRows: rows) + let run = try await Self.run(on: server, closingWith: nil) { driver in + var read = Read() + let options = StreamOptions(maxRows: 100_000, enforcesReadOnly: true) + for try await element in driver.executeStreaming(query: Self.scan, options: options) { + guard case .row = element else { continue } + read.rows += 1 + if read.rows == 1 { _ = try await driver.execute(query: "SELECT 2") } + } + return read + } + + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped(Self.scan)) + } + + private struct Read { + var rows = 0 + var cappedAt: Int? + } + + private struct Run { + let read: Value + let served: FakeMySQLServer.Session + } + + // Connects, runs `body`, sends `closingWith` so the server shows what came after it, and disconnects. + private static func run( + on server: FakeMySQLServer, + databaseType: DatabaseType = .mysql, + closingWith closing: String? = "SELECT 2", + _ body: (MySQLDriver) async throws -> Value + ) async throws -> Run { + defer { server.close() } + async let session = server.serveOneSession() + let driver = Self.driver(port: server.port, databaseType: databaseType) + try await driver.connect() + let read = try await body(driver) + if let closing { _ = try await driver.execute(query: closing) } + try await driver.disconnect() + return Run(read: read, served: await session) + } + + private static func read(_ stream: AsyncThrowingStream) async throws -> Read { + var read = Read() + for try await element in stream { + switch element { + case .row: + read.rows += 1 + case .truncated(let reason): + if case .rowCap(let cap) = reason { read.cappedAt = cap } + default: + break + } + } + return read + } + + private static func driver(port: Int, databaseType: DatabaseType = .mysql) -> MySQLDriver { + MySQLDriver( + host: "127.0.0.1", port: port, user: "app", password: "app-password", database: "", + databaseType: databaseType + ) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift index 910302a3d6..1bbe8b04d4 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift @@ -231,6 +231,34 @@ struct PluginBackedDriverTests { ]) } + @Test("A plugin that reads the default mark gets every left-out column marked") + func insertMarksOmittedColumnsForAMarkingPlugin() throws { + let plugin = StubPluginDriver() + plugin.declaredCapabilities = .columnDefaultMarks + let driver = makeWriter(plugin) + + _ = try driver.insertWrites( + table: "events", schema: nil, columns: ["id", "name", "note"], values: [(column: "name", value: "x")] + ) + + #expect(plugin.generatedChanges.first?.defaultColumns == [0, 2]) + } + + @Test("A plugin that reads the default mark stores the marker's text as a value") + func markingPluginStoresTheMarkerText() throws { + let plugin = StubPluginDriver() + plugin.declaredCapabilities = .columnDefaultMarks + let driver = makeWriter(plugin) + + _ = try driver.insertWrites( + table: "events", schema: nil, columns: ["id", "note"], + values: [(column: "id", value: "1"), (column: "note", value: PluginRowWritingDriver.defaultSentinel)] + ) + + #expect(plugin.generatedInserts[0] == ["1", .text(PluginRowWritingDriver.defaultSentinel)]) + #expect(plugin.generatedChanges.first?.defaultColumns == nil) + } + @Test("A value spelled as the default marker is refused, never stored as the default") func insertRefusesTheDefaultMarker() { let plugin = StubPluginDriver() diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLReadOnlyRunTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLReadOnlyRunTests.swift new file mode 100644 index 0000000000..02bb3d1594 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLReadOnlyRunTests.swift @@ -0,0 +1,179 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct PostgreSQLReadOnlyRunTests { + private static let report = "SELECT n FROM orders" + private static let purge = "SELECT purge_orders()" + private static let readOnly = StreamOptions(maxRows: 100, enforcesReadOnly: true) + private static let wrapped = [LibPQReadOnlyTransaction.begin, report, LibPQReadOnlyTransaction.end, "SELECT 2"] + + @Test("A Read-Only statement runs alone through the extended protocol inside a read-only transaction, then ROLLBACK") + func statementRunsInReadOnlyTransaction() async throws { + let server = try FakePostgreSQLServer(replies: [Self.report: .rows(3)]) + let run = try await Self.run(on: server) { driver in + try await Self.read(driver.executeStreaming(query: Self.report, options: Self.readOnly)) + } + + #expect(run.read.rows == 3) + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped) + #expect(run.served.extendedQueries == [Self.report]) + } + + @Test("Without Read-Only, and on Redshift, the statement goes out as a simple query with no transaction around it", arguments: [ + (DatabaseType.postgresql, false), (.redshift, true), + ]) + func otherRunsAreNotWrapped(type: DatabaseType, enforcesReadOnly: Bool) async throws { + let server = try FakePostgreSQLServer(replies: [Self.report: .rows(3)]) + let options = StreamOptions(maxRows: 100, enforcesReadOnly: enforcesReadOnly) + let run = try await Self.run(on: server, databaseType: type) { driver in + try await Self.read(driver.executeStreaming(query: Self.report, options: options)) + } + + #expect(run.read.rows == 3) + #expect(Array(run.served.queries.suffix(2)) == [Self.report, "SELECT 2"]) + #expect(run.served.extendedQueries.isEmpty) + } + + @Test("A statement that fails stays a query error, and its read-only transaction still ends") + func failedStatementEndsTheTransaction() async throws { + let server = try FakePostgreSQLServer(replies: [ + Self.report: .failure(sqlState: "42P01", message: "relation \"orders\" does not exist"), + ]) + let run = try await Self.run(on: server) { driver in + await #expect(throws: PostgreSQLError.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.report, options: Self.readOnly)) + } + } + + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped) + } + + @Test("The server refusing a write in the read-only transaction is Read-Only's refusal, before or after rows", arguments: [ + 0, 2, + ]) + func refusalStateIsReadOnlysRefusal(rowsFirst: Int) async throws { + let server = try FakePostgreSQLServer(replies: [ + Self.purge: .failure( + sqlState: "25006", message: "cannot execute DELETE in a read-only transaction", rowsFirst: rowsFirst + ), + ]) + let run = try await Self.run(on: server) { driver in + await #expect(throws: ReadOnlyModeRefusal.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.purge, options: Self.readOnly)) + } + } + + guard case .refused(let message, let serverMessage)? = run.read else { + Issue.record("expected Read-Only's refusal, got \(String(describing: run.read))") + return + } + #expect(message == LibPQReadOnlyTransaction.refusalMessage(.safeMode)) + #expect(serverMessage.contains("cannot execute DELETE in a read-only transaction")) + #expect(Array(run.served.queries.suffix(4)) == [ + LibPQReadOnlyTransaction.begin, Self.purge, LibPQReadOnlyTransaction.end, "SELECT 2", + ]) + } + + @Test("A session with a transaction open is refused before anything is sent") + func openTransactionIsRefused() async throws { + let server = try FakePostgreSQLServer(replies: [Self.report: .rows(3)]) + defer { server.close() } + async let session = server.serveOneSession() + let driver = Self.driver(port: server.port) + try await driver.connect() + + _ = try await driver.execute(query: "BEGIN") + let error = await #expect(throws: ReadOnlyModeRefusal.self) { + _ = try await Self.read(driver.executeStreaming(query: Self.report, options: Self.readOnly)) + } + _ = try await driver.execute(query: "ROLLBACK") + try await driver.disconnect() + + let served = await session + #expect(error == .sessionNotIdle(message: LibPQReadOnlyTransaction.notIdleMessage(.safeMode))) + #expect(Array(served.queries.suffix(2)) == ["BEGIN", "ROLLBACK"]) + #expect(served.extendedQueries.isEmpty) + } + + @Test("At the row cap the statement is cancelled and its read-only transaction ends") + func rowCapEndsTheTransaction() async throws { + let server = try FakePostgreSQLServer(replies: [Self.report: .rows(1_000)]) + let run = try await Self.run(on: server) { driver in + try await Self.read(driver.executeStreaming(query: Self.report, options: Self.readOnly)) + } + + #expect(run.read.rows == 100) + #expect(run.read.cappedAt == 100) + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped) + } + + @Test("A query sent while a Read-Only read streams reaches the server after its ROLLBACK") + func replacedStreamEndsTheTransaction() async throws { + let rows = 50_000 + let server = try FakePostgreSQLServer(replies: [Self.report: .rows(rows)]) + let run = try await Self.run(on: server, closingWith: nil) { driver in + var read = Read() + let options = StreamOptions(maxRows: 100_000, enforcesReadOnly: true) + for try await element in driver.executeStreaming(query: Self.report, options: options) { + guard case .row = element else { continue } + read.rows += 1 + if read.rows == 1 { _ = try await driver.execute(query: "SELECT 2") } + } + return read + } + + #expect(Array(run.served.queries.suffix(4)) == Self.wrapped) + } + + private struct Read { + var rows = 0 + var cappedAt: Int? + } + + private struct Run { + let read: Value + let served: FakePostgreSQLServer.Session + } + + // Connects, runs `body`, sends `closingWith` so the server shows what came after it, and disconnects. + private static func run( + on server: FakePostgreSQLServer, + databaseType: DatabaseType = .postgresql, + closingWith closing: String? = "SELECT 2", + _ body: (PostgreSQLDriver) async throws -> Value + ) async throws -> Run { + defer { server.close() } + async let session = server.serveOneSession() + let driver = Self.driver(port: server.port, databaseType: databaseType) + try await driver.connect() + let read = try await body(driver) + if let closing { _ = try await driver.execute(query: closing) } + try await driver.disconnect() + return Run(read: read, served: await session) + } + + private static func read(_ stream: AsyncThrowingStream) async throws -> Read { + var read = Read() + for try await element in stream { + switch element { + case .row: + read.rows += 1 + case .truncated(let reason): + if case .rowCap(let cap) = reason { read.cappedAt = cap } + default: + break + } + } + return read + } + + private static func driver(port: Int, databaseType: DatabaseType = .postgresql) -> PostgreSQLDriver { + PostgreSQLDriver( + host: "127.0.0.1", port: port, user: "app", password: "app-password", database: "app", + databaseType: databaseType + ) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLStreamTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLStreamTests.swift index 349d3758e5..405043ed68 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLStreamTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/PostgreSQLStreamTests.swift @@ -85,6 +85,37 @@ struct PostgreSQLStreamTests { #expect(drained.end == .cancelled) } + @Test("The first failure keeps its SQLSTATE") + func firstFailureKeepsItsState() { + let drained = drain([.failed("ERROR: cannot execute DELETE\n", sqlState: "25006"), .failed("ERROR: later\n", sqlState: "XX000")]) + #expect(drained.end == .failed("ERROR: cannot execute DELETE\n", sqlState: "25006")) + } + + @Test("In the read-only transaction a refusal state is Read-Only's refusal; anywhere else it is a query error", arguments: [ + "25006", "25001", + ]) + func refusalStateInReadOnlyTransaction(sqlState: String) { + let end = PostgreSQLStreamEnd.failed("ERROR: cannot execute DELETE in a read-only transaction\n", sqlState: sqlState) + let refusal = end.error(inReadOnlyTransaction: true) as? ReadOnlyModeRefusal + #expect(refusal == .refused( + message: LibPQReadOnlyTransaction.refusalMessage(.safeMode), + serverMessage: "ERROR: cannot execute DELETE in a read-only transaction\n" + )) + #expect(refusal?.errorDescription == """ + \(LibPQReadOnlyTransaction.refusalMessage(.safeMode)) + ERROR: cannot execute DELETE in a read-only transaction + """) + #expect(end.error(inReadOnlyTransaction: false) is PostgreSQLError) + #expect(end.error is PostgreSQLError) + } + + @Test("In the read-only transaction any other failure stays a query error") + func otherStateInReadOnlyTransaction() { + let end = PostgreSQLStreamEnd.failed("ERROR: relation \"orders\" does not exist\n", sqlState: "42P01") + #expect(end.error(inReadOnlyTransaction: true) is PostgreSQLError) + #expect(PostgreSQLStreamEnd.failed("ERROR: lost\n").error(inReadOnlyTransaction: true) is PostgreSQLError) + } + @Test("A failed end carries the server's message, and a cancelled end is a cancellation") func endErrors() { let failure = PostgreSQLStreamEnd.failed("ERROR: division by zero\n").error as? PostgreSQLError diff --git a/apple/TableProMobile/TableProMobileTests/ForeignKeyPreviewModelTests.swift b/apple/TableProMobile/TableProMobileTests/ForeignKeyPreviewModelTests.swift index 64201edb24..0b9aa94273 100644 --- a/apple/TableProMobile/TableProMobileTests/ForeignKeyPreviewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ForeignKeyPreviewModelTests.swift @@ -79,6 +79,25 @@ struct ForeignKeyPreviewModelTests { #expect(query.contains("\"id\" = '5'")) } + @Test("On SQL Server the lookup and the referenced table's reads name the database the key was read in") + func sqlServerLookupNamesTheDatabase() async throws { + let driver = Self.driver(rows: [Self.userRow]) + let reference = ForeignKeyReference(table: "tenant_users", schema: "dbo", database: "shop", parts: [ + ForeignKeyReference.Part(column: "tenant_id", value: "2"), + ForeignKeyReference.Part(column: "id", value: "5") + ]) + let model = ForeignKeyPreviewModel(reference: reference, session: Self.session(driver), databaseType: .mssql) + await model.load() + + let query = try #require(driver.executedQueries.first) + #expect(query.hasPrefix("SELECT * FROM [shop].[dbo].[tenant_users]")) + #expect(driver.fetchColumnsDatabases == ["shop"]) + #expect(driver.fetchForeignKeysDatabases == ["shop"]) + let linked = model.fields.compactMap { $0.reference } + #expect(!linked.isEmpty) + #expect(linked.allSatisfy { $0.database == "shop" }) + } + // MySQL finds no row for `bit_col = '5'`, and the browser's filter compares the same column bare. @Test("The lookup reads the referenced table's columns first and takes each literal from their types") func lookupTakesLiteralsFromTheColumns() async throws { diff --git a/apple/TableProMobile/TableProMobileTests/ForeignKeyReferenceTests.swift b/apple/TableProMobile/TableProMobileTests/ForeignKeyReferenceTests.swift index 5071833516..2bff67f4aa 100644 --- a/apple/TableProMobile/TableProMobileTests/ForeignKeyReferenceTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ForeignKeyReferenceTests.swift @@ -35,6 +35,22 @@ struct ForeignKeyReferenceTests { #expect(reference.parts == [ForeignKeyReference.Part(column: "id", value: "5")]) } + @Test("A reference keeps the database its key was read in, from one key or from all of them") + func referenceKeepsTheSourceDatabase() throws { + let cells = Self.cells(.text("1"), .text("2"), .text("5"), .null) + let reference = try #require(ForeignKeyReference.make( + forColumn: "user_id", foreignKeys: [Self.userKey], columns: Self.orderColumns, cells: cells, + sourceSchema: "dbo", sourceDatabase: "shop" + )) + let all = ForeignKeyReference.all( + foreignKeys: [Self.userKey], columns: Self.orderColumns, cells: cells, + sourceSchema: "dbo", sourceDatabase: "shop" + ) + + #expect(reference.database == "shop") + #expect(all.map { $0.reference.database } == ["shop"]) + } + @Test("The key's own schema wins over the source schema") func referencedSchemaWins() throws { let key = ForeignKeyInfo( diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift index fe8b181201..0f68fa77de 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift @@ -76,12 +76,54 @@ struct ClipboardExporterFidelityTests { } } - @Test("real numbers are still emitted unquoted") + @Test("numbers in a numeric column are emitted unquoted") func realNumbersStayNumbers() { - #expect(json(["1", "0", "x"]).contains("\"zip\": 0")) - #expect(json(["1", "-12", "x"]).contains("\"zip\": -12")) - #expect(json(["1", "1.5", "x"]).contains("\"zip\": 1.5")) - #expect(json(["1", "1e10", "x"]).contains("\"zip\": 1e10")) + #expect(json(["0", "1", "x"]).contains("\"id\": 0")) + #expect(json(["-12", "1", "x"]).contains("\"id\": -12")) + #expect(json(["1.5", "1", "x"]).contains("\"id\": 1.5")) + #expect(json(["1e10", "1", "x"]).contains("\"id\": 1e10")) + } + + @Test("text that reads like a number or a boolean stays a string in a text column") + func textColumnsStayStrings() throws { + for value in ["1e10", "12345678901234567890", "0", "true", "false"] { + let text = json(["1", value, value]) + #expect(text.contains("\"zip\": \"\(value)\""), "\(value) should stay quoted") + #expect(text.contains("\"note\": \"\(value)\""), "\(value) should stay quoted") + _ = try JSONSerialization.jsonObject(with: try #require(text.data(using: .utf8))) + } + } + + @Test("each column's declared type picks the JSON type") + func declaredTypePicksTheJSONType() throws { + func value(_ text: String, typeName: String) throws -> String { + let output = ClipboardExporter.exportRow( + columns: [ColumnInfo(name: "v", typeName: typeName, ordinalPosition: 0)], row: [text], format: .json, + tableName: "t", databaseType: .postgresql, driver: nil + ) + _ = try JSONSerialization.jsonObject(with: try #require(output.data(using: .utf8))) + let line = try #require(output.split(separator: "\n").first { $0.contains("\"v\"") }) + return String(line.drop { $0 != ":" }.dropFirst(2)) + } + #expect(try value("t", typeName: "bool") == "true") + #expect(try value("f", typeName: "bool") == "false") + #expect(try value("TRUE", typeName: "BOOLEAN") == "true") + #expect(try value("false", typeName: "Nullable(Bool)") == "false") + #expect(try value("yes", typeName: "bool") == "\"yes\"") + #expect(try value("42", typeName: "int4") == "42") + #expect(try value("12.50", typeName: "numeric(10,2)") == "12.50") + #expect(try value("-1.5e3", typeName: "Float64") == "-1.5e3") + #expect(try value("18446744073709551615", typeName: "UInt64") == "18446744073709551615") + #expect(try value("1", typeName: "bit") == "1") + #expect(try value("NaN", typeName: "float8") == "\"NaN\"") + #expect(try value("$5.00", typeName: "money") == "\"$5.00\"") + #expect(try value("t", typeName: "text") == "\"t\"") + #expect(try value("1", typeName: "BOOLEAN") == "true") + #expect(try value("0", typeName: "BOOLEAN") == "false") + // SQLite reports no type for an expression such as count(*), so its value decides. + #expect(try value("42", typeName: "") == "42") + #expect(try value("true", typeName: "") == "true") + #expect(try value("01234", typeName: "") == "\"01234\"") } @Test("a NULL is emitted as JSON null") diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/LiteralInputTraitsTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/LiteralInputTraitsTests.swift index 4094fa3120..9ec1769267 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/LiteralInputTraitsTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/LiteralInputTraitsTests.swift @@ -40,6 +40,20 @@ struct LiteralInputTraitsTests { #expect(textView.writingToolsBehavior == UIWritingToolsBehavior.none) } + @Test("The table search types what the keyboard sends, so a name with an apostrophe still matches") + func searchFieldTraits() { + let bar = UISearchBar() + + LiteralSearchField.configure(bar) + + let field = bar.searchTextField + #expect(field.smartQuotesType == .no) + #expect(field.smartDashesType == .no) + #expect(field.autocorrectionType == .no) + #expect(field.autocapitalizationType == UITextAutocapitalizationType.none) + #expect(bar.returnKeyType == .search) + } + @Test("The traits leave the keyboard type the field asked for") func keyboardTypeKept() { let field = UITextField() diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/ReadOnlyBareNameGuardTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/ReadOnlyBareNameGuardTests.swift new file mode 100644 index 0000000000..c5e839916a --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Helpers/ReadOnlyBareNameGuardTests.swift @@ -0,0 +1,40 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProSQLGrammar +import Testing + +struct ReadOnlyBareNameGuardTests { + private struct Unreachable: Error {} + + // The catalog Oracle 23ai answered for a schema with function PURGE, table ORDERS(ID, PURGE) and DUAL. + private static let catalog: SQLCallableCatalogCache.Reader = { query in + if query == SQLCallableCatalog.query { return [["F", "PURGE"]] } + if query.contains("'DUAL'") { return [["0", "3", nil, "DUMMY"]] } + return [["0", "1", nil, "ID"], ["0", "1", nil, "PURGE"]] + } + + private static func refuses(_ sql: String, level: SafeModeLevel = .readOnly, read: @escaping SQLCallableCatalogCache.Reader = catalog) async -> Bool { + await ReadOnlyFunctionGuard.refusesBareName( + sql, databaseType: .oracle, level: level, connectionId: UUID(), schema: nil, read: read + ) + } + + @Test("Oracle Read-Only refuses a function named bare and runs the column of the same name") + func functionNamedBare() async { + #expect(await Self.refuses("SELECT purge FROM dual")) + #expect(await Self.refuses("SELECT pkg.purge FROM dual")) + #expect(!(await Self.refuses("SELECT purge FROM orders"))) + #expect(!(await Self.refuses("SELECT id FROM orders"))) + } + + @Test("Only Read-Only refuses, and a catalog that cannot be read refuses") + func onlyReadOnly() async { + #expect(!(await Self.refuses("SELECT purge FROM dual", level: .confirmWrites))) + #expect(await Self.refuses("SELECT id FROM orders", read: { _ in throw Unreachable() })) + #expect(!(await ReadOnlyFunctionGuard.refusesBareName( + "SELECT purge FROM dual", databaseType: .postgresql, level: .readOnly, connectionId: UUID(), schema: nil, + read: Self.catalog + ))) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/ReadOnlyFunctionGuardTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/ReadOnlyFunctionGuardTests.swift new file mode 100644 index 0000000000..3e35c52a3f --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Helpers/ReadOnlyFunctionGuardTests.swift @@ -0,0 +1,114 @@ +import Foundation +@testable import TableProMobile +import TableProDatabase +import TableProModels +import Testing + +struct ReadOnlyFunctionGuardTests { + @Test("Without the engine's read-only mode, Read-Only refuses a call off the built-in list where functions can write", arguments: [ + DatabaseType.postgresql, .mysql, .mariadb, .oceanbase, .oracle, + ]) + func refusesAnUnlistedCallWithoutReadOnlyMode(type: DatabaseType) { + #expect(ReadOnlyFunctionGuard.refuses( + "SELECT purge_orders()", databaseType: type, level: .readOnly, enforcesReadOnlyMode: false + )) + #expect(ReadOnlyFunctionGuard.refuses( + "SELECT app.count(id) FROM orders", databaseType: type, level: .readOnly, enforcesReadOnlyMode: false + )) + } + + @Test("In the engine's read-only mode the call runs, since the server refuses what it writes", arguments: [ + DatabaseType.postgresql, .mysql, .mariadb, + ]) + func runsAnUnlistedCallInReadOnlyMode(type: DatabaseType) { + #expect(!ReadOnlyFunctionGuard.refuses( + "SELECT purge_orders()", databaseType: type, level: .readOnly, enforcesReadOnlyMode: true + )) + } + + @Test("PostgreSQL, MySQL and MariaDB drivers hold Read-Only in the engine; Oracle, OceanBase, TiDB, Redshift and old servers do not") + func driversSayWhichRunInReadOnlyMode() throws { + let factory = IOSDriverFactory() + func mode(_ type: DatabaseType) throws -> Bool { + let connection = DatabaseConnection(name: "Shop", type: type, host: "db.example.com") + return try factory.createDriver(for: connection, password: nil).enforcesReadOnlyMode + } + #expect(try mode(.postgresql)) + #expect(try mode(.mysql)) + #expect(try mode(.mariadb)) + #expect(try !mode(.redshift)) + #expect(try !mode(.oracle)) + #expect(try !mode(.oceanbase)) + #expect(try !mode(.tidb)) + + #expect(MySQLDriver.runsReadOnlyTransactions(for: .mysql, banner: "8.4.13")) + #expect(MySQLDriver.runsReadOnlyTransactions(for: .mysql, banner: "10.11.6-MariaDB")) + #expect(!MySQLDriver.runsReadOnlyTransactions(for: .mysql, banner: "5.6.4-log")) + #expect(!MySQLDriver.runsReadOnlyTransactions(for: .mariadb, banner: "5.5.64-MariaDB")) + #expect(!MySQLDriver.runsReadOnlyTransactions(for: .mysql, banner: "8.0.11-TiDB-v8.5.1")) + #expect(!MySQLDriver.runsReadOnlyTransactions(for: .oceanbase, banner: "5.7.25-OceanBase_CE-v4.2.1.0")) + } + + @Test("A server too old for READ ONLY keeps the refusal") + func oldServerKeepsTheRefusal() { + let mode = MySQLDriver.runsReadOnlyTransactions(for: .mysql, banner: "5.6.4-log") + #expect(ReadOnlyFunctionGuard.refuses( + "SELECT purge_orders()", databaseType: .mysql, level: .readOnly, enforcesReadOnlyMode: mode + )) + } + + @Test("Built-ins, a cast to the engine's own type and a plain EXPLAIN run", arguments: [ + (DatabaseType.postgresql, "SELECT COUNT(*), LOWER(name), NOW() FROM orders WHERE total::numeric(10, 2) > 1"), + (.postgresql, "EXPLAIN SELECT purge_orders()"), + (.mysql, "SELECT DATE_FORMAT(created_at, '%Y-%m'), CAST(total AS DECIMAL(10, 2)) FROM orders"), + (.oracle, "SELECT NVL(note, 'x'), TO_CHAR(created, 'YYYY') FROM orders"), + ]) + func runsBuiltins(type: DatabaseType, sql: String) { + #expect(!ReadOnlyFunctionGuard.refuses(sql, databaseType: type, level: .readOnly, enforcesReadOnlyMode: false)) + } + + @Test("Only Read-Only refuses, and only where a function a query calls can write") + func refusesOnlyAtReadOnlyWhereFunctionsWrite() { + let sql = "SELECT purge_orders()" + #expect(!ReadOnlyFunctionGuard.refuses(sql, databaseType: .postgresql, level: .confirmWrites, enforcesReadOnlyMode: false)) + #expect(!ReadOnlyFunctionGuard.refuses(sql, databaseType: .postgresql, level: .off, enforcesReadOnlyMode: false)) + #expect(!ReadOnlyFunctionGuard.refuses(sql, databaseType: .mssql, level: .readOnly, enforcesReadOnlyMode: false)) + #expect(!ReadOnlyFunctionGuard.refuses(sql, databaseType: .sqlite, level: .readOnly, enforcesReadOnlyMode: false)) + } + + @Test("A read at Read-Only on an idle session runs in the read-only mode") + func idleReadRunsInReadOnlyMode() async { + let driver = MockDatabaseDriver() + driver.enforcesReadOnlyMode = true + driver.scriptedTransactionState = .idle + + #expect(await ReadOnlyFunctionGuard.runsInReadOnlyMode(writes: false, level: .readOnly, driver: driver)) + #expect(driver.transactionStateReads == 1) + } + + @Test("A session holding a transaction, or one it cannot name, runs the read as it is", arguments: [ + DriverTransactionState.explicitTransaction, .implicitTransaction, .abortedTransaction, .unknown, + ]) + func busySessionRunsAsItIs(state: DriverTransactionState) async { + let driver = MockDatabaseDriver() + driver.enforcesReadOnlyMode = true + driver.scriptedTransactionState = state + + #expect(await !ReadOnlyFunctionGuard.runsInReadOnlyMode(writes: false, level: .readOnly, driver: driver)) + } + + @Test("A write, a level below Read-Only and a driver without the mode never ask the session") + func otherRunsNeverAskTheSession() async { + let withMode = MockDatabaseDriver() + withMode.enforcesReadOnlyMode = true + withMode.scriptedTransactionState = .idle + let withoutMode = MockDatabaseDriver() + withoutMode.scriptedTransactionState = .idle + + #expect(await !ReadOnlyFunctionGuard.runsInReadOnlyMode(writes: true, level: .readOnly, driver: withMode)) + #expect(await !ReadOnlyFunctionGuard.runsInReadOnlyMode(writes: false, level: .confirmWrites, driver: withMode)) + #expect(await !ReadOnlyFunctionGuard.runsInReadOnlyMode(writes: false, level: .readOnly, driver: withoutMode)) + #expect(withMode.transactionStateReads == 0) + #expect(withoutMode.transactionStateReads == 0) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift index 49c9af8d46..855e0d68f9 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift @@ -3,7 +3,6 @@ import Testing import TableProModels @testable import TableProMobile -@Suite("RowInsertPlanner") struct RowInsertPlannerTests { private let columns = [ ColumnInfo(name: "id", typeName: "integer", isPrimaryKey: true, isNullable: false, ordinalPosition: 0), @@ -69,6 +68,18 @@ struct RowInsertPlannerTests { #expect((mysql.first ?? "").contains("1890")) } + @Test("A SQL Server insert names the window's database in front of the schema") + func sqlServerInsertNamesTheDatabase() throws { + let write = PlannedRowWrite.insert( + table: "orders", schema: "dbo", database: "shop", type: .mssql, columns: columns, + values: ["name": .text("Ada")] + ) + + let batch = try #require(try write.batch(on: ansiDriver())) + + #expect(batch.statements == ["INSERT INTO [shop].[dbo].[orders] ([name]) VALUES (N'Ada')"]) + } + @Test("writes NULL for null values") func nullValue() throws { let row = PayloadRow(values: ["name": .text("Ada"), "note": .null]) diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/RowWriteRoutingTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/RowWriteRoutingTests.swift index c429dc64c7..86b2433e47 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/RowWriteRoutingTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/RowWriteRoutingTests.swift @@ -417,6 +417,45 @@ struct RowWriteRoutingTests { #expect(plugin.generatedInserts[0] == [.text(PluginRowWritingDriver.defaultSentinel), "x"]) } + @Test("A SQL Server row save names the database the row was read in, in front of its schema") + func sqlServerSaveNamesTheDatabase() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(QueryResult(columns: [], rows: [], rowsAffected: 1, executionTime: 0))] + let details = [ + ColumnInfo(name: "id", typeName: "int", isPrimaryKey: true, isNullable: false, ordinalPosition: 0), + ColumnInfo(name: "note", typeName: "nvarchar", ordinalPosition: 1) + ] + let vm = RowDetailViewModel( + columns: details, + rows: [Row(cells: [.text("1"), .text("old")])], + initialIndex: 0, + table: TableInfo(name: "orders"), + session: ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "staging"), + columnDetails: details, + databaseType: .mssql, + schema: "dbo", + database: "shop" + ) + vm.startEditing() + vm.setEditedValue("new", at: 1) + + let saved = await vm.saveChanges() + + #expect(saved) + #expect(driver.executedQueries == ["UPDATE [shop].[dbo].[orders] SET [note] = N'new' WHERE [id] = 1"]) + } + + @Test("A SQL Server delete batch names the database in front of the schema") + func sqlServerDeleteBatchNamesTheDatabase() throws { + let details = [ColumnInfo(name: "id", typeName: "int", isPrimaryKey: true, isNullable: false, ordinalPosition: 0)] + let batch = try RowWriteBatch.delete( + table: "orders", schema: "dbo", database: "shop", type: .mssql, driver: MockDatabaseDriver(), + columns: ["id"], columnDetails: details, locator: RowLocator(row: ["1"], keyValues: [(column: "id", value: "1")]) + ) + + #expect(batch?.statements == ["DELETE FROM [shop].[dbo].[orders] WHERE [id] = 1"]) + } + // D1's plugin generates no row writes; behind a row writer every edit failed with "cannot write this change". @Test("A D1 edit, delete and insert become SQLite statements keyed on the primary key") func cloudflareD1WritesAreSQL() throws { diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderSchemaQualificationTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderSchemaQualificationTests.swift index 9e57a5b478..384f0b18b9 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderSchemaQualificationTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderSchemaQualificationTests.swift @@ -4,10 +4,8 @@ import TableProQuery import Testing @testable import TableProMobile -/// SQL Server has no session-level schema to set, so every data statement has to name the schema -/// the toolbar shows. An unqualified one resolves against the login's default schema, which is a -/// different table of the same name. -@Suite("SQLBuilder schema qualification") +// SQL Server has no session schema to set and resolves `[schema].[table]` in the session's database, which a `USE` +// moves, so every statement names the schema and the database the window shows. struct SQLBuilderSchemaQualificationTests { private func driver() -> MockDatabaseDriver { MockDatabaseDriver() } @@ -70,6 +68,92 @@ struct SQLBuilderSchemaQualificationTests { == "SELECT COUNT(*) FROM `orders`") } + @Test("SQL Server's row writes and truncate name the window's database in front of the schema") + func sqlServerWritesNameTheDatabase() { + let key = [(column: "id", value: "1")] + #expect(SQLBuilder.buildUpdate( + table: "orders", schema: "dbo", database: "shop", type: .mssql, driver: driver(), + changes: [(column: "note", value: "x")], primaryKeys: key, columns: [] + ).hasPrefix("UPDATE [shop].[dbo].[orders] SET")) + #expect(SQLBuilder.buildDelete( + table: "orders", schema: "dbo", database: "shop", type: .mssql, driver: driver(), + primaryKeys: key, columns: [] + ).hasPrefix("DELETE FROM [shop].[dbo].[orders] WHERE")) + #expect(SQLBuilder.buildInsert( + table: "orders", schema: "dbo", database: "shop", type: .mssql, driver: driver(), + columns: ["note"], values: ["x"] + ) == "INSERT INTO [shop].[dbo].[orders] ([note]) VALUES (N'x')") + #expect(SQLBuilder.buildInsert( + table: "orders", schema: "dbo", database: "shop", type: .mssql, driver: driver(), columns: [], values: [] + ) == "INSERT INTO [shop].[dbo].[orders] DEFAULT VALUES") + #expect(SQLBuilder.truncateStatement(table: "orders", schema: "dbo", database: "shop", type: .mssql) + == "TRUNCATE TABLE [shop].[dbo].[orders]") + } + + @Test("SQL Server's browse, count, filter, search and lookup reads name the window's database") + func sqlServerReadsNameTheDatabase() { + let columns = [ColumnInfo(name: "note", typeName: "nvarchar", ordinalPosition: 0)] + let sorted = SortState(columns: [SortColumn(name: "note", ascending: true)]) + let table = "SELECT * FROM [shop].[dbo].[orders] " + #expect(SQLBuilder.buildSelect(table: "orders", schema: "dbo", database: "shop", type: .mssql, limit: 10, offset: 0) + .hasPrefix(table)) + #expect(SQLBuilder.buildSelect( + table: "orders", schema: "dbo", database: "shop", type: .mssql, sortState: sorted, limit: 10, offset: 0 + ).hasPrefix(table)) + #expect(SQLBuilder.buildCount(table: "orders", schema: "dbo", database: "shop", type: .mssql) + == "SELECT COUNT(*) FROM [shop].[dbo].[orders]") + #expect(SQLBuilder.buildFilteredSelect( + table: "orders", schema: "dbo", database: "shop", type: .mssql, + filters: [], logicMode: .and, limit: 10, offset: 0 + ).hasPrefix(table)) + #expect(SQLBuilder.buildFilteredCount( + table: "orders", schema: "dbo", database: "shop", type: .mssql, filters: [], logicMode: .and + ) == "SELECT COUNT(*) FROM [shop].[dbo].[orders]") + #expect(SQLBuilder.buildSearchSelect( + table: "orders", schema: "dbo", database: "shop", type: .mssql, + searchText: "a", searchColumns: columns, limit: 10, offset: 0 + ).hasPrefix(table)) + #expect(SQLBuilder.buildSearchCount( + table: "orders", schema: "dbo", database: "shop", type: .mssql, searchText: "a", searchColumns: columns + ).hasPrefix("SELECT COUNT(*) FROM [shop].[dbo].[orders] WHERE")) + #expect(SQLBuilder.buildSingleRowLookup( + table: "orders", schema: "dbo", database: "shop", key: [(column: "id", value: "1")], columns: [], + type: .mssql, driver: driver() + ).hasPrefix("SELECT * FROM [shop].[dbo].[orders] WHERE")) + } + + @Test("SQL Server with a database and no schema reads the database's default schema") + func sqlServerDatabaseWithoutSchema() { + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: nil, database: "shop", for: .mssql) + == "[shop]..[orders]") + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "", database: "shop", for: .mssql) + == "[shop]..[orders]") + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "dbo", database: "", for: .mssql) + == "[dbo].[orders]") + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "dbo", database: nil, for: .mssql) + == "[dbo].[orders]") + } + + @Test("A closing bracket in a SQL Server database name is doubled") + func sqlServerQuotesTheDatabase() { + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "dbo", database: "shop]; DROP TABLE x --", for: .mssql) + == "[shop]]; DROP TABLE x --].[dbo].[orders]") + } + + @Test("MySQL and PostgreSQL ignore the database: MySQL's sits in the schema slot, PostgreSQL has no third part") + func otherEnginesIgnoreTheDatabase() { + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "shop", database: "shop", for: .mysql) + == "`shop`.`orders`") + #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "public", database: "shop", for: .postgresql) + == "\"public\".\"orders\"") + #expect(SQLBuilder.buildDelete( + table: "orders", schema: "shop", database: "staging", type: .mysql, driver: driver(), + primaryKeys: [(column: "id", value: "1")], columns: [] + ).hasPrefix("DELETE FROM `shop`.`orders` WHERE")) + #expect(SQLBuilder.truncateStatement(table: "orders", schema: "public", database: "shop", type: .postgresql) + == "TRUNCATE TABLE \"public\".\"orders\"") + } + @Test("the qualifier follows each dialect's own quoting") func dialectQuoting() { #expect(SQLBuilder.qualifiedIdentifier(table: "orders", schema: "sales", for: .postgresql) diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift index f227485b50..6936bedadb 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift @@ -70,6 +70,21 @@ struct SessionLocalStatementsTests { ) } + @Test("Only a USE moves the session's database, so only it makes the app read where the session went") + func onlyUseMovesTheDatabase() { + #expect(SessionLocalStatements.effect(of: ["USE archive"], databaseType: .mysql).usesDatabase) + #expect(SessionLocalStatements.effect(of: ["/*!40000 USE archive */"], databaseType: .mysql).usesDatabase) + #expect( + SessionLocalStatements.effect(of: ["SELECT 1\nUSE [archive]\nSELECT 2"], databaseType: .mssql).usesDatabase + ) + for statement in ["CALL make_scratch()", "EXECUTE stmt", "SET ROLE admin", "/*!40101 SET NAMES utf8 */"] { + let effect = SessionLocalStatements.effect(of: [statement], databaseType: .mysql) + #expect(effect.leavesCatalog, "\(statement)") + #expect(!effect.usesDatabase, "\(statement)") + } + #expect(!SessionLocalStatements.effect(of: ["SELECT * FROM t USE INDEX (i)"], databaseType: .mysql).usesDatabase) + } + @Test("A procedure call, dynamic SQL or an anonymous block may make a table only the session sees") func opaqueStatementsLeaveTheCatalog() { #expect(SessionLocalStatements.effect(of: ["CALL make_scratch()"], databaseType: .mysql).leavesCatalog) diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/InMemoryKeychainItems.swift b/apple/TableProMobile/TableProMobileTests/Mocks/InMemoryKeychainItems.swift index 037016add4..8e2f48e929 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/InMemoryKeychainItems.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/InMemoryKeychainItems.swift @@ -30,6 +30,23 @@ final class InMemoryKeychainItems: KeychainItemStore, @unchecked Sendable { private var refusedAdd: OSStatus? private var refusedUpdate: OSStatus? private var writes = 0 + private var callsOnMainThread = 0 + private var listingHook: (@Sendable () -> Void)? + + // Calls made on the main thread, where a real Keychain call blocks the UI. + var mainThreadCalls: Int { + lock.withLock { callsOnMainThread } + } + + // Runs inside the next listing, before it reads the items, as a save landing mid-listing would. + func duringNextListing(_ hook: @escaping @Sendable () -> Void) { + lock.withLock { listingHook = hook } + } + + private func countIfMainThread() { + guard Thread.isMainThread else { return } + lock.withLock { callsOnMainThread += 1 } + } // Each write is one tick later than the one before, as a later save on any device would be. private func nextDate() -> Date { @@ -134,8 +151,19 @@ final class InMemoryKeychainItems: KeychainItemStore, @unchecked Sendable { } } + func accounts(_ query: [String: Any]) -> [String] { + countIfMainThread() + let hook = lock.withLock { () -> (@Sendable () -> Void)? in + defer { listingHook = nil } + return listingHook + } + hook?() + return lock.withLock { matches(query).map { items[$0].account } } + } + func delete(_ query: [String: Any]) -> OSStatus { - lock.withLock { () -> OSStatus in + countIfMainThread() + return lock.withLock { () -> OSStatus in let matched = Set(matches(query)) guard !matched.isEmpty else { return errSecItemNotFound } items = items.enumerated().filter { !matched.contains($0.offset) }.map { $0.element } diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/MockDatabaseDriver.swift b/apple/TableProMobile/TableProMobileTests/Mocks/MockDatabaseDriver.swift index 5718af7b1f..4f6120180f 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/MockDatabaseDriver.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/MockDatabaseDriver.swift @@ -3,6 +3,7 @@ import TableProDatabase import TableProModels import TableProPluginKit import TableProSQLGrammar +@testable import TableProMobile final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { enum MockError: Error { case scripted } @@ -16,6 +17,10 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { var scriptedDatabases: [String] = [] // The database the driver says it is in, which may be named differently from the one the connection stores. var scriptedCurrentDatabase: String? + // What `SELECT DATABASE()` and `current_schema()` answer; nil keeps the driver where it is. + var scriptedSessionDatabase: String? + var scriptedSessionSchema: String? + private(set) var sessionDatabaseReads = 0 var scriptedSchemas: [String] = [] var scriptedTransactionState: DriverTransactionState = .unknown // Answered in order, one per call, before falling back to `scriptedTransactionState`. @@ -26,6 +31,7 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { var connectError: Error? var switchDatabaseError: Error? var fetchTablesError: Error? + var fetchDatabasesError: Error? var commitError: Error? var rollbackError: Error? @@ -34,12 +40,19 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { private(set) var fetchForeignKeysCalls: Int = 0 private(set) var fetchColumnsSchemas: [String?] = [] private(set) var fetchForeignKeysSchemas: [String?] = [] + private(set) var fetchColumnsDatabases: [String?] = [] + private(set) var fetchForeignKeysDatabases: [String?] = [] private(set) var beganTransactionModes: [PluginTransactionAccessMode] = [] private(set) var didBeginTransaction = false private(set) var didCommitTransaction = false private(set) var didRollbackTransaction = false private(set) var pingCount = 0 private(set) var switchedDatabases: [String] = [] + private(set) var switchedSchemas: [String] = [] + private(set) var fetchTablesSchemas: [String?] = [] + private(set) var fetchSchemasCalls = 0 + private(set) var fetchDatabasesCalls = 0 + private(set) var transactionStateReads = 0 var supportsSchemas: Bool = false var currentSchema: String? @@ -48,6 +61,7 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { var holdsSuspensionBlockingResource: Bool = false var sendsBatchesWhole: Bool = false var usesBackslashEscaping: Bool = false + var enforcesReadOnlyMode: Bool = false func escapeStringLiteral(_ value: String) -> String { usesBackslashEscaping @@ -90,14 +104,20 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { } func fetchTables(schema: String?) async throws -> [TableInfo] { + fetchTablesSchemas.append(schema) await beforeFetchTables?() if let fetchTablesError { throw fetchTablesError } return scriptedTables } func fetchColumns(table: String, schema: String?) async throws -> [ColumnInfo] { + try await fetchColumns(table: table, schema: schema, database: nil) + } + + func fetchColumns(table: String, schema: String?, database: String?) async throws -> [ColumnInfo] { fetchColumnsCalls += 1 fetchColumnsSchemas.append(schema) + fetchColumnsDatabases.append(database) if let fetchColumnsError { throw fetchColumnsError } return scriptedColumns } @@ -105,21 +125,38 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { func fetchIndexes(table: String, schema: String?) async throws -> [IndexInfo] { [] } func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] { + try await fetchForeignKeys(table: table, schema: schema, database: nil) + } + + func fetchForeignKeys(table: String, schema: String?, database: String?) async throws -> [ForeignKeyInfo] { fetchForeignKeysCalls += 1 fetchForeignKeysSchemas.append(schema) + fetchForeignKeysDatabases.append(database) if let fetchForeignKeysError { throw fetchForeignKeysError } return scriptedForeignKeys } - func fetchDatabases() async throws -> [String] { scriptedDatabases } - func fetchSchemas() async throws -> [String] { scriptedSchemas } + func fetchDatabases() async throws -> [String] { + fetchDatabasesCalls += 1 + if let fetchDatabasesError { throw fetchDatabasesError } + return scriptedDatabases + } + + func fetchSchemas() async throws -> [String] { + fetchSchemasCalls += 1 + return scriptedSchemas + } var currentDatabase: String? { scriptedCurrentDatabase } + var refusedDatabases: Set = [] + func switchDatabase(to name: String) async throws { if let switchDatabaseError { throw switchDatabaseError } + if refusedDatabases.contains(name) { throw MockError.scripted } switchedDatabases.append(name) } func switchSchema(to name: String) async throws { + switchedSchemas.append(name) currentSchema = name } func beginTransaction() async throws { didBeginTransaction = true } @@ -145,6 +182,7 @@ final class MockDatabaseDriver: DatabaseDriver, @unchecked Sendable { } func sessionTransactionState() async -> DriverTransactionState { + transactionStateReads += 1 await beforeTransactionState?() guard !scriptedTransactionStates.isEmpty else { return scriptedTransactionState } return scriptedTransactionStates.removeFirst() @@ -179,3 +217,17 @@ final class MockSecureStore: SecureStore, @unchecked Sendable { storage[key] = value } } + +extension MockDatabaseDriver: SessionDatabaseReporting, SessionSchemaReporting { + var reportsDatabaseWithoutStatement: Bool { false } + + func sessionDatabase() async throws -> String? { + sessionDatabaseReads += 1 + return scriptedSessionDatabase + } + + func settleCurrentSchema() async { + guard let scriptedSessionSchema else { return } + currentSchema = scriptedSessionSchema + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/MockQueryBuildingDriver.swift b/apple/TableProMobile/TableProMobileTests/Mocks/MockQueryBuildingDriver.swift index f4d22c9079..944f55f234 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/MockQueryBuildingDriver.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/MockQueryBuildingDriver.swift @@ -8,6 +8,7 @@ final class MockQueryBuildingDriver: DriverQueryBuilding, @unchecked Sendable { var scriptedRows: [[String?]] = [] var scriptedCount: Int? = 0 var scriptedValues: [Result] = [] + var beforeCount: (@Sendable () async -> Void)? private(set) var pageRequests: [(request: TableReadRequest, limit: Int, offset: Int)] = [] private(set) var countRequests: [TableReadRequest] = [] @@ -25,6 +26,7 @@ final class MockQueryBuildingDriver: DriverQueryBuilding, @unchecked Sendable { func rowCount(_ request: TableReadRequest) async throws -> Int? { countRequests.append(request) + await beforeCount?() return scriptedCount } diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/RecordingPushRegistrar.swift b/apple/TableProMobile/TableProMobileTests/Mocks/RecordingPushRegistrar.swift new file mode 100644 index 0000000000..a05145c22a --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Mocks/RecordingPushRegistrar.swift @@ -0,0 +1,19 @@ +@testable import TableProMobile +import TableProSyncTransport + +@MainActor +final class RecordingPushRegistrar { + private(set) var registrations = 0 + private(set) var unregistrations = 0 + + var registrar: SyncPushRegistrar { + SyncPushRegistrar( + register: { [weak self] in self?.registrations += 1 }, + unregister: { [weak self] in self?.unregistrations += 1 } + ) + } +} + +extension SyncPushRegistrar { + static let inert = SyncPushRegistrar(register: {}, unregister: {}) +} diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSearchIndex.swift b/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSearchIndex.swift index 3b2a8bdf14..3f24600fba 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSearchIndex.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSearchIndex.swift @@ -9,9 +9,13 @@ final class RecordingSearchIndex: ConnectionSearchIndexing { private(set) var replacements: [[SearchableConnection]] = [] private(set) var contents: [SearchableConnection] = [] + // Kept by the index across launches, as CoreSpotlight keeps a client state. + private(set) var state: Data? + private(set) var stateReads = 0 private(set) var maxConcurrent = 0 private var inFlight = 0 private var failuresRemaining = 0 + private var stateReadFailures = 0 private var holdsNext = false private var held: CheckedContinuation? private var heldWaiter: CheckedContinuation? @@ -24,6 +28,10 @@ final class RecordingSearchIndex: ConnectionSearchIndexing { failuresRemaining += 1 } + func failNextStateRead() { + stateReadFailures += 1 + } + func holdNextReplace() { holdsNext = true } @@ -39,7 +47,17 @@ final class RecordingSearchIndex: ConnectionSearchIndexing { } @MainActor - func replaceConnections(with connections: [SearchableConnection]) async throws { + func storedState() async throws -> Data? { + stateReads += 1 + if stateReadFailures > 0 { + stateReadFailures -= 1 + throw Failure.scripted + } + return state + } + + @MainActor + func replaceConnections(with connections: [SearchableConnection], state: Data) async throws { inFlight += 1 maxConcurrent = max(maxConcurrent, inFlight) defer { inFlight -= 1 } @@ -57,5 +75,6 @@ final class RecordingSearchIndex: ConnectionSearchIndexing { throw Failure.scripted } contents = connections + self.state = state } } diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSyncTransport.swift b/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSyncTransport.swift index fa68820f00..0b7da0990f 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSyncTransport.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/RecordingSyncTransport.swift @@ -12,6 +12,10 @@ actor RecordingSyncTransport: IOSSyncTransport { private(set) var pushedRecords: [CKRecord] = [] private(set) var pushedDeletions: [CKRecord.ID] = [] private(set) var pullCount = 0 + private(set) var subscriptionSaves = 0 + private var subscriptionError: (any Error)? + private var beforeAccountRead: (@Sendable () async -> Void)? + private var beforePull: (@Sendable () async -> Void)? init( remoteRecords: [CKRecord] = [], @@ -28,18 +32,41 @@ actor RecordingSyncTransport: IOSSyncTransport { remoteDeletions = deletions } + // Runs inside every account read, before any download starts. + func holdAccountChecks(_ hold: @escaping @Sendable () async -> Void) { + beforeAccountRead = hold + } + + // Runs inside every pull, once it is counted. + func holdPulls(_ hold: @escaping @Sendable () async -> Void) { + beforePull = hold + } + func accountStatus() async throws -> CKAccountStatus { .available } func currentAccountId() async throws -> String { - "recording-account" + await beforeAccountRead?() + return "recording-account" } func ensureZoneExists() async throws {} + func failSubscriptionSaves(with error: (any Error)?) { + subscriptionError = error + } + + func saveSubscription() async throws { + subscriptionSaves += 1 + if let subscriptionError { + throw subscriptionError + } + } + func pull(since token: CKServerChangeToken?) async throws -> PullResult { pullCount += 1 + await beforePull?() return PullResult(changedRecords: remoteRecords, deletedRecordIDs: remoteDeletions, newToken: newToken) } diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift b/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift index 376ba32e9c..e5ede52867 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift @@ -12,8 +12,11 @@ final class StubPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var checkRows: [Int] = [] var onFetchTables: (@Sendable () async -> Void)? var currentDb = "" + var declaredCapabilities: PluginCapabilities = [] private(set) var databasesAtExecute: [String] = [] + var capabilities: PluginCapabilities { declaredCapabilities } + private(set) var boundedQueries: [(query: String, rowCap: Int)] = [] private(set) var plainQueries: [String] = [] private(set) var parameterizedQueries: [(query: String, parameters: [PluginCellValue])] = [] diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/UnreachableSyncTransport.swift b/apple/TableProMobile/TableProMobileTests/Mocks/UnreachableSyncTransport.swift index 69ff123e1f..a14e8a02cb 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/UnreachableSyncTransport.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/UnreachableSyncTransport.swift @@ -19,6 +19,10 @@ struct UnreachableSyncTransport: IOSSyncTransport { func ensureZoneExists() async throws {} + func saveSubscription() async throws { + throw CKError(.notAuthenticated) + } + func pull(since token: CKServerChangeToken?) async throws -> PullResult { PullResult(changedRecords: [], deletedRecordIDs: [], newToken: nil) } diff --git a/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateFixture.swift b/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateFixture.swift index 85d24b749d..2fbfbdddbb 100644 --- a/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateFixture.swift +++ b/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateFixture.swift @@ -58,7 +58,8 @@ struct AppStateFixture { recordCache: SyncRecordCache(directory: root.appendingPathComponent("Cache"), defaults: nil), makeTransport: { transport }, isEnabled: { syncEnabled }, - notificationCenter: NotificationCenter() + notificationCenter: NotificationCenter(), + pushRegistrar: .inert ) return AppState( libraryDirectory: libraryDirectory, @@ -73,8 +74,10 @@ struct AppStateFixture { bookmarkStore: bookmarkStore, libraryPublisher: libraryPublisher ?? ConnectionLibraryPublisher( searchIndex: RecordingSearchIndex(), + readWidgetItems: { [] }, writeWidgetItems: { _ in }, - refreshShortcutParameters: {} + refreshShortcutParameters: {}, + defaults: defaults ), driverFactory: driverFactory ) diff --git a/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateLibraryTests.swift b/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateLibraryTests.swift index 17fb2705ef..f1c9db870f 100644 --- a/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateLibraryTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Onboarding/AppStateLibraryTests.swift @@ -23,8 +23,10 @@ struct AppStateLibraryTests { let widgetWrites = widgetWrites return ConnectionLibraryPublisher( searchIndex: searchIndex, + readWidgetItems: { [] }, writeWidgetItems: { _ in widgetWrites.total += 1 }, - refreshShortcutParameters: {} + refreshShortcutParameters: {}, + defaults: fixture.defaults ) } diff --git a/apple/TableProMobile/TableProMobileTests/Platform/KeychainSecureStoreTests.swift b/apple/TableProMobile/TableProMobileTests/Platform/KeychainSecureStoreTests.swift index c608f163b7..d68128cece 100644 --- a/apple/TableProMobile/TableProMobileTests/Platform/KeychainSecureStoreTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Platform/KeychainSecureStoreTests.swift @@ -1,4 +1,5 @@ import Foundation +import os import Security @testable import TableProMobile import Testing @@ -122,4 +123,69 @@ struct KeychainSecureStoreTests { try store.store("new-pem", role: .clientCertificate, for: connectionId) #expect(store.pem(role: .clientCertificate, for: connectionId) == "new-pem") } + + // MARK: - Launch cleanup + + @Test("The launch cleanup removes a deleted connection's device-only secrets and keeps current, synced and Mac-owned ones") + func cleanupRemovesOnlyOrphans() async { + let kept = UUID() + let deleted = UUID() + let keptPassword = ConnectionSecretKind.password.account(for: kept) + let deletedPassword = ConnectionSecretKind.password.account(for: deleted) + let deletedSSHPassword = ConnectionSecretKind.sshPassword.account(for: deleted) + let macOwnedKey = ConnectionSecretKind.sshPrivateKey.account(for: deleted) + items.seed("kept", account: keptPassword, synchronizable: false) + items.seed("orphan", account: deletedPassword, synchronizable: false) + items.seed("orphan", account: deletedSSHPassword, synchronizable: false) + items.seed("synced", account: deletedPassword, synchronizable: true) + items.seed("mac", account: macOwnedKey, synchronizable: false) + + await KeychainSecureStore.cleanOrphanedCredentials(items: items) { Set([kept]) } + + #expect(items.value(account: keptPassword, synchronizable: false) == "kept") + #expect(items.value(account: deletedPassword, synchronizable: false) == nil) + #expect(items.value(account: deletedSSHPassword, synchronizable: false) == nil) + #expect(items.value(account: deletedPassword, synchronizable: true) == "synced") + #expect(items.value(account: macOwnedKey, synchronizable: false) == "mac") + } + + @MainActor + @Test("Started from the main actor, the launch cleanup makes no Keychain call on the main thread") + func cleanupStaysOffTheMainThread() async { + let orphan = ConnectionSecretKind.password.account(for: UUID()) + items.seed("orphan", account: orphan, synchronizable: false) + + await KeychainSecureStore.cleanOrphanedCredentials(items: items) { Set([UUID()]) } + + #expect(items.value(account: orphan, synchronizable: false) == nil) + #expect(items.mainThreadCalls == 0) + } + + @Test("A connection saved while the Keychain is listed keeps its secrets, since the library is read after the listing") + func connectionSavedDuringListingKeepsItsSecrets() async { + let existing = UUID() + let saved = UUID() + let savedPassword = ConnectionSecretKind.password.account(for: saved) + let library = OSAllocatedUnfairLock(initialState: Set([existing])) + items.duringNextListing { [items] in + // A save adds the connection, then writes its secrets. + library.withLock { _ = $0.insert(saved) } + items.seed("new", account: savedPassword, synchronizable: false) + } + + await KeychainSecureStore.cleanOrphanedCredentials(items: items) { library.withLock { $0 } } + + #expect(items.value(account: savedPassword, synchronizable: false) == "new") + } + + @Test("A library that is empty or not loaded removes nothing") + func unknownLibraryRemovesNothing() async { + let account = ConnectionSecretKind.password.account(for: UUID()) + items.seed("secret", account: account, synchronizable: false) + + await KeychainSecureStore.cleanOrphanedCredentials(items: items) { nil } + await KeychainSecureStore.cleanOrphanedCredentials(items: items) { Set() } + + #expect(items.value(account: account, synchronizable: false) == "secret") + } } diff --git a/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift b/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift index 5815991060..be0bde871b 100644 --- a/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift @@ -1,6 +1,7 @@ import Foundation import TableProDatabase @testable import TableProMobile +import TableProTestSupport import Testing @MainActor @@ -29,6 +30,57 @@ struct SessionTransactionMonitorTests { return driver } + @Test("A run that moves the database waits for a write already on the session") + func moveWaitsForAWriteInFlight() async throws { + attach() + await monitor.statementsFinished(for: id) + let log = EventLog() + let gate = TestGate() + let write = Task { + await monitor.track(id) { + await gate.enter() + log.events.append("write") + } + } + await gate.waitUntilEntered() + let run = Task { + log.events.append("waiting") + await monitor.statementsFinished(for: id) + log.events.append("run") + } + try await waitUntil { log.events.contains("waiting") } + + await gate.open() + await write.value + await run.value + + #expect(log.events == ["waiting", "write", "run"]) + } + + @Test("Stop ends the wait for a write that is still running") + func stopEndsTheWait() async throws { + attach() + let log = EventLog() + let gate = TestGate() + let write = Task { await monitor.track(id) { await gate.enter() } } + await gate.waitUntilEntered() + let run = Task { + log.events.append("waiting") + await monitor.statementsFinished(for: id) + log.events.append("run") + } + try await waitUntil { log.events.contains("waiting") } + + run.cancel() + await run.value + + #expect(log.events == ["waiting", "run"]) + #expect(monitor.isRunningStatements(id)) + await gate.open() + await write.value + #expect(!monitor.isRunningStatements(id)) + } + @Test("A run that leaves a transaction open shows it, explicit or implicit") func runLeavesTheTransactionOpen() async { let driver = attach(.explicitTransaction) @@ -332,3 +384,8 @@ struct SessionTransactionMonitorTests { #expect(serverState == .idle) } } + +@MainActor +private final class EventLog { + var events: [String] = [] +} diff --git a/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift index 4a135755a6..49c61a02c0 100644 --- a/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift @@ -24,9 +24,9 @@ struct QueryEditorViewModelTests { let vm = QueryEditorViewModel(windowCapacity: 100) await vm.run(driver: driver, script: .statements("SELECT * FROM t"), maxRows: 3) - #expect(vm.legacyRows.count == 3) - #expect(vm.legacyRows.first?.first == "0") - #expect(vm.legacyRows.last?.first == "2") + #expect(vm.window.shownValues.count == 3) + #expect(vm.window.shownValues.first?.first == "0") + #expect(vm.window.shownValues.last?.first == "2") #expect(vm.truncationReason != nil) #expect(vm.truncationMessage != nil) if case .truncated(let reason) = vm.phase, case .rowCap(let cap) = reason { @@ -36,6 +36,23 @@ struct QueryEditorViewModelTests { } } + @Test("Share and Copy Results format the rows on screen, and refuse a result the free memory cannot hold") + func exportResultsFormatsTheRows() async throws { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [ + .success(QueryResult(columns: makeColumns(), rows: [["1"], ["2"]], rowsAffected: 0, executionTime: 0)) + ] + let vm = QueryEditorViewModel(windowCapacity: 100) + await vm.run(driver: driver, script: .statements("SELECT id FROM t"), maxRows: 100) + + let csv = try await vm.exportResults(as: .csv, databaseType: .mysql, driver: driver, availableBytes: 0) + #expect(csv == "id\n1\n2") + + await #expect(throws: ClipboardExporter.TooLarge(source: .queryResult)) { + try await vm.exportResults(as: .csv, databaseType: .mysql, driver: driver, availableBytes: 1) + } + } + @Test("stop marks the run cancelled rather than letting it settle as finished") func stopRecordsCancellation() async { let driver = MockDatabaseDriver() @@ -106,7 +123,7 @@ struct QueryEditorViewModelTests { let vm = QueryEditorViewModel(windowCapacity: 100) await vm.run(driver: driver, script: .statements("SELECT id FROM t"), maxRows: 100) - #expect(vm.legacyRows.count == 2) + #expect(vm.window.shownValues.count == 2) #expect(vm.truncationReason == nil) #expect(vm.truncationMessage == nil) if case .finished = vm.phase {} else { @@ -127,7 +144,7 @@ struct QueryEditorViewModelTests { await vm.handlePressure(.warning) - #expect(vm.legacyRows.count == 2) + #expect(vm.window.shownValues.count == 2) #expect(vm.truncationReason == nil) #expect(vm.truncationMessage == nil) if case .finished = vm.phase {} else { @@ -144,11 +161,11 @@ struct QueryEditorViewModelTests { let vm = QueryEditorViewModel(windowCapacity: 100) await vm.run(driver: driver, script: .statements("SELECT id FROM t"), maxRows: 100) - #expect(vm.legacyRows.count == 1) + #expect(vm.window.shownValues.count == 1) vm.reset() - #expect(vm.legacyRows.isEmpty) + #expect(vm.window.shownValues.isEmpty) #expect(vm.columns.isEmpty) if case .idle = vm.phase {} else { Issue.record("expected idle phase after reset") @@ -401,6 +418,109 @@ struct QueryEditorViewModelTests { ]) #expect(vm.requestCount == 3) } + + private static let longValue = String(repeating: "x", count: 10_000) + + private func longValueResult(rows: Int) -> QueryResult { + QueryResult( + columns: [ColumnInfo(name: "body", typeName: "TEXT", ordinalPosition: 0)], + rows: Array(repeating: [Self.longValue], count: rows), + rowsAffected: 0, + executionTime: 0 + ) + } + + @Test("a long value comes back whole, and one past the budget keeps a preview and says so") + func longValuesStayWholeWithinTheBudget() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 2))] + + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + await vm.run(driver: driver, script: .statements("SELECT body FROM notes")) + + let cells = vm.window.rows.compactMap { $0.cells.first } + guard cells.count == 2, case .text(let kept) = cells[0], case .truncatedText(_, let totalBytes, let ref) = cells[1] else { + Issue.record("expected the first value whole and the second cut, got \(cells)") + return + } + #expect(kept == Self.longValue) + #expect(totalBytes == 10_000) + #expect(ref == nil) + #expect(vm.longValuesNotice == String(localized: "Some long values are shown in part to stay within memory.")) + let shown = vm.window.shownValues.first?.first ?? nil + #expect(shown?.utf8.count == Cell.previewByteLimit) + } + + @Test("each request of a script starts with a budget of its own") + func eachRequestGetsAFreshBudget() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 1)), .success(longValueResult(rows: 1))] + + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + await vm.run(driver: driver, script: .statements("SELECT body FROM a", "SELECT body FROM b")) + + #expect(driver.executedQueries == ["SELECT body FROM a", "SELECT body FROM b"]) + guard case .text(let kept)? = vm.window.rows.first?.cells.first else { + Issue.record("the second request's value should be whole, got \(vm.window.rows)") + return + } + #expect(kept == Self.longValue) + #expect(vm.longValuesNotice == nil) + } + + @Test("a new run clears the notice of the one before") + func newRunClearsTheNotice() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 2)), .success(longValueResult(rows: 1))] + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + + await vm.run(driver: driver, script: .statements("SELECT body FROM notes")) + #expect(vm.longValuesNotice != nil) + await vm.run(driver: driver, script: .statements("SELECT body FROM notes LIMIT 1")) + + #expect(vm.longValuesNotice == nil) + } + + @Test("Copy and Share Results ask for the memory the data browser's export asks for") + func resultsExportChecksMemory() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 1))] + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + await vm.run(driver: driver, script: .statements("SELECT body FROM notes")) + let margin = 64 * 1_024 * 1_024 + + let rows = vm.window.rows + #expect(QueryEditorViewModel.resultsFitExport(rows: rows, availableBytes: 0)) + #expect(QueryEditorViewModel.resultsFitExport(rows: rows, availableBytes: margin + 40_001)) + #expect(!QueryEditorViewModel.resultsFitExport(rows: rows, availableBytes: margin + 40_000)) + } + + @Test("a Read-Only run asks every request for the engine's read-only mode, and any other run asks none") + func readOnlyReachesEveryRequest() async { + let driver = StreamOptionsRecorder() + let vm = QueryEditorViewModel(windowCapacity: 100) + + await vm.run(driver: driver, script: .statements("SELECT 1", "SELECT 2"), enforcesReadOnly: true) + let readOnlyRun = driver.options.map { $0.enforcesReadOnly } + await vm.run(driver: driver, script: .statements("SELECT 3")) + let plainRun = driver.options.dropFirst(2).map { $0.enforcesReadOnly } + + #expect(readOnlyRun == [true, true]) + #expect(plainRun == [false]) + } + + @Test("the budget is a quarter of the headroom past 64 MB, at most 64 MB, and 64 MB when the headroom is unknown") + func textBudgetFollowsTheHeadroom() { + let megabyte = 1_024 * 1_024 + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 0) == 64 * megabyte) + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 32 * megabyte) == 0) + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 100 * megabyte) == 6 * megabyte) + // A result that spent its whole budget still exports, with the kept text off the headroom. + let available = 200 * megabyte + let spent = MemoryPressureMonitor.textBudgetBytes(availableBytes: available) + #expect(DataBrowserViewModel.exportFits(pendingBytes: spent, availableBytes: available - spent)) + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 4_096 * megabyte) == 64 * megabyte) + } } private extension QueryScript { @@ -409,6 +529,40 @@ private extension QueryScript { } } +// Keeps the options each request streams with and answers every one with nothing. +private final class StreamOptionsRecorder: DatabaseDriver, @unchecked Sendable { + private(set) var options: [StreamOptions] = [] + + var supportsSchemas: Bool { false } + var currentSchema: String? { nil } + var supportsTransactions: Bool { true } + var serverVersion: String? { nil } + + func executeStreaming(query: String, options: StreamOptions) -> AsyncThrowingStream { + self.options.append(options) + return AsyncThrowingStream { $0.finish() } + } + + func connect() async throws {} + func disconnect() async throws {} + func ping() async throws -> Bool { true } + func execute(query: String) async throws -> QueryResult { + QueryResult(columns: [], rows: [], rowsAffected: 0, executionTime: 0) + } + func cancelCurrentQuery() async throws {} + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [ColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] { [] } + func fetchDatabases() async throws -> [String] { [] } + func switchDatabase(to name: String) async throws {} + func switchSchema(to name: String) async throws {} + func fetchSchemas() async throws -> [String] { [] } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} + private actor QueryGate { private var continuation: CheckedContinuation? private var isOpen = false diff --git a/apple/TableProMobile/TableProMobileTests/QueryHistoryStorageDamageTests.swift b/apple/TableProMobile/TableProMobileTests/QueryHistoryStorageDamageTests.swift new file mode 100644 index 0000000000..11b45ce935 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/QueryHistoryStorageDamageTests.swift @@ -0,0 +1,91 @@ +import Foundation +import Testing +@testable import TableProMobile + +struct QueryHistoryStorageDamageTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("ios-history-damage-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + private var historyURL: URL { + directory.appendingPathComponent("query-history.json") + } + + private func setAsideCopies() throws -> [URL] { + try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + .filter { $0.lastPathComponent.hasPrefix("query-history.damaged-") } + } + + @Test("A history file that does not decode is set aside, not written over") + func undecodableFileIsSetAside() throws { + let damaged = Data("[{\"id\": \"not-a-uuid\"".utf8) + try damaged.write(to: historyURL) + let storage = QueryHistoryStorage(directory: directory) + + #expect(storage.save(QueryHistoryItem(query: "SELECT 1", connectionId: UUID()))) + + let copies = try setAsideCopies() + #expect(copies.count == 1) + #expect(try copies.first.map { try Data(contentsOf: $0) } == damaged) + #expect(storage.loadAll().map(\.query) == ["SELECT 1"]) + } + + @Test("Only a clear that asks for it deletes the set-aside copy, even once no history is left") + func onlyAnExplicitClearDeletesTheSetAsideCopy() throws { + try Data("not json".utf8).write(to: historyURL) + let storage = QueryHistoryStorage(directory: directory) + let first = UUID() + let second = UUID() + #expect(storage.save(QueryHistoryItem(query: "SELECT 1", connectionId: first))) + #expect(storage.save(QueryHistoryItem(query: "SELECT 2", connectionId: second))) + #expect(storage.hasSetAsideCopies) + + storage.clearAll(for: [first]) + storage.clearAll(for: [second]) + #expect(storage.loadAll().isEmpty) + #expect(storage.hasSetAsideCopies) + + storage.clearAll(for: [first], removingSetAsideCopies: true) + #expect(!storage.hasSetAsideCopies) + #expect(try setAsideCopies().isEmpty) + } + + @Test("Deleting a connection keeps the set-aside copy, which holds every connection's history") + @MainActor + func deletingAConnectionKeepsTheSetAsideCopy() throws { + try Data("not json".utf8).write(to: historyURL) + let storage = QueryHistoryStorage(directory: directory) + let deleted = UUID() + #expect(storage.save(QueryHistoryItem(query: "SELECT 1", connectionId: deleted))) + let suite = "QueryHistoryStorageDamageTests.\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let localState = ConnectionLocalState( + secrets: ConnectionSecrets(secureStore: MockSecureStore()), + queryHistory: storage, + defaults: defaults + ) + + localState.purge([deleted]) + + #expect(storage.loadAll().isEmpty) + #expect(storage.hasSetAsideCopies) + #expect(try setAsideCopies().count == 1) + } + + @Test("A history file that decodes is kept and appended to") + func readableFileIsKept() throws { + let storage = QueryHistoryStorage(directory: directory) + #expect(storage.save(QueryHistoryItem(query: "SELECT 1", connectionId: UUID()))) + + #expect(storage.save(QueryHistoryItem(query: "SELECT 2", connectionId: UUID()))) + + #expect(try setAsideCopies().isEmpty) + #expect(storage.loadAll().map(\.query) == ["SELECT 1", "SELECT 2"]) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/QueryHistoryStorageTests.swift b/apple/TableProMobile/TableProMobileTests/QueryHistoryStorageTests.swift new file mode 100644 index 0000000000..d35d206be2 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/QueryHistoryStorageTests.swift @@ -0,0 +1,97 @@ +import Foundation +import Testing +@testable import TableProMobile + +struct QueryHistoryStorageTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("ios-history-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + private var historyURL: URL { + directory.appendingPathComponent("query-history.json") + } + + @Test("A recorded run returns its connection's list, so the window needs no second read") + func recordReturnsTheConnectionsList() throws { + let storage = QueryHistoryStorage(directory: directory) + let mine = UUID() + storage.save(QueryHistoryItem(query: "SELECT 1", connectionId: mine)) + storage.save(QueryHistoryItem(query: "SELECT 2", connectionId: UUID())) + + let listed = try #require(storage.record(QueryHistoryItem(query: "SELECT 3", connectionId: mine))) + + #expect(listed.items.map(\.query) == ["SELECT 1", "SELECT 3"]) + #expect(listed.generation == storage.generation) + } + + @Test("The same run twice in a row records nothing the second time") + func duplicateRecordsNothing() { + let storage = QueryHistoryStorage(directory: directory) + let connectionId = UUID() + #expect(storage.record(QueryHistoryItem(query: "SELECT 1", connectionId: connectionId)) != nil) + + #expect(storage.record(QueryHistoryItem(query: "SELECT 1", connectionId: connectionId)) == nil) + } + + @Test("Text past the budget drops the oldest entries whole, and the newest entry always stays") + func byteBudgetDropsTheOldestWhole() { + let connectionId = UUID() + let half = String(repeating: "x", count: QueryHistoryStorage.maxTextBytes / 2) + let items = [ + QueryHistoryItem(query: "SELECT 1", connectionId: connectionId), + QueryHistoryItem(query: half, connectionId: connectionId), + QueryHistoryItem(query: half, connectionId: connectionId), + ] + + let kept = QueryHistoryStorage.withinBudget(items) + #expect(kept.map(\.query.utf8.count) == [half.utf8.count, half.utf8.count]) + + let huge = String(repeating: "y", count: QueryHistoryStorage.maxTextBytes + 1) + let alone = QueryHistoryStorage.withinBudget(items + [QueryHistoryItem(query: huge, connectionId: connectionId)]) + #expect(alone.map(\.query) == [huge]) + } + + @Test("At most 200 entries are kept") + func entryCapStillHolds() { + let connectionId = UUID() + let items = (0..<250).map { QueryHistoryItem(query: "SELECT \($0)", connectionId: connectionId) } + + let kept = QueryHistoryStorage.withinBudget(items) + + #expect(kept.count == 200) + #expect(kept.first?.query == "SELECT 50") + #expect(kept.last?.query == "SELECT 249") + } + + @Test("A write that fails leaves the listed history as the file holds it") + func failedWriteKeepsTheList() throws { + let storage = QueryHistoryStorage(directory: directory) + #expect(storage.save(QueryHistoryItem(query: "SELECT 1", connectionId: UUID()))) + let generation = storage.generation + try FileManager.default.removeItem(at: historyURL) + try FileManager.default.createDirectory(at: historyURL, withIntermediateDirectories: false) + + #expect(!storage.save(QueryHistoryItem(query: "SELECT 2", connectionId: UUID()))) + + #expect(storage.loadAll().map(\.query) == ["SELECT 1"]) + #expect(storage.generation == generation) + } + + @Test("The file follows memory, so the next launch reads what this one recorded") + func fileFollowsMemory() { + let first = QueryHistoryStorage(directory: directory) + let connectionId = UUID() + first.save(QueryHistoryItem(query: "SELECT 1", connectionId: connectionId)) + first.save(QueryHistoryItem(query: "SELECT 2", connectionId: connectionId)) + first.delete(first.loadAll()[0].id) + + let relaunched = QueryHistoryStorage(directory: directory) + + #expect(relaunched.load(for: connectionId).map(\.query) == ["SELECT 2"]) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift index 2f0824d27a..005d70f295 100644 --- a/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift @@ -26,6 +26,17 @@ struct RowDetailViewModelTests { ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "test") } + @Test("A row opened before the browser read its foreign keys shows them once they arrive") + func foreignKeysArriveAfterTheRowOpened() { + let source = ForeignKeySource() + let vm = RowDetailViewModel(columns: makeColumns(), rows: makeRows(), initialIndex: 0, foreignKeys: source.keys) + #expect(vm.foreignKeys.isEmpty) + + source.keys = [ForeignKeyInfo(name: "fk_owner", column: "id", referencedTable: "owners", referencedColumn: "id")] + + #expect(vm.foreignKeys.map { $0.referencedTable } == ["owners"]) + } + @Test("canEdit requires session, table, primary key, and not safe-mode-blocked") func canEditPreconditions() { let driver = MockDatabaseDriver() @@ -1125,4 +1136,64 @@ struct RowDetailViewModelTests { #expect(vm.exportCurrentRow(as: .csv)?.contains("abc and the rest") == true) #expect(vm.operationError == nil) } + + @Test("A query result's row copies as an INSERT in its engine's quoting, not SQLite's") + func queryResultRowExportsInTheEnginesQuoting() throws { + let vm = RowDetailViewModel( + columns: makeColumns(), rows: makeRows(), initialIndex: 0, + session: makeSession(driver: MockDatabaseDriver()), databaseType: .mysql + ) + + let insert = try #require(vm.exportCurrentRow(as: .sqlInsert)) + + #expect(insert.hasPrefix("INSERT INTO `table` (`id`, `name`) VALUES")) + #expect(!vm.canEdit) + } + + @Test("A long value shows an excerpt, while Copy Value and the full view get all of it") + func longValueShowsAnExcerpt() { + let long = String(repeating: "y", count: 10_000) + let vm = RowDetailViewModel( + columns: makeColumns(), rows: [Row(cells: [.text("1"), .text(long)])], initialIndex: 0 + ) + + #expect(vm.longValue(forRow: 0, at: 1) == long) + #expect(vm.completeValue(forRow: 0, at: 1) == long) + #expect(vm.longValue(forRow: 0, at: 0) == nil) + #expect(RowDetailViewModel.excerpt(of: long).utf8.count == RowDetailViewModel.inlineByteLimit) + #expect(!vm.isShownInPart(forRow: 0, at: 1)) + } + + @Test("A table value loaded in full is long enough to need the full view too") + func loadedValueShowsAnExcerpt() async { + let long = String(repeating: "z", count: 10_000) + let vm = makeViewModel( + lastCell: .truncatedText(prefix: "zzz", totalBytes: 10_000, ref: bodyRef(id: "1")), + loadFullValue: { _ in long } + ) + #expect(vm.longValue(forRow: 0, at: 2) == nil) + + await vm.loadFullValue(ref: bodyRef(id: "1"), forRow: 0, cellIndex: 2) + + #expect(vm.longValue(forRow: 0, at: 2) == long) + } + + @Test("Only a value cut with no key to load it by is shown in part") + func cutValueWithoutRefIsShownInPart() { + let vm = RowDetailViewModel( + columns: makeColumns(), + rows: [Row(cells: [.text("1"), .truncatedText(prefix: "a", totalBytes: 5_000, ref: nil)])], + initialIndex: 0 + ) + let loadable = makeViewModel(lastCell: .truncatedText(prefix: "a", totalBytes: 5_000, ref: bodyRef(id: "1"))) + + #expect(vm.isShownInPart(forRow: 0, at: 1)) + #expect(vm.longValue(forRow: 0, at: 1) == nil) + #expect(!loadable.isShownInPart(forRow: 0, at: 2)) + } +} + +@MainActor +private final class ForeignKeySource { + var keys: [ForeignKeyInfo] = [] } diff --git a/apple/TableProMobile/TableProMobileTests/SavedQueries/SavedQueryLibraryTests.swift b/apple/TableProMobile/TableProMobileTests/SavedQueries/SavedQueryLibraryTests.swift index fa4e64e86c..d2bf024349 100644 --- a/apple/TableProMobile/TableProMobileTests/SavedQueries/SavedQueryLibraryTests.swift +++ b/apple/TableProMobile/TableProMobileTests/SavedQueries/SavedQueryLibraryTests.swift @@ -23,7 +23,8 @@ struct SavedQueryLibraryTests { makeTransport: { UnreachableSyncTransport() }, isEnabled: { false }, notificationCenter: NotificationCenter(), - networkMonitor: nil + networkMonitor: nil, + pushRegistrar: .inert ) } diff --git a/apple/TableProMobile/TableProMobileTests/Security/CertificateMaterializerTests.swift b/apple/TableProMobile/TableProMobileTests/Security/CertificateMaterializerTests.swift new file mode 100644 index 0000000000..3219b7c7a7 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Security/CertificateMaterializerTests.swift @@ -0,0 +1,81 @@ +import Foundation +@testable import TableProMobile +import TableProDatabase +import Testing + +struct CertificateMaterializerTests { + private let root: URL + private let certificates = InMemoryCertificateStore() + + init() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("ios-certificates-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + + private var materializer: CertificateMaterializer { + CertificateMaterializer(store: certificates, root: root) + } + + private func folder(_ connectionId: UUID) -> URL { + root.appendingPathComponent(connectionId.uuidString, isDirectory: true) + } + + private func exists(_ url: URL) -> Bool { + FileManager.default.fileExists(atPath: url.path(percentEncoded: false)) + } + + @Test("Deleting a connection removes the client key a connect wrote to disk") + func deleteRemovesTheKeyFile() throws { + let connectionId = UUID() + try certificates.store("key", role: .clientKey, for: connectionId) + let written = try #require(try materializer.materialize(for: connectionId).clientKeyPath) + #expect(exists(URL(filePath: written))) + + let secrets = ConnectionSecrets( + secureStore: MockSecureStore(), + certificateStore: certificates, + bookmarkStore: FileBookmarkStore(suiteName: "com.TablePro.tests.CertificateMaterializer.\(UUID().uuidString)"), + materializer: materializer + ) + secrets.delete(for: connectionId) + + #expect(!exists(URL(filePath: written))) + #expect(!exists(folder(connectionId))) + } + + @Test("A certificate removed from the connection loses its file on the next connect") + func removedRoleLosesItsFile() throws { + let connectionId = UUID() + try certificates.store("ca", role: .certificateAuthority, for: connectionId) + try certificates.store("key", role: .clientKey, for: connectionId) + let first = try materializer.materialize(for: connectionId) + let keyPath = try #require(first.clientKeyPath) + + certificates.delete(role: .clientKey, for: connectionId) + let second = try materializer.materialize(for: connectionId) + + #expect(second.clientKeyPath == nil) + #expect(second.caCertificatePath != nil) + #expect(!exists(URL(filePath: keyPath))) + } + + @Test("The launch sweep removes the folders of connections that are gone and keeps the rest") + func sweepKeepsKnownConnections() throws { + let kept = UUID() + let gone = UUID() + for connectionId in [kept, gone] { + try certificates.store("key", role: .clientKey, for: connectionId) + _ = try materializer.materialize(for: connectionId) + } + let unrelated = root.appendingPathComponent("not-a-connection", isDirectory: true) + try FileManager.default.createDirectory(at: unrelated, withIntermediateDirectories: true) + + materializer.releaseAll(except: [kept]) + + #expect(exists(folder(kept))) + #expect(!exists(folder(gone))) + #expect(exists(unrelated)) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/SignIn/IAMErrorClassifierTests.swift b/apple/TableProMobile/TableProMobileTests/SignIn/IAMErrorClassifierTests.swift index deefd96c8f..206b6a0f35 100644 --- a/apple/TableProMobile/TableProMobileTests/SignIn/IAMErrorClassifierTests.swift +++ b/apple/TableProMobile/TableProMobileTests/SignIn/IAMErrorClassifierTests.swift @@ -29,6 +29,27 @@ struct IAMErrorClassifierTests { #expect(error.opensEditor) } + @Test("A DNS lookup that failed reads as a network problem") + func endpointLookupFailed() { + let error = classify(AWSAuthError.endpointLookupFailed(host: "reader.db.example.com")) + + #expect(error.title == String(localized: "DNS Lookup Failed")) + #expect(error.category == .network) + #expect(!error.opensEditor) + } + + @Test("An RDS Endpoint that names another database than Host says to clear it") + func rdsEndpointNotDialed() { + let error = classify(AWSAuthError.rdsEndpointNotDialed( + endpoint: "prod.abc123.us-east-1.rds.amazonaws.com", + host: "db.internal.example.com" + )) + + #expect(error.title == String(localized: "RDS Endpoint Does Not Match")) + #expect(error.category == .config) + #expect(error.opensEditor) + } + @Test("The PluginKit missing-key error reads as the same problem, though its message says authentication") func pluginKitMissingAccessKey() { let error = classify(AWSAuthError.missingAccessKey) diff --git a/apple/TableProMobile/TableProMobileTests/SignIn/IAMPasswordMintTests.swift b/apple/TableProMobile/TableProMobileTests/SignIn/IAMPasswordMintTests.swift index 0e004aea96..ee1cde7e7b 100644 --- a/apple/TableProMobile/TableProMobileTests/SignIn/IAMPasswordMintTests.swift +++ b/apple/TableProMobile/TableProMobileTests/SignIn/IAMPasswordMintTests.swift @@ -146,7 +146,8 @@ struct IAMPasswordMintTests { fields: [IAMSignIn.Field.rdsEndpoint: "\(Self.rdsHost):5432"] ) - await #expect(throws: AWSAuthError.rdsEndpointUnresolved(host: "db.internal.example.com")) { + let refusal = AWSAuthError.rdsEndpointNotDialed(endpoint: Self.rdsHost, host: "db.internal.example.com") + await #expect(throws: refusal) { try await mint.password(for: connection, method: .awsAccessKey, secureStore: store) } } diff --git a/apple/TableProMobile/TableProMobileTests/SignIn/IOSDriverFactoryEndpointAliasTests.swift b/apple/TableProMobile/TableProMobileTests/SignIn/IOSDriverFactoryEndpointAliasTests.swift new file mode 100644 index 0000000000..6429ce3418 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/SignIn/IOSDriverFactoryEndpointAliasTests.swift @@ -0,0 +1,85 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import Testing + +private final class FixedCNAMEResolver: CNAMEResolving, @unchecked Sendable { + private let lock = NSLock() + private let records: [String: String] + private var asked: [String] = [] + + init(_ records: [String: String]) { + self.records = records + } + + var queried: [String] { + lock.withLock { asked } + } + + func cnameTarget(of name: String, within timeout: Duration?) async -> CNAMELookup { + lock.withLock { + asked.append(name) + return records[name].map(CNAMELookup.found) ?? .failed + } + } +} + +struct IOSDriverFactoryEndpointAliasTests { + private static let alias = "reader.db.example.com" + private static let endpoint = "replica-a.abc123.us-east-1.rds.amazonaws.com" + + private func connection( + type: DatabaseType = .mysql, + method: IAMSignIn? = .awsProfile, + sshEnabled: Bool = false + ) -> DatabaseConnection { + var fields: [String: String] = [:] + fields[IAMSignIn.Field.method] = method?.pickerValue + return DatabaseConnection( + type: type, + host: Self.alias, + port: 3_306, + username: "iam_user", + additionalFields: fields, + sshEnabled: sshEnabled, + sshConfiguration: sshEnabled ? SSHConfiguration(host: "bastion.example.com", username: "ec2-user") : nil + ) + } + + @Test("An AWS IAM connection to a CNAME dials the RDS endpoint it stands for") + func awsConnectionDialsTheEndpoint() async throws { + let resolver = FixedCNAMEResolver([Self.alias: Self.endpoint]) + let factory = IOSDriverFactory(cnameResolver: resolver) + + for type in [DatabaseType.mysql, .mariadb, .postgresql] { + let resolved = try await factory.resolvingEndpointAlias(for: connection(type: type)) + #expect(resolved.host == Self.endpoint, "\(type)") + } + } + + @Test("Nothing is looked up behind SSH, without AWS IAM, or for ElastiCache") + func otherConnectionsAreNotLookedUp() async throws { + let resolver = FixedCNAMEResolver([Self.alias: Self.endpoint]) + let factory = IOSDriverFactory(cnameResolver: resolver) + + for saved in [ + connection(sshEnabled: true), + connection(method: nil), + connection(method: .googleApplicationDefault), + connection(type: .redis) + ] { + #expect(try await factory.resolvingEndpointAlias(for: saved).host == Self.alias) + } + #expect(resolver.queried.isEmpty) + } + + @Test("A lookup that fails stops the connect with an error naming the host") + func failedLookupThrows() async { + let factory = IOSDriverFactory(cnameResolver: FixedCNAMEResolver([:])) + + await #expect(throws: AWSAuthError.endpointLookupFailed(host: Self.alias)) { + _ = try await factory.resolvingEndpointAlias(for: connection()) + } + } +} diff --git a/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift b/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift index 141c959b6c..050ca4ec97 100644 --- a/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift +++ b/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift @@ -15,25 +15,42 @@ struct StreamingResultBufferTests { buffer.apply(.row(Row(cells: [.text("2")]))) buffer.flush() - #expect(buffer.legacyRows.count == 2) + #expect(buffer.window.shownValues.count == 2) #expect(buffer.window.count == 2) #expect(buffer.columns.count == 1) - #expect(buffer.legacyRows.first?.first == "1") + #expect(buffer.window.shownValues.first?.first == "1") } - @Test("shrink keeps window and legacy rows in lockstep") - func shrinkLockstep() { + @Test("shrink keeps the newest rows") + func shrinkKeepsNewest() { let buffer = StreamingResultBuffer(capacity: 100) for index in 0..<10 { buffer.apply(.row(Row(cells: [.text("\(index)")]))) } buffer.flush() - #expect(buffer.legacyRows.count == 10) + #expect(buffer.count == 10) buffer.shrink(to: 4) - #expect(buffer.legacyRows.count == 4) - #expect(buffer.window.count == 4) + #expect(buffer.count == 4) + #expect(buffer.window.shownValues.map { $0.first ?? nil } == ["6", "7", "8", "9"]) + } + + @Test("the grid gets a preview of a long value the row keeps whole") + func gridTextOfKeptValueIsCapped() throws { + let buffer = StreamingResultBuffer(capacity: 10) + let long = String(repeating: "界", count: 5_000) + buffer.apply(.row(Row(cells: [.text(long)]))) + buffer.flush() + + let shown = try #require(buffer.window.shownValues.first?.first ?? nil) + #expect(shown.utf8.count <= Cell.previewByteLimit) + #expect(long.hasPrefix(shown)) + guard case .text(let kept)? = buffer.window.rows.first?.cells.first else { + Issue.record("the row should keep the whole value") + return + } + #expect(kept == long) } @Test("reset clears all state") diff --git a/apple/TableProMobile/TableProMobileTests/Support/RowWindow+Shown.swift b/apple/TableProMobile/TableProMobileTests/Support/RowWindow+Shown.swift new file mode 100644 index 0000000000..9c3aff9b56 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Support/RowWindow+Shown.swift @@ -0,0 +1,9 @@ +@testable import TableProMobile +import TableProModels + +extension RowWindow { + // The text each row shows, derived the way the grid and the list derive it. + var shownValues: [[String?]] { + rows.map { $0.displayValues() } + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Sync/AppStateSyncWiringTests.swift b/apple/TableProMobile/TableProMobileTests/Sync/AppStateSyncWiringTests.swift index 345bdfc222..ece5284717 100644 --- a/apple/TableProMobile/TableProMobileTests/Sync/AppStateSyncWiringTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Sync/AppStateSyncWiringTests.swift @@ -21,6 +21,19 @@ struct AppStateSyncWiringTests { return state } + @Test("A CloudKit notice downloads through the app's wiring only while sync is on") + func noticeDownloadsOnlyWithSyncOn() async { + let transport = RecordingSyncTransport() + let state = makeSyncedState(transport: transport) + state.onboarding.setSyncChoice(false) + #expect(await state.syncFromBackground(.remoteChange) == false) + #expect(await transport.pullCount == 0) + + state.onboarding.setSyncChoice(true) + #expect(await state.syncFromBackground(.remoteChange)) + #expect(await transport.pullCount == 1) + } + @Test("A saved query made on the Mac reaches the saved queries and the file through the app's sync wiring") func pulledSavedQueryReachesTheLibrary() async { let folder = SavedQueryFolder(name: "Reports") diff --git a/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncActivationTests.swift b/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncActivationTests.swift new file mode 100644 index 0000000000..aec6399396 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncActivationTests.swift @@ -0,0 +1,169 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProSync +import TableProSyncTransport +import TableProTestSupport +import Testing + +@MainActor +struct IOSSyncActivationTests { + private let metadata: SyncMetadataStorage + private let cacheDirectory: URL + private let clock = ManualClock() + private let transport: RecordingSyncTransport + + init() throws { + transport = RecordingSyncTransport() + let defaults = try #require(UserDefaults(suiteName: "com.TablePro.tests.IOSSyncActivation.\(UUID().uuidString)")) + metadata = SyncMetadataStorage(userDefaults: defaults) + _ = metadata.adoptReadableRecordTypes(IOSSyncCoordinator.readableRecordTypes) + cacheDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("ios-sync-activation-\(UUID().uuidString)", isDirectory: true) + } + + private func makeCoordinator() -> IOSSyncCoordinator { + let transport = transport + let coordinator = IOSSyncCoordinator( + metadata: metadata, + recordCache: SyncRecordCache(directory: cacheDirectory, defaults: nil), + makeTransport: { transport }, + isEnabled: { true }, + notificationCenter: NotificationCenter(), + networkMonitor: nil, + activationClock: clock, + pushRegistrar: .inert + ) + coordinator.getCurrentState = { ([], [], []) } + coordinator.onConnectionsChanged = { _ in true } + coordinator.onGroupsChanged = { _ in true } + coordinator.onTagsChanged = { _ in true } + coordinator.getSavedQueries = { SavedQueryContents() } + coordinator.onSavedQueriesChanged = { _ in true } + return coordinator + } + + private func endInterval() async { + await clock.waitUntilSleeping() + clock.advance(by: IOSSyncCoordinator.activationInterval) + } + + @Test("Ten activations within a minute of a download start one run, and the held one runs once when the minute ends") + func activationsInsideTheIntervalRunOnce() async throws { + let coordinator = makeCoordinator() + + for _ in 0..<10 { + await coordinator.syncOnSceneActivation() + } + #expect(await transport.pullCount == 1) + #expect(coordinator.isHoldingActivations) + + await endInterval() + try await waitUntil { await transport.pullCount == 2 } + + // Waits for any run still going, so a second held run would show as a fourth pull. + await coordinator.sync(.userRequest) + #expect(await transport.pullCount == 3) + } + + @Test("With no activation held, the end of the interval starts no run") + func quietIntervalStartsNothing() async throws { + let coordinator = makeCoordinator() + await coordinator.syncOnSceneActivation() + + await endInterval() + try await waitUntil { !coordinator.isHoldingActivations } + await coordinator.sync(.userRequest) + + #expect(await transport.pullCount == 2) + } + + @Test("An edit and the person's request run at once inside the interval") + func editsAndRequestsAreNeverHeld() async { + let coordinator = makeCoordinator() + await coordinator.syncOnSceneActivation() + + await coordinator.sync(.localChange) + #expect(await transport.pullCount == 2) + await coordinator.sync(.userRequest) + #expect(await transport.pullCount == 3) + } + + @Test("A background refresh runs inside the interval") + func backgroundRefreshIsNeverHeld() async { + let coordinator = makeCoordinator() + await coordinator.syncOnSceneActivation() + + // What the BGAppRefresh task calls. + await coordinator.sync(.activation) + + #expect(await transport.pullCount == 2) + } + + @Test("An activation while a run has not started its download is answered by that run") + func activationBeforeTheDownloadIsDropped() async throws { + let coordinator = makeCoordinator() + let gate = AsyncGate() + await transport.holdAccountChecks { await gate.wait() } + let edit = Task { await coordinator.sync(.localChange) } + await gate.arrived(1) + + // Runs only once the activation below has returned or waits on the run, since both share the main actor. + let opener = Task { await gate.open() } + await coordinator.syncOnSceneActivation() + await opener.value + await edit.value + #expect(await transport.pullCount == 1) + + await endInterval() + try await waitUntil { !coordinator.isHoldingActivations } + await coordinator.sync(.userRequest) + #expect(await transport.pullCount == 2) + } + + @Test("An activation during a download runs once the interval ends, since the download began before it") + func activationDuringTheDownloadRunsAfterTheInterval() async throws { + let coordinator = makeCoordinator() + let gate = AsyncGate() + await transport.holdPulls { await gate.wait() } + let edit = Task { await coordinator.sync(.localChange) } + await gate.arrived(1) + + // Runs only once the activation below waits on the run, since both share the main actor. + let opener = Task { await gate.open() } + await coordinator.syncOnSceneActivation() + await opener.value + await edit.value + #expect(await transport.pullCount == 1) + #expect(coordinator.isHoldingActivations) + + await endInterval() + try await waitUntil { await transport.pullCount == 2 } + } + + @Test("A download that fails starts no interval, so the next activation runs") + func failedDownloadHoldsNothing() async { + let coordinator = makeCoordinator() + coordinator.onConnectionsChanged = { _ in false } + await coordinator.syncOnSceneActivation() + #expect(!coordinator.isHoldingActivations) + + coordinator.onConnectionsChanged = { _ in true } + await coordinator.syncOnSceneActivation() + + #expect(await transport.pullCount == 2) + } + + @Test("Turning sync off drops a held activation") + func turningSyncOffDropsTheHold() async throws { + let coordinator = makeCoordinator() + await coordinator.syncOnSceneActivation() + await coordinator.syncOnSceneActivation() + await clock.waitUntilSleeping() + + coordinator.setEnabled(false) + + #expect(!coordinator.isHoldingActivations) + #expect(clock.pendingSleeperCount == 0) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncCoordinatorTests.swift b/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncCoordinatorTests.swift index 801413e2e2..d4e08f766c 100644 --- a/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncCoordinatorTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncCoordinatorTests.swift @@ -129,6 +129,8 @@ private actor FakeSyncTransport: IOSSyncTransport { return accountId } + func saveSubscription() async throws {} + func ensureZoneExists() async throws { zoneSaves += 1 } @@ -211,7 +213,8 @@ struct IOSSyncCoordinatorTests { makeTransport: { transport }, isEnabled: { box.syncEnabled }, notificationCenter: notificationCenter, - networkMonitor: nil + networkMonitor: nil, + pushRegistrar: .inert ) coordinator.getCurrentState = { (box.connections, box.groups, box.tags) } coordinator.onConnectionsChanged = { box.connections = $0; return box.saves } diff --git a/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncPushTests.swift b/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncPushTests.swift new file mode 100644 index 0000000000..6fb1d2668b --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Sync/IOSSyncPushTests.swift @@ -0,0 +1,158 @@ +import CloudKit +import Foundation +@testable import TableProMobile +import TableProModels +import TableProSync +import TableProSyncTransport +import TableProTestSupport +import Testing + +@MainActor +struct IOSSyncPushTests { + private let metadata: SyncMetadataStorage + private let cacheDirectory: URL + private let clock = ManualClock() + private let transport: RecordingSyncTransport + private let registrar = RecordingPushRegistrar() + + init() throws { + transport = RecordingSyncTransport() + let defaults = try #require(UserDefaults(suiteName: "com.TablePro.tests.IOSSyncPush.\(UUID().uuidString)")) + metadata = SyncMetadataStorage(userDefaults: defaults) + _ = metadata.adoptReadableRecordTypes(IOSSyncCoordinator.readableRecordTypes) + cacheDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("ios-sync-push-\(UUID().uuidString)", isDirectory: true) + } + + private func makeCoordinator(isEnabled: @escaping () -> Bool = { true }) -> IOSSyncCoordinator { + let transport = transport + let coordinator = IOSSyncCoordinator( + metadata: metadata, + recordCache: SyncRecordCache(directory: cacheDirectory, defaults: nil), + makeTransport: { transport }, + isEnabled: isEnabled, + notificationCenter: NotificationCenter(), + networkMonitor: nil, + activationClock: clock, + pushRegistrar: registrar.registrar + ) + coordinator.getCurrentState = { ([], [], []) } + coordinator.onConnectionsChanged = { _ in true } + coordinator.onGroupsChanged = { _ in true } + coordinator.onTagsChanged = { _ in true } + coordinator.getSavedQueries = { SavedQueryContents() } + coordinator.onSavedQueriesChanged = { _ in true } + return coordinator + } + + @Test("A change notice downloads at once, inside the minute that holds activations back") + func noticeDownloadsInsideTheInterval() async { + let coordinator = makeCoordinator() + await coordinator.syncOnSceneActivation() + await coordinator.syncOnSceneActivation() + #expect(coordinator.isHoldingActivations) + #expect(await transport.pullCount == 1) + + await coordinator.sync(.remoteChange) + + #expect(await transport.pullCount == 2) + } + + @Test("Notices before a run's download are answered by it") + func noticesBeforeTheDownloadAddNoRun() async { + let coordinator = makeCoordinator() + let gate = AsyncGate() + await transport.holdAccountChecks { await gate.wait() } + let edit = Task { await coordinator.sync(.localChange) } + await gate.arrived(1) + + // Each notice runs before the opener, since all of them share the main actor. + let notices = (0..<5).map { _ in Task { await coordinator.sync(.remoteChange) } } + let opener = Task { await gate.open() } + for notice in notices { + await notice.value + } + await opener.value + await edit.value + + #expect(await transport.pullCount == 1) + } + + @Test("Notices during a run's download add one more run, never one each") + func noticesDuringTheDownloadAddOneRun() async { + let coordinator = makeCoordinator() + let gate = AsyncGate() + await transport.holdPulls { await gate.wait() } + let edit = Task { await coordinator.sync(.localChange) } + await gate.arrived(1) + + await transport.holdPulls {} + let notices = (0..<5).map { _ in Task { await coordinator.sync(.remoteChange) } } + let opener = Task { await gate.open() } + for notice in notices { + await notice.value + } + await opener.value + await edit.value + + #expect(await transport.pullCount == 2) + } + + @Test("With sync off nothing registers, saves a subscription or downloads") + func syncOffDoesNothing() async { + let coordinator = makeCoordinator(isEnabled: { false }) + coordinator.registerForPushesIfEnabled() + await coordinator.sync(.remoteChange) + + #expect(registrar.registrations == 0) + #expect(await transport.subscriptionSaves == 0) + #expect(await transport.pullCount == 0) + } + + @Test("Turning sync on registers once, and turning it off unregisters") + func registrationFollowsTheSwitch() async { + let switchBox = SwitchBox() + let coordinator = makeCoordinator(isEnabled: { switchBox.isOn }) + coordinator.registerForPushesIfEnabled() + coordinator.registerForPushesIfEnabled() + #expect(registrar.registrations == 1) + + switchBox.isOn = false + coordinator.setEnabled(false) + #expect(registrar.unregistrations == 1) + + switchBox.isOn = true + coordinator.setEnabled(true) + #expect(registrar.registrations == 2) + } + + @Test("The subscription is saved once for the account") + func subscriptionIsSavedOnce() async { + let coordinator = makeCoordinator() + await coordinator.sync(.userRequest) + await coordinator.sync(.userRequest) + + #expect(await transport.subscriptionSaves == 1) + #expect(metadata.subscribedAccountId == "recording-account") + } + + @Test("A save that fails on the network leaves the run alone and is tried on the next run") + func transientSaveFailureRetries() async { + await transport.failSubscriptionSaves(with: CKError(.networkFailure)) + let coordinator = makeCoordinator() + await coordinator.sync(.userRequest) + #expect(coordinator.status == .idle) + #expect(await transport.pullCount == 1) + #expect(metadata.subscribedAccountId == nil) + + await transport.failSubscriptionSaves(with: nil) + await coordinator.sync(.userRequest) + #expect(await transport.subscriptionSaves == 2) + #expect(metadata.subscribedAccountId == "recording-account") + } +} + +@MainActor +private final class SwitchBox { + var isOn = true +} diff --git a/apple/TableProMobile/TableProWidget/Localizable.xcstrings b/apple/TableProMobile/TableProWidget/Localizable.xcstrings new file mode 100644 index 0000000000..2abd97dc73 --- /dev/null +++ b/apple/TableProMobile/TableProWidget/Localizable.xcstrings @@ -0,0 +1,286 @@ +{ + "sourceLanguage" : "en", + "strings" : { + "%lld rows" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "%lld개 행" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "%lld dòng" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "%lld行" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "%lld 列" + } + } + } + }, + "1 row" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "1개 행" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "1 dòng" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "1 行" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "1 列" + } + } + } + }, + "Add a connection in TablePro" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "TablePro에서 연결 추가" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Thêm một kết nối trong TablePro" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "在 TablePro 中添加连接" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "在 TablePro 中新增連線" + } + } + } + }, + "Done" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "완료" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Xong" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "完成" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "完成" + } + } + } + }, + "Failed" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "실패" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Thất bại" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "失败" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "失敗" + } + } + } + }, + "Interrupted" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "중단됨" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Bị gián đoạn" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "已中断" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "已中斷" + } + } + } + }, + "Quick Connect" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "빠른 연결" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Kết nối nhanh" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "快速连接" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "快速連線" + } + } + } + }, + "Quickly connect to your databases." : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "데이터베이스에 빠르게 연결합니다." + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Kết nối nhanh tới các cơ sở dữ liệu của bạn." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "快速连接到你的数据库。" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "快速連線到你的資料庫。" + } + } + } + }, + "Running" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "실행 중" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Đang chạy" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "正在运行" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "執行中" + } + } + } + }, + "Stopped" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "중지됨" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Đã dừng" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "已停止" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "已停止" + } + } + } + } + }, + "version" : "1.0" +} \ No newline at end of file diff --git a/apple/TableProMobile/project.yml b/apple/TableProMobile/project.yml index 9116d12957..d1671a8228 100644 --- a/apple/TableProMobile/project.yml +++ b/apple/TableProMobile/project.yml @@ -67,6 +67,13 @@ targets: - ../Plugins/MySQLDriverPlugin/MySQLErrorText.swift - ../Plugins/MySQLDriverPlugin/MySQLForeignKeyClause.swift - ../Plugins/MySQLDriverPlugin/MySQLServerFlavor.swift + # The read-only transaction statements, refusal codes and messages both apps run a Read-Only statement with. + - ../Plugins/MySQLDriverPlugin/MySQLReadOnlyTransaction.swift + - ../Plugins/PostgreSQLDriverPlugin/LibPQReadOnlyTransaction.swift + # What a MySQL session holds that the read-only transaction would end, its table locks among them. + - ../Plugins/MySQLDriverPlugin/MySQLSessionFootprint.swift + - ../Plugins/MySQLDriverPlugin/MySQLTemporaryTableNames.swift + - ../Plugins/MySQLDriverPlugin/MySQLLexicalFeatures.swift # The socket key a Mac connection stores, so iOS refuses exactly what the Mac driver dials # through a socket. - ../Plugins/MySQLDriverPlugin/MySQLLocalSocket.swift @@ -231,11 +238,11 @@ targets: TableProWidgetExtension: type: app-extension platform: iOS + # The folder holds the widget's own Localizable.xcstrings, so the extension ships only its strings. sources: - path: TableProWidget excludes: - Info.plist - - TableProMobile/Localizable.xcstrings configFiles: Debug: ../Configs/Version-iOS.xcconfig Release: ../Configs/Version-iOS.xcconfig diff --git a/apple/TableProTests/AWS/RedshiftIAMSignInTests.swift b/apple/TableProTests/AWS/RedshiftIAMSignInTests.swift index 7d42e1d7f2..26a8297152 100644 --- a/apple/TableProTests/AWS/RedshiftIAMSignInTests.swift +++ b/apple/TableProTests/AWS/RedshiftIAMSignInTests.swift @@ -24,7 +24,8 @@ struct RedshiftIAMSignInTests { user: String = "", host: String = clusterHost, preTunnelHost: String? = nil, - fields: [String: String] = ["awsAuth": "profile"] + fields: [String: String] = ["awsAuth": "profile"], + aliases: FakeCNAMEResolver = FakeCNAMEResolver() ) -> RedshiftIAMSignIn { RedshiftIAMSignIn( source: "profile", @@ -32,7 +33,8 @@ struct RedshiftIAMSignInTests { database: "dev", fields: fields, configuredHost: host, - preTunnelHost: preTunnelHost + preTunnelHost: preTunnelHost, + aliasResolver: aliases ) } @@ -85,35 +87,106 @@ struct RedshiftIAMSignInTests { } @Test("A tunneled connection signs in for the endpoint typed before the tunnel") - func tunneledConnectionUsesPreTunnelHost() throws { + func tunneledConnectionUsesPreTunnelHost() async throws { let tunneled = Self.signIn(host: "127.0.0.1", preTunnelHost: Self.clusterHost) - #expect(try tunneled.resolveEndpoint() == .cluster(identifier: "analytics", region: "us-east-1")) + #expect(try await tunneled.resolveEndpoint(within: nil) == .cluster(identifier: "analytics", region: "us-east-1")) } @Test("The Redshift Endpoint field names a forward's target, and a host that names nothing says what to enter") - func endpointOverride() throws { + func endpointOverride() async throws { let forwarded = Self.signIn( host: "127.0.0.1", fields: ["awsAuth": "sso", RedshiftIAMSignIn.endpointFieldId: "https://\(Self.serverlessHost):5439/dev"] ) - #expect(try forwarded.resolveEndpoint() == .serverless(workgroup: "reporting", region: "eu-west-1")) + #expect(try await forwarded.resolveEndpoint(within: nil) == .serverless(workgroup: "reporting", region: "eu-west-1")) - let elsewhere = Self.signIn( - host: "warehouse.corp.example", + let otherCluster = Self.signIn( + host: "other.abc123xyz.us-east-1.redshift.amazonaws.com", fields: [RedshiftIAMSignIn.endpointFieldId: Self.serverlessHost] ) - #expect(throws: RedshiftIAMError.endpointNotDialed(host: "warehouse.corp.example")) { - try elsewhere.resolveEndpoint() + await #expect(throws: RedshiftIAMError.endpointNotDialed(host: "other.abc123xyz.us-east-1.redshift.amazonaws.com")) { + try await otherCluster.resolveEndpoint(within: nil) } - #expect(throws: RedshiftIAMError.endpointUnresolved(host: "warehouse.corp.example")) { - try Self.signIn(host: "warehouse.corp.example").resolveEndpoint() + await #expect(throws: RedshiftIAMError.endpointNotDialed(host: "10.0.0.5")) { + try await Self.signIn(host: "10.0.0.5", fields: [RedshiftIAMSignIn.endpointFieldId: Self.clusterHost]) + .resolveEndpoint(within: nil) } - #expect(throws: RedshiftIAMError.endpointInvalid("not-an-endpoint")) { - try Self.signIn(host: "localhost", fields: [RedshiftIAMSignIn.endpointFieldId: "not-an-endpoint"]).resolveEndpoint() + + await #expect(throws: RedshiftIAMError.endpointUnresolved(host: "warehouse.corp.example")) { + try await Self.signIn(host: "warehouse.corp.example").resolveEndpoint(within: nil) + } + await #expect(throws: RedshiftIAMError.endpointInvalid("not-an-endpoint")) { + try await Self.signIn(host: "localhost", fields: [RedshiftIAMSignIn.endpointFieldId: "not-an-endpoint"]) + .resolveEndpoint(within: nil) } } + @Test("A custom domain Host is asked for by name, with the kind and region from the Redshift Endpoint field") + func customDomainWithEndpointField() async throws { + let serverless = Self.signIn( + host: "Warehouse.Corp.Example.", + fields: [RedshiftIAMSignIn.endpointFieldId: Self.serverlessHost] + ) + #expect(try await serverless.resolveEndpoint(within: nil) + == .serverlessDomain(name: "warehouse.corp.example", region: "eu-west-1")) + + let cluster = Self.signIn(host: "warehouse.corp.example", fields: [RedshiftIAMSignIn.endpointFieldId: Self.clusterHost]) + #expect(try await cluster.resolveEndpoint(within: nil) == .clusterDomain(name: "warehouse.corp.example", region: "us-east-1")) + } + + @Test("A custom domain Host finds its kind and region through its CNAME, and is still asked for by name") + func customDomainThroughCNAME() async throws { + let aliases = FakeCNAMEResolver([ + "warehouse.corp.example": "edge.corp.example", + "edge.corp.example": Self.clusterHost + ]) + let direct = Self.signIn(host: "warehouse.corp.example", aliases: aliases) + #expect(try await direct.resolveEndpoint(within: nil) == .clusterDomain(name: "warehouse.corp.example", region: "us-east-1")) + + let tunneled = Self.signIn( + host: "127.0.0.1", + preTunnelHost: "bi.corp.example", + aliases: FakeCNAMEResolver(["bi.corp.example": Self.serverlessHost]) + ) + #expect(try await tunneled.resolveEndpoint(within: nil) == .serverlessDomain(name: "bi.corp.example", region: "eu-west-1")) + } + + @Test("A DNS name that starts like 127.0.0.1 is not a forward, so the endpoint field cannot point it at a real cluster") + func loopbackLookalikeIsACustomDomain() async throws { + let fields = [RedshiftIAMSignIn.endpointFieldId: Self.clusterHost] + #expect(try await Self.signIn(host: "127.0.0.1.attacker.example", fields: fields).resolveEndpoint(within: nil) + == .clusterDomain(name: "127.0.0.1.attacker.example", region: "us-east-1")) + await #expect(throws: RedshiftIAMError.endpointNotDialed(host: "127.0.0.1.")) { + try await Self.signIn(host: "127.0.0.1.", fields: fields).resolveEndpoint(within: nil) + } + } + + @Test("A custom domain asks by CustomDomainName, never by the cluster or workgroup name") + func customDomainRequests() throws { + let cluster = try Self.signIn(user: "reporting").request( + for: .clusterDomain(name: "warehouse.corp.example", region: "us-east-1"), + region: "us-east-1", + credentials: Self.credentials + ) + #expect(Self.formParameters(cluster) == [ + "Action": "GetClusterCredentials", + "Version": "2012-12-01", + "CustomDomainName": "warehouse.corp.example", + "DbUser": "reporting", + "DbName": "dev" + ]) + + let serverless = try Self.signIn().request( + for: .serverlessDomain(name: "bi.corp.example", region: "eu-west-1"), + region: "eu-west-1", + credentials: Self.credentials, + now: Self.signedAt + ) + #expect(serverless.value(forHTTPHeaderField: "x-amz-target") == "RedshiftServerless.GetCredentials") + #expect(String(data: serverless.httpBody ?? Data(), encoding: .utf8) == #"{"customDomainName":"bi.corp.example","dbName":"dev"}"#) + } + @Test("A cluster with a database user asks GetClusterCredentials, without one GetClusterCredentialsWithIAM") func clusterRequests() throws { let endpoint = RedshiftIAMSignIn.Endpoint.cluster(identifier: "analytics", region: "us-east-1") diff --git a/apple/TableProTests/Core/AI/AIConnectionAccessGateTests.swift b/apple/TableProTests/Core/AI/AIConnectionAccessGateTests.swift index 88d10d09a6..0eadf3f2cc 100644 --- a/apple/TableProTests/Core/AI/AIConnectionAccessGateTests.swift +++ b/apple/TableProTests/Core/AI/AIConnectionAccessGateTests.swift @@ -300,7 +300,8 @@ internal struct AIConnectionAccessGateSavedPolicyTests { return ChatToolContext( connectionId: connectionId, bridge: MCPConnectionBridge(), - authPolicy: MCPAuthPolicy() + authPolicy: MCPAuthPolicy(), + chatMode: .agent ) { id in await MainActor.run { AIConnectionAccessGate.policy( diff --git a/apple/TableProTests/Core/AI/AIModelCatalogTests.swift b/apple/TableProTests/Core/AI/AIModelCatalogTests.swift index 4efc395aaf..67fcdce1eb 100644 --- a/apple/TableProTests/Core/AI/AIModelCatalogTests.swift +++ b/apple/TableProTests/Core/AI/AIModelCatalogTests.swift @@ -113,6 +113,37 @@ struct AIModelCatalogTests { #expect(catalog.fetchedInfo(providerID: provider, modelID: "m") == nil) } + /// The chat panel reads model lists while a Save in Settings refreshes them. Taking a refresh + /// token there cancelled the Save's refresh, and a failed panel fetch then left no list at all. + @Test("A chat panel answer lands when nothing changed since it started, and never cancels a refresh") + func panelAnswerLandsBesideARefresh() { + let catalog = AIModelCatalog() + let provider = UUID() + let token = catalog.beginRefresh(providerID: provider) + let revision = catalog.revision(providerID: provider) + catalog.store(providerID: provider, models: [AIModelInfo(id: "panel")], ifUnchangedSince: revision) + #expect(catalog.fetchedInfo(providerID: provider, modelID: "panel") != nil) + catalog.finishRefresh(providerID: provider, token: token, models: [AIModelInfo(id: "saved")]) + #expect(catalog.fetchedInfo(providerID: provider, modelID: "saved") != nil) + } + + @Test("A chat panel answer is dropped once a Save stored, removed or began a refresh after it started") + func panelAnswerIsDroppedAfterASave() { + let catalog = AIModelCatalog() + let provider = UUID() + let writes: [(AIModelCatalog) -> Void] = [ + { $0.store(providerID: provider, models: [AIModelInfo(id: "saved")]) }, + { $0.remove(providerID: provider) }, + { _ = $0.beginRefresh(providerID: provider) } + ] + for write in writes { + let revision = catalog.revision(providerID: provider) + write(catalog) + catalog.store(providerID: provider, models: [AIModelInfo(id: "late")], ifUnchangedSince: revision) + #expect(catalog.fetchedInfo(providerID: provider, modelID: "late") == nil) + } + } + @Test("A refresh fetches through the transport it is given") func refreshFetchesThroughTheTransport() async { let catalog = AIModelCatalog() diff --git a/apple/TableProTests/Core/AI/AIProviderCapabilitiesTests.swift b/apple/TableProTests/Core/AI/AIProviderCapabilitiesTests.swift index 9c28638cef..e0e800a123 100644 --- a/apple/TableProTests/Core/AI/AIProviderCapabilitiesTests.swift +++ b/apple/TableProTests/Core/AI/AIProviderCapabilitiesTests.swift @@ -115,4 +115,90 @@ struct AIProviderCapabilitiesTests { #expect(descriptor(type)?.showsTelemetryToggle == false, "\(type.rawValue) must not show telemetry") } } + + /// The effort is saved with the provider's own model, and the chat panel can send another. + /// GPT-5.6 offers Minimal, and picking GPT-5.5 in the panel sent `minimal` to a model that + /// takes low to extra high. + @Test("The effort sent is fitted to the model the chat panel picked") + func effortIsFittedToTheSentModel() { + let chatGPT = AIProviderConfig(type: .chatgptCodex, model: "gpt-5.6-sol", reasoningEffort: .minimal) + let catalog = AIModelCatalog() + #expect(AIProviderFactory.reasoningEffort(.minimal, config: chatGPT, model: "gpt-5.6-sol", catalog: catalog) == .minimal) + #expect(AIProviderFactory.reasoningEffort(.minimal, config: chatGPT, model: "gpt-5.5", catalog: catalog) == .low) + } + + @Test("No effort goes to a model its server lists as not reasoning, or to a provider without reasoning") + func noEffortWhereNoneIsTaken() { + let catalog = AIModelCatalog() + let router = AIProviderConfig(preset: .requesty) + var configured = router + configured.reasoningEffort = .high + catalog.store(providerID: router.id, models: [AIModelInfo(id: "alibaba/qwen-max", reasoning: .unsupported)]) + #expect(AIProviderFactory.reasoningEffort(.high, config: configured, model: "alibaba/qwen-max", catalog: catalog) == nil) + + let copilot = AIProviderConfig(type: .copilot, model: "gpt-5.5", reasoningEffort: .high) + #expect(AIProviderFactory.reasoningEffort(.high, config: copilot, model: "gpt-5.5", catalog: catalog) == nil) + } + + @Test("A Meta provider sends its effort to any Muse chat model the panel picks") + func metaEffortReachesEveryMuseModel() { + var meta = AIProviderConfig(preset: .meta) + meta.model = "muse-spark-1.3" + meta.reasoningEffort = .xhigh + let catalog = AIModelCatalog() + for model in ["muse-spark-1.3", "muse-spark-1.2", "muse-wonder-1.0", "muse-ember-1.0"] { + #expect(AIProviderFactory.reasoningEffort(.xhigh, config: meta, model: model, catalog: catalog) == .xhigh, "\(model)") + } + } + + /// Meta runs its most expensive level when no effort is sent, and inline suggestions and a + /// provider synced from an older build send none. + @Test("A Muse model sent no effort gets its default, and other models stay without one") + func mandatoryReasoningGetsItsDefault() { + let meta = AIProviderConfig(preset: .meta) + let catalog = AIModelCatalog() + #expect(AIProviderFactory.reasoningEffort(nil, config: meta, model: "muse-wonder-1.0", catalog: catalog) == .high) + let openAI = AIProviderConfig(type: .openAI, model: "gpt-5.5") + #expect(AIProviderFactory.reasoningEffort(nil, config: openAI, model: "gpt-5.5", catalog: catalog) == nil) + } + + @Test("Effort levels for every provider without a preset are what they were") + func presetFreeLevelsAreUnchanged() throws { + #expect(try #require(descriptor(.openRouter)).supportedEffortLevels(forModelID: "m") == [.low, .medium, .high]) + #expect(try #require(descriptor(.custom)).supportedEffortLevels(forModelID: "m") == [.low, .medium, .high]) + #expect(try #require(descriptor(.openAI)).supportedEffortLevels(forModelID: "gpt-5.5") == [.low, .medium, .high, .xhigh]) + #expect(try #require(descriptor(.copilot)).supportedEffortLevels(forModelID: "gpt-5.5").isEmpty) + } + + /// Inline suggestions send no effort where a model allows it. A Muse model has no such choice, + /// and sending none there ran Meta's most expensive level on every keystroke pause. + @Test("Inline suggestions send the saved effort only to a model that always reasons") + func inlineEffortOnlyWhereRequired() { + var meta = AIProviderConfig(preset: .meta) + meta.reasoningEffort = .minimal + let openAI = AIProviderConfig(type: .openAI, model: "gpt-5.5", reasoningEffort: .high) + let catalog = AIModelCatalog() + #expect(AIProviderFactory.reasoningEffort( + .minimal, config: meta, model: "muse-spark-1.3", onlyWhereRequired: true, catalog: catalog + ) == .minimal) + #expect(AIProviderFactory.reasoningEffort( + nil, config: meta, model: "muse-ember-1.0", onlyWhereRequired: true, catalog: catalog + ) == .high) + #expect(AIProviderFactory.reasoningEffort( + .high, config: openAI, model: "gpt-5.5", onlyWhereRequired: true, catalog: catalog + ) == nil) + } + + /// OpenAI rejects a reasoning effort for its non-reasoning models, which the chat panel lists + /// beside the reasoning ones. + @Test("No effort goes to an OpenAI model that does not reason") + func noEffortForOpenAINonReasoningModels() throws { + let openAI = AIProviderConfig(type: .openAI, model: "gpt-5.5", reasoningEffort: .high) + let catalog = AIModelCatalog() + for model in ["gpt-4.1", "gpt-4o-mini", "chatgpt-4o-latest", "gpt-5-chat-latest", "gpt-3.5-turbo"] { + #expect(AIProviderFactory.reasoningEffort(.high, config: openAI, model: model, catalog: catalog) == nil, "\(model)") + } + #expect(AIProviderFactory.reasoningEffort(.high, config: openAI, model: "gpt-5.5", catalog: catalog) == .high) + #expect(try #require(descriptor(.openAI)).supportedEffortLevels(forModelID: "gpt-4.1").isEmpty) + } } diff --git a/apple/TableProTests/Core/AI/AIProviderDraftRulesTests.swift b/apple/TableProTests/Core/AI/AIProviderDraftRulesTests.swift index b3845fbfe1..65f13bf00d 100644 --- a/apple/TableProTests/Core/AI/AIProviderDraftRulesTests.swift +++ b/apple/TableProTests/Core/AI/AIProviderDraftRulesTests.swift @@ -136,4 +136,56 @@ struct AIProviderDraftRulesTests { == .keep ) } + + private let optional = AIReasoningSupport(mode: .effortOnly, effortLevels: [.low, .medium, .high]) + private let mandatory = AIReasoningSupport( + mode: .effortOnly, effortLevels: [.minimal, .low, .medium, .high, .xhigh], defaultEffort: .high, isMandatory: true + ) + private let curated = CuratedModel( + id: "gpt-5.5", displayName: "GPT-5.5", supportedEffortLevels: [.low, .medium, .high], defaultEffort: .medium + ) + + @Test("An effort the next model takes is kept") + func effortTheModelTakesIsKept() { + #expect(AIProviderDraftRules.effort(afterPicking: .high, reasoning: optional, curated: nil) == .high) + #expect(AIProviderDraftRules.effort(afterPicking: .low, reasoning: mandatory, curated: nil) == .low) + } + + @Test("A curated model starts on its own default, and replaces an effort it does not take") + func curatedModelStartsOnItsDefault() { + #expect(AIProviderDraftRules.effort(afterPicking: nil, reasoning: optional, curated: curated) == .medium) + #expect(AIProviderDraftRules.effort(afterPicking: .xhigh, reasoning: optional, curated: curated) == .medium) + } + + @Test("A listed or typed model that reasons optionally keeps Off, and drops an effort it does not take") + func optionalReasoningKeepsOff() { + #expect(AIProviderDraftRules.effort(afterPicking: nil, reasoning: optional, curated: nil) == nil) + #expect(AIProviderDraftRules.effort(afterPicking: .xhigh, reasoning: optional, curated: nil) == nil) + } + + /// Meta runs its most expensive level when the effort is left out, so Off there was the + /// costliest choice in the picker. + @Test("A model that always reasons never ends up with no effort") + func mandatoryReasoningAlwaysHasAnEffort() { + #expect(AIProviderDraftRules.effort(afterPicking: nil, reasoning: mandatory, curated: nil) == .high) + let narrow = AIReasoningSupport( + mode: .effortOnly, effortLevels: [.low, .medium], defaultEffort: .low, isMandatory: true + ) + #expect(AIProviderDraftRules.effort(afterPicking: .xhigh, reasoning: narrow, curated: nil) == .medium) + } + + /// Typing a model id under Other… used to keep the effort as it stood. A rule that cleared it + /// on every keystroke turned Extra High into Off before the id was complete. + @Test("Typing, preselecting or reopening keeps the effort, except that a model that always reasons gets one") + func keptEffortChangesOnlyForMandatoryReasoning() { + #expect(AIProviderDraftRules.effort(keeping: .xhigh, reasoning: optional) == .xhigh) + #expect(AIProviderDraftRules.effort(keeping: nil, reasoning: optional) == nil) + #expect(AIProviderDraftRules.effort(keeping: nil, reasoning: mandatory) == .high) + #expect(AIProviderDraftRules.effort(keeping: .minimal, reasoning: mandatory) == .minimal) + } + + @Test("A model that does not reason has no effort") + func noReasoningNoEffort() { + #expect(AIProviderDraftRules.effort(afterPicking: .high, reasoning: .unsupported, curated: curated) == nil) + } } diff --git a/apple/TableProTests/Core/AI/AIProviderFactoryResolveTests.swift b/apple/TableProTests/Core/AI/AIProviderFactoryResolveTests.swift index 38621ed64f..b56163cb23 100644 --- a/apple/TableProTests/Core/AI/AIProviderFactoryResolveTests.swift +++ b/apple/TableProTests/Core/AI/AIProviderFactoryResolveTests.swift @@ -147,4 +147,33 @@ struct AIProviderFactoryResolveTests { let resolved = AIProviderFactory.resolve(settings: settings) #expect(resolved?.model == "") } + + /// The chat panel keeps its pick after the picked provider is removed in Settings, and the + /// next turn went to the active provider under the removed provider's model id. + @Test("A model picked for a removed provider is not sent to the active one") + func removedOverrideProviderDropsItsModel() { + let active = makeProvider(model: "claude-sonnet-5") + let settings = AISettings(enabled: true, providers: [active], activeProviderID: active.id) + let resolved = AIProviderFactory.resolve( + settings: settings, + overrideProviderId: UUID(), + overrideModel: "muse-spark-1.3" + ) + #expect(resolved?.config.id == active.id) + #expect(resolved?.model == "claude-sonnet-5") + } + + @Test("A model picked for a provider that still exists is sent to it") + func overrideModelGoesToItsProvider() { + let active = makeProvider(model: "claude-sonnet-5") + let other = makeProvider(type: .custom, model: "muse-spark-1.3") + let settings = AISettings(enabled: true, providers: [active, other], activeProviderID: active.id) + let resolved = AIProviderFactory.resolve( + settings: settings, + overrideProviderId: other.id, + overrideModel: "muse-wonder-1.0" + ) + #expect(resolved?.config.id == other.id) + #expect(resolved?.model == "muse-wonder-1.0") + } } diff --git a/apple/TableProTests/Core/AI/AIProviderModelFetchTests.swift b/apple/TableProTests/Core/AI/AIProviderModelFetchTests.swift index 97dfb1587c..212c7d235c 100644 --- a/apple/TableProTests/Core/AI/AIProviderModelFetchTests.swift +++ b/apple/TableProTests/Core/AI/AIProviderModelFetchTests.swift @@ -195,4 +195,112 @@ struct AIProviderModelFetchTests { == "https://generativelanguage.googleapis.com/v1beta/models" ) } + + /// OpenAI's list holds embedding, speech, image and realtime models beside the chat ones, and + /// picking one in the chat panel failed the turn. + @Test("OpenAI's own model list keeps only the models Responses can chat with") + func openAIListKeepsChatModels() async throws { + StubModelListProtocol.respond(status: 200, body: #""" + {"object":"list","data":[ + {"id":"gpt-5.5","object":"model"},{"id":"text-embedding-3-small","object":"model"}, + {"id":"whisper-1","object":"model"},{"id":"gpt-image-1","object":"model"}, + {"id":"o3","object":"model"},{"id":"gpt-4o-realtime-preview","object":"model"}, + {"id":"tts-1","object":"model"},{"id":"omni-moderation-latest","object":"model"}, + {"id":"ft:gpt-4.1:acme::abc123","object":"model"} + ]} + """#) + let provider = OpenAIResponsesProvider(endpoint: "https://api.openai.com", apiKey: "key", session: stubSession()) + let models = try await provider.fetchAvailableModels() + #expect(models.map(\.id) == ["ft:gpt-4.1:acme::abc123", "gpt-5.5", "o3"]) + } + + @Test("A gateway in OpenAI's place keeps every model it lists") + func gatewayListIsNotFiltered() async throws { + StubModelListProtocol.respond(status: 200, body: #"{"data":[{"id":"text-embedding-3-small"},{"id":"gpt-5.5"}]}"#) + let provider = OpenAIResponsesProvider(endpoint: "https://gateway.internal/openai", apiKey: "key", session: stubSession()) + let models = try await provider.fetchAvailableModels() + #expect(models.map(\.id) == ["gpt-5.5", "text-embedding-3-small"]) + } + + @Test("xAI lists its chat models from language-models, with the images each one reads") + func xaiListsLanguageModels() async throws { + StubModelListProtocol.respond(status: 200, body: #""" + {"models":[ + {"id":"grok-4.5","input_modalities":["text","image"],"output_modalities":["text"]}, + {"id":"grok-4.20-0309-non-reasoning","input_modalities":["text"],"output_modalities":["text"]}, + {"id":"grok-imagine-image","input_modalities":["text","image"],"output_modalities":["image"]} + ]} + """#) + let provider = OpenAIResponsesProvider( + endpoint: "https://api.x.ai", apiKey: "key", dialect: .xai, session: stubSession() + ) + let models = try await provider.fetchAvailableModels() + #expect(StubModelListProtocol.lastRequestedURL() == "https://api.x.ai/v1/language-models") + #expect(models.map(\.id) == ["grok-4.20-0309-non-reasoning", "grok-4.5"]) + #expect(models.first?.supportsImages == false) + #expect(models.last?.supportsImages == true) + } + + @Test("Without language-models, an xAI list drops image, video and voice models by what it says") + func xaiFallbackDropsNonChatModels() { + let keep: [[String: Any]] = [["id": "grok-4.5"], ["id": "grok-4.3"]] + let drop: [[String: Any]] = [ + ["id": "grok-studio-1", "image_price": 200], + ["id": "grok-imagine-video-1.5"], + ["id": "grok-voice-3.0"] + ] + for entry in keep { + #expect(OpenAIResponsesProvider.listsChatModel(entry, dialect: .xai, host: "api.x.ai")) + } + for entry in drop { + #expect(!OpenAIResponsesProvider.listsChatModel(entry, dialect: .xai, host: "api.x.ai")) + } + } + + @Test("OpenAI's regional hosts are filtered like its main one; a fine-tune is judged by its base model") + func openAIRegionalHostAndFineTunes() { + let whisper: [String: Any] = ["id": "whisper-1"] + #expect(!OpenAIResponsesProvider.listsChatModel(whisper, dialect: .openAI, host: "eu.api.openai.com")) + #expect(OpenAIResponsesProvider.listsChatModel(whisper, dialect: .openAI, host: "gateway.internal")) + #expect(OpenAIResponsesProvider.isOpenAIChatModel("ft:gpt-4o-mini-2024-07-18:watts-energy::AbC123")) + #expect(OpenAIResponsesProvider.isOpenAIChatModel("ft:gpt-4.1:acme:doc-search:x1")) + #expect(!OpenAIResponsesProvider.isOpenAIChatModel("ft:whisper-1:acme::x2")) + } + + @Test("Without a usable language-models answer, xAI's list comes from its model list minus image models") + func xaiFallsBackToTheModelList() async throws { + RoutedModelListProtocol.routes = [ + "/v1/language-models": (200, #"{"models":[{"id":"grok-only-listed-here","input_modalities":["text"]}]}"#), + "/v1/models": (200, #"{"data":[{"id":"grok-4.5"},{"id":"grok-2-image-1212"},{"id":"grok-imagine-image","image_price":200}]}"#) + ] + let config = URLSessionConfiguration.ephemeral + config.protocolClasses = [RoutedModelListProtocol.self] + let provider = OpenAIResponsesProvider( + endpoint: "https://api.x.ai", apiKey: "key", dialect: .xai, session: URLSession(configuration: config) + ) + #expect(try await provider.fetchAvailableModels().map(\.id) == ["grok-4.5"]) + + RoutedModelListProtocol.routes["/v1/language-models"] = (404, "") + #expect(try await provider.fetchAvailableModels().map(\.id) == ["grok-4.5"]) + } +} + +private final class RoutedModelListProtocol: URLProtocol, @unchecked Sendable { + nonisolated(unsafe) static var routes: [String: (status: Int, body: String)] = [:] + + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func stopLoading() {} + + override func startLoading() { + guard let url = request.url else { return } + let route = Self.routes[url.path] ?? (404, "") + guard let response = HTTPURLResponse( + url: url, statusCode: route.status, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + ) else { return } + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: Data(route.body.utf8)) + client?.urlProtocolDidFinishLoading(self) + } } diff --git a/apple/TableProTests/Core/AI/AIProviderPresetTests.swift b/apple/TableProTests/Core/AI/AIProviderPresetTests.swift index a13b06ec59..1ba8d7a150 100644 --- a/apple/TableProTests/Core/AI/AIProviderPresetTests.swift +++ b/apple/TableProTests/Core/AI/AIProviderPresetTests.swift @@ -52,7 +52,7 @@ struct AIProviderPresetTests { #expect(config.name == "Requesty") #expect(config.endpoint == "https://router.requesty.ai") #expect(config.kindName == "Requesty") - #expect(config.symbolName == AIProviderPreset.requesty.symbolName) + #expect(config.icon == AIProviderPreset.requesty.icon) #expect(config.authStyle == .apiKey) } @@ -68,7 +68,7 @@ struct AIProviderPresetTests { let config = AIProviderConfig(type: .custom, endpoint: "https://api.z.ai/api/paas/v4") #expect(config.preset == nil) #expect(config.kindName == AIProviderType.custom.displayName) - #expect(config.symbolName == AIProviderType.custom.symbolName) + #expect(config.icon == AIProviderType.custom.icon) #expect(config.authStyle == .optionalApiKey) #expect(config.defaultEndpoint.isEmpty) } @@ -162,4 +162,116 @@ struct AIProviderPresetTests { ) == nil ) } + + @Test("Meta needs a key, reads a 401 as a bad key, and resolves to its chat completions route") + func metaPreset() { + let preset = AIProviderPreset.meta + #expect(preset.authStyle == .apiKey) + #expect(!preset.rejectsBadKeyWithForbidden) + + let config = AIProviderConfig(preset: preset) + let style = config.type.endpointStyle + #expect(config.kindName == "Meta") + #expect( + AIEndpoint(config.endpoint, style: style)?.chatURL(model: "muse-spark-1.3", style: style)?.absoluteString + == "https://api.meta.ai/v1/chat/completions" + ) + #expect( + AIEndpoint(config.endpoint, style: style)?.url(appending: style.modelsResource)?.absoluteString + == "https://api.meta.ai/v1/models" + ) + } + + @Test("A new Meta provider starts on Muse Spark 1.3 before its model list loads") + func metaStartsOnMuseSpark() throws { + let descriptor = try #require(AIProviderRegistry.shared.descriptor(for: AIProviderType.custom.rawValue)) + let curated = descriptor.curatedModels(for: AIProviderPreset.meta) + #expect(curated.map(\.id) == ["muse-spark-1.3", "muse-wonder-1.0", "muse-ember-1.0"]) + #expect(AIProviderDraftRules.initialModel(curated: curated, fetched: []) == "muse-spark-1.3") + #expect(descriptor.curatedModels(for: AIProviderPreset.requesty).isEmpty) + #expect(descriptor.curatedModels(for: nil).isEmpty) + } + + /// Meta may train on what a contributor model is sent, so a list click never picks one. + @Test("Meta lists only its chat models, without the contributor ones; a preset with no filter lists every model") + func metaListsChatModels() { + let listed = ["muse-image-1.0", "muse-spark-1.3", "muse-spark-1.2-contributor", "muse-voice-transcribe-1.0", + "sam-3.1", "muse-ember-1.0", "muse-wonder-1.0"] + #expect(listed.filter { AIProviderPreset.meta.modelIDs.lists($0) } + == ["muse-spark-1.3", "muse-ember-1.0", "muse-wonder-1.0"]) + #expect(AIProviderPreset.meta.modelIDs.isChatFamily("muse-spark-1.2-contributor")) + #expect(listed.allSatisfy { AIProviderPreset.requesty.modelIDs.lists($0) }) + } + + @Test("A contributor model stays out of Meta's list wherever its id carries the word") + func contributorModelsStayUnlisted() { + for id in ["muse-spark-1.4-contributor-preview", "muse-spark-1.3-contributor"] { + #expect(!AIProviderPreset.meta.modelIDs.lists(id), "\(id)") + } + } + + @Test("Meta's model filter applies on Meta's host only; a gateway keeps its whole list") + func metaFilterFollowsTheHost() { + #expect(AIProviderPreset.meta.modelIDFilter(forEndpoint: "https://api.meta.ai") == AIProviderPreset.meta.modelIDs) + #expect(AIProviderPreset.meta.modelIDFilter(forEndpoint: " https://API.meta.ai/v1 ") == AIProviderPreset.meta.modelIDs) + #expect(AIProviderPreset.meta.modelIDFilter(forEndpoint: "https://gateway.example/meta") == .all) + } + + @Test("Every Meta chat model always reasons, from minimal to extra high, starting at high") + func metaReasoningIsMandatory() throws { + let descriptor = try #require(AIProviderRegistry.shared.descriptor(for: AIProviderType.custom.rawValue)) + for model in ["muse-spark-1.3", "muse-spark-1.1", "muse-wonder-1.0", "muse-ember-1.0"] { + let reasoning = descriptor.reasoning(forModelID: model, fetched: AIModelInfo(id: model), preset: .meta) + #expect(reasoning.isMandatory, "\(model)") + #expect(reasoning.effortLevels == [.minimal, .low, .medium, .high, .xhigh], "\(model)") + #expect(reasoning.defaultEffort == .high, "\(model)") + } + let typed = descriptor.reasoning(forModelID: "llama-4", preset: .meta) + #expect(!typed.isMandatory) + #expect(typed.effortLevels == [.low, .medium, .high]) + #expect(!descriptor.reasoning(forModelID: "muse-spark-1.3").isMandatory) + } + + @Test("A Meta provider asks the server only for chat models") + func metaModelListIsFiltered() async throws { + MetaModelListProtocol.body = #""" + {"object":"list","data":[ + {"id":"muse-image-1.0","object":"model","created":1788000003,"owned_by":"meta"}, + {"id":"muse-spark-1.3","object":"model","created":1788000002,"owned_by":"meta"}, + {"id":"sam-3.1","object":"model","created":1788000001,"owned_by":"meta"}, + {"id":"muse-voice-transcribe-1.0","object":"model","created":1788000000,"owned_by":"meta"}, + {"id":"muse-spark-1.3-contributor","object":"model","created":1787000001,"owned_by":"meta"}, + {"id":"muse-ember-1.0","object":"model","created":1787000000,"owned_by":"meta"} + ]} + """# + let sessionConfig = URLSessionConfiguration.ephemeral + sessionConfig.protocolClasses = [MetaModelListProtocol.self] + let provider = OpenAICompatibleProvider( + config: AIProviderConfig(preset: .meta), + apiKey: "LLM_1_key", + session: URLSession(configuration: sessionConfig) + ) + let models = try await provider.fetchAvailableModels() + #expect(models.map(\.id) == ["muse-ember-1.0", "muse-spark-1.3"]) + } +} + +private final class MetaModelListProtocol: URLProtocol, @unchecked Sendable { + nonisolated(unsafe) static var body = "" + + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + override func stopLoading() {} + + override func startLoading() { + guard let url = request.url, + let response = HTTPURLResponse( + url: url, statusCode: 200, httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + ) + else { return } + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: Data(Self.body.utf8)) + client?.urlProtocolDidFinishLoading(self) + } } diff --git a/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift b/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift index a2480ec0d5..8ae559471d 100644 --- a/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift +++ b/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift @@ -1,14 +1,38 @@ import Foundation @testable import TablePro +import TableProTestSupport import Testing struct AgentCLIProcessTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + private final class Lines: @unchecked Sendable { + private let lock = NSLock() + private var received: [String] = [] + + func append(_ line: String) { + lock.withLock { received.append(line) } + } + + var all: [String] { + lock.withLock { received } + } + } + private let shellLaunch = AgentCLILaunch( executable: URL(fileURLWithPath: "/bin/sh"), arguments: ["-c", "exit 0"], environment: [:] ) + private static func perlLaunch(_ script: String) -> AgentCLILaunch { + AgentCLILaunch(executable: URL(fileURLWithPath: perl), arguments: ["-e", script], environment: [:]) + } + @Test("A launch the launcher refuses fails as a launch failure instead of running") func refusedLaunchFailsTheRun() async { await #expect(throws: AgentCLIError.self) { @@ -22,4 +46,123 @@ struct AgentCLIProcessTests { for try await _ in AgentCLIProcess(label: "claude").streamLines(shellLaunch) {} } } + + @Test( + "Every stdout line arrives while the child holds stderr open and never writes to it", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func manyLinesArriveBesideSilentStderr() async throws { + let stream = AgentCLIProcess(label: "test").streamLines( + Self.perlLaunch(#"$| = 1; print "line $_\n" for 1 .. 30000;"#) + ) + var lines: [String] = [] + for try await line in stream { + lines.append(line) + } + #expect(lines == (1 ... 30_000).map { "line \($0)" }) + } + + @Test( + "A line arrives while the child keeps running, and ending the stream stops the child", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func lineArrivesBeforeTheChildExits() async throws { + let received = Lines() + let stream = AgentCLIProcess(label: "test").streamLines( + Self.perlLaunch(#"$| = 1; print "$$\n"; select(undef, undef, undef, 0.3); print "running\n"; sleep 30;"#) + ) + let consumer = Task { + for try await line in stream { + received.append(line) + } + } + defer { consumer.cancel() } + + try await waitUntil { received.all.count == 2 } + #expect(received.all.last == "running") + let child = try #require(received.all.first.flatMap { pid_t($0) }) + + consumer.cancel() + _ = await consumer.result + try await waitUntil { kill(child, 0) != 0 } + } + + @Test( + "A last line without a newline still arrives", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func unterminatedLastLineArrives() async throws { + var lines: [String] = [] + for try await line in AgentCLIProcess(label: "test").streamLines(Self.perlLaunch(#"print "first\nlast";"#)) { + lines.append(line) + } + #expect(lines == ["first", "last"]) + } + + @Test( + "A run returns everything a child writes past the pipe's capacity", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func runReturnsOutputPastPipeCapacity() async throws { + let result = try await AgentCLIProcess(label: "test").run( + Self.perlLaunch(#"$| = 1; print "x" x 200000; print STDERR "done"; exit 3;"#) + ) + #expect(result.code == 3) + #expect(result.output == String(repeating: "x", count: 200_000) + "done") + } + + @Test( + "Cancelling a run stops the child, even when the cancel lands before launch", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func cancelledRunStopsTheChild() async { + let cancelled = AsyncGate() + let run = Task { + await cancelled.wait() + return try await AgentCLIProcess(label: "test").run(Self.perlLaunch("sleep 30")) + } + run.cancel() + await cancelled.open() + + let ended = await BoundedCall.result { () async -> Bool in + _ = try? await run.value + return true + } + #expect(ended == true) + } + + @Test("Lines end at LF alone, lose a trailing CR, skip empty lines and keep a character split across chunks") + func splitterJoinsChunks() { + var splitter = AgentCLIProcess.LineSplitter() + let accented = Array("\u{E9}\n".utf8) + let chunks = [ + Data("a\r\nb".utf8), + Data("c\n\n\r\n".utf8), + Data(accented.prefix(1)), + Data(accented.dropFirst()), + Data("x\u{2028}y\r\n".utf8) + ] + + let lines = chunks.map { splitter.append($0) } + + #expect(lines == [["a"], ["bc"], [], ["\u{E9}"], ["x\u{2028}y"]]) + } + + @Test("A line that is not UTF-8 is dropped, and a tail without a newline comes out at the end") + func splitterDropsInvalidLinesAndFlushesTheTail() { + var splitter = AgentCLIProcess.LineSplitter() + + let lines = splitter.append(Data([0x6F, 0xFF, 0x6B, 0x0A]) + Data("ok\n".utf8) + Data("tail".utf8)) + let tail = splitter.finish() + let afterTail = splitter.finish() + + #expect(lines == ["ok"]) + #expect(tail == ["tail"]) + #expect(afterTail.isEmpty) + } } diff --git a/apple/TableProTests/Core/AI/AssembleToolUseBlocksTests.swift b/apple/TableProTests/Core/AI/AssembleToolUseBlocksTests.swift index 31b68b6a9e..e2008f4af7 100644 --- a/apple/TableProTests/Core/AI/AssembleToolUseBlocksTests.swift +++ b/apple/TableProTests/Core/AI/AssembleToolUseBlocksTests.swift @@ -62,7 +62,7 @@ struct AssembleToolUseBlocksTests { #expect(blocks[0].id == "call_2") } - @Test("Malformed JSON falls back to empty object") + @Test("Malformed JSON is denied with an explanation instead of running with empty arguments") func malformedJSON() { let blocks = AIChatViewModel.assembleToolUseBlocks( order: ["call_1"], @@ -71,5 +71,42 @@ struct AssembleToolUseBlocksTests { ) #expect(blocks.count == 1) #expect(blocks[0].input == .object([:])) + #expect(blocks[0].approvalState == .denied(reason: AIChatViewModel.unparsableToolArgumentsContent)) + } + + @Test("Arguments cut off at the output limit are denied, and the calls around them are not") + func truncatedArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "execute_query", "call_2": "list_tables"], + inputs: ["call_1": #"{"query":"SELECT * FROM orders WHERE"#, "call_2": #"{}"#] + ) + #expect(blocks.count == 2) + #expect(blocks[0].approvalState == .denied(reason: AIChatViewModel.unparsableToolArgumentsContent)) + #expect(blocks[1].approvalState == .approved) + #expect(blocks[1].input == .object([:])) + } + + @Test("Arguments that parse to something other than an object are denied", arguments: [#""users""#, "42", "[1]", "true"]) + func nonObjectArguments(raw: String) { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "describe_table"], + inputs: ["call_1": raw] + ) + #expect(blocks.count == 1) + #expect(blocks[0].approvalState == .denied(reason: AIChatViewModel.unparsableToolArgumentsContent)) + } + + @Test("A null or blank argument string reads as a call with no arguments", arguments: ["null", " "]) + func nullArgumentsAreEmpty(raw: String) { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "list_tables"], + inputs: ["call_1": raw] + ) + #expect(blocks.count == 1) + #expect(blocks[0].input == .object([:])) + #expect(blocks[0].approvalState == .approved) } } diff --git a/apple/TableProTests/Core/AI/Chat/ChatToolTargetTests.swift b/apple/TableProTests/Core/AI/Chat/ChatToolTargetTests.swift index 46128d57cf..c647f63ddb 100644 --- a/apple/TableProTests/Core/AI/Chat/ChatToolTargetTests.swift +++ b/apple/TableProTests/Core/AI/Chat/ChatToolTargetTests.swift @@ -21,6 +21,7 @@ struct ChatToolTargetTests { connectionId: connectionId, bridge: MCPConnectionBridge(), authPolicy: MCPAuthPolicy(), + chatMode: .agent, approvalWasExplicit: approvalWasExplicit ) } diff --git a/apple/TableProTests/Core/AI/Chat/ListConnectionsChatToolTests.swift b/apple/TableProTests/Core/AI/Chat/ListConnectionsChatToolTests.swift new file mode 100644 index 0000000000..4b133fbe4a --- /dev/null +++ b/apple/TableProTests/Core/AI/Chat/ListConnectionsChatToolTests.swift @@ -0,0 +1,98 @@ +// +// ListConnectionsChatToolTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProPluginKit +import Testing + +struct ListConnectionsChatToolTests { + private let session = UUID() + private let other = UUID() + private let third = UUID() + + private func context( + connectionId: UUID?, + policy: AIConnectionPolicy = .askEachTime + ) -> ChatToolContext { + ChatToolContext( + connectionId: connectionId, + bridge: MCPConnectionBridge(), + authPolicy: MCPAuthPolicy(), + chatMode: .agent, + policyLookup: { _ in policy } + ) + } + + private func connection(id: UUID, name: String, host: String) -> DatabaseConnection { + var connection = TestFixtures.makeConnection(id: id, name: name, database: "\(name)_db", type: .postgresql) + connection.host = host + return connection + } + + @Test("A conversation with no connection attached lists nothing") + func noSessionConnectionIsRefused() async { + await #expect(throws: ChatToolTargetError.noConnection) { + _ = try await ListConnectionsChatTool().execute(input: .object([:]), context: context(connectionId: nil)) + } + } + + @Test("A connection whose AI policy is Never is refused") + func neverPolicyIsRefused() async { + await #expect(throws: ChatToolAuthorizationError.self) { + _ = try await ListConnectionsChatTool().execute( + input: .object([:]), + context: context(connectionId: session, policy: .never) + ) + } + } + + @Test("The tool lists no connection other than the conversation's own") + func toolListsOnlyTheSessionConnection() async throws { + let result = try await ListConnectionsChatTool().execute( + input: .object([:]), + context: context(connectionId: session) + ) + let data = try #require(result.content.data(using: .utf8)) + let payload = try JSONDecoder().decode(JsonValue.self, from: data) + let listed = try #require(payload["connections"]?.arrayValue) + + for entry in listed { + #expect(entry["id"]?.stringValue == session.uuidString) + } + } + + @Test("The listing carries the session's connection and no other host or name") + func listingHoldsOnlyTheSessionEntry() throws { + let listings = ExternalConnectionDirectory.listings( + connections: [ + connection(id: session, name: "dev", host: "localhost"), + connection(id: other, name: "prod_payments", host: "db1.internal.corp"), + connection(id: third, name: "ledger", host: "10.0.0.5") + ], + groups: [], + tags: [], + live: [:], + defaultPolicy: .askEachTime + ) + + let payload = MCPConnectionBridge.connectionListing(listings, id: session) + let listed = try #require(payload["connections"]?.arrayValue) + let text = payload.jsonString() + + #expect(listed.count == 1) + #expect(listed.first?["id"]?.stringValue == session.uuidString) + #expect(!text.contains("prod_payments")) + #expect(!text.contains("db1.internal.corp")) + #expect(!text.contains("ledger")) + #expect(!text.contains("10.0.0.5")) + } + + @Test("The tool's description no longer promises every saved connection") + func descriptionNamesTheSessionConnection() { + #expect(!ListConnectionsChatTool().description.localizedCaseInsensitiveContains("all saved")) + } +} diff --git a/apple/TableProTests/Core/AI/ChatGPTCodexCallbackServerTests.swift b/apple/TableProTests/Core/AI/ChatGPTCodexCallbackServerTests.swift new file mode 100644 index 0000000000..03160cd745 --- /dev/null +++ b/apple/TableProTests/Core/AI/ChatGPTCodexCallbackServerTests.swift @@ -0,0 +1,137 @@ +// +// ChatGPTCodexCallbackServerTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct ChatGPTCodexCallbackTests { + private static let state = "expected-state" + + private static func head(_ target: String, method: String = "GET") -> String { + "\(method) \(target) HTTP/1.1\r\nHost: localhost:1455\r\n\r\n" + } + + private static func parse(_ head: String) -> ChatGPTCodexCallback { + ChatGPTCodexCallback.parse(requestHead: head, expectedState: state) + } + + @Test("The redirect carrying this sign-in's state yields its code") + func matchingStateYieldsTheCode() { + #expect(Self.parse(Self.head("/auth/callback?code=abc&state=expected-state")) == .code("abc")) + } + + @Test("A request that is not this sign-in's redirect is ignored", arguments: [ + "/auth/callback?code=abc&state=forged", + "/auth/callback?code=abc", + "/auth/callback?code=abc&state=expected-state&state=expected-state", + "/auth/callback?error=access_denied&state=forged", + "/auth/callback?error=access_denied", + "/auth/callback?state=expected-state", + "/favicon.ico", + "/?code=abc&state=expected-state", + "/auth/callback/extra?code=abc&state=expected-state" + ]) + func strayRequestIsIgnored(target: String) { + #expect(Self.parse(Self.head(target)) == .ignore) + } + + @Test("Only a GET in origin form is read") + func otherRequestShapesAreIgnored() { + #expect(Self.parse(Self.head("/auth/callback?code=abc&state=expected-state", method: "POST")) == .ignore) + #expect(Self.parse("GET http://localhost:1455/auth/callback?code=abc&state=expected-state HTTP/1.1\r\n\r\n") == .ignore) + #expect(Self.parse("") == .ignore) + } + + @Test("An empty expected state matches nothing") + func emptyExpectedStateMatchesNothing() { + let head = Self.head("/auth/callback?code=abc&state=") + #expect(ChatGPTCodexCallback.parse(requestHead: head, expectedState: "") == .ignore) + } + + @Test("An error redirect with this sign-in's state is a denial carrying the server's reason") + func errorRedirectIsADenial() { + let head = Self.head("/auth/callback?error=access_denied&error_description=User+denied+access%2E&state=expected-state") + #expect(Self.parse(head) == .denied(error: "access_denied", description: "User denied access.")) + } + + @Test("An error sent beside a code wins, so the code is never exchanged") + func errorWinsOverCode() { + let head = Self.head("/auth/callback?code=abc&error=server_error&state=expected-state") + #expect(Self.parse(head) == .denied(error: "server_error", description: nil)) + } + + @Test("The server's error text keeps only the characters RFC 6749 allows") + func errorTextIsSanitized() { + let head = Self.head("/auth/callback?error=%22%0Abad%5C&error_description=a%0D%0Ab%22c&state=expected-state") + #expect(Self.parse(head) == .denied(error: "bad", description: "abc")) + } + + @Test("A denial tells the user what the server said") + func denialMessages() { + typealias ServerError = ChatGPTCodexCallbackServer.ServerError + let bare = ServerError.authorizationDenied(error: "access_denied", description: nil).errorDescription + let explained = ServerError.authorizationDenied(error: "access_denied", description: "No Codex access").errorDescription + let other = ServerError.authorizationDenied(error: "server_error", description: nil).errorDescription + #expect(bare?.isEmpty == false) + #expect(explained?.contains("No Codex access") == true) + #expect(other?.contains("server_error") == true) + } +} + +struct ChatGPTCodexCallbackServerTests { + private static let state = "expected-state" + + private static func status(of target: String, port: UInt16) async -> Int? { + guard let url = URL(string: "http://127.0.0.1:\(port)\(target)") else { return nil } + let session = URLSession(configuration: .ephemeral) + defer { session.finishTasksAndInvalidate() } + let request = URLRequest(url: url, timeoutInterval: 5) + guard let (_, response) = try? await session.data(for: request), + let httpResponse = response as? HTTPURLResponse + else { + return nil + } + return httpResponse.statusCode + } + + @Test("Stray requests are answered and ignored, and the real redirect still completes the sign-in") + func strayRequestsDoNotEndTheSignIn() async throws { + let server = ChatGPTCodexCallbackServer(expectedState: Self.state, port: 0) + let port = try await server.start() + defer { server.stop() } + let code = Task { try await server.waitForCode() } + + let forged = await Self.status(of: "/auth/callback?code=forged&state=wrong", port: port) + let deniedElsewhere = await Self.status(of: "/auth/callback?error=access_denied&state=wrong", port: port) + let favicon = await Self.status(of: "/favicon.ico", port: port) + let redirect = await Self.status(of: "/auth/callback?code=real-code&state=\(Self.state)", port: port) + + #expect([forged, deniedElsewhere, favicon, redirect] == [400, 400, 400, 200]) + #expect(try await code.value == "real-code") + } + + @Test("An error redirect ends the sign-in at once with the server's reason") + func errorRedirectFailsAtOnce() async throws { + let server = ChatGPTCodexCallbackServer(expectedState: Self.state, port: 0) + let port = try await server.start() + defer { server.stop() } + let code = Task { try await server.waitForCode() } + + let redirect = await Self.status( + of: "/auth/callback?error=access_denied&error_description=User+denied+access&state=\(Self.state)", + port: port + ) + + #expect(redirect == 200) + let expected = ChatGPTCodexCallbackServer.ServerError.authorizationDenied( + error: "access_denied", + description: "User denied access" + ) + await #expect(throws: expected) { + try await code.value + } + } +} diff --git a/apple/TableProTests/Core/AI/ChatToolRegistryTests.swift b/apple/TableProTests/Core/AI/ChatToolRegistryTests.swift index bfe51320d7..6c02460b83 100644 --- a/apple/TableProTests/Core/AI/ChatToolRegistryTests.swift +++ b/apple/TableProTests/Core/AI/ChatToolRegistryTests.swift @@ -33,7 +33,8 @@ struct ChatToolRegistryTests { private static let stubContext = ChatToolContext( connectionId: nil, bridge: MCPConnectionBridge(), - authPolicy: MCPAuthPolicy() + authPolicy: MCPAuthPolicy(), + chatMode: .agent ) @Test("Registered tool can be looked up by name") diff --git a/apple/TableProTests/Core/AI/ClaudeAgentAskProfileTests.swift b/apple/TableProTests/Core/AI/ClaudeAgentAskProfileTests.swift new file mode 100644 index 0000000000..0198e9a29e --- /dev/null +++ b/apple/TableProTests/Core/AI/ClaudeAgentAskProfileTests.swift @@ -0,0 +1,225 @@ +// +// ClaudeAgentAskProfileTests.swift +// TableProTests +// + +import Foundation +import TableProMCP +import TableProMCPTestSupport +import Testing + +@testable import TablePro + +private final class AskProfileCredentials: MCPTokenCredentialStoring, @unchecked Sendable { + private let lock = NSLock() + private var stored: Data? + + init(stored: Data? = Data("[]".utf8)) { + self.stored = stored + } + + func read() -> MCPTokenBlob { + lock.withLock { stored.map(MCPTokenBlob.found) ?? .missing } + } + + @discardableResult + func write(_ data: Data) -> Bool { + lock.withLock { stored = data } + return true + } + + func delete() { + lock.withLock { stored = nil } + } +} + +struct ClaudeAgentAskProfileTests { + private static let askToolNames: Set = [ + "list_connections", + "get_connection_status", + "list_databases", + "list_schemas", + "list_tables", + "describe_table", + "get_table_ddl" + ] + + private let connectionId = UUID() + + private func mint( + in store: MCPTokenStore, + toolProfile: MCPToolProfile? + ) async throws -> (token: MCPAuthToken, plaintext: String) { + try await store.generate( + name: "__claude_agent__", + permissions: .readOnly, + connectionAccess: .limited([connectionId]), + expiresAt: Date.now.addingTimeInterval(900), + isBridgeCredential: false, + isClaudeAgentCredential: true, + extraScopes: [], + toolProfile: toolProfile + ) + } + + private func authenticate(_ plaintext: String, against store: MCPTokenStore) async -> MCPAuthDecision { + let clock = MCPTestClock() + let authenticator = MCPBearerTokenAuthenticator( + tokenStore: store, + rateLimiter: MCPRateLimiter(clock: clock), + clock: clock + ) + return await authenticator.authenticate(authorizationHeader: "Bearer \(plaintext)", clientAddress: .loopback) + } + + private func principal(presenting plaintext: String, to store: MCPTokenStore) async throws -> MCPPrincipal { + let decision = await authenticate(plaintext, against: store) + guard case .allow(let principal) = decision else { + Issue.record("Expected allow, got \(decision)") + throw CancellationError() + } + return principal + } + + private func storedEntries(_ tokens: [MCPAuthToken]) throws -> [[String: Any]] { + let data = try MCPTokenStore.encodeStored(tokens: tokens, undecoded: []) + guard let entries = try JSONSerialization.jsonObject(with: data) as? [[String: Any]] else { + Issue.record("The stored tokens are not a JSON list") + throw CancellationError() + } + return entries + } + + private func reloaded(entries: [[String: Any]]) throws -> MCPTokenStore { + let data = try JSONSerialization.data(withJSONObject: entries, options: [.sortedKeys]) + return MCPTokenStore(credentialStore: AskProfileCredentials(stored: data)) + } + + @Test("Ask mode gets the schema lookup profile, Edit and Agent keep today's token") + func profileFollowsTheChatMode() { + #expect(ClaudeAgentMCPBridge.toolProfile(for: .ask) == .schemaLookup) + #expect(ClaudeAgentMCPBridge.toolProfile(for: .edit) == nil) + #expect(ClaudeAgentMCPBridge.toolProfile(for: .agent) == nil) + } + + @Test("The profile's tools are the seven documented Ask tools") + @MainActor + func profileMatchesTheDocumentedAskSet() throws { + let chatAskTools = Set(ChatToolBootstrap.makeTools().filter { $0.mode.isAllowed(in: .ask) }.map { $0.name }) + + #expect(MCPToolProfile.schemaLookup.toolNames == Self.askToolNames) + #expect(MCPToolProfile.schemaLookup.toolNames == chatAskTools) + #expect(MCPToolProfile.schemaLookup.scopes == [.toolsRead]) + for name in MCPToolProfile.schemaLookup.toolNames { + let tool = try #require(MCPToolRegistry.tool(named: name), "\(name) is not an MCP tool") + #expect(type(of: tool).requiredScopes.isSubset(of: MCPToolProfile.schemaLookup.scopes)) + } + } + + @Test("An Ask token authenticates as tools:read only, held to the seven tools") + func askTokenIsNarrowed() async throws { + let store = MCPTokenStore(credentialStore: AskProfileCredentials()) + let minted = try await mint(in: store, toolProfile: .schemaLookup) + + let principal = try await principal(presenting: minted.plaintext, to: store) + + #expect(principal.scopes == [.toolsRead]) + #expect(!principal.has(.resourcesRead)) + #expect(principal.toolAllowlist == Self.askToolNames) + #expect(principal.allowsTool("describe_table")) + #expect(!principal.allowsTool("execute_query")) + #expect(!principal.allowsTool("browse_table")) + #expect(!principal.allowsTool("search_query_history")) + } + + @Test("A token minted without a profile, as pairing and Settings mint, keeps its tier and every tool") + func mintWithoutProfileIsUnchanged() async throws { + let store = MCPTokenStore(credentialStore: AskProfileCredentials()) + let minted = try await store.generate( + name: "Raycast", + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + + let principal = try await principal(presenting: minted.plaintext, to: store) + + #expect(minted.token.toolProfile == nil) + #expect(principal.scopes == MCPScope.readOnlySet) + #expect(principal.toolAllowlist == nil) + } + + @Test("The profile survives a save and reload, and a token without one writes no key") + func profileRoundTrips() async throws { + let store = MCPTokenStore(credentialStore: AskProfileCredentials()) + let ask = try await mint(in: store, toolProfile: .schemaLookup).token + let edit = try await mint(in: store, toolProfile: nil).token + + let data = try MCPTokenStore.encodeStored(tokens: [ask, edit], undecoded: []) + let decoded = MCPTokenStore.decodeStored(data) + let profiles = decoded.tokens.map { $0.toolProfile } + let mentions = String(bytes: data, encoding: .utf8)?.components(separatedBy: "toolProfile").count + + #expect(profiles == [.schemaLookup, nil]) + #expect(decoded.undecoded.isEmpty) + #expect(mentions == 2) + #expect(decoded.tokens.first?.scopes == [.toolsRead]) + } + + @Test("A stored profile this build cannot read keeps the entry undecoded and its token stops working") + func unknownProfileFailsClosed() async throws { + let original = MCPTokenStore(credentialStore: AskProfileCredentials()) + let minted = try await mint(in: original, toolProfile: .schemaLookup) + + let damagedValues: [Any] = ["everything", "", NSNull(), 7] + for damaged in damagedValues { + var entries = try storedEntries([minted.token]) + entries[0]["toolProfile"] = damaged + let data = try JSONSerialization.data(withJSONObject: entries, options: [.sortedKeys]) + + let decoded = MCPTokenStore.decodeStored(data) + #expect(decoded.tokens.isEmpty, "a damaged profile \(damaged) must not decode") + #expect(decoded.undecoded.count == 1) + + let store = try reloaded(entries: entries) + await store.loadFromDisk() + let decision = await authenticate(minted.plaintext, against: store) + guard case .deny = decision else { + Issue.record("A token with a damaged profile \(damaged) authenticated: \(decision)") + continue + } + } + } + + @Test("An Ask token is refused resources/read and prompts/get at the dispatcher") + func askTokenIsRefusedResourcesAndPrompts() async throws { + let store = MCPTokenStore(credentialStore: AskProfileCredentials()) + let minted = try await mint(in: store, toolProfile: .schemaLookup) + let principal = try await principal(presenting: minted.plaintext, to: store) + let services = MCPProtocolHandlerTestSupport.makeToolServices() + + for method in [ResourcesReadHandler.method, PromptsGetHandler.method, ResourcesListHandler.method] { + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [ + ResourcesReadHandler(services: services), + ResourcesListHandler(services: services), + PromptsGetHandler(services: services) + ]) + let (exchange, sink) = MCPProtocolTestSupport.makeExchange( + message: MCPProtocolTestSupport.makeModernRequest( + method: method, + payload: ["uri": .string("tablepro://connections/\(connectionId.uuidString)/history")] + ), + principal: principal + ) + await dispatcher.dispatch(exchange) + await sink.waitForCompletion() + + let errorValue = try await sink.errorEnvelope() + let error = try #require(errorValue, "\(method) answered an Ask token") + #expect(error.code == JsonRpcErrorCode.forbidden) + } + } +} diff --git a/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift b/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift index c589970a09..645b6b46f9 100644 --- a/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift +++ b/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift @@ -1,8 +1,3 @@ -// -// ClaudeAgentConnectionScopeTests.swift -// TableProTests -// - import Foundation @testable import TablePro import TableProMCP @@ -10,15 +5,33 @@ import Testing private actor RecordingToolBridge: ClaudeAgentMCPBridging { private(set) var requested: [UUID?] = [] + private(set) var modes: [AIChatMode] = [] - func makeGrant(connectionId: UUID?) async -> ClaudeAgentToolGrant? { + func makeGrant(connectionId: UUID?, mode: AIChatMode) async -> ClaudeAgentToolGrant? { requested.append(connectionId) + modes.append(mode) return nil } func release(_ grant: ClaudeAgentToolGrant?) async {} } +private struct FailingToolBridge: ClaudeAgentMCPBridging { + func makeGrant(connectionId: UUID?, mode: AIChatMode) async throws -> ClaudeAgentToolGrant? { + throw ClaudeAgentToolGrantError.tokenNotSaved + } + + func release(_ grant: ClaudeAgentToolGrant?) async {} +} + +private struct RefusingCredentialStore: MCPTokenCredentialStoring { + func read() -> MCPTokenBlob { .found(Data("[]".utf8)) } + + func write(_ data: Data) -> Bool { false } + + func delete() {} +} + struct ClaudeAgentConnectionScopeTests { @Test("The token reaches the conversation's connection and no other") func tokenIsLimitedToTheConversationConnection() throws { @@ -31,9 +44,39 @@ struct ClaudeAgentConnectionScopeTests { } @Test("A conversation with no connection gets no token") - func noConnectionMeansNoToken() async { + func noConnectionMeansNoToken() async throws { #expect(ClaudeAgentMCPBridge.connectionAccess(for: nil) == nil) - #expect(await ClaudeAgentMCPBridge().makeGrant(connectionId: nil) == nil) + #expect(try await ClaudeAgentMCPBridge().makeGrant(connectionId: nil, mode: .agent) == nil) + } + + @Test("A token the Keychain refused stops the grant with a reason instead of dropping the tools") + func unsavedTokenIsAnError() async throws { + let store = MCPTokenStore(credentialStore: RefusingCredentialStore()) + let endpoint = try #require(URL(string: "http://127.0.0.1:1/mcp")) + let bridge = ClaudeAgentMCPBridge(endpoint: { endpoint }, tokenStore: { store }) + + await #expect(throws: ClaudeAgentToolGrantError.tokenNotSaved) { + _ = try await bridge.makeGrant(connectionId: UUID(), mode: .agent) + } + #expect(await store.list().isEmpty) + #expect(ClaudeAgentToolGrantError.tokenNotSaved.localizedDescription.contains("Keychain")) + } + + @Test("The chat gets the grant's failure as the turn's error") + func grantFailureEndsTheTurn() async throws { + let provider = ClaudeAgentProvider( + model: "sonnet", + cli: ClaudeAgentCLI(executableOverride: "/usr/bin/true"), + toolBridge: FailingToolBridge() + ) + let stream = provider.streamChat( + turns: [ChatTurnWire(role: .user, blocks: [.text("hi")])], + options: ChatTransportOptions(model: "sonnet", connectionId: UUID()) + ) + + await #expect(throws: ClaudeAgentToolGrantError.tokenNotSaved) { + for try await _ in stream {} + } } @Test("Transport options carry no connection unless the caller names one") @@ -41,6 +84,31 @@ struct ClaudeAgentConnectionScopeTests { #expect(ChatTransportOptions(model: "sonnet").connectionId == nil) } + @Test("Transport options that name no mode are Ask, the narrowest") + func optionsDefaultToAsk() { + #expect(ChatTransportOptions(model: "sonnet").chatMode == .ask) + } + + @Test("The provider asks for a grant in the chat mode its options carry") + func providerPassesTheModeToTheBridge() async throws { + let bridge = RecordingToolBridge() + let provider = ClaudeAgentProvider( + model: "sonnet", + cli: ClaudeAgentCLI(executableOverride: "/usr/bin/true"), + toolBridge: bridge + ) + + for mode in AIChatMode.allCases { + let stream = provider.streamChat( + turns: [ChatTurnWire(role: .user, blocks: [.text("hi")])], + options: ChatTransportOptions(model: "sonnet", connectionId: UUID(), chatMode: mode) + ) + for try await _ in stream {} + } + + #expect(await bridge.modes == AIChatMode.allCases) + } + @Test("The provider asks for a grant on the connection in its options") func providerPassesTheConnectionToTheBridge() async throws { let bridge = RecordingToolBridge() diff --git a/apple/TableProTests/Core/AI/ClaudeAgentProviderTests.swift b/apple/TableProTests/Core/AI/ClaudeAgentProviderTests.swift index 0df630ac06..90bd6bde51 100644 --- a/apple/TableProTests/Core/AI/ClaudeAgentProviderTests.swift +++ b/apple/TableProTests/Core/AI/ClaudeAgentProviderTests.swift @@ -14,7 +14,8 @@ struct ClaudeAgentProviderTests { prompt: "count the users", model: "sonnet", systemPrompt: "be terse", - mcpConfigPath: nil + mcpConfigPath: nil, + allowedToolNames: nil ) #expect(args.contains("-p")) #expect(args.contains("stream-json")) @@ -35,7 +36,8 @@ struct ClaudeAgentProviderTests { prompt: "hi", model: "", systemPrompt: nil, - mcpConfigPath: nil + mcpConfigPath: nil, + allowedToolNames: nil ) guard let index = args.firstIndex(of: "--tools") else { Issue.record("--tools flag missing") @@ -52,7 +54,8 @@ struct ClaudeAgentProviderTests { prompt: "--dangerously-skip-permissions", model: "", systemPrompt: nil, - mcpConfigPath: nil + mcpConfigPath: nil, + allowedToolNames: nil ) #expect(args[args.count - 2] == "--") #expect(args.last == "--dangerously-skip-permissions") @@ -64,7 +67,8 @@ struct ClaudeAgentProviderTests { prompt: "hi", model: "", systemPrompt: nil, - mcpConfigPath: "/tmp/config.json" + mcpConfigPath: "/tmp/config.json", + allowedToolNames: nil ) #expect(args.contains("--mcp-config")) #expect(args.contains("/tmp/config.json")) @@ -75,6 +79,31 @@ struct ClaudeAgentProviderTests { #expect(args[index + 1] == "mcp__tablepro__*") } + @Test("An Ask grant names the seven schema tools to the CLI instead of the whole server") + func askGrantNamesItsTools() { + let args = ClaudeAgentProvider.inferenceArguments( + prompt: "hi", + model: "", + systemPrompt: nil, + mcpConfigPath: "/tmp/config.json", + allowedToolNames: MCPToolProfile.schemaLookup.toolNames.sorted() + ) + guard let index = args.firstIndex(of: "--allowedTools") else { + Issue.record("--allowedTools flag missing") + return + } + #expect(args[index + 1] == [ + "mcp__tablepro__describe_table", + "mcp__tablepro__get_connection_status", + "mcp__tablepro__get_table_ddl", + "mcp__tablepro__list_connections", + "mcp__tablepro__list_databases", + "mcp__tablepro__list_schemas", + "mcp__tablepro__list_tables" + ].joined(separator: ",")) + #expect(!args.contains("mcp__tablepro__*")) + } + @Test("Partial message deltas decode as incremental text") func partialMessageDeltasDecodeAsText() { let line = """ diff --git a/apple/TableProTests/Core/AI/CopilotDocumentSyncTests.swift b/apple/TableProTests/Core/AI/CopilotDocumentSyncTests.swift new file mode 100644 index 0000000000..19e060ebf0 --- /dev/null +++ b/apple/TableProTests/Core/AI/CopilotDocumentSyncTests.swift @@ -0,0 +1,555 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProEditorKit +import TableProTestSupport +import TableProTextEngine +import Testing +import TextStory + +@MainActor +private final class RecordingCopilotServer: CopilotDocumentServer { + enum Message { + case open(LSPTextDocumentItem) + case change(LSPDidChangeParams) + case focus(String) + case close(String) + } + + var generation = 1 + var hasClient = true + private(set) var messages: [Message] = [] + + var changes: [LSPDidChangeParams] { + messages.compactMap { message in + guard case .change(let params) = message else { return nil } + return params + } + } + + func didOpen(_ document: LSPTextDocumentItem) async { + messages.append(.open(document)) + } + + func didChange(_ change: LSPDidChangeParams) async { + messages.append(.change(change)) + } + + func didFocus(uri: String) async { + messages.append(.focus(uri)) + } + + func didClose(uri: String) async { + messages.append(.close(uri)) + } +} + +@MainActor +private final class TabSelection { + var id = UUID() +} + +@MainActor +private final class SchemaSelection { + var provider: SQLSchemaProvider + var budget: Int + + init(provider: SQLSchemaProvider, budget: Int) { + self.provider = provider + self.budget = budget + } + + var source: CopilotPreambleBuilder.Source { + CopilotPreambleBuilder.Source( + schemaProvider: provider, + databaseName: "shop", + databaseType: .mysql, + grammar: DatabaseType.mysql.lexicalGrammar, + budget: budget + ) + } +} + +private final class FetchCounter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func increment() { + lock.withLock { value += 1 } + } +} + +// The server's document model (vscode-languageserver-textdocument): CR, LF and CR LF end a line, and a position past +// a line's content is clamped to before its line break. +private struct LSPDocumentModel { + private var units: [UInt16] + + init(_ text: String) { + units = Array(text.utf16) + } + + var text: String { + String(decoding: units, as: UTF16.self) + } + + mutating func apply(_ change: LSPTextDocumentContentChangeEvent) { + let replacement = Array(change.text.utf16) + guard let range = change.range else { + units = replacement + return + } + let start = offset(at: range.start) + let end = max(start, offset(at: range.end)) + units.replaceSubrange(start.. Int { + let lineStarts = lineStarts() + guard position.line < lineStarts.count else { return units.count } + guard position.line >= 0 else { return 0 } + let lineStart = lineStarts[position.line] + guard position.character > 0 else { return lineStart } + let nextLineStart = position.line + 1 < lineStarts.count ? lineStarts[position.line + 1] : units.count + var offset = min(lineStart + position.character, nextLineStart) + while offset > lineStart, units[offset - 1] == 0x0D || units[offset - 1] == 0x0A { + offset -= 1 + } + return offset + } + + private func lineStarts() -> [Int] { + var starts = [0] + var index = 0 + while index < units.count { + let unit = units[index] + if unit == 0x0D || unit == 0x0A { + if unit == 0x0D, index + 1 < units.count, units[index + 1] == 0x0A { + index += 1 + } + starts.append(index + 1) + } + index += 1 + } + return starts + } +} + +@MainActor +struct CopilotDocumentSyncTests { + @MainActor + private struct Harness { + let controller: TextViewController + let server: RecordingCopilotServer + let sync: CopilotDocumentSync + let tab: TabSelection + let schema: SchemaSelection + + var textView: TextView { controller.textView } + var uri: String { sync.documentURI(for: tab.id) } + var expectedText: String { sync.preambleBuilder.preamble + controller.textView.string } + + func replayed(uri: String? = nil) -> String? { + CopilotDocumentSyncTests.replayedText(of: server.messages, uri: uri ?? self.uri) + } + } + + @Test("random edits, typing filter edits and a multi-range replace replay to the editor's text as range changes") + func incrementalEditsReplayToTheEditorText() async throws { + let harness = await Self.makeHarness(text: "SELECT id, name\nFROM table_0\nWHERE id = 1;\n") + var generator = SeededGenerator(seed: 0x5EED) + let snippets = ["x", "SELECT ", "\n", "a\nb", "é", "😀", " ", "\n\n", "(", "'", ";\n"] + + for step in 0..<120 { + let text = harness.textView.string as NSString + let location = Self.boundary(generator.index(below: text.length + 1), in: text) + let end = Self.boundary(min(text.length, location + 1 + generator.index(below: 8)), in: text) + let snippet = snippets[generator.index(below: snippets.count)] + switch generator.index(below: 3) { + case 0: + harness.textView.replaceCharacters(in: NSRange(location: location, length: 0), with: snippet) + case 1: + harness.textView.replaceCharacters(in: NSRange(location: location, length: end - location), with: "") + default: + harness.textView.replaceCharacters(in: NSRange(location: location, length: end - location), with: snippet) + } + if step.isMultiple(of: 10) { + await harness.sync.flush() + } + } + + let text = harness.textView.string as NSString + harness.textView.replaceCharacters([ + TextReplacement(range: NSRange(location: 0, length: 0), string: "-- head\n"), + TextReplacement(range: NSRange(location: Self.boundary(text.length / 2, in: text), length: 0), string: "mid\n"), + TextReplacement(range: NSRange(location: text.length, length: 0), string: "\n-- tail") + ]) + harness.textView.applyMutation( + TextMutation(string: "/* filter */\n", range: NSRange(location: 0, length: 0), limit: harness.textView.textStorage.length) + ) + await harness.sync.flush() + + let sent = harness.server.changes + try #require(!sent.isEmpty) + #expect(sent.allSatisfy { params in params.contentChanges.allSatisfy { $0.range != nil } }) + let versions = sent.map { $0.textDocument.version } + #expect(versions == Array(1...sent.count)) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a keystroke in a 130,000 unit script with a 300-table preamble sends under 1 KB") + func keystrokePayloadStaysSmall() async throws { + let document = Self.script(units: 130_000) + let harness = await Self.makeHarness(text: document, tableCount: 300) + let fullText = LSPDidChangeParams( + textDocument: LSPVersionedTextDocumentIdentifier(uri: harness.uri, version: 1), + contentChanges: [LSPTextDocumentContentChangeEvent(text: harness.sync.preambleBuilder.prependToText(document))] + ) + let fullTextBytes = try JSONEncoder().encode(fullText).count + + harness.textView.replaceCharacters(in: NSRange(location: 65_000, length: 0), with: "x") + await harness.sync.flush() + + let keystroke = try #require(harness.server.changes.last) + let keystrokeBytes = try JSONEncoder().encode(keystroke).count + #expect(keystroke.contentChanges.first?.range != nil) + #expect(keystrokeBytes < 1_024) + #expect(fullTextBytes > 200_000) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("typing in a CR LF script stays incremental; an edit inside a CR LF sends the full text until the lines agree again") + func editNextToCarriageReturnSendsFullText() async throws { + let harness = await Self.makeHarness(text: "SELECT id\r\nFROM table_0\r\nWHERE id = 1\r\n") + + harness.textView.replaceCharacters(in: NSRange(location: 13, length: 0), with: "x") + await harness.sync.flush() + let typed = try #require(harness.server.changes.last) + #expect(typed.contentChanges.allSatisfy { $0.range != nil }) + #expect(harness.replayed() == harness.expectedText) + + let lineFeed = (harness.textView.string as NSString).range(of: "\n", options: .literal).location + harness.textView.replaceCharacters(in: NSRange(location: lineFeed, length: 0), with: "y") + await harness.sync.flush() + let split = try #require(harness.server.changes.last) + #expect(split.contentChanges.count == 1) + #expect(split.contentChanges.first?.range == nil) + #expect(harness.replayed() == harness.expectedText) + + harness.textView.replaceCharacters(in: NSRange(location: harness.textView.textStorage.length, length: 0), with: "LIMIT 1") + await harness.sync.flush() + let later = try #require(harness.server.changes.last) + #expect(later.contentChanges.first?.range == nil) + #expect(harness.replayed() == harness.expectedText) + + harness.textView.replaceCharacters(in: NSRange(location: lineFeed, length: 1), with: "") + await harness.sync.flush() + let rejoined = try #require(harness.server.changes.last) + #expect(rejoined.contentChanges.first?.range == nil) + + harness.textView.replaceCharacters(in: NSRange(location: harness.textView.textStorage.length, length: 0), with: ";") + await harness.sync.flush() + let resumed = try #require(harness.server.changes.last) + #expect(resumed.contentChanges.allSatisfy { $0.range != nil }) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("switching tabs sends nothing; the first edit or request there closes the old document and opens the new one") + func tabSwitchSendsNothingUntilTheTabIsUsed() async { + let harness = await Self.makeHarness(text: "SELECT 1;\n") + let firstURI = harness.uri + let before = harness.server.messages.count + + harness.tab.id = UUID() + harness.controller.setText("SELECT name FROM table_1;\n") + harness.sync.textViewDidReplaceDocument() + #expect(harness.sync.currentDocumentInfo()?.uri == nil) + + harness.textView.replaceCharacters(in: NSRange(location: 7, length: 0), with: "id, ") + await harness.sync.flush() + let secondURI = harness.uri + #expect(secondURI != firstURI) + #expect(Self.summary(Array(harness.server.messages.dropFirst(before))) == [ + "close \(firstURI)", "open \(secondURI)", "focus \(secondURI)" + ]) + #expect(harness.sync.currentDocumentInfo()?.uri == secondURI) + #expect(harness.replayed(uri: secondURI) == harness.expectedText) + #expect(harness.replayed(uri: firstURI) == nil) + + let switched = harness.server.messages.count + harness.tab.id = UUID() + harness.controller.setText("SELECT 3;\n") + harness.sync.textViewDidReplaceDocument() + await harness.sync.flush() + #expect(Self.summary(Array(harness.server.messages.dropFirst(switched))) == [ + "close \(secondURI)", "open \(harness.uri)", "focus \(harness.uri)" + ]) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a request whose statement names an undescribed table puts it in the preamble and resends the full text once") + func requestAddsTheStatementsTableToThePreamble() async throws { + let fetches = FetchCounter() + let harness = await Self.makeHarness(text: "SELECT 1;\n", tableCount: 2_000, maxTables: 20, fetches: fetches) + #expect(!harness.sync.preambleBuilder.preamble.contains("-- table_1500(")) + harness.textView.replaceCharacters(in: NSRange(location: 0, length: 9), with: "SELECT * FROM table_1500;") + await harness.sync.flush() + let before = harness.server.messages.count + let fetchesBefore = fetches.count + + harness.sync.refreshPreamble(forStatementAt: 10, in: harness.textView.string) + try await waitUntil { harness.sync.preambleBuilder.preamble.contains("-- table_1500(") } + await harness.sync.flush() + + #expect(fetches.count - fetchesBefore <= 20) + let tail = Array(harness.server.messages.dropFirst(before)) + #expect(Self.summary(tail) == ["change \(harness.uri)"]) + let resent = try #require(harness.server.changes.last) + #expect(resent.contentChanges.count == 1) + #expect(resent.contentChanges.first?.range == nil) + #expect(resent.contentChanges.first?.text.contains("-- table_1500(") == true) + #expect(harness.replayed() == harness.expectedText) + + let settled = harness.server.messages.count + let described = await harness.sync.preambleBuilder.rebuiltPreamble( + forStatementAt: 10, + in: harness.textView.string, + from: harness.schema.source + ) + #expect(described?.preamble == nil) + await harness.sync.flush() + #expect(harness.server.messages.count == settled) + } + + @Test("an editor that had no schema when Copilot started gets the schema header on its first request") + func requestBuildsThePreambleAnEditorStartedWithout() async throws { + let harness = await Self.makeHarness(text: "SELECT * FROM table_1;\n", buildsPreamble: false) + #expect(harness.sync.preambleBuilder.preamble.isEmpty) + let before = harness.server.messages.count + + harness.sync.refreshPreamble(forStatementAt: 10, in: harness.textView.string) + try await waitUntil { harness.sync.preambleBuilder.preamble.contains("-- table_1(") } + await harness.sync.flush() + + #expect(harness.server.messages.count == before + 1) + let resent = try #require(harness.server.changes.last) + #expect(resent.contentChanges.first?.range == nil) + #expect(resent.contentChanges.first?.text.hasPrefix("-- Database: shop\n") == true) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("after the tab moves to another database, a request describes that database's table of the same name") + func requestFollowsTheTabsSchemaProvider() async throws { + let sales = await Self.makeProvider( + tables: [TestFixtures.makeTableInfo(name: "orders")], + columns: [TestFixtures.makeColumnInfo(name: "amount", dataType: "DECIMAL")] + ) + let staff = await Self.makeProvider( + tables: [TestFixtures.makeTableInfo(name: "orders")], + columns: [TestFixtures.makeColumnInfo(name: "salary", dataType: "INT")] + ) + let harness = await Self.makeHarness(text: "SELECT * FROM orders;\n", provider: sales) + #expect(harness.sync.preambleBuilder.preamble.contains("-- orders(amount DECIMAL PK NOT NULL)")) + let before = harness.server.messages.count + + harness.schema.provider = staff + harness.sync.refreshPreamble(forStatementAt: 10, in: harness.textView.string) + try await waitUntil { harness.sync.preambleBuilder.preamble.contains("salary") } + await harness.sync.flush() + + let resent = try #require(harness.server.changes.last) + #expect(harness.server.messages.count == before + 1) + #expect(resent.contentChanges.first?.range == nil) + let text = try #require(resent.contentChanges.first?.text) + #expect(text.contains("-- orders(salary INT PK NOT NULL)")) + #expect(!text.contains("amount")) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a lower Max schema tables applies on the next request") + func lowerBudgetAppliesOnTheNextRequest() async throws { + let harness = await Self.makeHarness(text: "SELECT 1;\n", tableCount: 10) + #expect(harness.sync.preambleBuilder.preamble.contains("-- table_9(")) + + harness.schema.budget = 3 + harness.sync.refreshPreamble(forStatementAt: 3, in: harness.textView.string) + try await waitUntil { !harness.sync.preambleBuilder.preamble.contains("-- table_9(") } + await harness.sync.flush() + + let text = try #require(harness.server.changes.last?.contentChanges.first?.text) + #expect(text.contains("-- … and 7 more tables")) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("a tab switch that keeps the same text moves the next edit to the new tab's document") + func tabSwitchWithTheSameTextMovesTheNextEdit() async { + let harness = await Self.makeHarness(text: "SELECT 1;\n") + let firstURI = harness.uri + let before = harness.server.messages.count + + harness.tab.id = UUID() + harness.textView.replaceCharacters(in: NSRange(location: 8, length: 0), with: ", 2") + await harness.sync.flush() + + #expect(Self.summary(Array(harness.server.messages.dropFirst(before))) == [ + "close \(firstURI)", "open \(harness.uri)", "focus \(harness.uri)" + ]) + #expect(harness.sync.currentDocumentInfo()?.uri == harness.uri) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("after the server restarts, the next edit or request reopens the document with its full text") + func serverRestartReopensTheDocument() async { + let harness = await Self.makeHarness(text: "SELECT 1;\n") + let before = harness.server.messages.count + + harness.server.hasClient = false + harness.server.generation += 1 + harness.textView.replaceCharacters(in: NSRange(location: 8, length: 0), with: ", 2") + await harness.sync.flush() + #expect(harness.server.messages.count == before) + #expect(harness.sync.currentDocumentInfo()?.uri == nil) + + harness.server.hasClient = true + harness.server.generation += 1 + harness.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "-- again\n") + await harness.sync.flush() + #expect(Self.summary(Array(harness.server.messages.dropFirst(before))) == ["open \(harness.uri)", "focus \(harness.uri)"]) + #expect(harness.sync.currentDocumentInfo()?.version == 0) + #expect(harness.replayed() == harness.expectedText) + + let reopened = harness.server.messages.count + harness.server.generation += 1 + await harness.sync.flush() + #expect(Self.summary(Array(harness.server.messages.dropFirst(reopened))) == ["open \(harness.uri)", "focus \(harness.uri)"]) + #expect(harness.replayed() == harness.expectedText) + } + + @Test("past the length limit nothing is sent, and back under it the full text is") + func overTheLengthLimitResyncsOnceUnder() async throws { + let harness = await Self.makeHarness(text: "SELECT 1;", lengthLimit: 20) + let before = harness.server.messages.count + + harness.textView.replaceCharacters(in: NSRange(location: 9, length: 0), with: " -- a long comment") + await harness.sync.flush() + #expect(harness.server.messages.count == before) + #expect(harness.sync.currentDocumentInfo()?.uri == nil) + + harness.textView.replaceCharacters(in: NSRange(location: 9, length: 18), with: "") + await harness.sync.flush() + let resync = try #require(harness.server.changes.last) + #expect(resync.contentChanges.count == 1) + #expect(resync.contentChanges.first?.range == nil) + #expect(harness.replayed() == harness.expectedText) + } + + // MARK: - Helpers + + private static func makeHarness( + text: String, + tableCount: Int = 3, + maxTables: Int? = nil, + lengthLimit: Int = 2_000_000, + fetches: FetchCounter? = nil, + provider given: SQLSchemaProvider? = nil, + buildsPreamble: Bool = true + ) async -> Harness { + let server = RecordingCopilotServer() + let sync = CopilotDocumentSync(server: server, lengthLimit: lengthLimit) + let provider: SQLSchemaProvider + if let given { + provider = given + } else { + provider = await makeProvider( + tables: (0.. SQLSchemaProvider { + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { _, _ in + fetches?.increment() + return columns + }, + fetchAllColumns: { [:] } + ) + let provider = SQLSchemaProvider(metadataSource: source) + await provider.resetForDatabase( + "shop", + tables: tables, + driver: MockDatabaseDriver(), + connection: TestFixtures.makeConnection() + ) + await provider.waitForEagerColumnLoad() + return provider + } + + private static func script(units: Int) -> String { + let block = "CREATE TABLE t (\n id INT,\n name TEXT\n);\nSELECT id, name\n FROM t\n WHERE id IN (\n 1, 2, 3\n );\n" + return String(repeating: block, count: units / (block as NSString).length + 1) + } + + private static func boundary(_ offset: Int, in text: NSString) -> Int { + guard offset > 0, offset < text.length else { return offset } + return text.rangeOfComposedCharacterSequence(at: offset).location + } + + private static func summary(_ messages: [RecordingCopilotServer.Message]) -> [String] { + messages.map { message -> String in + switch message { + case .open(let document): "open \(document.uri)" + case .change(let params): "change \(params.textDocument.uri)" + case .focus(let uri): "focus \(uri)" + case .close(let uri): "close \(uri)" + } + } + } + + private static func replayedText(of messages: [RecordingCopilotServer.Message], uri: String) -> String? { + var model: LSPDocumentModel? + for message in messages { + switch message { + case .open(let document) where document.uri == uri: + model = LSPDocumentModel(document.text) + case .change(let params) where params.textDocument.uri == uri: + for change in params.contentChanges { + model?.apply(change) + } + case .close(let closed) where closed == uri: + model = nil + default: + break + } + } + return model?.text + } +} diff --git a/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift b/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift index 027662991b..87935e50d9 100644 --- a/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift +++ b/apple/TableProTests/Core/AI/CopilotPreambleBuilderTests.swift @@ -8,10 +8,21 @@ import Foundation import TableProPluginKit import Testing +private final class FetchCounter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + var count: Int { + lock.withLock { value } + } + + func increment() { + lock.withLock { value += 1 } + } +} + @MainActor struct CopilotPreambleBuilderTests { - /// The preamble fetched every table's columns with no schema and keyed them by the bare name, - /// so a table outside the current schema listed nothing or its namesake's columns. @Test("same-named tables in two schemas each list their own columns under a qualified name") func sameNamedTablesInTwoSchemasListTheirOwnColumns() async { let columnsBySchema: [String: [ColumnInfo]] = [ @@ -35,10 +46,52 @@ struct CopilotPreambleBuilderTests { await provider.waitForEagerColumnLoad() let builder = CopilotPreambleBuilder() - await builder.buildPreamble(schemaProvider: provider, databaseName: "shop", databaseType: .mysql) + await builder.buildPreamble(schemaProvider: provider, databaseName: "shop", databaseType: .mysql, maxTables: 20) let lines = builder.preamble.split(separator: "\n").map(String.init) #expect(lines.contains("-- sales.orders(amount DECIMAL PK NOT NULL)")) #expect(lines.contains("-- hr.orders(salary INT PK NOT NULL)")) } + + @Test("a large catalog reads columns for the table budget only, the document's tables first, one line per table") + func largeCatalogReadsColumnsForTheBudgetOnly() async throws { + let fetches = FetchCounter() + let columns = [TestFixtures.makeColumnInfo(name: "id")] + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { _, _ in + fetches.increment() + return columns + }, + fetchAllColumns: { [:] } + ) + let provider = SQLSchemaProvider(metadataSource: source) + let tables = [TestFixtures.makeTableInfo(name: "line\nbreak")] + + (1..<2_000).map { TestFixtures.makeTableInfo(name: "table_\($0)") } + await provider.resetForDatabase( + "shop", + tables: tables, + driver: MockDatabaseDriver(), + connection: TestFixtures.makeConnection() + ) + await provider.waitForEagerColumnLoad() + let fetchesBefore = fetches.count + + let builder = CopilotPreambleBuilder() + await builder.buildPreamble( + schemaProvider: provider, + databaseName: "shop", + databaseType: .mysql, + referencing: "SELECT * FROM table_1500 t JOIN table_1999 u ON u.id = t.id;", + maxTables: 20 + ) + + #expect(fetches.count - fetchesBefore <= 20) + let lines = builder.preamble.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) + try #require(lines.count == 25) + #expect(lines[3].hasPrefix("-- table_1500(")) + #expect(lines[4].hasPrefix("-- table_1999(")) + #expect(lines[5] == "-- line break(id INT PK NOT NULL)") + #expect(lines[23] == "-- … and 1980 more tables") + #expect(builder.preambleLineCount == 24) + } } diff --git a/apple/TableProTests/Core/AI/ExecuteToolUsesTests.swift b/apple/TableProTests/Core/AI/ExecuteToolUsesTests.swift index 1fc171948a..432b006ae4 100644 --- a/apple/TableProTests/Core/AI/ExecuteToolUsesTests.swift +++ b/apple/TableProTests/Core/AI/ExecuteToolUsesTests.swift @@ -52,7 +52,8 @@ struct ExecuteToolUsesTests { ChatToolContext( connectionId: nil, bridge: MCPConnectionBridge(), - authPolicy: MCPAuthPolicy() + authPolicy: MCPAuthPolicy(), + chatMode: .agent ) } diff --git a/apple/TableProTests/Core/AI/OpenAICompatibleProviderConnectionTests.swift b/apple/TableProTests/Core/AI/OpenAICompatibleProviderConnectionTests.swift index f28b644773..014e48578b 100644 --- a/apple/TableProTests/Core/AI/OpenAICompatibleProviderConnectionTests.swift +++ b/apple/TableProTests/Core/AI/OpenAICompatibleProviderConnectionTests.swift @@ -62,6 +62,7 @@ private final class StubConnectionProtocol: URLProtocol, @unchecked Sendable { struct OpenAICompatibleProviderConnectionTests { private func makeProvider( endpoint: String, + model: String = "glm-4.6", treatsForbiddenAsAuthFailure: Bool = false ) -> OpenAICompatibleProvider { let config = URLSessionConfiguration.ephemeral @@ -70,7 +71,7 @@ struct OpenAICompatibleProviderConnectionTests { endpoint: endpoint, apiKey: "key", providerType: .custom, - model: "glm-4.6", + model: model, treatsForbiddenAsAuthFailure: treatsForbiddenAsAuthFailure, session: URLSession(configuration: config) ) @@ -237,4 +238,32 @@ struct OpenAICompatibleProviderConnectionTests { _ = try await makeProvider(endpoint: "https://api.z.ai/api/paas/v4").testConnection() #expect(StubConnectionProtocol.lastRequestedURL() == "https://api.z.ai/api/paas/v4/chat/completions") } + + private static let unknownModelBody = #""" + {"error":{"code":"model_not_found","message":"The model `test` does not exist or is not available","param":"model","type":"invalid_request_error"}} + """# + + /// With no model picked yet the probe names a placeholder, and a server that answers "no such + /// model" read as a wrong Base URL, failing a setup whose route and key were both right. + @Test("With no model picked, a server that says the placeholder model does not exist is reachable") + func unknownPlaceholderModelIsAWorkingConnection() async throws { + StubConnectionProtocol.respond(status: 404, body: Self.unknownModelBody) + #expect(try await makeProvider(endpoint: "https://api.meta.ai", model: "").testConnection()) + } + + @Test("A model the user picked that the server does not know is still a failure") + func unknownChosenModelFails() async { + StubConnectionProtocol.respond(status: 404, body: Self.unknownModelBody) + await #expect(throws: AIProviderError.self) { + _ = try await makeProvider(endpoint: "https://api.meta.ai", model: "muse-spark-9").testConnection() + } + } + + @Test("With no model picked, any other 404 is still a failure") + func otherNotFoundWithoutModelFails() async { + StubConnectionProtocol.respond(status: 404, body: #"{"error":{"code":"not_found","message":"no route"}}"#) + await #expect(throws: AIProviderError.self) { + _ = try await makeProvider(endpoint: "https://host/v1", model: "").testConnection() + } + } } diff --git a/apple/TableProTests/Core/AI/OpenAICompatibleProviderParserTests.swift b/apple/TableProTests/Core/AI/OpenAICompatibleProviderParserTests.swift index da5f8338bb..f036ece50c 100644 --- a/apple/TableProTests/Core/AI/OpenAICompatibleProviderParserTests.swift +++ b/apple/TableProTests/Core/AI/OpenAICompatibleProviderParserTests.swift @@ -313,4 +313,53 @@ struct OpenAICompatibleProviderParserTests { let ollamaEmpty = OpenAICompatibleProvider.decodeStreamLine("", providerType: .ollama) #expect(ollamaEmpty == nil) } + + /// Recorded from Meta's Chat Completions endpoint for `muse-spark-1.3`: text, then a tool call + /// whose id and name come first with empty arguments, and usage on the `finish_reason` chunk. + @Test("A Meta tool-call stream decodes to text, one tool call and its usage") + func metaToolCallStream() throws { + let lines = [ + #"data: {"id":"c1","choices":[{"delta":{"content":"I'll look up the weather.","role":"assistant"},"finish_reason":null,"index":0}],"model":"muse-spark-1.3","object":"chat.completion.chunk"}"#, + #"data: {"id":"c1","choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_01a0","type":"function","function":{"name":"lookup_weather","arguments":""}}]},"finish_reason":null,"index":0}],"model":"muse-spark-1.3","object":"chat.completion.chunk"}"#, + #"data: {"id":"c1","choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"city\":\"Paris\"}"}}]},"finish_reason":null,"index":0}],"model":"muse-spark-1.3","object":"chat.completion.chunk"}"#, + #"data: {"id":"c1","choices":[{"delta":{},"finish_reason":"tool_calls","index":0}],"model":"muse-spark-1.3","object":"chat.completion.chunk","usage":{"completion_tokens":139,"prompt_tokens":570,"total_tokens":709,"completion_tokens_details":{"reasoning_tokens":70}}}"#, + "data: [DONE]" + ] + var state = OpenAIStreamState() + var events: [ChatStreamEvent] = [] + for line in lines { + guard let json = OpenAICompatibleProvider.decodeStreamLine(line, providerType: .custom) else { continue } + events += try OpenAICompatibleProvider.streamEvents(from: json, state: &state) + } + events += try OpenAICompatibleProvider.finalStreamEvents(state) + + guard events.count == 5 else { + Issue.record("expected 5 events; got \(events)") + return + } + if case .textDelta(let text) = events[0] { #expect(text == "I'll look up the weather.") } else { + Issue.record("expected textDelta; got \(events[0])") + } + if case .toolUseStart(let id, let name, _) = events[1] { + #expect(id == "call_01a0") + #expect(name == "lookup_weather") + } else { + Issue.record("expected toolUseStart; got \(events[1])") + } + if case .toolUseDelta(let id, let delta) = events[2] { + #expect(id == "call_01a0") + #expect(delta == #"{"city":"Paris"}"#) + } else { + Issue.record("expected toolUseDelta; got \(events[2])") + } + if case .toolUseEnd(let id) = events[3] { #expect(id == "call_01a0") } else { + Issue.record("expected toolUseEnd; got \(events[3])") + } + if case .usage(let usage) = events[4] { + #expect(usage.inputTokens == 570) + #expect(usage.outputTokens == 139) + } else { + Issue.record("expected usage; got \(events[4])") + } + } } diff --git a/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift b/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift new file mode 100644 index 0000000000..cf91ae70ea --- /dev/null +++ b/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift @@ -0,0 +1,177 @@ +// +// OutputLimitSignalTests.swift +// TableProTests +// + +import Foundation +import TableProMCP +@testable import TablePro +import Testing + +private func limits(_ events: [ChatStreamEvent]) -> [ChatOutputLimit] { + events.compactMap { event in + if case .outputLimitReached(let limit) = event { return limit } + return nil + } +} + +struct OutputLimitSignalTests { + @Test("Anthropic message_delta stop reasons that cut the reply are reported", arguments: [ + ("max_tokens", [ChatOutputLimit.outputTokens]), + ("model_context_window_exceeded", [.contextWindow]), + ("end_turn", []), + ("tool_use", []), + ] as [(String, [ChatOutputLimit])]) + func anthropicStopReason(reason: String, expected: [ChatOutputLimit]) throws { + var state = AnthropicStreamState() + let events = try AnthropicProvider.parseChunk([ + "type": "message_delta", + "delta": ["stop_reason": reason], + "usage": ["output_tokens": 12], + ], state: &state) + #expect(limits(events) == expected) + #expect(state.outputTokens == 12) + } + + @Test("Chat Completions finish_reason length is reported, stop is not", arguments: [ + ("length", [ChatOutputLimit.outputTokens]), + ("stop", []), + ("tool_calls", []), + ] as [(String, [ChatOutputLimit])]) + func chatCompletionsFinishReason(reason: String, expected: [ChatOutputLimit]) { + var state = OpenAIStreamState() + let result = OpenAICompatibleProvider.parseChunk([ + "choices": [["delta": [:] as [String: Any], "finish_reason": reason]], + ], state: &state) + #expect(limits(result.events) == expected) + } + + @Test("Ollama's done_reason length is reported") + func ollamaDoneReason() { + var state = OpenAIStreamState() + let result = OpenAICompatibleProvider.parseChunk([ + "message": ["content": ""], + "done": true, + "done_reason": "length", + ], state: &state) + #expect(limits(result.events) == [.outputTokens]) + } + + @Test("A Responses stream that ends incomplete at max_output_tokens is reported, with its usage") + func responsesIncomplete() throws { + var state = ResponsesStreamState() + let events = try OpenAIResponsesProvider.parseEvent([ + "type": "response.incomplete", + "response": [ + "status": "incomplete", + "incomplete_details": ["reason": "max_output_tokens"], + "usage": ["input_tokens": 30, "output_tokens": 64], + ], + ], state: &state) + #expect(limits(events) == [.outputTokens]) + #expect(state.inputTokens == 30) + #expect(state.outputTokens == 64) + } + + @Test("A Responses stream cut by its content filter is not an output limit") + func responsesContentFilter() throws { + var state = ResponsesStreamState() + let events = try OpenAIResponsesProvider.parseEvent([ + "type": "response.incomplete", + "response": ["incomplete_details": ["reason": "content_filter"]], + ], state: &state) + #expect(limits(events).isEmpty) + } + + @Test("Gemini finishReason MAX_TOKENS is reported, STOP is not", arguments: [ + ("MAX_TOKENS", [ChatOutputLimit.outputTokens]), + ("STOP", []), + ] as [(String, [ChatOutputLimit])]) + func geminiFinishReason(reason: String, expected: [ChatOutputLimit]) { + var state = GeminiStreamState() + let events = GeminiProvider.parseChunk([ + "candidates": [["content": ["parts": [["text": "partial"]]], "finishReason": reason]], + ], state: &state, idGenerator: { "id" }) + #expect(limits(events) == expected) + } + + @Test("Claude Code's passed-through message_delta reports the cut") + func claudeAgentStopReason() { + let line = #"{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"max_tokens"}}}"# + #expect(ClaudeAgentProvider.decodeEvent(line) == .outputLimit(.outputTokens)) + let ended = #"{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"end_turn"}}}"# + #expect(ClaudeAgentProvider.decodeEvent(ended) == nil) + } + + @Test("At the output limit the last call is denied even with no arguments yet, and complete calls before it stand") + func cutLastCallIsDenied() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "list_tables", "call_2": "execute_query"], + inputs: ["call_1": #"{"schema":"public"}"#, "call_2": ""], + outputLimit: .outputTokens + ) + #expect(blocks.count == 2) + #expect(blocks[0].approvalState == .approved) + #expect(blocks[0].input == .object(["schema": .string("public")])) + #expect(blocks[1].approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit cut arguments name the limit in the error the model gets") + func cutArgumentsNameTheLimit() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "execute_query"], + inputs: ["call_1": #"{"query":"SELECT * FR"#], + outputLimit: .outputTokens + ) + #expect(blocks.first?.approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit the last call is denied even when its arguments parse, as Gemini and Ollama hand them over") + func cutLastCallIsDeniedWithParsedArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "execute_query"], + inputs: ["call_1": "{}"], + outputLimit: .outputTokens + ) + #expect(blocks.first?.approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit a malformed call before the last keeps the plain reason") + func earlierMalformedCallKeepsThePlainReason() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "describe_table", "call_2": "list_tables"], + inputs: ["call_1": #"{"table":"#, "call_2": "{}"], + outputLimit: .outputTokens + ) + #expect(blocks[0].approvalState == .denied(reason: AIChatViewModel.unparsableToolArgumentsContent)) + #expect(blocks[1].approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("Without the limit a call with no arguments still runs with none") + func noLimitKeepsEmptyArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "list_tables"], + inputs: ["call_1": ""] + ) + #expect(blocks.first?.approvalState == .approved) + } + + @Test("A turn keeps its output limit through the saved conversation, and an unknown value reads as none") + func outputLimitPersists() throws { + let wire = ChatTurnWire(role: .assistant, blocks: [.text("cut")], outputLimit: .contextWindow) + let data = try JSONEncoder().encode(wire) + #expect(try JSONDecoder().decode(ChatTurnWire.self, from: data).outputLimit == .contextWindow) + + var object = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + object["outputLimit"] = "somethingNew" + let future = try JSONSerialization.data(withJSONObject: object) + let decoded = try JSONDecoder().decode(ChatTurnWire.self, from: future) + #expect(decoded.outputLimit == nil) + #expect(decoded.plainText == "cut") + } +} diff --git a/apple/TableProTests/Core/AI/ToolApprovalCenterTests.swift b/apple/TableProTests/Core/AI/ToolApprovalCenterTests.swift index 1e309c3db8..84ea1808dd 100644 --- a/apple/TableProTests/Core/AI/ToolApprovalCenterTests.swift +++ b/apple/TableProTests/Core/AI/ToolApprovalCenterTests.swift @@ -125,6 +125,25 @@ struct ToolApprovalCenterTests { #expect(center.hasPending == false) } + @Test("A call is awaited from its announcement until its turn lets it go, and never in another session") + func isAwaitingFollowsTheTurn() async { + let center = ToolApprovalCenter() + #expect(!center.isAwaiting(sessionId: session, toolUseId: "call_0")) + + center.expect(sessionId: session, toolUseIds: ["call_0"]) + #expect(center.isAwaiting(sessionId: session, toolUseId: "call_0")) + #expect(!center.isAwaiting(sessionId: UUID(), toolUseId: "call_0")) + + let waiter = Task { await center.awaitDecision(sessionId: session, toolUseId: "call_0") } + await runQueuedMainActorWork() + center.forget(sessionId: session, toolUseIds: ["call_0"]) + #expect(center.isAwaiting(sessionId: session, toolUseId: "call_0")) + + center.resolve(sessionId: session, toolUseId: "call_0", decision: .run) + _ = await waiter.value + #expect(!center.isAwaiting(sessionId: session, toolUseId: "call_0")) + } + /// The main queue runs its jobs in order, so a job queued now runs after every waiter the case /// started has reached its await. private func runQueuedMainActorWork() async { diff --git a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift index 414aacc08c..c7b1794182 100644 --- a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift +++ b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderFallbackTests.swift @@ -59,6 +59,8 @@ private final class MockFallbackDriver: DatabaseDriver, @unchecked Sendable { return columnsPerTable } + var fetchesAllColumnsInOneRequest: Bool { true } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } func fetchApproximateRowCount(table: String) async throws -> Int? { nil } @@ -536,4 +538,62 @@ struct SQLSchemaProviderFallbackTests { // Documentation should reference the canonical table name "Users" #expect(items[0].documentation?.contains("Users") == true) } + + // MARK: - Built once per cache + + // Every keystroke before a FROM asks again; at 300 tables of 40 columns each request rebuilt 12,000 items. + @Test("Completions before a FROM reuse the column items while the cache and table list stand") + func columnItemsAreBuiltOncePerCache() async { + let driver = MockFallbackDriver() + var tables: [TableInfo] = [] + for table in 0..<30 { + let name = "table_\(table)" + tables.append(TestFixtures.makeTableInfo(name: name)) + driver.columnsPerTable[name] = (0..<10).map { + TestFixtures.makeColumnInfo(name: "col_\(table)_\($0)", isPrimaryKey: false) + } + } + driver.tablesToReturn = tables + + let provider = SQLSchemaProvider() + await provider.loadSchema(using: driver, connection: TestFixtures.makeConnection()) + await populateCache(provider: provider, driver: driver) + + let first = await provider.allColumnsFromCachedTables().map(\.label) + for _ in 0..<50 { + let again = await provider.allColumnsFromCachedTables().map(\.label) + #expect(again == first) + } + #expect(first.count == 300) + #expect(await provider.tableColumnItemsBuildCount == 1) + } + + @Test("A newly cached table, a new table list and a cleared cache each reach the column items") + func columnItemsFollowTheCacheAndTableList() async { + let driver = MockFallbackDriver() + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "users"), TestFixtures.makeTableInfo(name: "orders")] + driver.columnsPerTable = [ + "users": [TestFixtures.makeColumnInfo(name: "id")], + "orders": [TestFixtures.makeColumnInfo(name: "total", dataType: "DECIMAL", isPrimaryKey: false)] + ] + let provider = SQLSchemaProvider() + await provider.loadSchema(using: driver, connection: TestFixtures.makeConnection()) + _ = await provider.getColumns(for: "users") + let usersOnly = await provider.allColumnsFromCachedTables().map(\.label) + #expect(usersOnly == ["id"]) + + _ = await provider.getColumns(for: "orders") + let both = Set(await provider.allColumnsFromCachedTables().map(\.label)) + #expect(both == ["id", "total"]) + + await provider.updateTables([TestFixtures.makeTableInfo(name: "Users"), TestFixtures.makeTableInfo(name: "orders")]) + let renamed = await provider.allColumnsFromCachedTables() + let idDocumentation = renamed.first { $0.label == "id" }?.documentation + #expect(idDocumentation?.contains("Users") == true) + + await provider.clearColumnCache() + let cleared = await provider.allColumnsFromCachedTables() + #expect(cleared.isEmpty) + #expect(await provider.tableColumnItemsBuildCount == 3) + } } diff --git a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift index ad9c23b09f..ea12aa4994 100644 --- a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift +++ b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift @@ -123,15 +123,23 @@ final class MockDatabaseDriver: DatabaseDriver, SchemaSwitchable, @unchecked Sen } } + // Runs before the timeout is recorded, for a test that makes the statement wait on the connection's queue. + var applyQueryTimeoutGate: (@Sendable () async -> Void)? + func applyQueryTimeout(_ seconds: Int) async throws { + await applyQueryTimeoutGate?() applyQueryTimeoutValues.append(seconds) } // A statement holding this text fails, the way a server rejects one statement of a script. var failsStatementContaining: (text: String, error: any Error)? + // Holds `execute` until the test opens it, whether or not the task awaiting it is cancelled. + var executeGate: AsyncGate? + func execute(query: String) async throws -> QueryResult { executedQueries.append(query) + await executeGate?.wait() if let failure = failsStatementContaining, query.contains(failure.text) { throw failure.error } if executeDelaySeconds > 0 { try await Task.sleep(nanoseconds: UInt64(executeDelaySeconds * 1_000_000_000)) @@ -308,6 +316,9 @@ final class MockDatabaseDriver: DatabaseDriver, SchemaSwitchable, @unchecked Sen return allColumnsToReturn } + var oneRequestColumnRead = true + var fetchesAllColumnsInOneRequest: Bool { oneRequestColumnRead } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } func fetchApproximateRowCount(table: String) async throws -> Int? { nil } @@ -387,11 +398,15 @@ final class MockDatabaseDriver: DatabaseDriver, SchemaSwitchable, @unchecked Sen func commitTransaction() async throws { commitTransactionCallCount += 1 } func rollbackTransaction() async throws {} + // A switch to this schema fails, the way a server refuses a schema it does not have. + var switchSchemaError: (schema: String, error: any Error)? + func switchSchema(to schema: String) async throws { await onSwitchSchema?() if switchSchemaDelaySeconds > 0 { try await Task.sleep(nanoseconds: UInt64(switchSchemaDelaySeconds * 1_000_000_000)) } + if let failure = switchSchemaError, failure.schema == schema { throw failure.error } switchSchemaCallCount += 1 currentSchema = schema } @@ -888,6 +903,54 @@ struct SQLSchemaProviderTests { #expect(driver.fetchColumnsCallCount == 0) } + @Test("Only a driver that reads every table's columns in one request preloads them", arguments: [true, false]) + func preloadNeedsAOneRequestColumnRead(oneRequest: Bool) async { + let driver = MockDatabaseDriver() + driver.oneRequestColumnRead = oneRequest + driver.allColumnsToReturn = ["users": [TestFixtures.makeColumnInfo(name: "id")]] + driver.columnsToReturn = ["users": [TestFixtures.makeColumnInfo(name: "id")]] + let provider = SQLSchemaProvider() + await provider.resetForDatabase("db", tables: [TestFixtures.makeTableInfo(name: "users")], driver: driver) + + await provider.requestColumnPreload() + await provider.waitForEagerColumnLoad() + await provider.requestColumnPreload() + _ = await provider.allColumnsFromCachedTables() + await provider.waitForEagerColumnLoad() + + #expect(driver.fetchAllColumnsCallCount == (oneRequest ? 1 : 0)) + let columns = await provider.getColumns(for: "users") + #expect(columns.map { $0.name } == ["id"]) + #expect(driver.fetchColumnsCallCount == (oneRequest ? 0 : 1)) + } + + @Test("A source asks for every column only when the driver that filled the scope reads them in one request", arguments: [true, false]) + func sourcePreloadFollowsTheFillingDriver(oneRequest: Bool) async { + let recorder = CallRecorder() + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { table, _ in + recorder.record("columns.\(table)") + return [TestFixtures.makeColumnInfo(name: "id")] + }, + fetchAllColumns: { + recorder.record("allColumns") + return ["users": [TestFixtures.makeColumnInfo(name: "id")]] + } + ) + let driver = MockDatabaseDriver() + driver.oneRequestColumnRead = oneRequest + let provider = SQLSchemaProvider(metadataSource: source) + await provider.resetForDatabase("db", tables: [TestFixtures.makeTableInfo(name: "users")], driver: driver) + + await provider.requestColumnPreload() + await provider.waitForEagerColumnLoad() + let columns = await provider.getColumns(for: "users") + + #expect(recorder.calls == (oneRequest ? ["allColumns"] : ["columns.users"])) + #expect(columns.map { $0.name } == ["id"]) + #expect(driver.fetchAllColumnsCallCount == 0) + } + @Test("A table-list write after a preload sends nothing until the next request") func tableListWriteAfterPreloadWaitsForTheNextRequest() async { let driver = MockDatabaseDriver() @@ -1253,3 +1316,127 @@ private final class CallRecorder: @unchecked Sendable { return storage } } + +private struct ForeignKeyReadFailure: Error {} + +extension SQLSchemaProviderTests { + nonisolated private static let orderItemKey = [ + ForeignKeyInfo(name: "items_order_fk", column: "order_id", referencedTable: "orders", referencedColumn: "id"), + ForeignKeyInfo(name: "items_order_fk", column: "tenant_id", referencedTable: "orders", referencedColumn: "tenant_id"), + ] + + private static func keySource( + recorder: CallRecorder, + read: (@Sendable () async throws -> [ForeignKeyInfo])? = nil + ) -> SQLSchemaProvider.ColumnMetadataSource { + SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { _, _ in [] }, + fetchAllColumns: { [:] }, + fetchForeignKeys: { table, _ in + recorder.record(table) + return try await read?() ?? Self.orderItemKey + } + ) + } + + @Test("A table's foreign keys are read once across keystrokes, a composite key as one") + func foreignKeysReadOnce() async { + let recorder = CallRecorder() + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder)) + + let first = await provider.foreignKeys(of: "order_items") + _ = await provider.foreignKeys(of: "order_items") + _ = await provider.foreignKeys(of: "ORDER_ITEMS") + + #expect(first.count == 1) + #expect(first.first?.columns == ["order_id", "tenant_id"]) + #expect(first.first?.referencedColumns == ["id", "tenant_id"]) + #expect(recorder.calls == ["order_items"]) + } + + @Test("Concurrent requests for one table's foreign keys share one read") + func foreignKeysShareTheReadInFlight() async { + let recorder = CallRecorder() + let gate = AsyncGate() + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder) { + await gate.wait() + return Self.orderItemKey + }) + + let first = Task { await provider.foreignKeys(of: "order_items") } + await gate.arrived(1) + let second = Task { await provider.foreignKeys(of: "order_items") } + await gate.open() + + #expect(await first.value.count == 1) + #expect(await second.value.count == 1) + #expect(recorder.calls == ["order_items"]) + } + + @Test("resetForDatabase drops the foreign keys, so the next request reads again") + func resetClearsForeignKeys() async { + let recorder = CallRecorder() + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder)) + + _ = await provider.foreignKeys(of: "order_items") + await provider.resetForDatabase( + "db", tables: [TestFixtures.makeTableInfo(name: "order_items")], driver: MockDatabaseDriver() + ) + _ = await provider.foreignKeys(of: "order_items") + + #expect(recorder.calls == ["order_items", "order_items"]) + } + + @Test("A failed foreign key read is kept as no keys until the table list is read again") + func failedForeignKeyReadIsKeptUntilTheTablesAreReadAgain() async { + let recorder = CallRecorder() + let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .postgresql)) + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder) { + guard recorder.calls.count > 1 else { throw ForeignKeyReadFailure() } + return Self.orderItemKey + }) + + let failed = await provider.foreignKeys(of: "order_items") + let kept = await provider.foreignKeys(of: "order_items") + await provider.resetForDatabase("db", tables: [], driver: driver, connection: driver.connection) + let reread = await provider.foreignKeys(of: "order_items") + + #expect(failed.isEmpty) + #expect(kept.isEmpty) + #expect(reread.count == 1) + #expect(recorder.calls == ["order_items", "order_items"]) + } + + @Test("An engine without foreign keys is never asked for them") + func noForeignKeyReadWithoutSupport() async { + let recorder = CallRecorder() + let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .mongodb)) + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder)) + await provider.resetForDatabase("db", tables: [], driver: driver, connection: driver.connection) + + #expect(await provider.foreignKeys(of: "orders").isEmpty) + #expect(recorder.calls.isEmpty) + } + + @Test("A join's ON offers the condition from the cached keys") + func joinConditionThroughTheProvider() async { + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { table, _ in + table == "orders" + ? [TestFixtures.makeColumnInfo(name: "id"), TestFixtures.makeColumnInfo(name: "tenant_id")] + : [TestFixtures.makeColumnInfo(name: "order_id"), TestFixtures.makeColumnInfo(name: "tenant_id")] + }, + fetchAllColumns: { [:] }, + fetchForeignKeys: { table, _ in table == "order_items" ? Self.orderItemKey : [] } + ) + let schemaProvider = SQLSchemaProvider(metadataSource: source) + let provider = SQLCompletionProvider(schemaProvider: schemaProvider) + let text = "SELECT * FROM orders o JOIN order_items i ON " + + let (items, _) = await provider.getCompletions(text: text, cursorPosition: (text as NSString).length) + + #expect(items.first?.label == "o.id = i.order_id AND o.tenant_id = i.tenant_id") + #expect(items.first?.kind == .joinCondition) + #expect(items.filter { $0.label == "o.id" }.count == 1) + } +} diff --git a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift index 5abc551188..fff4ca6e57 100644 --- a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift @@ -249,4 +249,359 @@ struct UpdateSaveOrderTests { #expect(pending.change(forRow: .existing(0), type: .update)?.sequence == sequence) #expect(pending.isCellModified(rowID: .existing(0), columnIndex: 1)) } + + @Test("A row edited again after another row's delete is written after that delete") + func laterEditRunsAfterTheDeleteBeforeIt() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + let statements = try manager.generateSQL() + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + try #require(statements.count == 3) + #expect(EditAroundADelete.strings(statements[0]) == ["Cyd", "3"]) + #expect(EditAroundADelete.strings(statements[1]) == ["2"]) + #expect(EditAroundADelete.strings(statements[2]) == ["b@test.com", "3"]) + } + + @Test("A swap through a temporary value saves each step in the order it was made") + func swapThroughATemporaryValueSavesInOrder() throws { + let (manager, _) = EditAroundADelete.manager() + let rowA: [PluginCellValue] = ["1", "Al", "a@test.com"] + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "tmp", originalRow: rowA + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "a@test.com", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "tmp", newValue: "b@test.com", originalRow: ["1", "Al", "tmp"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "UPDATE", "UPDATE"]) + #expect(bound == [["tmp", "1"], ["a@test.com", "2"], ["b@test.com", "1"]]) + } + + @Test("A later edit of a row keys on the key an earlier edit of it wrote") + func laterRunKeysOnTheKeyTheEarlierRunWrote() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", + oldValue: "3", newValue: "6", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: ["6", "Cy", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound == [["6", "3"], ["2"], ["Cyd", "6"]]) + } + + @Test("A tab switch keeps the place of every edit of a row, so a later change sorts after them") + func aTabSwitchKeepsEveryRunsPlace() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + let state = manager.saveState() + + manager.restoreState(from: state, tableName: "users", databaseType: .mysql, generatedColumns: []) + manager.recordRowInsertion(rowID: .inserted(UUID()), values: ["9", "Di", "d@test.com"]) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE", "UPDATE", "INSERT"]) + } + + @Test("Undoing the delete of a row edited around another delete brings back both of its edits in place") + func undoingADeleteBringsBackEveryRun() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + manager.recordRowDeletion(rowID: .existing(2), originalRow: ["3", "Cyd", "b@test.com"]) + + manager.undoRowDeletion(rowID: .existing(2)) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE", "UPDATE"]) + } + + @Test("Two edits of a row with nothing between them stay one UPDATE") + func editsWithNothingBetweenStayOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(bound.first == ["Cyd", "x@test.com", "3"]) + } + + @Test("A cell typed back to its value as read is not written, whatever came between") + func aCellTypedBackToItsValueAsReadIsNotWritten() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "x@test.com", newValue: "c@test.com", originalRow: ["3", "Cy", "x@test.com"] + ) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["DELETE"]) + } + + @Test("Edits of a row around a change that was then cancelled are one UPDATE") + func aChangeCancelledBetweenRunsLeavesOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.undoRowDeletion(rowID: .existing(1)) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE"]) + #expect(bound == [["Cyd", "x@test.com", "3"]]) + } + + @Test("A row whose key an earlier edit set to SQL stays one UPDATE keyed on the key as read") + func aKeySetToSQLBeforeAnotherChangeKeepsTheRowWhole() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", oldValue: "3", newValue: "NOW()", + originalRow: EditAroundADelete.rowC, absence: FieldAbsence(), isExpression: true + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: ["NOW()", "Cy", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(statements.first?.sql.contains("= NOW()") == true) + #expect(bound.first == ["Cyd", "3"]) + } + + @Test("A key set to SQL by a row's last edit still saves that edit in its own place") + func aKeySetToSQLInTheLastEditStillSplits() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", oldValue: "3", newValue: "NOW()", + originalRow: ["3", "Cyd", "c@test.com"], absence: FieldAbsence(), isExpression: true + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound == [["Cyd", "3"], ["2"], ["3"]]) + } + + @Test("A row of a table with no primary key stays one UPDATE at its first edit's place") + func aKeylessRowStaysOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager(primaryKeyColumns: []) + EditAroundADelete.record(on: manager) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE"]) + } +} + +@MainActor +struct UpdateRunActionTests { + @Test("One command over several rows writes each row once, whatever its other rows' changes") + func oneCommandOverSeveralRowsIsOneUpdatePerRow() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordRowDeletion(rowID: .existing(3), originalRow: ["4", "Di", "d@test.com"]) + manager.recordAsOneAction { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "Alf", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "Bob", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "alf@test.com", originalRow: ["1", "Alf", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "bob@test.com", originalRow: ["2", "Bob", "b@test.com"] + ) + } + manager.recordRowDeletion(rowID: .existing(4), originalRow: ["5", "Ed", "e@test.com"]) + + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["DELETE", "UPDATE", "UPDATE", "DELETE"]) + } + + @Test("Typing into several rows at once writes each row once") + func typingIntoSeveralRowsIsOneUpdatePerRow() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordTypedCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "A", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordTypedCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "A", originalRow: EditAroundADelete.rowB + ) + manager.recordTypedCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "A", newValue: "Ab", originalRow: ["1", "A", "a@test.com"] + ) + manager.endCoalescedUndoRun() + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "UPDATE"]) + } +} + +@MainActor +struct UpdateRunUndoTests { + @Test("Redoing a row's later edit puts it back after the delete it followed") + func redoPutsTheLaterEditBackAfterTheDelete() throws { + let (manager, undoManager) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + undoManager.undo() + undoManager.redo() + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound.last == ["b@test.com", "3"]) + } + + @Test("Undoing a row's later edit leaves its earlier one where it was") + func undoingTheLaterEditLeavesTheEarlierUpdate() throws { + let (manager, undoManager) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + undoManager.undo() + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(bound.first == ["Cyd", "3"]) + } + + @Test("Undoing and redoing a command over two rows still writes each row once") + func undoAndRedoOfACommandKeepOneUpdatePerRow() throws { + let (manager, undoManager) = EditAroundADelete.manager() + manager.recordRowDeletion(rowID: .existing(3), originalRow: ["4", "Di", "d@test.com"]) + undoManager.beginUndoGrouping() + manager.recordAsOneAction { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "Alf", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "Bob", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "alf@test.com", originalRow: ["1", "Alf", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "bob@test.com", originalRow: ["2", "Bob", "b@test.com"] + ) + } + undoManager.endUndoGrouping() + + undoManager.undo() + let undone = try EditAroundADelete.verbs(manager.generateSQL()) + undoManager.redo() + let redone = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(undone == ["DELETE"]) + #expect(redone == ["DELETE", "UPDATE", "UPDATE"]) + } +} + +// Edit C, delete B, then give C the email B freed: each step is valid on a unique `email`. +@MainActor +enum EditAroundADelete { + static let columns = ["id", "name", "email"] + static let rowB: [PluginCellValue] = ["2", "Bo", "b@test.com"] + static let rowC: [PluginCellValue] = ["3", "Cy", "c@test.com"] + + static func manager( + table: String = "users", + primaryKeyColumns: [String] = ["id"], + databaseType: DatabaseType = .mysql + ) -> (DataChangeManager, UndoManager) { + let manager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + manager.undoManagerProvider = { undoManager } + manager.configureForTable( + tableName: table, + columns: columns, + primaryKeyColumns: primaryKeyColumns, + databaseType: databaseType, + generatedColumns: [] + ) + return (manager, undoManager) + } + + static func record(on manager: DataChangeManager) { + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "b@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + } + + nonisolated static func verbs(_ statements: [ParameterizedStatement]) -> [String] { + statements.map { String($0.sql.prefix(while: { $0 != " " })) } + } + + nonisolated static func strings(_ statement: ParameterizedStatement) -> [String] { + statement.parameters.compactMap { $0 as? String } + } } diff --git a/apple/TableProTests/Core/ChangeTracking/RowEditUndoStepTests.swift b/apple/TableProTests/Core/ChangeTracking/RowEditUndoStepTests.swift new file mode 100644 index 0000000000..d49243c64e --- /dev/null +++ b/apple/TableProTests/Core/ChangeTracking/RowEditUndoStepTests.swift @@ -0,0 +1,115 @@ +// +// RowEditUndoStepTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct RowEditUndoStepTests { + private static let columns = ["code", "status"] + + private func makeManager() -> (DataChangeManager, UndoManager) { + let manager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + manager.undoManagerProvider = { undoManager } + manager.configureForTable( + tableName: "tickets", + columns: Self.columns, + primaryKeyColumns: ["code"], + databaseType: .postgresql, + generatedColumns: [] + ) + return (manager, undoManager) + } + + @Test("Undoing a cell typed back to its value stages the row as it was read") + func undoAfterTypingBackKeepsTheRowAsRead() throws { + let (manager, undoManager) = makeManager() + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "status", + oldValue: "open", newValue: "closed", originalRow: ["A1", "open"] + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "status", + oldValue: "closed", newValue: "open", originalRow: ["A1", "closed"] + ) + #expect(manager.pendingUpdate(forRow: .existing(0)) == nil) + + undoManager.undo() + + let update = try #require(manager.pendingUpdate(forRow: .existing(0))) + #expect(update.originalRow == ["A1", "open"]) + #expect(update.cellChanges.first?.newValue == "closed") + } + + @Test("Undoing a key typed back keys the UPDATE on the key as read") + func undoAfterTypingTheKeyBackKeysOnTheReadKey() throws { + let (manager, undoManager) = makeManager() + manager.recordCellChange( + rowID: .existing(0), columnIndex: 0, columnName: "code", + oldValue: "A1", newValue: "A2", originalRow: ["A1", "open"] + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 0, columnName: "code", + oldValue: "A2", newValue: "A1", originalRow: ["A2", "open"] + ) + + undoManager.undo() + + let update = try #require(manager.pendingUpdate(forRow: .existing(0))) + #expect(update.originalRow == ["A1", "open"]) + let statement = try #require(try manager.generateSQL().first) + let parameters = statement.parameters.compactMap { $0 as? String } + #expect(parameters == ["A2", "A1"]) + } + + @Test("Undo Delete from the row menu is its own step, so Undo deletes the row again and then restores it") + func restoringADeletedRowIsUndoable() { + let (manager, undoManager) = makeManager() + manager.recordRowDeletion(rowID: .existing(0), originalRow: ["A1", "open"]) + + manager.restoreDeletedRows([.existing(0)]) + #expect(!manager.isRowDeleted(.existing(0))) + #expect(undoManager.undoActionName == String(localized: "Restore Row")) + + undoManager.undo() + #expect(manager.isRowDeleted(.existing(0))) + #expect(undoManager.redoActionName == String(localized: "Restore Row")) + #expect(undoManager.undoActionName == String(localized: "Delete Row")) + + undoManager.undo() + #expect(!manager.isRowDeleted(.existing(0))) + #expect(!undoManager.canUndo) + #expect(!manager.hasChanges) + + undoManager.redo() + #expect(manager.isRowDeleted(.existing(0))) + undoManager.redo() + #expect(!manager.isRowDeleted(.existing(0))) + } + + @Test("Restoring several deleted rows is one step over all of them") + func restoringSeveralRowsIsOneStep() { + let (manager, undoManager) = makeManager() + manager.recordBatchRowDeletion(rows: [ + (rowID: .existing(0), originalRow: ["A1", "open"]), + (rowID: .existing(1), originalRow: ["A2", "open"]), + ]) + + manager.restoreDeletedRows([.existing(0), .existing(1)]) + #expect(manager.deletedRowIDs.isEmpty) + #expect(undoManager.undoActionName == String(localized: "Restore Rows")) + + undoManager.undo() + #expect(manager.deletedRowIDs == [.existing(0), .existing(1)]) + + undoManager.undo() + #expect(manager.deletedRowIDs.isEmpty) + #expect(!undoManager.canUndo) + } +} diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBatchDeleteScaleTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBatchDeleteScaleTests.swift index 2c13f6db07..9621e1b4ed 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBatchDeleteScaleTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBatchDeleteScaleTests.swift @@ -76,6 +76,25 @@ struct SQLStatementGeneratorBatchDeleteScaleTests { #expect(statements.reduce(0) { $0 + $1.parameters.count } == cap + 1) } + @Test("Cloudflare D1 deletes in statements of at most 100 parameters") + func cloudflareD1ChunksAtItsLimit() throws { + let generator = try makeGenerator( + columns: ["id", "name"], primaryKeyColumns: ["id"], databaseType: .cloudflareD1 + ) + let changes = deleteChanges(count: 150, columns: ["id", "name"]) + + let statements = generator.generateStatements( + from: changes, + insertedRowData: [:], + deletedRowIDs: Set((0..<150).map(RowID.existing)), + insertedRowIDs: [] + ) + + #expect(statements.count == 2) + #expect(statements.allSatisfy { $0.parameters.count <= 100 }) + #expect(statements.reduce(0) { $0 + $1.parameters.count } == 150) + } + @Test("Primary-key selection exactly at the cap stays one statement") func testPKSelectionExactlyAtCapSingleStatement() throws { let generator = try makeGenerator( diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift index d3d4c2d300..aed8b0f01c 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift @@ -92,8 +92,8 @@ struct SQLStatementGeneratorBitColumnTests { let rowID = RowID.inserted(UUID()) let statements = generator.generateStatements( - from: [RowChange(rowID: rowID, type: .insert)], - insertedRowData: [rowID: ["__DEFAULT__", "5", "Ann"]], + from: [RowChange(rowID: rowID, type: .insert, expressionColumns: [0])], + insertedRowData: [rowID: [.columnDefault, "5", "Ann"]], deletedRowIDs: [], insertedRowIDs: [rowID] ) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift index 6945aab201..b08b19563b 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorImportTests.swift @@ -100,6 +100,9 @@ struct SQLStatementGeneratorImportTests { #expect(try makeGenerator(databaseType: .sqlite).maxBindParameters == 32_766) #expect(try makeGenerator(databaseType: .mssql).maxBindParameters == 2_100) #expect(try makeGenerator(databaseType: .mysql).maxBindParameters == 65_535) + #expect(try makeGenerator(databaseType: .libsql).maxBindParameters == 32_766) + #expect(try makeGenerator(databaseType: .turso).maxBindParameters == 32_766) + #expect(try makeGenerator(databaseType: .cloudflareD1).maxBindParameters == 100) } @Test("keyMatchCountStatement asks one count per key, tagged with its position") diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorParameterTypeTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorParameterTypeTests.swift new file mode 100644 index 0000000000..2f9aed77eb --- /dev/null +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorParameterTypeTests.swift @@ -0,0 +1,119 @@ +// +// SQLStatementGeneratorParameterTypeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +// Oracle stored an imported `00123` in a VARCHAR2 as 123: the driver was never told which column each value was for. +struct SQLStatementGeneratorParameterTypeTests { + private static let columns = ["ID", "CODE", "BORN"] + private static let typeNames = ["ID": "NUMBER(10)", "CODE": "VARCHAR2(20 BYTE)", "BORN": "DATE"] + + private func makeGenerator( + primaryKeyColumns: [String] = ["ID"], + databaseType: DatabaseType = .postgresql, + typeNames: [String: String] = SQLStatementGeneratorParameterTypeTests.typeNames + ) throws -> SQLStatementGenerator { + try SQLStatementGenerator( + tableName: "people", + columns: Self.columns, + primaryKeyColumns: primaryKeyColumns, + databaseType: databaseType, + columnTypeNames: typeNames + ) + } + + @Test + func anInsertNamesTheColumnTypeOfEachValue() throws { + let statement = try #require(try makeGenerator().insertStatement(columns: ["CODE", "ID"], values: ["00123", "7"])) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aMultiRowInsertNamesTheTypesForEveryRow() throws { + let statement = try #require(try makeGenerator().insertStatement( + columns: ["ID", "CODE"], + rows: [["1", "00123"], ["2", "0.50"]] + )) + + #expect(statement.parameterTypeNames == ["NUMBER(10)", "VARCHAR2(20 BYTE)", "NUMBER(10)", "VARCHAR2(20 BYTE)"]) + } + + @Test + func aColumnOfNoKnownTypeLeavesItsEntryEmpty() throws { + let statement = try #require(try makeGenerator().insertStatement(columns: ["CODE", "NOTE"], values: ["a", "b"])) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", nil]) + } + + @Test + func noKnownTypeLeavesTheListOut() throws { + let generator = try makeGenerator(typeNames: [:]) + + let insert = try #require(generator.insertStatement(columns: ["CODE"], values: ["00123"])) + let match = try #require(generator.keyMatchCountStatement(keyColumns: ["ID"], keys: [["7"]])) + + #expect(insert.parameterTypeNames == nil) + #expect(match.parameterTypeNames == nil) + } + + @Test + func aKeyedUpdateNamesItsAssignmentsThenItsKey() throws { + let statement = try #require(try makeGenerator().keyedUpdateStatement( + columns: ["CODE", "BORN"], + values: ["00123", "2024-01-15"], + keyColumns: ["ID"], + keyValues: ["7"] + )) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "DATE", "NUMBER(10)"]) + } + + @Test + func aKeyMatchNamesTheKeyTypeForEveryKey() throws { + let statement = try #require(try makeGenerator().keyMatchCountStatement( + keyColumns: ["CODE", "ID"], + keys: [["00123", "1"], ["00456", "2"]] + )) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)", "VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aGridEditNamesItsNewValueAndItsKey() throws { + let change = RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "CODE", oldValue: "00100", newValue: "00123")], + originalRow: ["7", "00100", .null] + ) + + let statement = try #require(try makeGenerator().generateUpdateSQL(for: change)) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aBitValueCastToANumberHasNoColumnType() throws { + let generator = try makeGenerator(databaseType: .mysql, typeNames: ["ID": "INT", "CODE": "BIT(8)"]) + + let statement = try #require(generator.insertStatement(columns: ["CODE", "ID"], values: ["5", "1"])) + + #expect(statement.sql == "INSERT INTO `people` (`CODE`, `ID`) VALUES (CAST(? AS UNSIGNED), ?)") + #expect(statement.parameterTypeNames == [nil, "INT"]) + } + + @Test + func theStatementContextCarriesTheTypesAndNothingElse() throws { + let statement = try #require(try makeGenerator().insertStatement(columns: ["CODE"], values: ["00123"])) + var expected = PluginStatementContext() + expected.parameterTypeNames = ["VARCHAR2(20 BYTE)"] + + #expect(statement.context == expected) + } +} diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift index 44d6a863ef..ea1bbb0f87 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift @@ -62,10 +62,12 @@ struct SQLStatementGeneratorRowMatchTests { @Test("A Databend row left entirely on defaults names one column with DEFAULT") func databendAllDefaultsInsert() throws { - let change = RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + let change = RowChange( + rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0, 1, 2, 3] + ) let statements = try generator(excluding: []).generateStatements( from: [change], - insertedRowData: [.existing(0): [.text("__DEFAULT__"), .text("__DEFAULT__"), .text("__DEFAULT__"), .text("__DEFAULT__")]], + insertedRowData: [.existing(0): [.columnDefault, .columnDefault, .columnDefault, .columnDefault]], deletedRowIDs: [], insertedRowIDs: [.existing(0)] ) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift index edeb219dea..731615a286 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift @@ -89,14 +89,14 @@ struct SQLStatementGeneratorTests { #expect(statements[0].parameters[2] == nil) } - @Test("Insert skips __DEFAULT__ columns") + @Test("Insert leaves out a column marked as its default") func testInsertSkipsDefaultColumns() throws { let generator = try makeGenerator() let insertedRowData: [RowID: [PluginCellValue]] = [ - .existing(0): ["__DEFAULT__", "John", "john@example.com"] + .existing(0): [.columnDefault, "John", "john@example.com"] ] let changes: [RowChange] = [ - RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0]) ] let statements = generator.generateStatements( @@ -114,11 +114,35 @@ struct SQLStatementGeneratorTests { #expect(stmt.parameters.count == 2) } + @Test("Insert binds text that only reads like the default, unmarked", arguments: ["__DEFAULT__", "DEFAULT"]) + func testInsertBindsUnmarkedDefaultText(text: String) throws { + let generator = try makeGenerator() + let insertedRowData: [RowID: [PluginCellValue]] = [ + .existing(0): [.columnDefault, .text(text), "john@example.com"] + ] + let changes: [RowChange] = [ + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0]) + ] + + let statements = generator.generateStatements( + from: changes, + insertedRowData: insertedRowData, + deletedRowIDs: [], + insertedRowIDs: [.existing(0)] + ) + + #expect(statements.count == 1) + let stmt = statements[0] + #expect(!stmt.sql.contains("`id`")) + #expect(stmt.sql.contains("`name`")) + #expect(stmt.parameters.compactMap { $0 as? String } == [text, "john@example.com"]) + } + /// This used to expect no statement at all, which dropped the row from the batch while the rest /// of the save committed and reported success. A table of nothing but an identity column and /// defaults is exactly the shape that produces it. @Test( - "Insert with all __DEFAULT__ names no column", + "Insert with every column marked as its default names no column", arguments: [ (DatabaseType.mysql, "() VALUES ()"), (DatabaseType.postgresql, "DEFAULT VALUES"), @@ -128,10 +152,10 @@ struct SQLStatementGeneratorTests { func testInsertAllDefaultNamesNoColumn(databaseType: DatabaseType, expected: String) throws { let generator = try makeGenerator(databaseType: databaseType) let insertedRowData: [RowID: [PluginCellValue]] = [ - .existing(0): ["__DEFAULT__", "__DEFAULT__", "__DEFAULT__"] + .existing(0): [.columnDefault, .columnDefault, .columnDefault] ] let changes: [RowChange] = [ - RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0, 1, 2]) ] let statements = generator.generateStatements( @@ -378,7 +402,7 @@ struct SQLStatementGeneratorTests { #expect(statements[0].parameters[0] == nil) } - @Test("Update with __DEFAULT__ value") + @Test("Update with a cell marked as its default") func testUpdateWithDefaultValue() throws { let generator = try makeGenerator() let changes: [RowChange] = [ @@ -386,7 +410,10 @@ struct SQLStatementGeneratorTests { rowID: .existing(0), type: .update, cellChanges: [ - CellChange(columnIndex: 1, columnName: "name", oldValue: "John", newValue: "__DEFAULT__") + CellChange( + columnIndex: 1, columnName: "name", oldValue: "John", newValue: .columnDefault, + newIsExpression: true + ) ], originalRow: ["1", "John", "john@example.com"] ) @@ -405,6 +432,32 @@ struct SQLStatementGeneratorTests { #expect(stmt.parameters.count == 1) } + @Test("Update binds typed text that only reads like the default", arguments: ["__DEFAULT__", "DEFAULT"]) + func testUpdateBindsUnmarkedDefaultText(text: String) throws { + let generator = try makeGenerator() + let changes: [RowChange] = [ + RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "name", oldValue: "John", newValue: .text(text))], + originalRow: ["1", "John", "john@example.com"] + ) + ] + + let statements = generator.generateStatements( + from: changes, + insertedRowData: [:], + deletedRowIDs: [], + insertedRowIDs: [] + ) + + #expect(statements.count == 1) + let stmt = statements[0] + #expect(stmt.sql.contains("`name` = ?")) + #expect(!stmt.sql.contains("DEFAULT")) + #expect(stmt.parameters.compactMap { $0 as? String } == [text, "1"]) + } + @Test("Update with a cell marked as SQL inlines it") func testUpdateWithSQLFunction() throws { let generator = try makeGenerator() diff --git a/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift b/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift new file mode 100644 index 0000000000..1d8668db4f --- /dev/null +++ b/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift @@ -0,0 +1,90 @@ +// +// CompareDriverRowCutTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +struct CompareDriverRowCutTests { + private func stream(rowLimit: Int?, ids: [Int]) -> AsyncThrowingStream { + var header = PluginStreamHeader(columns: ["id", "name"], columnTypeNames: ["integer", "text"]) + header.rowLimit = rowLimit + let rows: [PluginRow] = ids.map { [.text(String($0)), .text("n")] } + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + private func targetRows(_ ids: [Int]) -> [DataRow] { + ids.map { DataRow(values: ["id": .text(String($0)), "name": .text("n")]) } + } + + private func engine() -> DataDiffEngine { + DataDiffEngine( + options: DataCompareOptions(), + shape: DataComparisonShape( + keyColumns: ["id"], + keyOrders: [.numeric], + comparedColumns: ["name"], + digestColumns: ["id", "name"] + ) + ) + } + + @Test("A stream that ends at the limit its driver declares reads as cut there") + func streamEndingAtTheDeclaredLimitIsCut() async throws { + let provider = StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])) + while try await provider.nextRow() != nil {} + + #expect(provider.endedAtRowLimit) + #expect(provider.driverRowCut == 2) + } + + @Test("A stream that ends short of its declared limit read every row") + func streamShortOfTheDeclaredLimitIsWhole() async throws { + let provider = StreamingRowProvider(stream: stream(rowLimit: 5, ids: [1, 2])) + while try await provider.nextRow() != nil {} + + #expect(!provider.endedAtRowLimit) + #expect(provider.driverRowCut == nil) + } + + // A HANA source stops at 5,000,000 rows: every target row past them read as a delete, and a sync removed them. + @Test("A source the driver cut short reports no deletes past the cut") + func cutSourceProducesNoDeletesPastIt() async throws { + let summary = try await engine().compare( + source: StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])), + target: ArrayRowProvider(rows: targetRows([1, 2, 3, 4])) + ) + + #expect(summary.deleteCount == 0) + #expect(summary.identicalCount == 2) + #expect(summary.stoppedAtRowLimit) + #expect(summary.sourceRowCut == 2) + #expect(summary.targetRowCut == nil) + #expect(summary.resumeKey == [.text("2")]) + } + + @Test("A target the driver cut short reports no inserts past the cut") + func cutTargetProducesNoInsertsPastIt() async throws { + let summary = try await engine().compare( + source: ArrayRowProvider(rows: targetRows([1, 2, 3, 4])), + target: StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])) + ) + + #expect(summary.insertCount == 0) + #expect(summary.targetRowCut == 2) + #expect(summary.stoppedAtRowLimit) + } + + @Test("The cut is named in the wording the export uses") + func cutWordingMatchesTheExport() { + #expect(StreamRowCutWarning.text(table: "orders", limit: 5_000_000).hasPrefix("orders: ")) + } +} diff --git a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift index d6ba71ac85..d662447ad7 100644 --- a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift +++ b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift @@ -48,23 +48,34 @@ struct CompareSQLLiteralTests { @Test func mysqlBitIsWrittenAsItsNumber() { let driver = NoBackslashEscapesDriver() - let bit = ColumnType.boolean(rawType: "bit(8)") + let bits = ColumnTypeClassifier(family: .mysql).classify(rawTypeName: "BIT(8)") + #expect(bits == .integer(rawType: "BIT(8)")) for type in [DatabaseType.mysql, .mariadb] { - #expect(CompareSQLLiteral.textLiteral("5", columnType: bit, databaseType: type, driver: driver) == "5") + #expect(CompareSQLLiteral.textLiteral("5", columnType: bits, databaseType: type, driver: driver) == "5") #expect(CompareSQLLiteral.literal( - for: .text("170"), columnType: bit, databaseType: type, driver: driver, keptInScript: false + for: .text("170"), columnType: bits, databaseType: type, driver: driver, keptInScript: false ) == "170") } - #expect(CompareSQLLiteral.textLiteral("abc", columnType: bit, databaseType: .mysql, driver: driver) == "'abc'") - #expect(CompareSQLLiteral.textLiteral("+5", columnType: bit, databaseType: .mysql, driver: driver) == "'+5'") + #expect(CompareSQLLiteral.textLiteral("abc", columnType: bits, databaseType: .mysql, driver: driver) == "'abc'") + #expect(CompareSQLLiteral.textLiteral("+5", columnType: bits, databaseType: .mysql, driver: driver) == "'+5'") + #expect(CompareSQLLiteral.textLiteral("-1", columnType: bits, databaseType: .mysql, driver: driver) == "-1") + } + + @Test + func mysqlOneBitFlagIsWrittenAsItsNumber() { + let driver = NoBackslashEscapesDriver() + let flag = ColumnTypeClassifier(family: .mysql).classify(rawTypeName: "BIT") + #expect(flag.isBooleanType) + #expect(CompareSQLLiteral.textLiteral("1", columnType: flag, databaseType: .mysql, driver: driver) == "1") + #expect(CompareSQLLiteral.textLiteral("-1", columnType: flag, databaseType: .mysql, driver: driver) == "'-1'") } @Test func otherBooleansAndBitStringsStayQuoted() { let driver = NoBackslashEscapesDriver() - let tinyint = ColumnType.boolean(rawType: "tinyint(1)") + let tinyint = ColumnTypeClassifier(family: .mysql).classify(rawTypeName: "TINYINT(1)") #expect(CompareSQLLiteral.textLiteral("1", columnType: tinyint, databaseType: .mysql, driver: driver) == "'1'") - let bit = ColumnType.boolean(rawType: "bit(8)") + let bit = ColumnTypeClassifier(family: .postgres).classify(rawTypeName: "bit(8)") #expect(CompareSQLLiteral.textLiteral("5", columnType: bit, databaseType: .postgresql, driver: driver) != "5") } @@ -99,14 +110,27 @@ struct CompareSQLLiteralTests { #expect(CompareSQLLiteral.prefixed("0x89504E47", databaseType: .mssql) == "0x89504E47") } + // `'\x…'::bytea` reads its backslash by `standard_conforming_strings`, and with that off a sync stored other bytes. @Test - func postgresFamilyUsesAByteaCast() throws { - for type in [DatabaseType.postgresql, .cockroachdb, .redshift, .pglite, .kingbasees] { + func postgresFamilyDecodesHex() throws { + for type in [DatabaseType.postgresql, .cockroachdb, .pglite] { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: png, databaseType: type)) - #expect(literal == "'\\x89504e47'::bytea", "\(type.rawValue) uses a bytea literal") + #expect(literal == "decode('89504E47', 'hex')", "\(type.rawValue) decodes hex") } } + // KingbaseES documents `decode` only as Oracle's DECODE; its bytea decoder is not the PostgreSQL one. + @Test + func kingbaseKeepsTheByteaLiteral() { + #expect(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .kingbasees) == "'\\x89504E47'::bytea") + } + + // Redshift has no bytea, and its DECODE is the CASE-like one; a binary column there is VARBYTE. + @Test + func redshiftReadsHexIntoVarbyte() { + #expect(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .redshift) == "from_hex('89504E47')") + } + @Test func sQLServerUsesZeroX() throws { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .mssql)) @@ -121,6 +145,12 @@ struct CompareSQLLiteralTests { #expect(literal == "HEXTORAW('89504E47')") } + // `HEXTORAW('')` stores NULL, and a `BLOB NOT NULL` refuses it. + @Test + func oracleWritesAnEmptyValueAsAnEmptyBlob() { + #expect(CompareSQLLiteral.binaryLiteral(for: Data(), databaseType: .oracle) == "EMPTY_BLOB()") + } + /// An engine this build does not name falls back to the driver's own spelling rather than /// guessing at one, which is why the helper returns nil instead of a default. @Test @@ -132,6 +162,6 @@ struct CompareSQLLiteralTests { func emptyDataStillProducesAValidLiteral() throws { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: Data(), databaseType: .postgresql)) - #expect(literal == "'\\x'::bytea") + #expect(literal == "decode('', 'hex')") } } diff --git a/apple/TableProTests/Core/Compare/CompareTimeoutLiftOrderTests.swift b/apple/TableProTests/Core/Compare/CompareTimeoutLiftOrderTests.swift new file mode 100644 index 0000000000..eb6dcd9028 --- /dev/null +++ b/apple/TableProTests/Core/Compare/CompareTimeoutLiftOrderTests.swift @@ -0,0 +1,53 @@ +// +// CompareTimeoutLiftOrderTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@MainActor +struct CompareTimeoutLiftOrderTests { + // One queue runs the connection's statements in turn, as a libpq session's does, so a lift sent while a bounded + // stream waits at a full buffer would wait behind it, and the walk that would drain the buffer never starts. + @Test( + "A comparison lifts the query timeout before its streams start their statements", + .timeLimit(.minutes(1)) + ) + func timeoutIsLiftedBeforeTheStreamsStart() async throws { + let connectionQueue = DispatchQueue(label: "test.compare.connection") + let driver = MockDatabaseDriver() + driver.applyQueryTimeoutGate = { + await withCheckedContinuation { continuation in + connectionQueue.async { continuation.resume() } + } + } + let batches = PluginRowStream.boundedBufferBatches * 3 + + let rowsRead = try await CompareRowService.readingBothSides(liftingTimeoutOn: [driver]) { + PluginRowStream.makeBounded { continuation, _ in + connectionQueue.async { + continuation.yield(.header(PluginStreamHeader(columns: ["id"], columnTypeNames: ["integer"]))) + for index in 0.. QueryTab { + let fixture = CoordinatorFixture(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let tab = fixture.addTab(QueryTab(title: "Query", query: script, tabType: .query)) + let statements = SQLStatementScanner.executableStatements( + in: script, grammar: fixture.coordinator.lexicalGrammar + ) + fixture.coordinator.queryExecutionCoordinator.executeMultipleStatements(statements) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + await handle.value + return try #require(fixture.tabManager.tabs.first { $0.id == tab.id }) + } + + @Test("A run sends a COPY's rows through copy-in with its header, byte for byte, and never the \\. line") + func copyGoesThroughCopyIn() async throws { + let driver = CopyInRecordingDriver(connection: connection, copiesIn: true) + + let tab = try await run(Self.script, on: driver) + + #expect(driver.sentCopies == [ + CopyInRecordingDriver.Copy(statement: "COPY t (a, b) FROM stdin", data: Data("1\tone\n2\té\n".utf8)), + ]) + #expect(driver.sentStatements == ["CREATE TABLE t (a int, b text)", "ANALYZE t"]) + #expect(!tab.display.resultSets.isEmpty) + #expect(tab.display.resultSets.allSatisfy { $0.errorMessage == nil }) + } + + @Test("A driver without copy-in refuses the COPY before anything is sent for it, and the run fails there") + func driverWithoutCopyInRefuses() async throws { + let driver = CopyInRecordingDriver(connection: connection, copiesIn: false) + + let tab = try await run(Self.script, on: driver) + + #expect(driver.sentCopies.isEmpty) + #expect(driver.sentStatements == ["CREATE TABLE t (a int, b text)"]) + let failed = try #require(tab.display.resultSets.last) + #expect(failed.failedQuery == "COPY t (a, b) FROM stdin") + let refusal = try #require(PluginDriverUnsupportedOperation.copyFromStandardInput.errorDescription) + #expect(failed.errorMessage?.contains(refusal) == true) + } + + @Test("A COPY with no rows still goes through copy-in, as a copy of none") + func copyWithoutRows() async throws { + let driver = CopyInRecordingDriver(connection: connection, copiesIn: true) + + _ = try await run("COPY t FROM stdin;", on: driver) + + #expect(driver.sentCopies == [CopyInRecordingDriver.Copy(statement: "COPY t FROM stdin", data: Data())]) + #expect(driver.sentStatements.isEmpty) + } + + @Test("A COPY that ends a run cut again after a SET keeps its rows") + func replannedCopyKeepsItsRows() async throws { + let standard = PluginSessionLexicalState(determined: [.backslashEscapesInSingleQuotes], enabled: []) + let escapes = PluginSessionLexicalState( + determined: [.backslashEscapesInSingleQuotes], enabled: [.backslashEscapesInSingleQuotes] + ) + #expect(SQLLexicalResolver.readings(for: .postgresql, session: standard).execution + != SQLLexicalResolver.readings(for: .postgresql, session: escapes).execution) + let driver = CopyInRecordingDriver( + connection: connection, copiesIn: true, lexicalState: standard, lexicalStateAfterFirstStatement: escapes + ) + + _ = try await run("SET standard_conforming_strings = off;\nCOPY t FROM stdin;\n1\tC:\\\\dir\n\\.", on: driver) + + #expect(driver.sentStatements == ["SET standard_conforming_strings = off"]) + #expect(driver.sentCopies == [ + CopyInRecordingDriver.Copy(statement: "COPY t FROM stdin", data: Data("1\tC:\\\\dir\n".utf8)), + ]) + } +} diff --git a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift index 32b9aea657..a965934eae 100644 --- a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift +++ b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift @@ -95,12 +95,14 @@ struct FindCoordinatorRoutingTests { } @Test("Find Next and Find Previous step the selected tab's matches") - func stepsTheSelectedTabsMatches() throws { + func stepsTheSelectedTabsMatches() async throws { let (coordinator, first, _) = makeTabs() let model = coordinator.gridModel(forTab: first) coordinator.findCoordinator.showFind() model.find.term = "ali" model.findQueryDidChange() + #expect(coordinator.findCoordinator.hasActiveFind, "Find Next waits for the running search instead of going to the editor") + await model.findSearchSettled() #expect(model.find.matches.count == 2) #expect(coordinator.findCoordinator.hasActiveFind) let start = try #require(model.find.currentIndex) diff --git a/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift b/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift index a2dc916f67..1f5f9164fc 100644 --- a/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift +++ b/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift @@ -28,7 +28,7 @@ struct MultiStatementReplanTests { let last = try #require(planned.last) let rest = QueryExecutionCoordinator.remainingStatements( - in: Self.script, after: first.range, through: NSMaxRange(last.range), grammar: Self.noEscapes + in: Self.script, after: first, through: NSMaxRange(last.range), grammar: Self.noEscapes ) let restSQL = rest.map { $0.sql } let restText = rest.map { (Self.script as NSString).substring(with: $0.range) } @@ -39,6 +39,23 @@ struct MultiStatementReplanTests { #expect(!QueryExecutionCoordinator.scriptHolds(planned, in: "SELECT 1")) } + @Test("A replan starts past a COPY's rows and keeps the DELIMITER in force") + func replanStartsPastTheSegment() throws { + let postgres = "COPY t FROM stdin;\nDROP TABLE users; COMMIT;\n\\.\nSELECT 1;" + let copy = try #require(SQLStatementScanner.executableStatements(in: postgres, grammar: TestGrammar.postgres).first) + let afterCopy = QueryExecutionCoordinator.remainingStatements( + in: postgres, after: copy, through: (postgres as NSString).length, grammar: TestGrammar.postgres + ) + #expect(afterCopy.map { $0.sql } == ["SELECT 1"]) + + let mysql = "DELIMITER //\nSET sql_mode = 'NO_BACKSLASH_ESCAPES'//\nSELECT 'a\\'//" + let set = try #require(SQLStatementScanner.executableStatements(in: mysql, grammar: TestGrammar.mysql).first) + let afterSet = QueryExecutionCoordinator.remainingStatements( + in: mysql, after: set, through: (mysql as NSString).length, grammar: Self.noEscapes + ) + #expect(afterSet.map { $0.sql } == ["SELECT 'a\\'"]) + } + @Test("A run sends the replanned rest in place of what it planned") func runSendsTheReplannedRest() async { let driver = ScriptAnsweringDriver(connection: TestFixtures.makeConnection(type: .mysql), sendsBatchesWhole: false) diff --git a/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift b/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift index 71a31a2c8f..1386171d49 100644 --- a/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift +++ b/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift @@ -71,7 +71,7 @@ struct RowEditingCoordinatorUUIDKeyTests { coordinator.addNewRow() - #expect(try #require(lastRow(of: coordinator)).values[0] == .text("__DEFAULT__")) + #expect(try #require(lastRow(of: coordinator)).values[0] == .columnDefault) } @Test("Add Row on an integer key leaves it as before") diff --git a/apple/TableProTests/Core/Coordinators/RunResultLimitTests.swift b/apple/TableProTests/Core/Coordinators/RunResultLimitTests.swift new file mode 100644 index 0000000000..80a05afd60 --- /dev/null +++ b/apple/TableProTests/Core/Coordinators/RunResultLimitTests.swift @@ -0,0 +1,81 @@ +// +// RunResultLimitTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +extension GlobalStateSuites { + @MainActor + struct RunResultLimitTests { + // SQL Server: the grammar that reads GO lines as batch separators. + private let fixture = CoordinatorFixture(connection: TestFixtures.makeConnection(database: "shop", type: .mssql)) + + // Every batch answers `setsPerBatch` one-row result sets, numbered across the run. + private func run(_ text: String, setsPerBatch: Int) async throws -> [ResultSet] { + let connection = fixture.connection + let counter = BatchCounter() + let driver = ScriptAnsweringDriver(connection: connection, answer: { _ in + let first = counter.take(setsPerBatch) + return ScriptAnsweringDriver.batch((first.. Int { + lock.withLock { + defer { next += count } + return next + 1 + } + } +} diff --git a/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift b/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift new file mode 100644 index 0000000000..435a31ba59 --- /dev/null +++ b/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift @@ -0,0 +1,137 @@ +// +// RunResultPlanTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import TableProTestSupport +@testable import TablePro +import Testing + +struct RunResultPlanTests { + @Test("Statements that return no rows share one summary placed where the last of them ran") + func columnlessStatementsCollapse() { + let plan = RunResultPlan(producesRows: [false, false, true, false, true], keepingRowResults: nil) + #expect(plan.entries == [.statement(2), .summary([0, 1, 3]), .statement(4)]) + #expect(plan.droppedNotice == nil) + } + + @Test("A single statement without rows keeps its own result") + func singleColumnlessStatementStays() { + let plan = RunResultPlan(producesRows: [false, true], keepingRowResults: nil) + #expect(plan.entries == [.statement(0), .statement(1)]) + } + + @Test("A run that ends on a statement without rows ends on the summary") + func summaryIsLastWhenTheRunEndsWithoutRows() { + let plan = RunResultPlan(producesRows: [true, false, false], keepingRowResults: nil) + #expect(plan.entries.last == .summary([1, 2])) + } + + @Test("Only the last results that return rows are kept, and only their rows are built") + func rowResultsAreCapped() { + let plan = RunResultPlan(producesRows: Array(repeating: true, count: 5), keepingRowResults: 2) + #expect(plan.entries == [.statement(3), .statement(4)]) + #expect(plan.droppedRowResultCount == 3) + #expect(plan.showsRows(ofStatement: 4)) + #expect(!plan.showsRows(ofStatement: 0)) + #expect(plan.droppedNotice == "3 earlier results were dropped.") + } + + @Test("Without a limit every result that returns rows is kept") + func noLimitKeepsEveryRowResult() { + let plan = RunResultPlan(producesRows: Array(repeating: true, count: 1_000), keepingRowResults: nil) + #expect(plan.entries.count == 1_000) + #expect(plan.droppedRowResultCount == 0) + #expect(plan.droppedNotice == nil) + } + + @Test("One dropped result is named in the singular") + func oneDroppedResult() { + let plan = RunResultPlan(producesRows: [true, true, false], keepingRowResults: 1) + #expect(plan.entries == [.statement(1), .statement(2)]) + #expect(plan.droppedNotice == "1 earlier result was dropped.") + } + + @Test("Results already gone count as dropped and never take a kept place") + func unavailableResultsCountAsDropped() { + let plan = RunResultPlan(producesRows: [true, true, true, true], keepingRowResults: nil, unavailable: [0, 1]) + #expect(plan.entries == [.statement(2), .statement(3)]) + #expect(plan.position(ofStatement: 2) == 0) + #expect(plan.droppedNotice == "2 earlier results were dropped.") + + let limited = RunResultPlan(producesRows: [true, true, true, true], keepingRowResults: 1, unavailable: [0]) + #expect(limited.entries == [.statement(3)]) + #expect(limited.droppedRowResultCount == 3) + } + + @Test("A long script of writes becomes one result") + func longWriteScriptIsOneResult() { + let plan = RunResultPlan(producesRows: Array(repeating: false, count: 10_000), keepingRowResults: nil) + #expect(plan.entries.count == 1) + #expect(!plan.showsRows(ofStatement: 0)) + } +} + +@MainActor +struct RunAllSummaryResultTests { + @Test("Run All of statements that return no rows shows one summary result") + func runAllCollapsesWrites() async throws { + let connection = TestFixtures.makeConnection() + let driver = MockDatabaseDriver(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + let tabManager = QueryTabManager() + let coordinator = MainContentCoordinator( + connection: connection, + tabManager: tabManager, + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState() + ) + defer { + coordinator.cancelAllQueryTasks() + coordinator.teardown() + DatabaseManager.shared.removeSession(for: connection.id) + } + let script = "CREATE TABLE a (x int);\nINSERT INTO a VALUES (1);\nUPDATE a SET x = 2;" + let tab = QueryTab(title: "Query", query: script, tabType: .query) + tabManager.tabs.append(tab) + tabManager.selectedTabId = tab.id + + coordinator.runAllStatements() + try await waitUntil { + !coordinator.tabExecution.isExecuting(tab.id) && tabManager.tabs.first?.display.resultSets.isEmpty == false + } + + let results = try #require(tabManager.tabs.first?.display.resultSets) + #expect(results.count == 1) + let summary = try #require(results.first) + #expect(summary.resultColumns.isEmpty) + #expect(summary.statusMessage?.contains("3") == true) + #expect(summary.statementAnchor == nil) + } + + @Test("The summary adds up rows affected and time, keeps each driver message once and the output in order") + func summaryCarriesWhatTheStatementsReported() { + let fixture = CoordinatorFixture() + let parts = [ + RunSummaryPart(rowsAffected: 2, executionTime: 0.5, serverOutput: PluginServerOutput(lines: ["a"], isTruncated: false), statusMessage: "INSERT: 2 rows"), + RunSummaryPart(rowsAffected: 3, executionTime: 0.25, serverOutput: PluginServerOutput(lines: ["b"], isTruncated: true), statusMessage: "Still creating"), + RunSummaryPart(rowsAffected: 0, executionTime: 0.25, serverOutput: .none, statusMessage: "Still creating"), + ] + + let summary = fixture.coordinator.queryExecutionCoordinator.makeRunSummaryResultSet(parts) + + #expect(summary.rowsAffected == 5) + #expect(summary.executionTime == 1) + #expect(summary.serverOutput == PluginServerOutput(lines: ["a", "b"], isTruncated: true)) + let status = summary.statusMessage ?? "" + #expect(status.contains("INSERT: 2 rows")) + #expect(status.components(separatedBy: "Still creating").count == 2) + #expect(summary.resultColumns.isEmpty) + #expect(summary.origin == nil) + } +} diff --git a/apple/TableProTests/Core/Coordinators/StoppedRunKeepsWorkTests.swift b/apple/TableProTests/Core/Coordinators/StoppedRunKeepsWorkTests.swift new file mode 100644 index 0000000000..4ba335d3a8 --- /dev/null +++ b/apple/TableProTests/Core/Coordinators/StoppedRunKeepsWorkTests.swift @@ -0,0 +1,319 @@ +// +// StoppedRunKeepsWorkTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import TableProTestSupport +import Testing + +@MainActor +struct StoppedRunKeepsWorkTests { + private static let script = "INSERT INTO t VALUES (1);\nINSERT INTO t VALUES (2);\nINSERT INTO t VALUES (3);" + + private let connection: DatabaseConnection + private let storage: QueryHistoryStorage + private let fixture: CoordinatorFixture + + init() { + let connection = TestFixtures.makeConnection() + let storage = QueryHistoryStorage( + databaseURL: FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("stopped_run_\(UUID().uuidString).db"), + removeDatabaseOnDeinit: true + ) + self.connection = connection + self.storage = storage + fixture = CoordinatorFixture( + connection: connection, + services: .live.recordingHistory(in: QueryHistoryManager(storage: storage, isCapturePaused: { false })) + ) + } + + @Test("Stop on Run All in an open transaction keeps the statements that ran, and their history") + func stopKeepsTheStatementsThatRan() async throws { + let driver = HeldStatementDriver(connection: connection, holding: 2, onCancel: .keepsRunning) + defer { finish(driver) } + let run = try startRun(on: driver) + + try await waitUntil { driver.isHolding } + fixture.coordinator.stopExecution(for: run.tabId) + driver.finishHeldStatement() + await run.handle.value + + let tab = try #require(fixture.tabManager.tabs.first { $0.id == run.tabId }) + #expect(driver.sentStatements.count == 2) + #expect(tab.display.resultSets.count == 1) + #expect(tab.display.resultSets.first?.rowsAffected == 2) + #expect(tab.execution.rowsAffected == 2) + #expect(tab.execution.statusMessage == PluginSessionTransactionState.inTransaction.openTransactionNotice) + #expect(fixture.coordinator.tabExecution.isAnyExecuting == false) + + let entries = try await history(count: 2) + #expect(entries.map { $0.query } == ["INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (2);"]) + #expect(entries.allSatisfy { $0.wasSuccessful }) + } + + @Test("Stop that interrupts a statement reads as a stop: the statements before it are kept, with no error") + func stopThatInterruptsAStatementKeepsTheOnesBefore() async throws { + let driver = HeldStatementDriver(connection: connection, holding: 3, onCancel: .failsTheStatement) + defer { finish(driver) } + let run = try startRun(on: driver) + + try await waitUntil { driver.isHolding } + fixture.coordinator.stopExecution(for: run.tabId) + try await waitUntil { !driver.isHolding } + await run.handle.value + + let tab = try #require(fixture.tabManager.tabs.first { $0.id == run.tabId }) + let results = tab.display.resultSets + #expect(driver.sentStatements.count == 3) + #expect(results.count == 1) + #expect(results.first?.rowsAffected == 2) + #expect(results.allSatisfy { $0.errorMessage == nil }) + #expect(tab.execution.errorMessage == nil) + #expect(tab.execution.statusMessage == PluginSessionTransactionState.inTransaction.openTransactionNotice) + #expect(fixture.coordinator.tabExecution.isAnyExecuting == false) + + let entries = try await history(count: 2) + #expect(entries.map { $0.query } == ["INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (2);"]) + #expect(entries.allSatisfy { $0.wasSuccessful }) + } + + @Test("A stopped run that comes back after edits were staged keeps them and only writes history") + func stoppedRunKeepsEditsStagedAfterStop() async throws { + let driver = HeldStatementDriver(connection: connection, holding: 2, onCancel: .keepsRunning) + defer { finish(driver) } + let run = try startRun(on: driver) + + try await waitUntil { driver.isHolding } + fixture.coordinator.stopExecution(for: run.tabId) + let contentAfterStop = fixture.coordinator.tabExecution.contentEpoch(for: run.tabId) + fixture.changeManager.recordRowDeletion(rowID: .existing(0), originalRow: [.text("1")]) + driver.finishHeldStatement() + await run.handle.value + + let tab = try #require(fixture.tabManager.tabs.first { $0.id == run.tabId }) + #expect(fixture.changeManager.hasChanges) + #expect(tab.display.resultSets.isEmpty) + #expect(fixture.coordinator.tabExecution.isSameContent(contentAfterStop, for: run.tabId)) + #expect(fixture.coordinator.tabExecution.isAnyExecuting == false) + + let entries = try await history(count: 2) + #expect(entries.map { $0.query } == ["INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (2);"]) + } + + @Test("A run that a new run superseded after its Stop writes nothing") + func supersededStoppedRunWritesNothing() async throws { + let driver = HeldStatementDriver(connection: connection, holding: 2, onCancel: .keepsRunning) + defer { finish(driver) } + let run = try startRun(on: driver) + + try await waitUntil { driver.isHolding } + fixture.coordinator.stopExecution(for: run.tabId) + let successor = fixture.coordinator.beginTabExecution(for: run.tabId) + driver.finishHeldStatement() + await run.handle.value + + let tab = try #require(fixture.tabManager.tabs.first { $0.id == run.tabId }) + #expect(tab.display.resultSets.isEmpty) + #expect(tab.execution.statusMessage == nil) + #expect(fixture.coordinator.tabExecution.isCurrent(successor.claim)) + } + + private func startRun(on driver: HeldStatementDriver) throws -> (tabId: UUID, handle: Task) { + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + + let tab = fixture.addTab(QueryTab(title: "Query", query: Self.script, tabType: .query)) + let statements = SQLStatementScanner.executableStatements( + in: Self.script, grammar: fixture.coordinator.lexicalGrammar + ) + fixture.coordinator.queryExecutionCoordinator.executeMultipleStatements(statements) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + return (tab.id, handle) + } + + private func finish(_ driver: HeldStatementDriver) { + driver.finishHeldStatement() + DatabaseManager.shared.removeSession(for: connection.id) + } + + // History is written on tasks the run does not await. + private func history(count: Int) async throws -> [QueryHistoryEntry] { + let storage = self.storage + let scope = QueryHistoryScope.connection(connection.id) + try await waitUntil { await storage.count(scope: scope) == count } + let page = await storage.fetch(QueryHistoryFilter(scope: scope), after: nil, limit: count + 1) + return page.entries.sorted { $0.query < $1.query } + } +} + +// Holds the statement at `heldPosition` until the test lets it go, or until a Stop's driver cancel fails it. +private final class HeldStatementDriver: DatabaseDriver, @unchecked Sendable { + enum OnCancel { + case keepsRunning + case failsTheStatement + } + + struct Interrupted: LocalizedError { + var errorDescription: String? { "canceling statement due to user request" } + } + + let connection: DatabaseConnection + var status: ConnectionStatus = .connected + var serverVersion: String? { nil } + var sessionLexicalState: PluginSessionLexicalState? { nil } + + private let heldPosition: Int + private let onCancel: OnCancel + private let lock = NSLock() + private var sent: [String] = [] + private var held: CheckedContinuation? + + init(connection: DatabaseConnection, holding heldPosition: Int, onCancel: OnCancel) { + self.connection = connection + self.heldPosition = heldPosition + self.onCancel = onCancel + } + + var sentStatements: [String] { + lock.withLock { sent } + } + + var isHolding: Bool { + lock.withLock { held != nil } + } + + func finishHeldStatement() { + takeHeld()?.resume() + } + + func cancelQuery() throws { + guard onCancel == .failsTheStatement else { return } + takeHeld()?.resume(throwing: Interrupted()) + } + + func sessionTransactionState() async -> PluginSessionTransactionState { .inTransaction } + + private func takeHeld() -> CheckedContinuation? { + lock.withLock { + let continuation = held + held = nil + return continuation + } + } + + private func run(_ query: String) async throws -> QueryResult { + let position = lock.withLock { + sent.append(query) + return sent.count + } + if position == heldPosition { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + lock.withLock { held = continuation } + } + } + return QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 1, executionTime: 0, error: nil) + } + + func execute(query: String) async throws -> QueryResult { try await run(query) } + func executeParameterized(query: String, parameters: [Any?]) async throws -> QueryResult { try await run(query) } + func executeUserQuery(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryResult { + try await run(query) + } + + func connect() async throws {} + func disconnect() {} + func testConnection() async throws -> Bool { true } + func ping() async throws {} + func applyQueryTimeout(_ seconds: Int) async throws {} + + func fetchTables() async throws -> [TableInfo] { [] } + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String) async throws -> [ColumnInfo] { [] } + func fetchAllColumns() async throws -> [String: [ColumnInfo]] { [:] } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } + func fetchApproximateRowCount(table: String) async throws -> Int? { nil } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> DatabaseMetadata { + DatabaseMetadata( + id: database, + name: database, + tableCount: nil, + sizeBytes: nil, + lastAccessed: nil, + isSystemDatabase: false, + icon: "cylinder" + ) + } + + func fetchTableDDL(table: String) async throws -> String { "" } + func fetchTableMetadata(tableName: String) async throws -> TableMetadata { + TableMetadata( + tableName: tableName, + dataSize: nil, + indexSize: nil, + totalSize: nil, + avgRowLength: nil, + rowCount: nil, + comment: nil, + engine: nil, + collation: nil, + createTime: nil, + updateTime: nil + ) + } + + func fetchViewDefinition(view: String) async throws -> String { "" } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} + +@MainActor +private extension AppServices { + func recordingHistory(in queryHistoryManager: QueryHistoryManager) -> AppServices { + AppServices( + appEvents: appEvents, + appSettings: appSettings, + appSettingsStorage: appSettingsStorage, + connectionStorage: connectionStorage, + databaseManager: databaseManager, + pluginManager: pluginManager, + schemaService: schemaService, + schemaRefreshService: schemaRefreshService, + schemaProviderRegistry: schemaProviderRegistry, + catalogChangeService: catalogChangeService, + sqlFavoriteManager: sqlFavoriteManager, + favoriteTablesStorage: favoriteTablesStorage, + favoriteDatabasesStorage: favoriteDatabasesStorage, + tableFolderStorage: tableFolderStorage, + aiChatStorage: aiChatStorage, + aiKeyStorage: aiKeyStorage, + aiAccessApprovals: aiAccessApprovals, + groupStorage: groupStorage, + tagStorage: tagStorage, + sshProfileStorage: sshProfileStorage, + credentialProfileStorage: credentialProfileStorage, + featureAccess: featureAccess, + syncMetadataStorage: syncMetadataStorage, + favoritesExpansionState: favoritesExpansionState, + linkedFolderWatcher: linkedFolderWatcher, + queryHistoryManager: queryHistoryManager, + dateFormattingService: dateFormattingService, + copilotService: copilotService, + mcpServerManager: mcpServerManager, + syncTracker: syncTracker, + welcomeRouter: welcomeRouter, + telemetry: telemetry + ) + } +} diff --git a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift index 6c3ed73cdd..1017472355 100644 --- a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift +++ b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift @@ -169,6 +169,97 @@ struct GridFindEngineTests { #expect(search("a b", in: source(rows, revealsInvisibleCharacters: true)).matches.isEmpty) } + // MARK: - Refreshing + + @Test("Reading only the written cells and the added rows again finds what a full search finds") + func refreshMatchesAFullSearch() throws { + var rows = Self.people + let before = search("act", in: source(rows)).matches + rows.edit(row: 1, column: 2, value: .text("acting")) + rows.edit(row: 2, column: 0, value: .text("done")) + rows.appendInsertedRow(values: [.text("react"), .text("Dee"), .null]) + let inserted = try #require(rows.rows.last?.id) + rows.remove(at: IndexSet(integer: 0)) + + let refreshed = GridFindEngine.refreshed( + before, + writtenCells: [.existing(1): IndexSet(integer: 2), .existing(2): IndexSet(integer: 0)], + addedRows: [inserted], + term: "act", + options: GridFindState.Options(), + in: source(rows) + ) + + let expected = [ + match(1, Self.status), + match(1, Self.remark), + match(2, Self.remark), + GridFindState.Match(rowID: inserted, columnID: Self.status) + ] + #expect(refreshed?.matches == expected) + #expect(search("act", in: source(rows)).matches == expected) + } + + @Test("Under a value filter a refresh keeps the filter's order and leaves out a hidden row it rereads") + func refreshFollowsTheFilteredOrder() { + let order = DataGridRowOrder(filteredIDs: [.existing(2), .existing(0)], revision: 1) + var rows = Self.people + let before = search("act", in: source(rows, order: order)).matches + rows.edit(row: 0, column: 2, value: .text("react")) + rows.edit(row: 1, column: 1, value: .text("Actor")) + + let refreshed = GridFindEngine.refreshed( + before, + writtenCells: [.existing(0): IndexSet(integer: 2), .existing(1): IndexSet(integer: 1)], + addedRows: [], + term: "act", + options: GridFindState.Options(), + in: source(rows, order: order) + ) + + #expect(refreshed?.matches == search("act", in: source(rows, order: order)).matches) + #expect(refreshed?.matches == [match(2, Self.status), match(2, Self.remark), match(0, Self.status), match(0, Self.remark)]) + } + + @Test("A term that cannot match leaves the refresh to a full search") + func refreshWithoutATerm() { + let blank = GridFindEngine.refreshed( + [], writtenCells: [:], addedRows: [.existing(0)], term: " ", options: GridFindState.Options(), in: source() + ) + let invalid = GridFindEngine.refreshed( + [], writtenCells: [:], addedRows: [], term: "(", options: options(regex: true), in: source() + ) + + #expect(blank == nil) + #expect(invalid == nil) + } + + // The cost pin: the caller names the written cells, and every other cell keeps its match without being read. + @Test("Only the written cells of a row are read again") + func refreshReadsOnlyTheWrittenCells() { + var rows = Self.people + let before = search("act", in: source(rows)).matches + rows.edit(row: 2, column: 0, value: .text("done")) + rows.edit(row: 2, column: 1, value: .text("Actor")) + + let refreshed = GridFindEngine.refreshed( + before, + writtenCells: [.existing(2): IndexSet(integer: 1)], + addedRows: [], + term: "act", + options: GridFindState.Options(), + in: source(rows) + ) + + #expect(refreshed?.matches == [ + match(0, Self.status), + match(1, Self.status), + match(2, Self.status), + match(2, Self.name), + match(2, Self.remark) + ]) + } + // MARK: - Binary columns private static let payload = TableRows( @@ -203,6 +294,125 @@ struct GridFindEngineTests { #expect(search("8950", in: source).matches.isEmpty) } + // MARK: - Off the main actor + + private static let shownValues = TableRows( + rows: [ + Row(id: .existing(0), values: [ + .text("2024-03-15 09:30:00.123456"), + .text("2024-12-31 23:59:59+07"), + .text("2024-03-15"), + .bytes(Data("signup".utf8)), + .bytes(Data((0..<16).map { UInt8($0) })), + .text("1700000000"), + .text("1700000000000"), + .text("a\rb") + ]), + Row(id: .existing(1), values: [ + .text("2023-11-14 22:13:20"), + .null, + .text("1999-01-01"), + .bytes(Data([0x89, 0x50, 0x4E, 0x47])), + .bytes(Data(repeating: 0xAB, count: 16)), + .text("1700003600"), + .text("1700003600000"), + .text("plain") + ]) + ], + columns: ["created", "updated", "born", "payload", "token", "stamp", "stampMillis", "note"], + columnTypes: [ + .datetime(rawType: "DATETIME(6)"), + .timestamp(rawType: "TIMESTAMPTZ"), + .date(rawType: "DATE"), + .blob(rawType: "VARBINARY(255)"), + .blob(rawType: "BINARY(16)"), + .integer(rawType: "BIGINT"), + .integer(rawType: "BIGINT"), + .text(rawType: "TEXT") + ] + ) + + private static let shownFormats: [ValueDisplayFormat?] = [nil, nil, nil, .text, .uuid, .unixTimestamp, .unixTimestampMillis, nil] + + private static let shownTerms = [ + "2024", "2023", "09:30:00.123456", "+07", "/", "PM", "signup", "00010203-0405", "abab", "a b", "b", "11" + ] + + // Both sides use the same renderer code, so this pins the same cells on and off the main actor, not the shown text. + @Test("The same search finds the same cells on and off the main actor, under every date format") + func searchOffTheMainActorMatchesTheMainActor() async { + for revealsInvisibleCharacters in [false, true] { + let source = self.source(Self.shownValues, formats: Self.shownFormats, revealsInvisibleCharacters: revealsInvisibleCharacters) + for format in DateFormatOption.allCases { + for term in Self.shownTerms { + let onMain = GridFindEngine.search( + term, + options: GridFindState.Options(), + in: source, + dates: DateDisplayRenderer(format: format), + isCancelled: { false } + ) + let offMain = await GridFindEngine.searchOffTheMainActor( + term, + options: GridFindState.Options(), + in: source, + dateFormat: format + ) + #expect(offMain == onMain, "same cells on and off the main actor: \(format) \(term) reveals: \(revealsInvisibleCharacters)") + } + } + for term in Self.shownTerms { + let shared = await GridFindEngine.searchOffTheMainActor( + term, + options: GridFindState.Options(), + in: source, + dateFormat: DateFormattingService.shared.currentFormat + ) + #expect(shared == search(term, in: source), "\(term)") + } + } + } + + @Test("The off-main search renders dates, bytes and timestamps the way the cells show them") + func searchOffTheMainActorReadsTheShownText() async { + let source = self.source(Self.shownValues, formats: Self.shownFormats) + func column(_ name: String) -> DataGridColumnID { DataGridColumnID(name: name, occurrence: 0) } + func found(_ term: String, _ format: DateFormatOption = .iso8601) async -> [GridFindState.Match]? { + await GridFindEngine.searchOffTheMainActor(term, options: GridFindState.Options(), in: source, dateFormat: format)?.matches + } + + #expect(await found("15/03/2024", .euLong) == [match(0, column("created")), match(0, column("born"))]) + #expect(await found("09:30:00.123456 AM", .usLong) == [match(0, column("created"))]) + #expect(await found("23:59:59+07") == [match(0, column("updated"))]) + #expect(await found("signup") == [match(0, column("payload"))]) + #expect(await found("00010203-0405-0607-0809-0a0b0c0d0e0f") == [match(0, column("token"))]) + #expect(await found("a b") == [match(0, column("note"))]) + #expect(await found("8950")?.isEmpty == true) + } + + @Test("A search asks whether to stop every 4,096 cells and gives up when told to") + func searchChecksForCancellation() { + let rows = TableRows.from( + queryRows: (0..<10_000).map { [PluginCellValue.text("row \($0)")] }, + columns: ["v"], + columnTypes: [.text(rawType: "TEXT")] + ) + let dates = DateDisplayRenderer(format: .iso8601) + var checks = 0 + + let finished = GridFindEngine.search("row", options: GridFindState.Options(), in: source(rows), dates: dates) { + checks += 1 + return false + } + let stopped = GridFindEngine.search( + "row", options: GridFindState.Options(), in: source(rows), dates: dates, isCancelled: { true } + ) + + #expect(finished?.matches.count == 10_000) + #expect(checks == 3) + #expect(stopped == nil) + } + // MARK: - Replacing private static let contacts = TableRows.from( @@ -360,6 +570,11 @@ struct DataGridModelFindTests { ) private static let company = DataGridColumnID(name: "company", occurrence: 0) + private static let city = DataGridColumnID(name: "city", occurrence: 0) + + private func match(_ row: Int, _ column: DataGridColumnID) -> GridFindState.Match { + GridFindState.Match(rowID: .existing(row), columnID: column) + } private func mount( _ configure: (inout DataGridConfiguration) -> Void = { _ in } @@ -378,18 +593,19 @@ struct DataGridModelFindTests { GridSelectionReducer.single(GridCoord(row: row, displayColumn: column)) } - private func search(_ model: DataGridModel, _ term: String) { + private func search(_ model: DataGridModel, _ term: String) async { model.showFind() model.find.term = term model.findQueryDidChange() + await model.findSearchSettled() } @Test("A match becomes the active cell, column included") - func matchSetsActiveCell() { + func matchSetsActiveCell() async { let (model, owner) = mount() model.setSelection(cell(2, 0)) - search(model, "acme") + await search(model, "acme") #expect(model.find.matches.count == 4) #expect(model.find.currentIndex == 1) @@ -399,9 +615,9 @@ struct DataGridModelFindTests { } @Test("Find Next walks the matches and wraps") - func findNextWraps() { + func findNextWraps() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") #expect(model.selection.activeCell == GridCoord(row: 1, displayColumn: 2)) model.findNext() @@ -416,9 +632,9 @@ struct DataGridModelFindTests { } @Test("Find Previous starts from where the user moved the cursor") - func findPreviousFromCursor() { + func findPreviousFromCursor() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") model.setSelection(cell(3, 0)) model.findPrevious() @@ -428,14 +644,15 @@ struct DataGridModelFindTests { } @Test("A refresh keeps the current match by identity and leaves the selection alone") - func refreshKeepsCurrentMatch() { + func refreshKeepsCurrentMatch() async { let (model, owner) = mount() model.setSelection(cell(2, 0)) - search(model, "acme") + await search(model, "acme") owner.tableRows.rows[0].values[2] = .text("Acme Town") model.reloadRows() model.refreshFind() + await model.findSearchSettled() #expect(model.find.matches.count == 5) #expect(model.find.currentMatch == GridFindState.Match(rowID: .existing(2), columnID: Self.company)) @@ -444,9 +661,9 @@ struct DataGridModelFindTests { } @Test("Replace commits the current match as one edit and moves to the next match") - func replaceCurrent() { + func replaceCurrent() async { let (model, owner) = mount { $0.isEditable = true } - search(model, "acme") + await search(model, "acme") model.find.replacement = "Umbrella" model.replaceCurrentMatch() @@ -460,12 +677,12 @@ struct DataGridModelFindTests { } @Test("Replace All is one commit and leaves a column it may not write") - func replaceAllOneCommit() throws { + func replaceAllOneCommit() async throws { let (model, owner) = mount { $0.isEditable = true $0.lockedColumns = ["city"] } - search(model, "acme") + await search(model, "acme") model.find.replacement = "Umbrella" let outcome = try #require(model.replaceAllMatches()) @@ -492,11 +709,12 @@ struct DataGridModelFindTests { } @Test("Hiding the bar clears the term, the matches, the scope and the replace row") - func hideClears() { + func hideClears() async { let (model, owner) = mount { $0.isEditable = true } model.findInColumn(Self.company, replace: true) model.find.term = "acme" model.findQueryDidChange() + await model.findSearchSettled() #expect(model.find.matches.count == 2) model.hideFind() @@ -510,24 +728,24 @@ struct DataGridModelFindTests { } @Test("Search All Rows hands the trimmed term to the owner only where the grid offers it") - func searchAllRows() { + func searchAllRows() async { let (plain, plainOwner) = mount() - search(plain, " zzz ") + await search(plain, " zzz ") plain.searchAllRows() #expect(plainOwner.searchedTerms.isEmpty) #expect(plain.find.isVisible) let (offered, owner) = mount { $0.commands = [.searchAllRows] } - search(offered, " zzz ") + await search(offered, " zzz ") offered.searchAllRows() #expect(owner.searchedTerms == ["zzz"]) #expect(offered.find.isVisible == false) } @Test("Search All Rows waits for plain options, since the server matches the term as typed in any column") - func searchAllRowsNeedsPlainOptions() { + func searchAllRowsNeedsPlainOptions() async { let (model, owner) = mount { $0.commands = [.searchAllRows] } - search(model, "^zzz$") + await search(model, "^zzz$") model.find.options.usesRegularExpression = true model.searchAllRows() @@ -540,9 +758,9 @@ struct DataGridModelFindTests { } @Test("New rows under an open bar are searched again") - func reloadSearchesTheNewRows() { + func reloadSearchesTheNewRows() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") #expect(model.find.matches.count == 4) owner.tableRows = TableRows.from( @@ -551,10 +769,98 @@ struct DataGridModelFindTests { columnTypes: [.integer(rawType: "INT"), .text(rawType: "TEXT"), .text(rawType: "TEXT")] ) model.reloadRows() + await model.findSearchSettled() #expect(model.find.matches.count == 1) } + private func fullSearch(_ model: DataGridModel, _ owner: FindGridOwner) -> [GridFindState.Match] { + let source = GridFindEngine.Source(rows: owner.tableRows, order: model.order, columns: model.columns) + return GridFindEngine.search(model.find.term, options: model.find.options, in: source).matches + } + + @Test("Edits under an open bar keep the matches a full search finds") + func editsKeepTheMatchesInStep() async { + let (model, owner) = mount() + await search(model, "acme") + + owner.tableRows.rows[2].values[1] = .text("Umbrella") + model.apply(.cellChanged(row: 2, column: 1)) + owner.tableRows.rows[0].values[2] = .text("Acme Heights") + owner.tableRows.rows[1].values[2] = .text("Falls") + model.apply(.cellsChanged([CellPosition(row: 0, column: 2), CellPosition(row: 1, column: 2)])) + + #expect(model.find.matches == [match(0, Self.city), match(3, Self.company), match(3, Self.city)]) + #expect(model.find.matches == fullSearch(model, owner)) + } + + @Test("Rows added or removed under an open bar keep the matches a full search finds") + func insertsAndRemovalsKeepTheMatchesInStep() async { + let (model, owner) = mount() + await search(model, "acme") + + let inserted = owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")]) + model.apply(inserted) + let insertedID = owner.tableRows.rows[4].id + #expect(model.find.matches.last == GridFindState.Match(rowID: insertedID, columnID: Self.company)) + #expect(model.find.matches == fullSearch(model, owner)) + + let removed = owner.tableRows.remove(at: IndexSet(integer: 1)) + model.apply(removed) + + #expect(model.find.matches == [ + match(2, Self.company), + match(3, Self.company), + match(3, Self.city), + GridFindState.Match(rowID: insertedID, columnID: Self.company) + ]) + #expect(model.find.matches == fullSearch(model, owner)) + } + + // An edit keeps its row on screen (#2667); an insert hides it, and once undone and inserted again it shows again. + @Test("A row a value filter shows again after an insert is searched again") + func rowBackInTheFilteredOrderIsSearched() async { + let (model, owner) = mount() + var filter = GridValueFilterState() + filter.set( + ColumnValueFilter(selectedValues: ["Acme Falls", "Paris", "Acme"], includesNull: false), + columnName: "city", + forColumn: 2 + ) + model.setValueFilter(filter) + await search(model, "acme") + #expect(model.find.matches == [match(1, Self.city), match(2, Self.company), match(3, Self.company), match(3, Self.city)]) + + owner.tableRows.rows[2].values[2] = .text("Lyon") + model.apply(.cellChanged(row: 2, column: 2)) + let firstRow = owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Nova"), .text("Oslo")]) + model.apply(firstRow) + #expect(model.displayIndex(of: .existing(2)) == nil) + let firstRowID = owner.tableRows.rows[4].id + model.apply(owner.tableRows.remove(rowIDs: [firstRowID])) + owner.tableRows.rows[2].values[2] = .text("Paris") + model.apply(.cellChanged(row: 2, column: 2)) + let secondRow = owner.tableRows.appendInsertedRow(values: [.text("6"), .text("Vega"), .text("Oslo")]) + model.apply(secondRow) + + #expect(model.displayIndex(of: .existing(2)) != nil) + #expect(model.find.matches == [match(1, Self.city), match(2, Self.company), match(3, Self.company), match(3, Self.city)]) + #expect(model.find.matches == fullSearch(model, owner)) + } + + @Test("A term typed but not yet searched makes the next refresh search every row") + func changedTermSearchesEveryRow() async { + let (model, owner) = mount() + await search(model, "acme") + + model.find.term = "paris" + owner.tableRows.rows[0].values[2] = .text("Paris") + model.apply(.cellChanged(row: 3, column: 1)) + await model.findSearchSettled() + + #expect(model.find.matches == [match(0, Self.city), match(2, Self.city)]) + } + @Test("The counter names the page whenever the owner holds more rows, Search All Rows offered or not") func pageScopeFollowsTheOwnersRows() { let (model, owner) = mount() @@ -566,16 +872,218 @@ struct DataGridModelFindTests { } @Test("Use Selection for Find takes the cell's text, escaped for a regular expression") - func useSelectionForFind() { + func useSelectionForFind() async { let (model, owner) = mount() model.setSelection(cell(0, 2)) model.find.options.usesRegularExpression = true model.useSelectionForFind() + await model.findSearchSettled() #expect(model.find.isVisible) #expect(model.find.term == NSRegularExpression.escapedPattern(for: "St. Louis")) #expect(model.find.matches == [GridFindState.Match(rowID: .existing(0), columnID: DataGridColumnID(name: "city", occurrence: 0))]) #expect(owner.commits.isEmpty) } + + // MARK: - A search off the main actor + + // Each test ends on `owner`: the model holds its data source weakly, and without it the grid has no rows. + + @Test("A typed term drops the old term's matches at once, and a refresh keeps its matches until its search lands") + func typedTermDropsTheOldMatches() async { + let (model, owner) = mount() + await search(model, "paris") + + model.find.term = "acme" + model.findQueryDidChange() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.isSearching == false) + #expect(model.find.matches.isEmpty) + #expect(model.find.currentIndex == nil) + + await model.findSearchSettled() + + #expect(model.find.loadedRowsSearch == nil) + #expect(model.find.matches.count == 4) + #expect(model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) + + model.refreshFind() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.matches.count == 4) + #expect(model.find.currentIndex == 2) + + await model.findSearchSettled() + #expect(model.find.currentIndex == 2) + #expect(owner.commits.isEmpty) + } + + @Test("A typed term drops the old term's invalid-pattern message at once") + func typedTermDropsTheOldProblem() async { + let (model, owner) = mount() + model.showFind() + model.find.options.usesRegularExpression = true + model.find.term = "[" + model.findQueryDidChange() + #expect(model.find.problem != nil) + + model.find.term = "acme" + model.findQueryDidChange() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.problem == nil) + + await model.findSearchSettled() + #expect(model.find.problem == nil) + #expect(owner.commits.isEmpty) + } + + @Test("Of two terms typed in quick succession only the second one's matches land") + func onlyTheLastTermLands() async throws { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + let first = try #require(model.find.loadedRowsSearch) + + model.find.term = "paris" + model.findQueryDidChange() + await model.findSearchSettled() + + #expect(first.isCancelled) + #expect(model.find.matches == [match(2, Self.city)]) + #expect(model.selection.activeCell == GridCoord(row: 2, displayColumn: 2)) + #expect(owner.commits.isEmpty) + } + + @Test("A row added while a search runs is among the matches that land") + func rowAddedDuringASearchIsFound() async throws { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + let running = try #require(model.find.loadedRowsSearch) + + model.apply(owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")])) + await model.findSearchSettled() + + let insertedID = owner.tableRows.rows[4].id + #expect(running.isCancelled) + #expect(model.find.matches.last == GridFindState.Match(rowID: insertedID, columnID: Self.company)) + #expect(model.find.matches == fullSearch(model, owner)) + #expect(model.selection.activeCell == GridCoord(row: 1, displayColumn: 2), "the typed term still lands on its first match") + } + + @Test("A typed term's matches land without moving a cursor the user moved during the search") + func landingKeepsAMovedCursor() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.setSelection(cell(0, 0)) + await model.findSearchSettled() + + #expect(model.find.matches.count == 4) + #expect(model.find.currentIndex == 0) + #expect(model.selection.activeCell == GridCoord(row: 0, displayColumn: 0)) + #expect(owner.commits.isEmpty) + } + + @Test("Find Next pressed while a search runs steps once its matches land") + func findNextWaitsForTheSearch() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.findNext() + #expect(model.selection.activeCell == nil) + + await model.findSearchSettled() + + #expect(model.find.currentIndex == 1) + #expect(model.selection.activeCell == GridCoord(row: 2, displayColumn: 1)) + #expect(owner.commits.isEmpty) + } + + @Test("Every Find Next and Find Previous pressed while a search runs counts once its matches land") + func deferredStepsAreCounted() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.findNext() + model.findNext() + model.findNext() + model.findPrevious() + await model.findSearchSettled() + + #expect(model.find.currentIndex == 2) + #expect(model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) + #expect(owner.commits.isEmpty) + } + + @Test("Replace pressed while a search runs replaces the match that search lands on") + func replaceWaitsForTheRunningSearch() async { + let (model, owner) = mount { $0.isEditable = true } + await search(model, "paris") + model.find.term = "acme" + model.findQueryDidChange() + model.find.replacement = "Umbrella" + + model.replaceCurrentMatch() + + #expect(model.find.loadedRowsSearch == nil) + #expect(owner.commits.first?.edits.map { $0.rowID } == [.existing(3)]) + #expect(owner.commits.first?.edits.map { $0.newValue } == [.text("Umbrella Corp")]) + } + + @Test("The bar says No matches and offers Search All Rows only once the search lands") + func noMatchesWaitsForTheSearch() async { + let (model, owner) = mount { $0.commands = [.searchAllRows] } + model.showFind() + model.find.term = "zzz" + model.findQueryDidChange() + + #expect(model.find.isAwaitingLoadedRows) + #expect(DataGridFindBar.counterText(status: nil, model: model).isEmpty) + #expect(!DataGridFindBar.showsSearchAllRows(offered: model.offersSearchAllRows, model: model)) + + await model.findSearchSettled() + + #expect(!model.find.isAwaitingLoadedRows) + #expect(DataGridFindBar.counterText(status: nil, model: model) == String(localized: "No matches")) + #expect(DataGridFindBar.showsSearchAllRows(offered: model.offersSearchAllRows, model: model)) + + model.find.term = "" + model.findQueryDidChange() + + #expect(!model.find.isAwaitingLoadedRows, "an empty term is answered without a search") + #expect(model.find.loadedRowsSearch == nil) + #expect(owner.commits.isEmpty) + } + + @Test("A grid that goes away cancels the search still running") + func releasedModelCancelsTheSearch() throws { + let owner: FindGridOwner + let running: Task + weak var released: DataGridModel? + do { + let (model, mounted) = mount() + owner = mounted + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + running = try #require(model.find.loadedRowsSearch) + released = model + } + + #expect(released == nil) + #expect(running.isCancelled) + #expect(owner.commits.isEmpty) + } } diff --git a/apple/TableProTests/Core/DataGrid/GridViewportResolverTests.swift b/apple/TableProTests/Core/DataGrid/GridViewportResolverTests.swift index 13773fdddc..f14023d787 100644 --- a/apple/TableProTests/Core/DataGrid/GridViewportResolverTests.swift +++ b/apple/TableProTests/Core/DataGrid/GridViewportResolverTests.swift @@ -28,18 +28,53 @@ struct GridViewportResolverTests { selectedRows: [Int] = [], keyColumns: [String] = keyColumns, isCellModified: (RowID, Int) -> Bool = { _, _ in false } + ) -> GridViewportSnapshot { + let order = DataGridRowOrder(filteredIDs: displayIDs, revision: 0) + let selection = selectedRows.isEmpty + ? GridSelectionSnapshot.empty + : GridSelectionSnapshot( + rowIDs: selectedRows.compactMap { order.rowID(at: $0, in: tableRows) }, + rectangles: [], + activeCell: nil, + anchor: nil + ) + return snapshot( + of: tableRows, + displayIDs: displayIDs, + firstVisibleRow: firstVisibleRow, + offset: offset, + selection: selection, + keyColumns: keyColumns, + isCellModified: isCellModified + ) + } + + private static func snapshot( + of tableRows: TableRows, + displayIDs: [RowID]? = nil, + firstVisibleRow: Int, + offset: CGFloat = 0, + selection: GridSelectionSnapshot, + keyColumns: [String] = keyColumns, + isCellModified: (RowID, Int) -> Bool = { _, _ in false } ) -> GridViewportSnapshot { GridViewportResolver.snapshot( of: tableRows, displayIDs: displayIDs, firstVisibleDisplayRow: firstVisibleRow, firstVisibleOffset: offset, - selectedDisplayRows: selectedRows, + selection: selection, keyColumns: keyColumns, isCellModified: isCellModified ) } + private static let nameColumn = DataGridColumnID(name: "name", occurrence: 0) + + private static func cell(_ row: Int) -> GridSelectionSnapshot.Cell { + GridSelectionSnapshot.Cell(rowID: .existing(row), columnID: nameColumn) + } + private static func placement( _ intent: GridReloadIntent, from snapshot: GridViewportSnapshot, @@ -259,6 +294,49 @@ struct GridViewportResolverTests { #expect(Self.placement(.firstRow, from: snapshot, in: Self.rows(ids: [7, 7, 8])).selectedRows.isEmpty) } + @Test("A re-read keeps a cell cursor a cursor, on its own column, wherever its row lands") + func keepPlaceKeepsTheCursorAndItsColumn() { + var cursor = GridSelectionSnapshot(rowIDs: [.existing(2)], rectangles: [], activeCell: Self.cell(2), anchor: Self.cell(2)) + cursor.mode = .cursor + let snapshot = Self.snapshot(of: Self.rows(ids: Array(1 ... 10)), firstVisibleRow: 0, selection: cursor) + + let placement = Self.placement(.keepPlace, from: snapshot, in: Self.rows(ids: Array((1 ... 10).reversed()))) + + #expect(placement.selection.mode == .cursor) + #expect(placement.selection.rowIDs == [.existing(7)]) + #expect(placement.selection.activeCell == Self.cell(7)) + } + + @Test("A re-read keeps a cell range's columns rather than widening it to whole rows") + func keepPlaceKeepsACellRange() { + let idColumn = DataGridColumnID(name: "id", occurrence: 0) + let range = GridSelectionSnapshot( + rowIDs: [.existing(3), .existing(4)], + rectangles: [GridSelectionSnapshot.Rectangle(rowIDs: [.existing(3), .existing(4)], columnIDs: [Self.nameColumn])], + activeCell: Self.cell(3), + anchor: Self.cell(3) + ) + let snapshot = Self.snapshot(of: Self.rows(ids: Array(1 ... 10)), firstVisibleRow: 0, selection: range) + + let placement = Self.placement(.keepPlace, from: snapshot, in: Self.rows(ids: Array(0 ... 10))) + + #expect(placement.selection.mode == .cells) + #expect(placement.selection.rectangles == [ + GridSelectionSnapshot.Rectangle(rowIDs: [.existing(4), .existing(5)], columnIDs: [Self.nameColumn]) + ]) + #expect(placement.selection.activeCell == Self.cell(4)) + #expect(!placement.selection.rectangles.contains { $0.columnIDs.contains(idColumn) }) + } + + @Test("A cursor whose row is gone leaves nothing selected") + func cursorOnAGoneRowSelectsNothing() { + var cursor = GridSelectionSnapshot(rowIDs: [.existing(5)], rectangles: [], activeCell: Self.cell(5), anchor: Self.cell(5)) + cursor.mode = .cursor + let snapshot = Self.snapshot(of: Self.rows(ids: Array(1 ... 10)), firstVisibleRow: 0, selection: cursor) + + #expect(Self.placement(.keepPlace, from: snapshot, in: Self.rows(ids: [1, 2, 3])).selection == .empty) + } + @Test("Without a key the position is read through the incoming display order") func keepPlaceFallbackUsesTheDisplayOrder() { let snapshot = Self.snapshot(of: Self.rows(ids: Array(1 ... 100)), firstVisibleRow: 2, offset: 4, keyColumns: []) diff --git a/apple/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift b/apple/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift index 0710c71a8e..8dabe05674 100644 --- a/apple/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift +++ b/apple/TableProTests/Core/DataGrid/RowDisplayCacheTests.swift @@ -211,6 +211,36 @@ struct RowDisplayCacheTests { #expect(cache.highlight(forID: .existing(3)) == nil) } + @Test("A cell's formatted flag is written with its text and cleared with it") + func formattedFlagFollowsTheText() { + let cache = RowDisplayCache() + let box = makeBox(Array(repeating: nil, count: 70)) + box.setText("uuid", at: 1, isFormatted: true) + box.setText("hex", at: 2, isFormatted: false) + box.setText("date", at: 65, isFormatted: true) + cache.setBox(box, forID: .existing(0)) + + #expect(box.isFormatted(at: 1)) + #expect(!box.isFormatted(at: 2)) + #expect(box.isFormatted(at: 65)) + #expect(!box.isFormatted(at: 69)) + #expect(!box.isFormatted(at: 200)) + #expect(!box.isFormatted(at: -1)) + + box.setText("raw", at: 1, isFormatted: false) + #expect(!box.isFormatted(at: 1)) + box.setText("uuid", at: 1, isFormatted: true) + + cache.clearValues(forDataIndexes: [1]) + #expect(box.values[1] == nil) + #expect(!box.isFormatted(at: 1)) + #expect(box.isFormatted(at: 65)) + + cache.clearValues(forID: .existing(0)) + #expect(box.values[65] == nil) + #expect(!box.isFormatted(at: 65)) + } + @Test("Inserted row IDs of both kinds round-trip") func mixedRowIDKinds() { let cache = RowDisplayCache() diff --git a/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift b/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift index cc8ad67a43..5f1cefb55a 100644 --- a/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift +++ b/apple/TableProTests/Core/DataWrite/DataWriteExecutorLookupTests.swift @@ -83,7 +83,7 @@ struct DataWriteExecutorLookupTests { steps: steps ) let driver = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .clickhouse), pluginDriver: lookup) - _ = try await DataWriteExecutor.run(plan, on: driver) + _ = try await DataWriteExecutor.run(plan, commitMode: .automatic, on: driver) } @Test("A write runs on the rows its lookup found, their values bound after its own") @@ -97,6 +97,40 @@ struct DataWriteExecutorLookupTests { #expect(driver.trace.map { $0.query } == ["LOOKUP DELETE pinned", "LOOKUP DELETE pinned", "DELETE pinned", "CHECK DELETE pinned"]) } + private func heldStep(_ held: PluginCellValue) -> DataWriteStep { + DataWriteStep( + kind: .rowWrite, + statement: ParameterizedStatement(sql: "UPDATE held", parameters: ["a"]), + expectedRowCount: 1, + tableName: "events", + lookup: RowLookup( + query: ParameterizedStatement(sql: "LOOKUP UPDATE held", parameters: []), + rowCount: 1, + pinsRows: false, + heldValues: [0: held] + ) + ) + } + + @Test("A row that still holds the reviewed values is written") + func heldValuesThatStillMatchWrite() async throws { + let driver = LookupDriver(found: [1, 1], affected: 1) + + try await run([heldStep("part_0")], on: driver) + + #expect(driver.writes.map { $0.query } == ["UPDATE held"]) + } + + @Test("A row another writer changed since the review refuses the write before anything runs") + func changedHeldValueRefuses() async throws { + let driver = LookupDriver(found: [1, 1], affected: 1) + + await #expect(throws: DataWriteError.self) { + try await run([heldStep("reviewed")], on: driver) + } + #expect(driver.writes.isEmpty) + } + @Test("A row its lookup cannot tell apart from another refuses the save before anything is written") func identicalRowsRefuseBeforeAnyWrite() async { let driver = LookupDriver(found: [1, 2]) diff --git a/apple/TableProTests/Core/DataWrite/DataWriteExecutorTableOperationTests.swift b/apple/TableProTests/Core/DataWrite/DataWriteExecutorTableOperationTests.swift new file mode 100644 index 0000000000..e240154fb3 --- /dev/null +++ b/apple/TableProTests/Core/DataWrite/DataWriteExecutorTableOperationTests.swift @@ -0,0 +1,172 @@ +// +// DataWriteExecutorTableOperationTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class TableOperationDriver: PluginDriverStub, @unchecked Sendable { + struct Failure: Error {} + + private let lock = NSLock() + private let state: PluginSessionTransactionState + private let failing: String? + private var sent: [String] = [] + + init(state: PluginSessionTransactionState = .idle, failing: String? = nil) { + self.state = state + self.failing = failing + } + + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { true } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + var trace: [String] { lock.withLock { sent } } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { sent.append(query) } + if query == failing { throw Failure() } + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { state } + + func beginTransaction(mode: PluginTransactionAccessMode) async throws { lock.withLock { sent.append("BEGIN") } } + func commitTransaction() async throws { lock.withLock { sent.append("COMMIT") } } + func rollbackTransaction() async throws { lock.withLock { sent.append("ROLLBACK") } } + + func switchDatabase(to database: String) async throws {} +} + +struct DataWriteExecutorTableOperationTests { + private static let dropA = "DROP TABLE a" + private static let dropB = "DROP TABLE b" + + private static let truncateA = "DELETE FROM a ALL" + + private static func dropPlan(_ databaseType: DatabaseType) -> DataWritePlan { + tableOperationPlan([dropA, dropB], databaseType: databaseType) + } + + private static func tableOperationPlan(_ statements: [String], databaseType: DatabaseType = .teradata) -> DataWritePlan { + DataWritePlan( + scope: DatabaseScope(connectionId: UUID(), database: "sales", schema: nil), + databaseType: databaseType, + steps: statements.map { + DataWriteStep(kind: .tableOperation, statement: ParameterizedStatement(sql: $0, parameters: [])) + } + ) + } + + private static func adapter(_ driver: TableOperationDriver, type: DatabaseType) -> DatabaseDriver { + PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver) + } + + @Test("Two Teradata drops each end their own transaction, with no commit after the last") + func teradataDropsCommitOneByOne() async throws { + let driver = TableOperationDriver() + + let run = try await DataWriteExecutor.run( + Self.dropPlan(.teradata), commitMode: .automatic, on: Self.adapter(driver, type: .teradata) + ) + + #expect(run.results.count == 2) + #expect(driver.trace == ["BEGIN", Self.dropA, "COMMIT", "BEGIN", Self.dropB, "COMMIT"]) + } + + @Test("Drops on an engine without the rule keep one transaction") + func otherEnginesKeepOneTransaction() async throws { + let driver = TableOperationDriver() + + _ = try await DataWriteExecutor.run( + Self.dropPlan(.mysql), commitMode: .automatic, on: Self.adapter(driver, type: .mysql) + ) + + #expect(driver.trace == ["BEGIN", Self.dropA, Self.dropB, "COMMIT"]) + } + + @Test("Teradata drops are refused before anything runs while the session holds a transaction") + func teradataDropsRefusedInsideAnOpenTransaction() async throws { + let driver = TableOperationDriver(state: .inTransaction) + + await #expect(throws: DefinitionInOpenTransactionRefusal()) { + try await DataWriteExecutor.run( + Self.dropPlan(.teradata), commitMode: .automatic, on: Self.adapter(driver, type: .teradata) + ) + } + #expect(driver.trace.isEmpty) + } + + @Test("Teradata drops are refused before anything runs under Manual Commit") + func teradataDropsRefusedUnderManualCommit() async throws { + let driver = TableOperationDriver() + + await #expect(throws: ManualCommitRefusal()) { + try await DataWriteExecutor.run( + Self.dropPlan(.teradata), commitMode: .manual, on: Self.adapter(driver, type: .teradata) + ) + } + #expect(driver.trace.isEmpty) + } + + @Test("A Teradata drop that fails after another committed reports that one as written") + func failedSecondDropReportsTheFirstAsWritten() async throws { + let driver = TableOperationDriver(failing: Self.dropB) + + do { + _ = try await DataWriteExecutor.run( + Self.dropPlan(.teradata), commitMode: .automatic, on: Self.adapter(driver, type: .teradata) + ) + Issue.record("The save reported success") + } catch let partial as DataWritePartialCommitError { + #expect(partial.committed.count == 1) + #expect(partial.totalStatements == 2) + #expect(partial.disposition == .written) + } + #expect(driver.trace == ["BEGIN", Self.dropA, "COMMIT", "BEGIN", Self.dropB, "ROLLBACK"]) + } + + @Test("A Teradata truncate joins the drop's transaction, which the drop then ends") + func teradataTruncateJoinsTheDropsSegment() async throws { + let driver = TableOperationDriver() + + _ = try await DataWriteExecutor.run( + Self.tableOperationPlan([Self.truncateA, Self.dropB]), commitMode: .automatic, + on: Self.adapter(driver, type: .teradata) + ) + + #expect(driver.trace == ["BEGIN", Self.truncateA, Self.dropB, "COMMIT"]) + } + + @Test("A Teradata truncate after a drop runs in a transaction of its own, committed at the end") + func teradataTruncateAfterADropGetsTheFinalCommit() async throws { + let driver = TableOperationDriver() + + _ = try await DataWriteExecutor.run( + Self.tableOperationPlan([Self.dropB, Self.truncateA]), commitMode: .automatic, + on: Self.adapter(driver, type: .teradata) + ) + + #expect(driver.trace == ["BEGIN", Self.dropB, "COMMIT", "BEGIN", Self.truncateA, "COMMIT"]) + } + + @Test("A Teradata truncate alone is held under Manual Commit and joins an open session transaction") + func teradataTruncateAloneIsDML() async throws { + let held = TableOperationDriver() + _ = try await DataWriteExecutor.run( + Self.tableOperationPlan([Self.truncateA]), commitMode: .manual, on: Self.adapter(held, type: .teradata) + ) + #expect(held.trace == ["BEGIN", Self.truncateA]) + + let joined = TableOperationDriver(state: .inTransaction) + _ = try await DataWriteExecutor.run( + Self.tableOperationPlan([Self.truncateA]), commitMode: .automatic, on: Self.adapter(joined, type: .teradata) + ) + #expect(joined.trace == [Self.truncateA]) + } +} diff --git a/apple/TableProTests/Core/DataWrite/DataWriteExecutorTests.swift b/apple/TableProTests/Core/DataWrite/DataWriteExecutorTests.swift index 5234fead9b..3295752a56 100644 --- a/apple/TableProTests/Core/DataWrite/DataWriteExecutorTests.swift +++ b/apple/TableProTests/Core/DataWrite/DataWriteExecutorTests.swift @@ -87,7 +87,9 @@ struct DataWriteExecutorTests { epilogue: [String] = [], matchesRowsWithoutKey: Bool = false, refusesSharedKey: Bool = false, - databaseType: DatabaseType = .sqlite + rowKind: RowWriteKind? = nil, + databaseType: DatabaseType = .sqlite, + countsMatchedRows: Bool = false ) -> DataWritePlan { DataWritePlan( scope: DatabaseScope(connectionId: UUID(), database: "shop", schema: nil), @@ -99,7 +101,9 @@ struct DataWriteExecutorTests { expectedRowCount: expectedRowCount, tableName: "t", matchesRowsWithoutKey: matchesRowsWithoutKey, - refusesSharedKey: refusesSharedKey + refusesSharedKey: refusesSharedKey, + rowKind: rowKind, + countsMatchedRows: countsMatchedRows ) }, prologue: prologue, @@ -114,7 +118,7 @@ struct DataWriteExecutorTests { @Test("The driver's real affected-row count reaches the caller") func reportsRealRowCount() async throws { let counting = CountingDriver(affectedRows: 42) - let results = try await DataWriteExecutor.run(plan(expectedRowCount: 50), on: driver(counting)).results + let results = try await DataWriteExecutor.run(plan(expectedRowCount: 50), commitMode: .automatic, on: driver(counting)).results #expect(results.first?.rowsAffected == 42) #expect(counting.didCommit) @@ -124,7 +128,7 @@ struct DataWriteExecutorTests { func tooManyRowsRollsBack() async throws { let counting = CountingDriver(affectedRows: 2) await #expect(throws: DataWriteError.tooManyRowsAffected(table: "t", expected: 1, actual: 2)) { - try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)) + try await DataWriteExecutor.run(plan(expectedRowCount: 1), commitMode: .automatic, on: driver(counting)) } #expect(counting.didRollBack) #expect(counting.didCommit == false) @@ -136,28 +140,122 @@ struct DataWriteExecutorTests { await #expect( throws: DataWriteError.tooManyRowsAffectedUnrecoverable(table: "t", expected: 1, actual: 2) ) { - try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)) + try await DataWriteExecutor.run(plan(expectedRowCount: 1), commitMode: .automatic, on: driver(counting)) } #expect(counting.didRollBack == false) } - @Test("Fewer rows than expected is an ordinary save, because MySQL reports zero for an unchanged value") + @Test("A keyed UPDATE short of its rows is an ordinary save where nothing says the count is the rows matched") func fewerRowsIsNotAFailure() async throws { let counting = CountingDriver(affectedRows: 0) - let results = try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)).results + let results = try await DataWriteExecutor.run( + plan(expectedRowCount: 1, rowKind: .update, databaseType: .snowflake), + commitMode: .automatic, on: driver(counting) + ).results #expect(results.first?.rowsAffected == 0) #expect(counting.didCommit) #expect(counting.didRollBack == false) } + @Test("A keyed UPDATE that matched no row fails where the count is the rows matched") + func keyedUpdateThatMatchedNothingFails() async throws { + let engines: [DatabaseType] = [.postgresql, .sqlite, .duckdb, .mysql, .mariadb, .tidb, .oceanbase] + for databaseType in engines { + let counting = CountingDriver(affectedRows: 0) + await #expect(throws: DataWriteError.self) { + try await DataWriteExecutor.run( + plan(expectedRowCount: 1, rowKind: .update, databaseType: databaseType), + commitMode: .automatic, on: driver(counting) + ) + } + #expect(counting.didRollBack) + #expect(counting.didCommit == false) + } + } + + @Test("A driver's own matched-row count holds a keyed UPDATE on an engine the host keeps no count for") + func driverCountHoldsAKeyedUpdate() async throws { + let counting = CountingDriver(affectedRows: 0) + await #expect(throws: DataWriteError.self) { + try await DataWriteExecutor.run( + plan(expectedRowCount: 1, rowKind: .update, databaseType: .snowflake, countsMatchedRows: true), + commitMode: .automatic, on: driver(counting) + ) + } + #expect(counting.didRollBack) + #expect(counting.didCommit == false) + } + + @Test("A driver's own count leaves a keyed DELETE of a row already gone an ordinary save") + func driverCountLeavesAGoneRowsDeleteAlone() async throws { + let counting = CountingDriver(affectedRows: 0) + _ = try await DataWriteExecutor.run( + plan(expectedRowCount: 1, rowKind: .delete, databaseType: .snowflake, countsMatchedRows: true), + commitMode: .automatic, on: driver(counting) + ) + + #expect(counting.didCommit) + } + + @Test("A driver's own count through a view keeps the lenient rule") + func driverCountThroughAViewStaysLenient() async throws { + let counting = CountingDriver(affectedRows: 0) + var viewPlan = plan(expectedRowCount: 1, rowKind: .update, databaseType: .mssql, countsMatchedRows: true) + viewPlan.viewTarget = TestFixtures.viewTarget(keys: ["id"]) + + _ = try await DataWriteExecutor.run(viewPlan, commitMode: .automatic, on: driver(counting)) + + #expect(counting.didCommit) + } + + @Test("A keyed UPDATE through a view keeps the lenient rule: its count comes from the view's trigger") + func keyedUpdateThroughAViewStaysLenient() async throws { + let counting = CountingDriver(affectedRows: 0) + var viewPlan = plan(expectedRowCount: 1, rowKind: .update, databaseType: .postgresql) + viewPlan.viewTarget = TestFixtures.viewTarget(keys: ["id"]) + + _ = try await DataWriteExecutor.run(viewPlan, commitMode: .automatic, on: driver(counting)) + + #expect(counting.didCommit) + } + + @Test("An UPDATE after a DELETE in the same save may find its row gone to the cascade, which is no failure") + func updateAfterADeleteStaysLenient() async throws { + let counting = CountingDriver(affectedRows: 0) + let update = plan(expectedRowCount: 1, rowKind: .update, databaseType: .postgresql) + let delete = DataWriteStep( + kind: .rowWrite, + statement: ParameterizedStatement(sql: "DELETE FROM \"t\" WHERE \"id\" = 1", parameters: []), + expectedRowCount: 1, + tableName: "t", + rowKind: .delete + ) + let cascading = DataWritePlan(scope: update.scope, databaseType: .postgresql, steps: [delete] + update.steps) + + _ = try await DataWriteExecutor.run(cascading, commitMode: .automatic, on: driver(counting)) + + #expect(counting.didCommit) + } + + @Test("A keyed DELETE that found its row already gone stays an ordinary save") + func keyedDeleteOfAGoneRowIsNotAFailure() async throws { + let counting = CountingDriver(affectedRows: 0) + _ = try await DataWriteExecutor.run( + plan(expectedRowCount: 1, rowKind: .delete, databaseType: .postgresql), + commitMode: .automatic, on: driver(counting) + ) + + #expect(counting.didCommit) + } + @Test("A keyless write that found no row is rolled back rather than reported as a save") func keylessWriteThatMatchedNothingFails() async throws { let counting = CountingDriver(affectedRows: 0) await #expect(throws: DataWriteError.rowsNoLongerMatch(table: "t", expected: 1, actual: 0)) { try await DataWriteExecutor.run( plan(expectedRowCount: 1, matchesRowsWithoutKey: true), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) } #expect(counting.didRollBack) @@ -171,7 +269,7 @@ struct DataWriteExecutorTests { await #expect(throws: DataWriteError.keyNoLongerMatchesOneRow(table: "t", expected: 1, actual: 0)) { try await DataWriteExecutor.run( plan(expectedRowCount: 1, refusesSharedKey: true), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) } #expect(counting.didRollBack) @@ -184,7 +282,7 @@ struct DataWriteExecutorTests { await #expect(throws: DataWriteError.rowsNoLongerMatch(table: "t", expected: 3, actual: 2)) { try await DataWriteExecutor.run( plan(expectedRowCount: 3, matchesRowsWithoutKey: true), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) } #expect(counting.didRollBack) @@ -196,7 +294,7 @@ struct DataWriteExecutorTests { await #expect(throws: DataWriteError.rowsNoLongerMatch(table: "t", expected: 1, actual: 0)) { try await DataWriteExecutor.run( plan(expectedRowCount: 1, matchesRowsWithoutKey: true, databaseType: .kingbasees), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) } #expect(counting.didRollBack) @@ -207,7 +305,7 @@ struct DataWriteExecutorTests { let counting = CountingDriver(affectedRows: 0) let results = try await DataWriteExecutor.run( plan(expectedRowCount: 1, matchesRowsWithoutKey: true, databaseType: .clickhouse), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ).results #expect(results.first?.rowsAffected == 0) @@ -224,7 +322,7 @@ struct DataWriteExecutorTests { prologue: ["PRAGMA foreign_keys = OFF"], epilogue: ["PRAGMA foreign_keys = ON"] ), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) #expect(counting.trace == [ @@ -248,7 +346,7 @@ struct DataWriteExecutorTests { prologue: ["PRAGMA foreign_keys = OFF"], epilogue: ["PRAGMA foreign_keys = ON"] ), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) } @@ -269,7 +367,7 @@ struct DataWriteExecutorTests { prologue: ["SET session_replication_role = replica"], epilogue: ["SET session_replication_role = origin"] ), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) #expect(!counting.trace.contains("open preserve")) @@ -279,7 +377,7 @@ struct DataWriteExecutorTests { @Test("A plan without a prologue leaves the session's settings alone") func planWithoutPrologueOpensNoScope() async throws { let counting = CountingDriver(affectedRows: 1) - _ = try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)) + _ = try await DataWriteExecutor.run(plan(expectedRowCount: 1), commitMode: .automatic, on: driver(counting)) #expect(counting.trace == ["BEGIN", "UPDATE \"t\" SET \"b\" = 0", "COMMIT"]) } @@ -296,7 +394,7 @@ struct DataWriteExecutorTests { prologueIsRequired: true ) await #expect(throws: (any Error).self) { - try await DataWriteExecutor.run(required, on: driver(counting)) + try await DataWriteExecutor.run(required, commitMode: .automatic, on: driver(counting)) } #expect(counting.trace == [ @@ -312,7 +410,7 @@ struct DataWriteExecutorTests { func anOptionalPrologueThatFailsLetsTheStepsRun() async throws { let counting = CountingDriver(affectedRows: 1, failOnStatement: 1) _ = try await DataWriteExecutor.run( - plan(expectedRowCount: 1, prologue: ["PRAGMA foreign_keys = OFF"]), on: driver(counting) + plan(expectedRowCount: 1, prologue: ["PRAGMA foreign_keys = OFF"]), commitMode: .automatic, on: driver(counting) ) #expect(counting.didCommit) @@ -323,7 +421,7 @@ struct DataWriteExecutorTests { let counting = CountingDriver(affectedRows: 1) let run = try await DataWriteExecutor.run( plan(expectedRowCount: 1, prologue: ["A"], epilogue: ["B"]), - on: driver(counting) + commitMode: .automatic, on: driver(counting) ) #expect(run.sideStatements == ["A", "B"]) @@ -335,7 +433,7 @@ struct DataWriteExecutorTests { do { _ = try await DataWriteExecutor.run( - plan(expectedRowCount: 1, statementCount: 3), on: driver(counting) + plan(expectedRowCount: 1, statementCount: 3), commitMode: .automatic, on: driver(counting) ) Issue.record("expected the run to throw") } catch let error as DataWritePartialCommitError { @@ -350,7 +448,7 @@ struct DataWriteExecutorTests { do { _ = try await DataWriteExecutor.run( - plan(expectedRowCount: 1, statementCount: 3), on: driver(counting) + plan(expectedRowCount: 1, statementCount: 3), commitMode: .automatic, on: driver(counting) ) Issue.record("expected the run to throw") } catch is DataWritePartialCommitError { @@ -368,7 +466,7 @@ struct DataWriteExecutorTests { do { _ = try await DataWriteExecutor.run( - plan(expectedRowCount: 1, statementCount: 3), on: driver(counting) + plan(expectedRowCount: 1, statementCount: 3), commitMode: .automatic, on: driver(counting) ) Issue.record("expected the run to throw") } catch let error as DataWritePartialCommitError { @@ -379,7 +477,7 @@ struct DataWriteExecutorTests { @Test("A statement whose row count carries no meaning is not held to one") func unverifiedStepIsNotChecked() async throws { let counting = CountingDriver(affectedRows: 9_999) - let results = try await DataWriteExecutor.run(plan(expectedRowCount: nil), on: driver(counting)).results + let results = try await DataWriteExecutor.run(plan(expectedRowCount: nil), commitMode: .automatic, on: driver(counting)).results #expect(results.first?.wasVerified == false) #expect(counting.didCommit) @@ -391,7 +489,7 @@ struct DataWriteExecutorTests { ) func saveJoinsTheSessionTransaction(state: PluginSessionTransactionState) async throws { let counting = CountingDriver(affectedRows: 1, sessionState: state) - let results = try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)).results + let results = try await DataWriteExecutor.run(plan(expectedRowCount: 1), commitMode: .automatic, on: driver(counting)).results #expect(results.first?.rowsAffected == 1) #expect(counting.trace == ["UPDATE \"t\" SET \"b\" = 0"]) @@ -401,7 +499,7 @@ struct DataWriteExecutorTests { func saveWrapsWhenNothingIsOpen() async throws { for state in [PluginSessionTransactionState.idle, .unknown] { let counting = CountingDriver(affectedRows: 1, sessionState: state) - _ = try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)) + _ = try await DataWriteExecutor.run(plan(expectedRowCount: 1), commitMode: .automatic, on: driver(counting)) #expect(counting.trace.first == "BEGIN") #expect(counting.didCommit) } @@ -413,7 +511,7 @@ struct DataWriteExecutorTests { do { _ = try await DataWriteExecutor.run( - plan(expectedRowCount: 1, statementCount: 3), on: driver(counting) + plan(expectedRowCount: 1, statementCount: 3), commitMode: .automatic, on: driver(counting) ) Issue.record("expected the run to throw") } catch let error as DataWritePartialCommitError { @@ -430,7 +528,7 @@ struct DataWriteExecutorTests { await #expect( throws: DataWriteError.tooManyRowsAffectedInSessionTransaction(table: "t", expected: 1, actual: 2) ) { - try await DataWriteExecutor.run(plan(expectedRowCount: 1), on: driver(counting)) + try await DataWriteExecutor.run(plan(expectedRowCount: 1), commitMode: .automatic, on: driver(counting)) } #expect(counting.didRollBack == false) } @@ -442,10 +540,12 @@ struct WriteTransactionOwnerTests { for state in [ PluginSessionTransactionState.idle, .inTransaction, .abortedTransaction, .holdsSessionLocks, .unknown, ] { - let owner = WriteTransactionOwner.resolve(supportsTransactions: false, sessionState: state) - #expect(owner == WriteTransactionOwner.none) - #expect(owner.opensTransaction == false) - #expect(owner.canRollBack == false) + for mode in ConnectionCommitMode.allCases { + let owner = WriteTransactionOwner.resolve(supportsTransactions: false, sessionState: state, commitMode: mode) + #expect(owner == WriteTransactionOwner.none) + #expect(owner.sendsBegin == false) + #expect(owner.canRollBack == false) + } } } @@ -454,19 +554,128 @@ struct WriteTransactionOwnerTests { arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction, .holdsSessionLocks] ) func sessionOwnsWhatItHolds(state: PluginSessionTransactionState) { - let owner = WriteTransactionOwner.resolve(supportsTransactions: true, sessionState: state) - #expect(owner == .session) - #expect(owner.opensTransaction == false) - #expect(owner.canRollBack == false) + for mode in ConnectionCommitMode.allCases { + let owner = WriteTransactionOwner.resolve(supportsTransactions: true, sessionState: state, commitMode: mode) + #expect(owner == .session) + #expect(owner.sendsBegin == false) + #expect(owner.canRollBack == false) + } } @Test("The app owns the transaction when nothing is open, and when the driver cannot say") func appOwnsTheRest() { for state in [PluginSessionTransactionState.idle, .unknown] { - let owner = WriteTransactionOwner.resolve(supportsTransactions: true, sessionState: state) + let owner = WriteTransactionOwner.resolve(supportsTransactions: true, sessionState: state, commitMode: .automatic) #expect(owner == .app) - #expect(owner.opensTransaction) + #expect(owner.sendsBegin) + #expect(owner.commitsAtEnd) #expect(owner.canRollBack) } } + + @Test("Manual Commit holds an idle session's write: BEGIN, no COMMIT, a rollback on failure") + func manualCommitHoldsAnIdleSession() { + let owner = WriteTransactionOwner.resolve(supportsTransactions: true, sessionState: .idle, commitMode: .manual) + #expect(owner == .held) + #expect(owner.sendsBegin) + #expect(owner.commitsAtEnd == false) + #expect(owner.canRollBack) + #expect(WriteTransactionOwner.resolve(supportsTransactions: true, sessionState: .unknown, commitMode: .manual) == .app) + } +} + +struct DataWriteExecutorHeldTests { + private func plan(prologue: [String] = []) -> DataWritePlan { + DataWritePlan( + scope: DatabaseScope(connectionId: UUID(), database: "shop", schema: nil), + databaseType: .sqlite, + steps: [ + DataWriteStep( + kind: .rowWrite, + statement: ParameterizedStatement(sql: "UPDATE \"t\" SET \"b\" = 1", parameters: []), + expectedRowCount: 1, + tableName: "t" + ), + ], + prologue: prologue + ) + } + + private func driver(_ counting: CountingDriver) -> DatabaseDriver { + PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .sqlite), pluginDriver: counting) + } + + @Test("A held save sends BEGIN and the write, and leaves the transaction open") + func heldSaveLeavesTheTransactionOpen() async throws { + let counting = CountingDriver(affectedRows: 1) + _ = try await DataWriteExecutor.run(plan(), commitMode: .manual, on: driver(counting)) + #expect(counting.trace == ["BEGIN", "UPDATE \"t\" SET \"b\" = 1"]) + } + + @Test("A held save that fails rolls back the transaction it opened") + func heldSaveRollsBackAFailure() async { + let counting = CountingDriver(affectedRows: 1, failOnStatement: 1) + await #expect(throws: (any Error).self) { + try await DataWriteExecutor.run(plan(), commitMode: .manual, on: driver(counting)) + } + #expect(counting.trace.first == "BEGIN") + #expect(counting.didRollBack) + #expect(counting.didCommit == false) + } + + @Test("A save with session toggles is refused in Manual Commit before anything runs") + func toggledSaveIsRefused() async { + let counting = CountingDriver(affectedRows: 1) + await #expect(throws: ManualCommitSessionOptionsRefusal()) { + try await DataWriteExecutor.run( + plan(prologue: ["PRAGMA foreign_keys = OFF"]), commitMode: .manual, on: driver(counting) + ) + } + #expect(counting.trace.isEmpty) + } + + @Test( + "A save the session can't hold is refused in Manual Commit before anything runs", + arguments: [PluginSessionTransactionState.unknown, .holdsSessionLocks] + ) + func unholdableSaveIsRefused(state: PluginSessionTransactionState) async { + let counting = CountingDriver(affectedRows: 1, sessionState: state) + await #expect(throws: ManualCommitRefusal()) { + try await DataWriteExecutor.run(plan(), commitMode: .manual, on: driver(counting)) + } + #expect(counting.trace.isEmpty) + } + + @Test("A save on an engine without transactions is refused in Manual Commit") + func saveWithoutTransactionsIsRefused() async { + let counting = CountingDriver(affectedRows: 1, transactional: false) + await #expect(throws: ManualCommitRefusal()) { + try await DataWriteExecutor.run(plan(), commitMode: .manual, on: driver(counting)) + } + #expect(counting.trace.isEmpty) + } + + @Test("A held save marks the turn's hold, and a rolled-back one clears it") + func heldSaveMarksTheTurn() async throws { + let held = ManualCommitTurnHold() + _ = try await ManualCommit.$turnHold.withValue(held) { + try await DataWriteExecutor.run(plan(), commitMode: .manual, on: driver(CountingDriver(affectedRows: 1))) + } + #expect(held.isHeld) + + let failed = ManualCommitTurnHold() + await ManualCommit.$turnHold.withValue(failed) { + _ = try? await DataWriteExecutor.run( + plan(), commitMode: .manual, on: driver(CountingDriver(affectedRows: 1, failOnStatement: 1)) + ) + } + #expect(!failed.isHeld) + } + + @Test("Manual Commit joins a transaction already open and sends no BEGIN") + func heldSaveJoinsAnOpenTransaction() async throws { + let counting = CountingDriver(affectedRows: 1, sessionState: .inTransaction) + _ = try await DataWriteExecutor.run(plan(), commitMode: .manual, on: driver(counting)) + #expect(counting.trace == ["UPDATE \"t\" SET \"b\" = 1"]) + } } diff --git a/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift b/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift index e09362a589..697ed53c68 100644 --- a/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift +++ b/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift @@ -10,7 +10,7 @@ import Testing @testable import TablePro struct PluginKeyedChangesTests { - @Test("Every row gets its own key, and the key agrees across the changes and the sets") + @Test("Every change gets its own key, and the key agrees across the changes and the sets") func keysAgreeAcrossCollections() { let insertedA = RowID.inserted(UUID()) let insertedB = RowID.inserted(UUID()) @@ -93,4 +93,31 @@ struct PluginKeyedChangesTests { #expect(crossed?.cellChanges.first?.newValue == "b") #expect(crossed?.originalRow == ["3", "a"]) } + + @MainActor + @Test("A row edited around another change reaches the driver once per edit, each with the row as it then stands") + func aRowEditedAroundAnotherChangeReachesTheDriverTwice() throws { + let handed = HandedChanges() + let driver = RowWriteStubDriver { changes, _, _, _ in + handed.changes = changes + return changes.map { PluginRowWrite(statement: "write(\($0.rowIndex))", rowIndices: [$0.rowIndex]) } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + _ = try manager.generateSQL() + let types = handed.changes.map { $0.type } + let keys = Set(handed.changes.map { $0.rowIndex }) + let laterColumns = handed.changes.last?.cellChanges.map { $0.columnName } + + #expect(types == [.update, .delete, .update]) + #expect(keys.count == 3) + #expect(handed.changes.last?.originalRow == ["3", "Cyd", "c@test.com"]) + #expect(laterColumns == ["email"]) + } +} + +private final class HandedChanges { + var changes: [PluginRowChange] = [] } diff --git a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift index fb7a319753..dd88b9d3b4 100644 --- a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift @@ -134,6 +134,30 @@ struct RewindPlannerTests { #expect(plan.statements.count == 1) } + @Test("An edit's restore reads its row again inside the write and holds it to what the save left") + func restoredEditHoldsTheRowAsReviewed() throws { + let plan = try planner(operations: [ + operation(kind: .update, preImage: ["7", "Ada"], postImage: ["7", "Grace"], writtenColumns: ["name"]), + ]).plan(currentRows: [RewindCurrentRow(values: ["7", "Grace"])]) + + let lookup = try #require(plan.lookups.first ?? nil) + #expect(lookup.rowCount == 1) + #expect(lookup.heldValues == [1: "Grace"]) + #expect(lookup.query.sql.contains("\"id\"")) + #expect(lookup.query.sql.contains("'7'")) + } + + @Test("Removing an inserted row holds it to what the save wrote; restoring a deleted one does not read") + func insertAndDeleteRestoreLookups() throws { + let insert = operation(kind: .insert, preImage: nil, postImage: ["9", "Lin"], writtenColumns: columns) + let delete = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) + let plan = try planner(operations: [insert, delete]).plan(currentRows: [RewindCurrentRow(values: ["9", "Lin"])]) + + #expect(plan.statements.count == 2) + let held = plan.lookups.compactMap { $0?.heldValues } + #expect(held == [[0: "9", 1: "Lin"]]) + } + @Test("A row someone else changed is left alone, and no statement is generated for it") func changedRowIsSkipped() throws { let plan = try planner(operations: [ @@ -277,6 +301,220 @@ struct RewindPlannerTests { #expect(queries.first?.contains("SELECT *") == false) } + @Test("An edit that changed the key is read and restored under the key the save gave it") + func movedKeyIsFoundUnderItsNewKey() throws { + let moved = operation(kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Ada"], writtenColumns: ["id"]) + let rewind = planner(operations: [moved]) + + let query = try #require(rewind.readQueries().first) + #expect(query.contains("k6")) + #expect(query.contains("k5")) + + let plan = try rewind.plan(currentRows: [RewindCurrentRow(values: ["k6", "Ada"])]) + #expect(plan.rows.map { $0.outcome } == [.willRestore]) + #expect(plan.statements.first?.sql.hasPrefix("UPDATE") == true) + #expect(plan.statements.first?.parameters.compactMap { $0 as? String } == ["k5", "k6"]) + } + + @Test("A moved row a rewind already put back reads as restored, and another row on its old key does not") + func movedKeyAlreadyRestored() throws { + let moved = operation( + kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Grace"], writtenColumns: ["id", "name"] + ) + + let restored = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k5", "Ada"])]) + #expect(restored.rows.map { $0.outcome } == [.alreadyRestored]) + #expect(restored.statements.isEmpty) + + let taken = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k5", "Bo"])]) + #expect(taken.rows.map { $0.outcome } == [.rowMissing]) + #expect(taken.statements.isEmpty) + } + + @Test("A row that took the key a save moved to is left alone, though the save wrote only the key") + func rowOnTheMovedKeyIsNotRewritten() throws { + let moved = operation(kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Ada"], writtenColumns: ["id"]) + + let plan = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k6", "Bo"])]) + + #expect(plan.rows.map { $0.outcome } == [.changedSinceSave]) + #expect(plan.statements.isEmpty) + } + + @Test("A moved row is not held to a column the server computes") + func movedKeyIgnoresGeneratedColumns() throws { + let columns = ["id", "name", "updated_at"] + let moved = RowWriteOperation( + kind: .update, target: target, columns: columns, primaryKeyColumns: ["id"], + preImage: ["k5", "Ada", "2026-01-01"], postImage: ["k6", "Ada", "2026-01-01"], + writtenColumns: ["id"], refusal: nil + ) + let record = RewindRecord( + id: UUID(), historyId: nil, connectionId: UUID(), databaseType: .sqlite, + target: target, capturedAt: Date(timeIntervalSince1970: 0), + generatedColumns: ["updated_at"], identityColumns: [], operations: [moved] + ) + let planner = RewindPlanner( + record: record, + factory: RowChangeStatementFactory( + tableName: target.table, schemaName: nil, columns: columns, primaryKeyColumns: ["id"], + generatedColumns: ["updated_at"], databaseType: .sqlite, pluginDriver: nil + ), + queryBuilder: sqliteQueryBuilder + ) + + let plan = try planner.plan(currentRows: [RewindCurrentRow(values: ["k6", "Ada", "2026-06-30"])]) + + #expect(plan.rows.map { $0.outcome } == [.willRestore]) + } + + @Test("Undoing a restore takes back what the restore did last first") + func inverseRecordRunsBackwards() throws { + let deleted = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) + let inserted = operation(kind: .insert, preImage: nil, postImage: ["8", "Ada"], writtenColumns: columns) + let plan = try planner(operations: [deleted, inserted]) + .plan(currentRows: [RewindCurrentRow(values: ["8", "Ada"])]) + + let inverse = RewindExecutor.inverseOperations(of: plan) + #expect(inverse.map { $0.kind } == [.delete, .insert]) + + let undo = try planner(operations: inverse).plan(currentRows: [RewindCurrentRow(values: ["7", "Ada"])]) + #expect(undo.statements.map { String($0.sql.prefix(6)) } == ["DELETE", "INSERT"]) + } + + private func recordedOperations(_ edits: (inout PendingChanges) -> Void) -> [RowWriteOperation] { + var pending = PendingChanges() + edits(&pending) + return RowWriteOperationBuilder.operations( + from: pending.changes, + insertedRowData: pending.insertedRowData, + deletedRowIDs: pending.deletedRowIDs, + insertedRowIDs: pending.insertedRowIDs, + target: target, + columns: columns, + primaryKeyColumns: ["id"], + generatedColumns: [], + containsTableOperation: false + ) + } + + // A.name a to tmp, B.name b to a, A.name tmp to b: valid step by step on a unique `name`. + private func swapOperations() -> [RowWriteOperation] { + recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "a", newValue: "tmp", + originalRow: ["k1", "a"] + ) + pending.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", oldValue: "b", newValue: "a", + originalRow: ["k2", "b"] + ) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "tmp", newValue: "b", + originalRow: ["k1", "tmp"] + ) + } + } + + private func bound(_ statements: [ParameterizedStatement]) -> [[String]] { + statements.map { statement in statement.parameters.compactMap { $0 as? String } } + } + + @Test("A swap rewinds its newest step first, so no step takes a value another still holds") + func aSwapRewindsNewestStepFirst() throws { + let plan = try planner(operations: swapOperations()).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "b"]), RewindCurrentRow(values: ["k2", "a"]) + ]) + + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(bound(plan.statements) == [["tmp", "k1"], ["b", "k2"], ["a", "k1"]]) + } + + @Test("Undoing the rewind of a swap replays the save in its own order") + func undoingASwapRewindReplaysTheSave() throws { + let rewind = try planner(operations: swapOperations()).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "b"]), RewindCurrentRow(values: ["k2", "a"]) + ]) + + let undo = try planner(operations: RewindExecutor.inverseOperations(of: rewind)).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "a"]), RewindCurrentRow(values: ["k2", "b"]) + ]) + + #expect(bound(undo.statements) == [["tmp", "k1"], ["a", "k2"], ["b", "k1"]]) + } + + @Test("A key moved in an earlier step is rewound under the key each step left") + func aKeyMovedInTheFirstStepRewindsUnderEachStepsKey() throws { + let operations = recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 0, columnName: "id", oldValue: "k5", newValue: "k6", + originalRow: ["k5", "Ada"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["k7", "Bo"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Ada", newValue: "Grace", + originalRow: ["k6", "Ada"] + ) + } + + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k6", "Grace"])]) + let verbs = plan.statements.map { String($0.sql.prefix(6)) } + + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(verbs == ["UPDATE", "INSERT", "UPDATE"]) + #expect(bound(plan.statements).first == ["Ada", "k6"]) + #expect(bound(plan.statements).last == ["k5", "k6"]) + } + + @Test("A step that wrote SQL refuses the row, though a later step replaced it with a value") + func aStepThatWroteSQLRefusesTheRow() throws { + let operations = recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Ada", newValue: "now()", + originalRow: ["k1", "Ada"], isExpression: true + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["k2", "Bo"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "now()", newValue: "Grace", + originalRow: ["k1", "now()"] + ) + } + + let update = try #require(operations.first { $0.kind == .update }) + #expect(update.refusal == .serverComputedValue) + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k1", "Grace"])]) + #expect(plan.rows.first?.outcome == .notReversible(.serverComputedValue)) + } + + @Test("A record saved before steps were kept rewinds newest first by list position") + func aRecordWithoutStepsRewindsAsBefore() throws { + let first = operation(kind: .update, preImage: ["k1", "Ada"], postImage: ["k1", "Grace"], writtenColumns: ["name"]) + let second = operation(kind: .update, preImage: ["k2", "Bo"], postImage: ["k2", "Cy"], writtenColumns: ["name"]) + + let plan = try planner(operations: [first, second]).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "Grace"]), RewindCurrentRow(values: ["k2", "Cy"]) + ]) + + #expect(bound(plan.statements) == [["Bo", "k2"], ["Ada", "k1"]]) + } + + @Test("A deleted row comes back once the new row that took its key is removed, and not while that row was changed") + func aDeletedRowComesBackAfterTheNewRowThatTookItsKey() throws { + let operations = recordedOperations { pending in + pending.recordRowDeletion(rowID: .existing(0), originalRow: ["k5", "Ada"]) + pending.recordRowInsertion(rowID: .inserted(UUID()), values: ["k5", "Bo"]) + } + + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k5", "Bo"])]) + let verbs = plan.statements.map { String($0.sql.prefix(6)) } + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(verbs == ["DELETE", "INSERT"]) + + let changed = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k5", "Cy"])]) + #expect(changed.rows.map { $0.outcome } == [.rowAlreadyPresent, .changedSinceSave]) + #expect(changed.statements.isEmpty) + } + @Test("The inverse runs newest first, so a value one row freed is not taken before the other gives it up") func inverseRunsInReverseOrder() throws { let first = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) diff --git a/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift b/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift index 5480185201..87d1d1b8d0 100644 --- a/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing struct RewindRecordIdentityDecodingTests { @@ -42,6 +43,62 @@ struct RewindRecordIdentityDecodingTests { } } +struct RewindRecordStepDecodingTests { + private func record() -> RewindRecord { + let target = DataWriteTarget(database: "shop", schema: nil, table: "users") + let operation = RowWriteOperation( + kind: .update, + target: target, + columns: ["id", "name"], + primaryKeyColumns: ["id"], + preImage: ["1", "a"], + postImage: ["1", "b"], + writtenColumns: ["name"], + refusal: nil, + order: 3, + steps: [ + RowWriteStep(order: 3, columns: [1], before: ["a"], after: ["tmp"]), + RowWriteStep(order: 5, columns: [1], before: ["tmp"], after: ["b"], afterAbsent: [1]) + ] + ) + return RewindRecord( + id: UUID(), + historyId: nil, + connectionId: UUID(), + databaseType: .postgresql, + target: target, + capturedAt: Date(timeIntervalSince1970: 1_700_000_000), + generatedColumns: [], + identityColumns: [], + operations: [operation] + ) + } + + @Test + func aRecordSavedBeforeStepsDecodes() throws { + let encoded = try JSONEncoder().encode(record()) + var object = try #require(try JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + var operations = try #require(object["operations"] as? [[String: Any]]) + operations[0].removeValue(forKey: "order") + operations[0].removeValue(forKey: "steps") + object["operations"] = operations + let legacy = try JSONSerialization.data(withJSONObject: object) + + let decoded = try JSONDecoder().decode(RewindRecord.self, from: legacy) + let operation = try #require(decoded.operations.first) + #expect(operation.order == nil) + #expect(operation.steps == nil) + #expect(operation.postImage == ["1", "b"]) + } + + @Test + func stepsRoundTrip() throws { + let original = record() + let decoded = try JSONDecoder().decode(RewindRecord.self, from: JSONEncoder().encode(original)) + #expect(decoded.operations == original.operations) + } +} + struct RewindRecordColumnTypeDecodingTests { private func record(columnTypeNames: [String: String]?) -> RewindRecord { RewindRecord( diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift index e688dff4ff..f4df13db8a 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift @@ -255,8 +255,8 @@ struct RowChangeStatementFactoryCoverageTests { let engine = DatabaseType(rawValue: "ProbeEngine") let hostOnly = LegacyStatementStubDriver { _, _, _, _ in nil } let inserted = RowID.inserted(UUID()) - let allDefaults: [PluginCellValue] = [.text("__DEFAULT__"), .text("__DEFAULT__")] - let changes = [nameEdit(), RowChange(rowID: inserted, type: .insert)] + let allDefaults: [PluginCellValue] = [.columnDefault, .columnDefault] + let changes = [nameEdit(), RowChange(rowID: inserted, type: .insert, expressionColumns: [0, 1])] let generator = try SQLStatementGenerator( tableName: "t", columns: columns, primaryKeyColumns: ["_id"], databaseType: engine, @@ -319,7 +319,42 @@ struct RowChangeStatementFactoryCoverageTests { let build = try manager.buildRowWrites(database: "d", schema: nil, containsTableOperation: false) #expect(build.steps.map(\.statement.sql) == ["updateOne(0)"]) - #expect(build.steps.allSatisfy { $0.expectedRowCount == nil }) + #expect(build.steps.allSatisfy { $0.expectedRowCount == nil && !$0.countsMatchedRows }) + } + + @Test("A driver's own matched-row count reaches the save plan as the count the step is held to") + func buildRowWritesTakesTheDriversMatchedRowCount() throws { + let manager = DataChangeManager() + manager.configureForTable( + tableName: "items", columns: columns, primaryKeyColumns: ["_id"], + databaseType: .snowflake, generatedColumns: [] + ) + manager.pluginDriver = RowWriteStubDriver { changes, _, _, _ in + changes.map { PluginRowWrite(statement: "updateOne(\($0.rowIndex))", rowIndices: [$0.rowIndex], expectedRowCount: 1) } + } + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["0", "a"] + ) + + let build = try manager.buildRowWrites(database: "d", schema: nil, containsTableOperation: false) + + #expect(build.steps.map(\.expectedRowCount) == [1]) + #expect(build.steps.allSatisfy { $0.countsMatchedRows && !$0.matchesRowsWithoutKey }) + } + + @Test("A driver's count never raises the one row the host holds a keyless statement to") + func driverCountNeverLoosensTheKeylessHold() throws { + let driver = RowWriteStubDriver { changes, _, _, _ in + changes.map { PluginRowWrite(statement: "updateOne(\($0.rowIndex))", rowIndices: [$0.rowIndex], expectedRowCount: 3) } + } + let written = try factory(primaryKeyColumns: [], databaseType: .mssql, driver: driver) + .rowWriteStatements(for: [nameEdit()]) + + let steps = DataWriteStep.rowWrites(written, tableName: "items", primaryKeyColumns: []) + + #expect(steps.map(\.expectedRowCount) == [1]) + #expect(steps.allSatisfy { $0.matchesRowsWithoutKey && $0.countsMatchedRows }) } @Test("A driver's keyed statement through a view is held to its one row") @@ -371,6 +406,40 @@ struct RowChangeStatementFactoryCoverageTests { #expect(statements.allSatisfy { $0.keylessRowCount == nil }) } + @Test("A driver that writes only the first edit of a row edited around a delete refuses the save") + func aDriverThatWritesOneRunOfARowRefusesTheSave() { + let driver = RowWriteStubDriver { changes, _, _, _ in + let firstUpdate = changes.first { $0.type == .update }?.rowIndex + return changes.compactMap { change in + guard change.type != .update || change.rowIndex == firstUpdate else { return nil } + return PluginRowWrite(statement: "write(\(change.rowIndex))", rowIndices: [change.rowIndex]) + } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + #expect(throws: DataWriteError.changesNotWritable(table: "items", unwritten: UnwrittenRowCounts(updates: 1))) { + _ = try manager.generateSQL() + } + } + + @Test("A refusal counts the rows it cannot write, not the edits of each") + func aRefusalCountsRowsNotRuns() { + let driver = RowWriteStubDriver { changes, _, _, _ in + changes.filter { $0.type == .delete }.map { + PluginRowWrite(statement: "deleteOne(\($0.rowIndex))", rowIndices: [$0.rowIndex]) + } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + #expect(throws: DataWriteError.changesNotWritable(table: "items", unwritten: UnwrittenRowCounts(updates: 1))) { + _ = try manager.generateSQL() + } + } + @Test("Preview SQL and Save both refuse the same mixed set") func buildRowWritesRefusesWhatGenerateSQLRefuses() { let manager = DataChangeManager() diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift index 93c24b8d32..376fdb75e2 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift @@ -36,6 +36,27 @@ private final class ExpressionRecordingDriver: PluginDriverStub, @unchecked Send func quoteIdentifier(_ name: String) -> String { "[\(name)]" } } +private final class RestoringDriver: PluginDriverStub, @unchecked Sendable { + let capabilities: PluginCapabilities + + init(capabilities: PluginCapabilities = []) { + self.capabilities = capabilities + } + + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + absentCells: [Int: Set] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + rows.map { (statement: "INSERT INTO [events] VALUES (?, ?, ?)", parameters: $0) } + } + + func quoteIdentifier(_ name: String) -> String { "[\(name)]" } +} + @MainActor struct RowChangeStatementFactoryExpressionTests { private let columns = ["id", "name", "stamped_at"] @@ -192,7 +213,7 @@ struct RowChangeStatementFactoryExpressionTests { @Test("SQLite refuses DEFAULT on a saved row, naming the column") func sqliteDefaultUpdateIsRefused() { - let change = edit(to: .text(PluginCellValue.defaultMarkerText), isExpression: false, column: 1) + let change = edit(to: .columnDefault, isExpression: true, column: 1) let refused = refusal { _ = try factory(databaseType: .sqlite).statements(for: [change]) } #expect(refused?.kind == .update) #expect(refused?.reason.contains("name") == true) @@ -202,8 +223,8 @@ struct RowChangeStatementFactoryExpressionTests { func sqliteDefaultInsertIsWritten() throws { let rowID = RowID.inserted(UUID()) let statements = try factory(databaseType: .sqlite).statements( - for: [RowChange(rowID: rowID, type: .insert)], - insertedRowData: [rowID: ["1", .text(PluginCellValue.defaultMarkerText), "2026-01-01"]], + for: [RowChange(rowID: rowID, type: .insert, expressionColumns: [1])], + insertedRowData: [rowID: ["1", .columnDefault, "2026-01-01"]], insertedRowIDs: [rowID] ) #expect(statements.count == 1) @@ -212,8 +233,129 @@ struct RowChangeStatementFactoryExpressionTests { @Test("PostgreSQL takes DEFAULT on a saved row") func postgresDefaultUpdate() throws { - let change = edit(to: .text(PluginCellValue.defaultMarkerText), isExpression: false, column: 1) + let change = edit(to: .columnDefault, isExpression: true, column: 1) let statements = try factory().statements(for: [change]) #expect(statements.first?.sql.contains("\"name\" = DEFAULT") == true) } + + @Test("Text a row holds or a user typed that reads like the default is written as that text", arguments: [ + "__DEFAULT__", "DEFAULT", + ]) + func unmarkedDefaultTextIsAValue(text: String) throws { + let saved = try factory(databaseType: .sqlite).statements(for: [edit(to: .text(text), isExpression: false, column: 1)]) + #expect(saved.first?.sql.contains("DEFAULT") == false) + #expect(saved.first?.parameters.first as? String == text) + + let rowID = RowID.inserted(UUID()) + let inserted = try factory(databaseType: .sqlite).statements( + for: [RowChange(rowID: rowID, type: .insert)], + insertedRowData: [rowID: ["1", .text(text), "2026-01-01"]], + insertedRowIDs: [rowID] + ) + #expect(inserted.first?.parameters.compactMap { $0 as? String } == ["1", text, "2026-01-01"]) + } + + @Test("A driver is handed a default in the spelling it reads, with no SQL mark, whatever it declares") + func defaultReachesADriverInItsSpelling() throws { + let driver = ExpressionRecordingDriver(capabilities: []) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: .columnDefault, isExpression: true, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == .pluginDefaultMarker) + #expect(driver.changes.first?.expressionColumns == nil) + } + + // A driver may write DEFAULT for it, and none can say it will not, so every one that writes its own rows is refused. + @Test("Text spelled the way drivers read a default is refused by a driver that writes its own rows", arguments: [ + DatabaseType.mssql, DatabaseType.redis, + ]) + func textADriverMayReadAsDefaultIsRefused(databaseType: DatabaseType) { + let driver = ExpressionRecordingDriver(capabilities: []) + let refused = refusal { + _ = try factory(databaseType: databaseType, driver: driver).statements( + for: [edit(to: "__DEFAULT__", isExpression: false, column: 1)] + ) + } + #expect(refused?.kind == .update) + #expect(refused?.reason.contains("name") == true) + } + + @Test("A deleted row a driver puts back is refused when it holds that text", arguments: [ + DatabaseType.dameng, DatabaseType.redis, + ]) + func restoredTextADriverMayReadAsDefaultIsRefused(databaseType: DatabaseType) { + let refused = refusal { + _ = try factory(databaseType: databaseType, driver: RestoringDriver()) + .restoreStatements(rows: [["1", "__DEFAULT__", "2026-01-01"]]) + } + #expect(refused?.kind == .insert) + #expect(refused?.reason.contains("name") == true) + } + + @Test("A driver that reads the default mark gets the mark beside its old spelling") + func defaultReachesAMarkingDriverWithItsMark() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: .columnDefault, isExpression: true, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == .pluginDefaultMarker) + #expect(driver.changes.first?.defaultColumns == [1]) + #expect(driver.changes.first?.expressionColumns == nil) + } + + @Test("A driver that reads the default mark saves the text __DEFAULT__ as text") + func textAMarkingDriverReadsIsSaved() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: "__DEFAULT__", isExpression: false, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == "__DEFAULT__") + #expect(driver.changes.first?.defaultColumns == nil) + } + + @Test("A new row's default cells are marked for the driver, and its text cells are not") + func newRowDefaultsAreMarked() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + let rowID = RowID.inserted(UUID()) + let change = RowChange(rowID: rowID, type: .insert, expressionColumns: [0]) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [change], + insertedRowData: [rowID: [.columnDefault, "__DEFAULT__", "2026-01-01"]], + insertedRowIDs: [rowID] + ) + #expect(driver.changes.first?.defaultColumns == [0]) + } + + @Test("A new row staged only as cell edits marks its default cells by column") + func newRowCellEditDefaultsAreMarked() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + let change = RowChange( + rowID: .inserted(UUID()), + type: .insert, + cellChanges: [ + CellChange(columnIndex: 1, columnName: "name", oldValue: .null, newValue: "__DEFAULT__"), + CellChange( + columnIndex: 2, columnName: "stamped_at", oldValue: .null, newValue: .columnDefault, + newIsExpression: true + ), + ] + ) + _ = try factory(databaseType: .mssql, driver: driver).statements(for: [change]) + #expect(driver.changes.first?.defaultColumns == [2]) + #expect(driver.changes.first?.cellChanges.first?.newValue == "__DEFAULT__") + } + + @Test("A deleted row holding that text is put back by a driver that reads the default mark") + func restoredTextAMarkingDriverReadsIsWritten() throws { + let restored = try factory(databaseType: .dameng, driver: RestoringDriver(capabilities: .columnDefaultMarks)) + .restoreStatements(rows: [["1", "__DEFAULT__", "2026-01-01"]]) + #expect(restored.statements.first?.parameters.compactMap { $0 as? String } == ["1", "__DEFAULT__", "2026-01-01"]) + } + + @Test("A deleted row a driver puts back with no such text is restored") + func restoredRowWithoutThatTextIsWritten() throws { + let restored = try factory(databaseType: .dameng, driver: RestoringDriver()) + .restoreStatements(rows: [["1", "Ada", "2026-01-01"]]) + #expect(restored.statements.first?.parameters.compactMap { $0 as? String } == ["1", "Ada", "2026-01-01"]) + } } diff --git a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift index d4c40a9899..c98576f909 100644 --- a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift @@ -78,14 +78,14 @@ struct RowWriteOperationBuilderTests { #expect(now.first?.refusal == .serverComputedValue) let marker = operations( - changes: [ - cellEdit( - column: "updated_at", index: 2, from: "2026-01-01", - to: .text(PluginCellValue.defaultMarkerText) - ), - ] + changes: [cellEdit(column: "updated_at", index: 2, from: "2026-01-01", to: .columnDefault, isExpression: true)] ) #expect(marker.first?.refusal == .serverComputedValue) + + let text = operations( + changes: [cellEdit(column: "updated_at", index: 2, from: "2026-01-01", to: "__DEFAULT__")] + ) + #expect(text.first?.refusal == nil) } @Test("A generated column cannot be written back") @@ -136,8 +136,8 @@ struct RowWriteOperationBuilderTests { func serverAssignedKeyIsRefused() { let change = RowChange(rowID: .existing(0), type: .insert) let withMarker = operations( - changes: [change], - insertedRowData: [.existing(0): [.text(PluginCellValue.defaultMarkerText), "Ada", "2026-01-01"]], + changes: [RowChange(rowID: .existing(0), type: .insert, expressionColumns: [0])], + insertedRowData: [.existing(0): [.columnDefault, "Ada", "2026-01-01"]], insertedRowIDs: [.existing(0)] ) #expect(withMarker.first?.refusal == .serverAssignedKey) @@ -184,6 +184,62 @@ struct RowWriteOperationBuilderTests { #expect(literal.first?.refusal == nil) } + @Test("The record keeps the order the rows were changed in after an earlier change is cancelled") + func recordFollowsEditOrder() { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ada", newValue: "Grace", originalRow: ["7", "Ada", "2026-01-01"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["8", "Bo", "2026-01-01"]) + let inserted = RowID.inserted(UUID()) + pending.recordRowInsertion(rowID: inserted, values: ["9", "Bo", "2026-01-01"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Grace", newValue: "Ada" + ) + #expect(pending.changes.map { $0.type } == [.insert, .delete]) + + let result = operations( + changes: pending.changes, + insertedRowData: pending.insertedRowData, + deletedRowIDs: pending.deletedRowIDs, + insertedRowIDs: pending.insertedRowIDs + ) + + #expect(result.map { $0.kind } == [.delete, .insert]) + } + + @Test("A row edited around another change is one operation with its net images and both steps") + func aRowEditedInTwoRunsIsOneOperationWithBothSteps() throws { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ada", newValue: "Grace", originalRow: ["7", "Ada", "2026-01-01"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["8", "Bo", "2026-01-01"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "updated_at", + oldValue: "2026-01-01", newValue: "2026-02-02", originalRow: ["7", "Grace", "2026-01-01"] + ) + + let result = operations( + changes: pending.changes, + deletedRowIDs: pending.deletedRowIDs + ) + + #expect(result.map { $0.kind } == [.update, .delete]) + let update = try #require(result.first) + #expect(update.preImage == ["7", "Ada", "2026-01-01"]) + #expect(update.postImage == ["7", "Grace", "2026-02-02"]) + #expect(update.order == 0) + let steps = try #require(update.steps) + #expect(steps.map { $0.order } == [0, 2]) + #expect(steps.map { $0.columns } == [[1], [2]]) + #expect(steps.map { $0.after } == [["Grace"], ["2026-02-02"]]) + #expect(result.last?.order == 1) + #expect(result.last?.steps == nil) + } + @Test("An insert carrying its own key is reversible") func userSuppliedKeyIsReversible() { let result = operations( diff --git a/apple/TableProTests/Core/Database/AppDefinitionCommitTests.swift b/apple/TableProTests/Core/Database/AppDefinitionCommitTests.swift new file mode 100644 index 0000000000..e841072e3a --- /dev/null +++ b/apple/TableProTests/Core/Database/AppDefinitionCommitTests.swift @@ -0,0 +1,348 @@ +// +// AppDefinitionCommitTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class DefinitionLedger: @unchecked Sendable { + struct Failure: Error, Equatable { + let step: String + } + + private let lock = NSLock() + private var current: PluginSessionTransactionState + private var sent: [String] = [] + + init(state: PluginSessionTransactionState = .idle) { + current = state + } + + var state: PluginSessionTransactionState { + get { lock.withLock { current } } + set { lock.withLock { current = newValue } } + } + + var calls: [String] { lock.withLock { sent } } + + func record(_ call: String) { + lock.withLock { sent.append(call) } + } +} + +struct AppDefinitionCommitTests { + private static func run( + _ ledger: DefinitionLedger, + family: TransactionEngineFamily = .teradata, + refuses: Bool = false, + commitFails: Bool = false, + body: () async throws -> Void + ) async throws { + try await AppDefinitionCommit.run( + family: family, + refusesOpenTransaction: refuses, + readState: { ledger.state }, + commit: { + // A driver's cancel handler refuses a statement sent from a cancelled task. + try Task.checkCancellation() + ledger.record("COMMIT") + if commitFails { throw DefinitionLedger.Failure(step: "COMMIT") } + ledger.state = .idle + }, + rollback: { + try Task.checkCancellation() + ledger.record("ROLLBACK") + ledger.state = .idle + }, + body: body + ) + } + + private static func definition(_ ledger: DefinitionLedger) { + ledger.record("CREATE") + ledger.state = .inTransaction + } + + @Test("A statement that leaves the session in a transaction is committed") + func commitsWhatTheStatementLeftOpen() async throws { + let ledger = DefinitionLedger() + + try await Self.run(ledger) { Self.definition(ledger) } + + #expect(ledger.calls == ["CREATE", "COMMIT"]) + #expect(ledger.state == .idle) + } + + @Test("A statement that leaves the session idle sends no commit") + func noCommitWhenIdleAfter() async throws { + let ledger = DefinitionLedger() + + try await Self.run(ledger) { ledger.record("CREATE") } + + #expect(ledger.calls == ["CREATE"]) + } + + @Test( + "On the session route an open transaction refuses the statement before it runs", + arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction] + ) + func refusesAnOpenTransactionOnTheSessionRoute(state: PluginSessionTransactionState) async throws { + let ledger = DefinitionLedger(state: state) + + await #expect(throws: DefinitionInOpenTransactionRefusal()) { + try await Self.run(ledger, refuses: true) { Self.definition(ledger) } + } + #expect(ledger.calls.isEmpty) + } + + @Test("On a pooled connection a transaction left pending is no reason to refuse, and is committed") + func pooledRouteNeverRefuses() async throws { + let ledger = DefinitionLedger(state: .inTransaction) + + try await Self.run(ledger, refuses: false) { Self.definition(ledger) } + + #expect(ledger.calls == ["CREATE", "COMMIT"]) + } + + @Test("A failed statement on an idle session is rolled back, and its own error is thrown") + func failedStatementOnAnIdleSessionRollsBack() async throws { + let ledger = DefinitionLedger() + + await #expect(throws: DefinitionLedger.Failure(step: "CREATE")) { + try await Self.run(ledger) { + ledger.record("CREATE") + throw DefinitionLedger.Failure(step: "CREATE") + } + } + #expect(ledger.calls == ["CREATE", "ROLLBACK"]) + } + + @Test("A failed statement on a session already in a transaction leaves that transaction alone") + func failedStatementInsideATransactionSendsNoRollback() async throws { + let ledger = DefinitionLedger(state: .inTransaction) + + await #expect(throws: DefinitionLedger.Failure(step: "CREATE")) { + try await Self.run(ledger) { + ledger.record("CREATE") + throw DefinitionLedger.Failure(step: "CREATE") + } + } + #expect(ledger.calls == ["CREATE"]) + } + + @Test("A commit that fails on a session idle before the statement is rolled back") + func failedCommitRollsBack() async throws { + let ledger = DefinitionLedger() + + await #expect(throws: DefinitionLedger.Failure(step: "COMMIT")) { + try await Self.run(ledger, commitFails: true) { Self.definition(ledger) } + } + #expect(ledger.calls == ["CREATE", "COMMIT", "ROLLBACK"]) + } + + @Test("An engine without the rule runs the statement as it is: no refusal, commit or rollback") + func otherFamiliesPassThrough() async throws { + let ledger = DefinitionLedger(state: .inTransaction) + + try await Self.run(ledger, family: .postgres, refuses: true) { Self.definition(ledger) } + await #expect(throws: DefinitionLedger.Failure(step: "CREATE")) { + try await Self.run(ledger, family: .other) { throw DefinitionLedger.Failure(step: "CREATE") } + } + + #expect(ledger.calls == ["CREATE"]) + } + + @Test("The commit still goes out when the task is cancelled once the statement ran") + func commitSurvivesCancellation() async throws { + let ledger = DefinitionLedger() + + let task = Task { + try await Self.run(ledger) { + Self.definition(ledger) + withUnsafeCurrentTask { $0?.cancel() } + } + } + try await task.value + + #expect(ledger.calls == ["CREATE", "COMMIT"]) + } +} + +private final class AnsiSessionDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var current: PluginSessionTransactionState + private var sent: [String] = [] + + init(state: PluginSessionTransactionState = .idle) { + current = state + } + + var supportsSchemas: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + var supportsTransactions: Bool { true } + + var state: PluginSessionTransactionState { + get { lock.withLock { current } } + set { lock.withLock { current = newValue } } + } + + var statements: [String] { lock.withLock { sent } } + + // ANSI: any request opens a transaction, and only COMMIT or ROLLBACK ends it. + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { + sent.append(query) + current = query == "COMMIT" || query == "ROLLBACK" ? .idle : .inTransaction + } + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { state } + + func switchDatabase(to database: String) async throws {} + + func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { + "CREATE TABLE \(definition.tableName) (id INTEGER)" + } +} + +@Suite(.serialized) +@MainActor +struct TeradataDefinitionCommitTests { + private static let first = "ALTER TABLE orders ADD a INTEGER;" + private static let second = "ALTER TABLE orders ADD b INTEGER;" + + private static func inject(database: String) -> (DatabaseConnection, AnsiSessionDriver) { + let connection = TestFixtures.makeConnection(database: "orders", type: .teradata) + let driver = AnsiSessionDriver() + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.browseDatabase = database + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return (connection, driver) + } + + private static func injectPooled(_ connection: DatabaseConnection) async throws -> (DatabaseScope, AnsiSessionDriver) { + let scope = DatabaseScope(connectionId: connection.id, database: "orders", schema: nil) + let pooled = AnsiSessionDriver() + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: pooled) + try await adapter.connect() + MetadataConnectionPool.shared.injectEntry(adapter, scope: scope) + return (scope, pooled) + } + + private static func tearDown(_ connection: DatabaseConnection) { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + private static func script() -> SchemaChangeScript { + SchemaChangeScript( + tableName: "orders", + statements: [first, second].map { SchemaStatement(sql: $0, description: $0, isDestructive: false) }, + operations: [], + review: PluginSchemaChangeReview() + ) + } + + @Test("A Teradata Structure save commits each statement before the next") + func structureSaveCommitsEachStatement() async throws { + let (connection, _) = Self.inject(database: "orders") + defer { Self.tearDown(connection) } + let (scope, pooled) = try await Self.injectPooled(connection) + + try await DatabaseManager.shared.executeSchemaChanges( + Self.script(), databaseType: .teradata, scope: scope, gate: AlwaysAllowGate() + ) + + #expect(pooled.statements == [Self.first, "COMMIT", Self.second, "COMMIT"]) + #expect(pooled.state == .idle) + } + + @Test("Teradata Create Table commits each statement before the next") + func createTableCommitsEachStatement() async throws { + let (connection, _) = Self.inject(database: "orders") + defer { Self.tearDown(connection) } + let (scope, pooled) = try await Self.injectPooled(connection) + + try await DatabaseManager.shared.executeCreateTable( + statements: [Self.first, Self.second], databaseType: .teradata, scope: scope, gate: AlwaysAllowGate() + ) + + #expect(pooled.statements == [Self.first, "COMMIT", Self.second, "COMMIT"]) + } + + @Test("A Teradata Structure save on the session driver is refused while the session holds a transaction") + func sessionRouteSaveRefusedInsideAnOpenTransaction() async throws { + let (connection, driver) = Self.inject(database: "") + defer { Self.tearDown(connection) } + driver.state = .inTransaction + let scope = DatabaseScope(connectionId: connection.id, database: "", schema: nil) + + await #expect(throws: DefinitionInOpenTransactionRefusal()) { + try await DatabaseManager.shared.executeSchemaChanges( + Self.script(), databaseType: .teradata, scope: scope, gate: AlwaysAllowGate() + ) + } + #expect(driver.statements.isEmpty) + #expect(driver.state == .inTransaction) + } + + @Test("A Teradata batch import makes its new tables before it clears a reused one") + func importCreatesBeforeItClears() async throws { + let (connection, driver) = Self.inject(database: "orders") + defer { Self.tearDown(connection) } + let scope = DatabaseScope(connectionId: connection.id, database: "orders", schema: nil) + var planner = NewTableImportPlanner() + planner.recordCreated(TableScope(table: "reused", in: scope), createTableSQL: "CREATE TABLE reused (id INTEGER)") + let definitions = ["reused", "fresh"].map { + PluginCreateTableDefinition(tableName: $0, columns: [], primaryKeyColumns: []) + } + var created: [String] = [] + + _ = try await RowImportTableMaker(connection: connection, gate: AlwaysAllowGate()) + .prepare(definitions, scope: scope, planner: planner) { table, _ in created.append(table.table) } + + #expect(created == ["fresh"]) + #expect(driver.statements.count == 3) + #expect(Array(driver.statements.prefix(2)) == ["CREATE TABLE fresh (id INTEGER)", "COMMIT"]) + #expect(driver.statements.last?.hasPrefix("DELETE") == true) + } + + @Test("Teradata startup commands are committed one by one") + func startupCommandsAreCommitted() async throws { + let connection = TestFixtures.makeConnection(type: .teradata) + let driver = AnsiSessionDriver() + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + let commands = "DATABASE sales\nSET SESSION DATEFORM = ANSIDATE" + let statements = DatabaseManager.startupStatements(from: commands, databaseType: .teradata) + + let failures = await DatabaseManager.shared.executeStartupCommands( + commands, on: adapter, connectionName: "Test" + ) + + #expect(failures.isEmpty) + #expect(statements.count == 2) + #expect(driver.statements == statements.flatMap { [$0, "COMMIT"] }) + #expect(driver.state == .idle) + } + + @Test("Startup commands on an engine without the rule send no commit") + func startupCommandsElsewhereSendNoCommit() async { + let connection = TestFixtures.makeConnection(type: .mysql) + let driver = AnsiSessionDriver() + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + let commands = "SET NAMES utf8mb4" + + _ = await DatabaseManager.shared.executeStartupCommands(commands, on: adapter, connectionName: "Test") + + #expect(driver.statements == DatabaseManager.startupStatements(from: commands, databaseType: .mysql)) + } +} diff --git a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift index 81ec4f95e9..6eb56754f3 100644 --- a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift +++ b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift @@ -5,6 +5,8 @@ import Foundation @testable import TablePro +import TableProPluginKit +import TableProSQLGrammar import Testing @Suite("Credential lease routing", .serialized) @@ -60,6 +62,35 @@ struct CredentialLeaseRoutingTests { #expect(manager.activeSessions[connection.id]?.driver === home) } + @Test("A lane closed for its ending lease reports the session state the user set up on it") + func vaultRotationReportsSessionState() async { + let connection = TestFixtures.makeConnection(database: "app") + let home = MockDatabaseDriver(connection: connection) + let lane = MockDatabaseDriver(connection: connection) + lane.sessionTransactionStateToReturn = .idle + var session = ConnectionSession(connection: connection) + session.driver = home + session.status = .connected + let manager = DatabaseManager.shared + manager.injectSession(session, for: connection.id) + manager.sessionLanes.park(lane, database: "logs", for: connection.id) + defer { + manager.sessionLanes.closeAll(for: connection.id) + manager.removeSession(for: connection.id) + } + let key = SessionTransactionTracker.Key(connectionId: connection.id, lane: "logs") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .preparedStatements), on: lane, for: key) + + await manager.credentialLeaseEnding(ending(connection.id, lane)) + + #expect(lane.disconnectCallCount == 1) + #expect(manager.sessionStateLedger.state(heldBy: lane, for: connection.id).isEmpty) + #expect( + manager.sessionLanes.takeLoss(database: "logs", for: connection.id) + == SessionLoss(transaction: false, state: .preparedStatements) + ) + } + @Test("A browsed driver replaced while it held a transaction reports the loss to the next statement, once") func reportsLostBrowsedTransaction() async throws { let connection = TestFixtures.makeConnection(database: "app", type: .mysql) @@ -69,7 +100,7 @@ struct CredentialLeaseRoutingTests { let manager = DatabaseManager.shared manager.injectSession(session, for: connection.id) defer { manager.removeSession(for: connection.id) } - manager.sessionLanes.markTransactionLost(database: "app", for: connection.id) + manager.sessionLanes.markLost(database: "app", transaction: true, for: connection.id) let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) let error = await #expect(throws: DatabaseError.self) { @@ -89,14 +120,14 @@ struct CredentialLeaseRoutingTests { defer { manager.removeSession(for: connection.id) } let scope = DatabaseScope(connectionId: connection.id, database: "", schema: nil) #expect(manager.metadataRoute(for: scope) == .sessionDriver) - manager.sessionLanes.markTransactionLost(database: "", for: connection.id) + manager.sessionLanes.markLost(database: "", transaction: true, for: connection.id) try await manager.withMetadataDriver(scope: scope) { _ in } - #expect(manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "", for: connection.id)) await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in } } - #expect(!manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + #expect(!manager.sessionLanes.hasLoss(database: "", for: connection.id)) } } diff --git a/apple/TableProTests/Core/Database/DatabaseAccessBridgeAnswerBudgetTests.swift b/apple/TableProTests/Core/Database/DatabaseAccessBridgeAnswerBudgetTests.swift new file mode 100644 index 0000000000..1975a0f28e --- /dev/null +++ b/apple/TableProTests/Core/Database/DatabaseAccessBridgeAnswerBudgetTests.swift @@ -0,0 +1,257 @@ +// +// DatabaseAccessBridgeAnswerBudgetTests.swift +// TableProTests +// +// MCP and the assistant read rows through DatabaseAccessBridge, which held them to a row count and never to a size. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct DatabaseAccessBridgeAnswerBudgetTests { + private func approved( + _ sql: String, + scope: DatabaseScope, + databaseType: DatabaseType, + maxRows: Int = 500 + ) async throws -> ApprovedOperation { + let statement = try await DatabaseAccessBridge().plannedStatement(scope: scope, query: sql, maxRows: maxRows) + return try await ApprovedOperationFixture.approve([statement], scope: scope, databaseType: databaseType) + } + + @Test("A read of 10,000 rows of 2 KB answers at most 4 MiB, marked truncated") + func largeReadIsHeldToTheBudget() async throws { + let driver = AnsweringPluginDriver { _ in + AnsweringPluginDriver.textRows(column: "body", count: 10_000, bytes: 2_048) + } + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + let operation = try await approved("SELECT body FROM documents", scope: scope, databaseType: .postgresql, maxRows: 10_000) + + let outcome = try await DatabaseAccessBridge().runStatement( + operation, maxRows: 10_000, timeoutSeconds: 30, cancellation: nil, budget: .mcpAnswer + ) + + let budget = try #require(outcome.budget) + #expect(!outcome.result.rows.isEmpty) + #expect(outcome.result.rows.count * 2_048 <= ExternalAnswerBudget.answerBytes) + #expect(outcome.result.rows.count < 10_000) + #expect(outcome.result.isTruncated) + #expect(budget.sizeLimited) + #expect(budget.spentBytes <= ExternalAnswerBudget.answerBytes) + } + + @Test("Without a budget, as AppleScript and Shortcuts run, every row comes back") + func noBudgetKeepsEveryRow() async throws { + let driver = AnsweringPluginDriver { _ in + AnsweringPluginDriver.textRows(column: "body", count: 3_000, bytes: 2_048) + } + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + let operation = try await approved("SELECT body FROM documents", scope: scope, databaseType: .postgresql, maxRows: 3_000) + + let outcome = try await DatabaseAccessBridge().runStatement( + operation, maxRows: 3_000, timeoutSeconds: 30, cancellation: nil + ) + + #expect(outcome.result.rows.count == 3_000) + #expect(outcome.budget == nil) + #expect(driver.sent.last?.context.resultByteLimit == nil) + } + + @Test("A read reaches the driver with the answer's byte limit, and a write reaches it with none") + func onlyReadsCarryTheByteLimit() async throws { + let driver = AnsweringPluginDriver() + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + let bridge = DatabaseAccessBridge() + + _ = try await bridge.runStatement( + try await approved("SELECT 1", scope: scope, databaseType: .postgresql), + maxRows: 500, timeoutSeconds: 30, cancellation: nil, budget: .assistantAnswer + ) + _ = try await bridge.runStatement( + try await approved("UPDATE orders SET paid = true WHERE id = 1", scope: scope, databaseType: .postgresql), + maxRows: 500, timeoutSeconds: 30, cancellation: nil, budget: .assistantAnswer + ) + + #expect(driver.sent.map { $0.context.resultByteLimit } == [ExternalAnswerBudget.assistantAnswerBytes, nil]) + #expect(driver.sent.map { $0.context.readOnly } == [true, false]) + } + + @Test("A read the driver stopped at its byte limit says the size limit left rows out") + func driverStopIsReported() async throws { + let driver = AnsweringPluginDriver { _ in + AnsweringPluginDriver.textRows(column: "body", count: 10, bytes: 10, isTruncated: true) + } + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + + let outcome = try await DatabaseAccessBridge().runStatement( + try await approved("SELECT body FROM documents", scope: scope, databaseType: .postgresql), + maxRows: 500, timeoutSeconds: 30, cancellation: nil, budget: .mcpAnswer + ) + + #expect(outcome.result.rows.count == 10) + #expect(outcome.result.isTruncated) + #expect(outcome.budget?.sizeLimited == true) + #expect(outcome.budget?.notes.count == 1) + } + + // If the cut ran first, the last four characters shown would be the end of the cut mark. + @Test("A partial mask reads the whole value before any cut") + func maskRunsBeforeTheCut() async throws { + var connection = TestFixtures.makeConnection(database: "shop", type: .mysql) + connection.maskingRules = [MaskingRule(pattern: "*.card", style: .lastFour)] + let long = String(repeating: "1", count: 200_000) + "WXYZ" + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: { _ in + var result = ScriptAnsweringDriver.resultSet(columns: ["card"], rows: [[long]]) + result.columnOrigins = [PluginColumnOrigin(schema: "shop", table: "payments", column: "card")] + return result + }) + var session = ConnectionSession(connection: connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "shop", schema: nil) + + let outcome = try await DatabaseAccessBridge().runStatement( + try await approved("SELECT card FROM payments", scope: scope, databaseType: .mysql), + maxRows: 500, timeoutSeconds: 30, cancellation: nil, budget: .mcpAnswer + ) + + let cell = try #require(outcome.result.rows.first?.first?.asText) + #expect(cell.hasSuffix("WXYZ")) + #expect(!cell.contains("TablePro cut")) + #expect(outcome.budget?.valuesCut == false) + } +} + +@Suite("SQL Server scripts held to one answer budget", .serialized) +@MainActor +struct DatabaseAccessBridgeScriptBudgetTests { + private func run( + _ text: String, + on driver: ScriptAnsweringDriver, + budget: ExternalAnswerBudget? + ) async throws -> DatabaseAccessBridge.ScriptOutcome { + var session = ConnectionSession(connection: driver.connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: driver.connection.id) + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + let scope = DatabaseScope(connectionId: driver.connection.id, database: driver.connection.database, schema: nil) + let bridge = DatabaseAccessBridge() + let planned = try await bridge.plannedScript(scope: scope, query: text, maxRows: 500) + let operation = try await ApprovedOperationFixture.approve( + [planned.statement], scope: scope, databaseType: .mssql, unit: planned.unit + ) + return try await bridge.runScript(operation, maxRows: 500, timeoutSeconds: 30, cancellation: nil, budget: budget) + } + + nonisolated private static func rows(_ column: String, count: Int, bytes: Int) -> QueryResult { + ScriptAnsweringDriver.resultSet( + columns: [column], + rows: Array(repeating: [String(repeating: "x", count: bytes)], count: count) + ) + } + + private func makeDriver(answer: @escaping @Sendable (String) -> QueryBatchResult) -> ScriptAnsweringDriver { + ScriptAnsweringDriver( + connection: TestFixtures.makeConnection(database: "warehouse", type: .mssql), + answer: answer + ) + } + + nonisolated private static let threeBatches = "SELECT 1 AS a\nGO\nSELECT 2 AS b\nGO\nSELECT 3 AS c" + + nonisolated private static func firstBatchFillsTheBudget(_ batch: String) -> QueryBatchResult { + batch.contains("AS a") + ? ScriptAnsweringDriver.batch([rows("a", count: 50, bytes: 1_000)]) + : ScriptAnsweringDriver.batch([rows("b", count: 5, bytes: 10)]) + } + + @Test("Once the budget is spent, later batches still run, reading one row and keeping none") + func laterBatchesRunWithoutRows() async throws { + let driver = makeDriver(answer: Self.firstBatchFillsTheBudget) + + let outcome = try await run(Self.threeBatches, on: driver, budget: ExternalAnswerBudget(bytes: 10_000)) + + #expect(driver.sentBatches.map { $0.rowCap } == [500, 1, 1]) + #expect(outcome.resultSets.count == 3) + let first = try #require(outcome.resultSets.first) + #expect(!first.rows.isEmpty && first.rows.count < 50) + #expect(first.isTruncated) + #expect(outcome.resultSets.dropFirst().allSatisfy { $0.rows.isEmpty && $0.isTruncated }) + #expect(outcome.primary.rows.count == first.rows.count) + #expect(outcome.primary.isTruncated) + #expect(outcome.budget?.sizeLimited == true) + } + + @Test("Without a budget, as AppleScript and Shortcuts run, every batch keeps its rows") + func noBudgetKeepsEveryBatch() async throws { + let driver = makeDriver(answer: Self.firstBatchFillsTheBudget) + + let outcome = try await run(Self.threeBatches, on: driver, budget: nil) + + #expect(driver.sentBatches.map { $0.rowCap } == [500, 500, 500]) + #expect(outcome.resultSets.map { $0.rows.count } == [50, 5, 5]) + #expect(outcome.budget == nil) + } + + @Test("Each batch's answer is cut to the budget before the next batch runs, so rows past it are not held") + func answersAreHeldToTheBudgetBatchByBatch() async throws { + let driver = makeDriver(answer: Self.firstBatchFillsTheBudget) + let batches = QueryBatchPlanner.batches( + in: Self.threeBatches, + model: QueryStatementModel.forDatabaseType(.mssql), + grammar: DatabaseType.mssql.lexicalGrammar + ) + + let answered = try await ScriptBatchRun.run( + batches, startLines: [1, 3, 5], rowCap: 500, driver: driver, progress: ScriptBatchProgress(), + budget: ExternalAnswerBudget(bytes: 10_000) + ) + + let heldRows = answered.answers.map { answer in answer.resultSets.map { set in set.rows.count } } + #expect(heldRows == [[9], [0], [0]]) + #expect(driver.sentBatches.map { $0.rowCap } == [500, 1, 1]) + #expect(answered.budget?.sizeLimited == true) + #expect((answered.budget?.spentBytes ?? .max) <= 10_000) + } + + // If a value were cut before the mask, the last four characters shown would be the end of the cut mark. + @Test("A script's partial mask reads the whole value before any cut") + func scriptMaskRunsBeforeTheCut() async throws { + var connection = TestFixtures.makeConnection(database: "warehouse", type: .mssql) + connection.maskingRules = [MaskingRule(pattern: "*.card", style: .lastFour)] + let long = String(repeating: "1", count: 200_000) + "WXYZ" + let driver = ScriptAnsweringDriver(connection: connection, answer: { _ in + var set = ScriptAnsweringDriver.resultSet(columns: ["card"], rows: [[long]]) + set.columnOrigins = [PluginColumnOrigin(schema: "dbo", table: "payments", column: "card")] + return ScriptAnsweringDriver.batch([set]) + }) + + let outcome = try await run("SELECT card FROM payments\nGO\nSELECT 2 AS b", on: driver, budget: .mcpAnswer) + + let cell = try #require(outcome.resultSets.first?.rows.first?.first?.asText) + #expect(cell.hasSuffix("WXYZ")) + #expect(!cell.contains("TablePro cut")) + } + + @Test("With more than one result set the first costs twice, since the answer writes it twice") + func firstSetCostsTwice() async throws { + let twoSets = makeDriver { _ in + ScriptAnsweringDriver.batch([Self.rows("a", count: 6, bytes: 1_000), Self.rows("b", count: 1, bytes: 1)]) + } + let oneSet = makeDriver { _ in ScriptAnsweringDriver.batch([Self.rows("a", count: 6, bytes: 1_000)]) } + + let both = try await run("EXEC report", on: twoSets, budget: ExternalAnswerBudget(bytes: 10_000)) + let single = try await run("EXEC report", on: oneSet, budget: ExternalAnswerBudget(bytes: 10_000)) + + #expect(both.resultSets.first?.rows.count == 4) + #expect(single.resultSets.first?.rows.count == 6) + } +} diff --git a/apple/TableProTests/Core/Database/DatabaseAccessBridgeCopyTests.swift b/apple/TableProTests/Core/Database/DatabaseAccessBridgeCopyTests.swift new file mode 100644 index 0000000000..842b5f1064 --- /dev/null +++ b/apple/TableProTests/Core/Database/DatabaseAccessBridgeCopyTests.swift @@ -0,0 +1,37 @@ +// +// DatabaseAccessBridgeCopyTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite(.serialized) +@MainActor +struct DatabaseAccessBridgeCopyTests { + @Test("An external caller's COPY FROM STDIN is refused before anything is sent, rows or not", arguments: [ + "COPY t FROM stdin;\n1\tone\n\\.\n", + "COPY t FROM stdin;", + ]) + func copyIsRefused(sql: String) async throws { + let driver = ScriptAnsweringDriver( + connection: TestFixtures.makeConnection(database: "app", type: .postgresql), + sendsBatchesWhole: false + ) + var session = ConnectionSession(connection: driver.connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: driver.connection.id) + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + let scope = DatabaseScope(connectionId: driver.connection.id, database: "app", schema: nil) + + await #expect { + _ = try await DatabaseAccessBridge().plannedScript(scope: scope, query: sql, maxRows: 10) + } throws: { error in + guard case .invalidArgument(let message) = error as? DatabaseAccessError else { return false } + return message.contains("COPY FROM STDIN") + } + #expect(driver.sentStatements.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Database/DatabaseAccessBridgeScriptTests.swift b/apple/TableProTests/Core/Database/DatabaseAccessBridgeScriptTests.swift index 6b6d12a5d8..2d630470e2 100644 --- a/apple/TableProTests/Core/Database/DatabaseAccessBridgeScriptTests.swift +++ b/apple/TableProTests/Core/Database/DatabaseAccessBridgeScriptTests.swift @@ -53,12 +53,14 @@ struct DatabaseAccessBridgeScriptTests { private func makeDriver( sendsBatchesWhole: Bool = true, transactionState: PluginSessionTransactionState = .idle, + transactionStateAfterBatches: PluginSessionTransactionState? = nil, answer: @escaping @Sendable (String) -> QueryBatchResult = { _ in .empty } ) -> ScriptAnsweringDriver { ScriptAnsweringDriver( connection: TestFixtures.makeConnection(database: "warehouse", type: .mssql), sendsBatchesWhole: sendsBatchesWhole, transactionState: transactionState, + transactionStateAfterBatches: transactionStateAfterBatches, answer: answer ) } @@ -134,7 +136,8 @@ struct DatabaseAccessBridgeScriptTests { @Test("Rows the script changed and a transaction it left open are reported on the first result") func scriptTotalsAndNoticesAreReported() async throws { let notice = try #require(PluginSessionTransactionState.inTransaction.openTransactionNotice) - let driver = makeDriver(transactionState: .inTransaction) { _ in + // Idle before the script, as a session is; a BEGIN TRAN over an open transaction is refused. + let driver = makeDriver(transactionStateAfterBatches: .inTransaction) { _ in ScriptAnsweringDriver.batch( [ScriptAnsweringDriver.resultSet(columns: ["id"], rows: [["1"]], isTruncated: true)], rowsAffected: 3 diff --git a/apple/TableProTests/Core/Database/DatabaseManagerRDSEndpointTests.swift b/apple/TableProTests/Core/Database/DatabaseManagerRDSEndpointTests.swift new file mode 100644 index 0000000000..a10f369e57 --- /dev/null +++ b/apple/TableProTests/Core/Database/DatabaseManagerRDSEndpointTests.swift @@ -0,0 +1,264 @@ +// +// DatabaseManagerRDSEndpointTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSSHConfig +import TableProSSHTransport +import Testing + +@MainActor +private final class EndpointSignInSpy { + var answer: ConsentAnswer = .approved + private(set) var destinations: [String] = [] + + func ask(_ question: LocalCredentialQuestion) async -> ConsentAnswer { + destinations.append(question.destination) + return answer + } +} + +@MainActor +struct DatabaseManagerRDSEndpointTests { + private static let alias = "reader.db.example.com" + private static let replicaA = "replica-a.abc123.us-east-1.rds.amazonaws.com" + private static let replicaB = "replica-b.abc123.us-east-1.rds.amazonaws.com" + + private func connection( + type: DatabaseType = .mysql, + host: String = alias, + signIn: String? = "profile" + ) -> DatabaseConnection { + var connection = DatabaseConnection(name: "RDS", host: host, port: 3_306, type: type) + connection.additionalFields[DatabaseConnection.iamSignInFieldId] = signIn + return connection + } + + @Test("The CNAME host of an AWS IAM connection becomes the RDS endpoint it stands for") + func directConnectionDialsTheEndpoint() async throws { + let resolver = FakeCNAMEResolver([Self.alias: Self.replicaA]) + let manager = DatabaseManager(cnameResolver: resolver) + + let effective = try await manager.resolvingRDSEndpointAlias(of: connection(), trigger: .request) + + #expect(effective.host == Self.replicaA) + #expect(effective.port == 3_306) + #expect(resolver.queried == [Self.alias]) + } + + @Test("Every lookup follows the CNAME again, so a moved CNAME gives the new endpoint and the saved row keeps its host") + func movedCNAMEIsFollowed() async throws { + let resolver = FakeCNAMEResolver([Self.alias: Self.replicaA]) + let manager = DatabaseManager(cnameResolver: resolver) + let saved = connection() + + let first = try await manager.resolvingRDSEndpointAlias(of: saved, trigger: .request) + resolver.point(Self.alias, at: Self.replicaB) + let second = try await manager.resolvingRDSEndpointAlias(of: saved, trigger: .request) + + #expect(first.host == Self.replicaA) + #expect(second.host == Self.replicaB) + #expect(saved.host == Self.alias) + } + + @Test("A host with no RDS endpoint behind it is dialled and signed as typed") + func unresolvedHostIsUnchanged() async throws { + let manager = DatabaseManager(cnameResolver: FakeCNAMEResolver()) + + let effective = try await manager.resolvingRDSEndpointAlias(of: connection(), trigger: .request) + + #expect(effective.host == Self.alias) + } + + @Test("Nothing is looked up without an RDS token: password sign-in, Google IAM, Redshift, ElastiCache, Keyspaces") + func otherSignInsAreNotLookedUp() async throws { + let resolver = FakeCNAMEResolver([Self.alias: Self.replicaA]) + let manager = DatabaseManager(cnameResolver: resolver) + let untouched = [ + connection(signIn: nil), + connection(signIn: GoogleCloudSQLAuthFields.applicationDefault), + connection(type: .redshift), + connection(type: .redis), + connection(type: .cassandra) + ] + + for saved in untouched { + let effective = try await manager.resolvingRDSEndpointAlias(of: saved, trigger: .request) + #expect(effective.host == Self.alias) + } + #expect(resolver.queried.isEmpty) + } + + @Test("Behind a tunnel the far end resolves the name, so the forward keeps the name as typed") + func tunnelKeepsTheTypedName() async throws { + let resolver = FakeCNAMEResolver([Self.alias: Self.replicaA]) + let manager = DatabaseManager(cnameResolver: resolver) + var saved = connection(type: .postgresql) + saved.port = 5_432 + saved.sshTunnelMode = .inline(SSHConfiguration(enabled: true, host: "bastion.example.com")) + + let resolved = try await manager.resolvingRDSEndpointAlias(of: saved, trigger: .request) + + #expect(resolved.sshForwardDestination == .tcp(host: Self.alias, port: 5_432)) + #expect(resolver.queried.isEmpty) + } + + @Test("The lookup runs for exactly the connections the minter gives an RDS token") + func lookupMatchesTheMinter() { + for type in [DatabaseType.mysql, .mariadb, .postgresql, .redshift, .redis, .cassandra, .scylladb] { + for signIn in ["profile", "accessKey", "sso", GoogleCloudSQLAuthFields.applicationDefault, nil] { + let saved = connection(type: type, signIn: signIn) + let minter = ConnectionCredentialResolver.iamPasswordMinter(for: saved, fields: saved.additionalFields) + var mintsRDSToken = false + if case .aws(let signer) = minter, case .rds = signer.target { + mintsRDSToken = true + } + #expect(ConnectionCredentialResolver.signsRDSToken(for: saved) == mintsRDSToken, "\(type) \(signIn ?? "password")") + } + } + } + + @Test("An endpoint counts as moved only when the dialled host changes, whatever its case") + func movedEndpointIsTheDialledHost() { + let manager = DatabaseManager(cnameResolver: FakeCNAMEResolver()) + var dialled = connection() + dialled.host = Self.replicaA + var session = ConnectionSession(connection: connection()) + session.effectiveConnection = dialled + + var moved = dialled + moved.host = Self.replicaB + var sameInCapitals = dialled + sameInCapitals.host = Self.replicaA.uppercased() + + #expect(manager.movesEndpoint(of: session, to: moved)) + #expect(!manager.movesEndpoint(of: session, to: sameInCapitals)) + #expect(!manager.movesEndpoint(of: ConnectionSession(connection: connection()), to: connection())) + } + + @Test("A failed lookup is an error that names the host, not a token signed for the CNAME") + func failedLookupThrows() async { + let manager = DatabaseManager(cnameResolver: FakeCNAMEResolver(failing: [Self.alias])) + + await #expect(throws: AWSAuthError.endpointLookupFailed(host: Self.alias)) { + _ = try await manager.resolvingRDSEndpointAlias(of: connection(), trigger: .request) + } + } + + @Test("AWS IAM signs in only with a value the type's own Authentication picker offers") + func signInNeedsTheTypesOwnPicker() { + #expect(connection(type: .mysql, signIn: "profile").iamSignIn == .aws(source: "profile")) + #expect(connection(type: .postgresql, signIn: GoogleCloudSQLAuthFields.applicationDefault).iamSignIn + == .googleCloud(.applicationDefault)) + // ScyllaDB has no AWS picker, but its plugin signs in to Keyspaces with the field, so TLS is still raised. + #expect(connection(type: .scylladb, signIn: "profile").iamSignIn == .aws(source: "profile")) + for (type, signIn) in [ + (DatabaseType.redis, GoogleCloudSQLAuthFields.applicationDefault), + (DatabaseType.mariadb, GoogleCloudSQLAuthFields.serviceAccount), + (DatabaseType(rawValue: "Elasticsearch"), "profile"), + (DatabaseType.mssql, "profile"), + (DatabaseType.mysql, "someday"), + (DatabaseType.scylladb, GoogleCloudSQLAuthFields.applicationDefault) + ] { + let saved = connection(type: type, signIn: signIn) + #expect(saved.iamSignIn == nil, "\(type) \(signIn)") + #expect(ConnectionCredentialResolver.iamPasswordMinter(for: saved, fields: saved.additionalFields) == nil) + } + } + + @Test("A row from someone else asks again for each RDS endpoint its CNAME leads to, and never from the background") + func sharedRowApprovesEachEndpoint() async throws { + let suite = try EphemeralDefaults() + let spy = EndpointSignInSpy() + let consent = LocalCredentialConsent( + approvals: ConnectionApprovalStore(kind: .localCredentials, defaults: suite.defaults, savedConnections: { [] }), + ask: { [spy] question in await spy.ask(question) } + ) + let resolver = FakeCNAMEResolver([Self.alias: Self.replicaA]) + let manager = DatabaseManager(localCredentialConsent: consent, cnameResolver: resolver) + var shared = connection() + shared.origin = .teamLibrary + + let first = try await manager.resolvingRDSEndpointAlias(of: shared, trigger: .request) + let again = try await manager.resolvingRDSEndpointAlias(of: shared, trigger: .background) + resolver.point(Self.alias, at: Self.replicaB) + await #expect(throws: LocalCredentialConsentError.needsPerson) { + _ = try await manager.resolvingRDSEndpointAlias(of: shared, trigger: .background) + } + spy.answer = .declined + await #expect(throws: LocalCredentialConsentError.declined) { + _ = try await manager.resolvingRDSEndpointAlias(of: shared, trigger: .request) + } + + #expect(first.host == Self.replicaA) + #expect(again.host == Self.replicaA) + #expect(spy.destinations.count == 2) + #expect(spy.destinations.allSatisfy { !$0.contains(Self.alias) }) + #expect(spy.destinations.first?.contains(Self.replicaA) == true) + #expect(spy.destinations.last?.contains(Self.replicaB) == true) + } + + @Test("A row made on this Mac follows its own CNAME without asking") + func ownRowDoesNotAsk() async throws { + let suite = try EphemeralDefaults() + let spy = EndpointSignInSpy() + let consent = LocalCredentialConsent( + approvals: ConnectionApprovalStore(kind: .localCredentials, defaults: suite.defaults, savedConnections: { [] }), + ask: { [spy] question in await spy.ask(question) } + ) + let manager = DatabaseManager( + localCredentialConsent: consent, + cnameResolver: FakeCNAMEResolver([Self.alias: Self.replicaA]) + ) + + let effective = try await manager.resolvingRDSEndpointAlias(of: connection(), trigger: .background) + + #expect(effective.host == Self.replicaA) + #expect(spy.destinations.isEmpty) + } + + private func approvingManager(failingLookup: Bool) throws -> (DatabaseManager, EphemeralDefaults) { + let suite = try EphemeralDefaults() + let consent = LocalCredentialConsent( + approvals: ConnectionApprovalStore(kind: .localCredentials, defaults: suite.defaults, savedConnections: { [] }), + ask: { _ in .approved } + ) + let resolver = FakeCNAMEResolver(failing: failingLookup ? [Self.alias] : []) + return (DatabaseManager(localCredentialConsent: consent, cnameResolver: resolver), suite) + } + + @Test("Connect, Test Connection and both reconnects look the CNAME up before they dial") + func everyDialPathLooksUp() async throws { + let (manager, suite) = try approvingManager(failingLookup: true) + defer { withExtendedLifetime(suite) {} } + let lookupFailed = AWSAuthError.endpointLookupFailed(host: Self.alias) + let saved = connection() + + await #expect(throws: lookupFailed) { + try await manager.connectToSession(saved, trigger: .request) + } + manager.removeSession(for: saved.id) + + await #expect(throws: lookupFailed) { + _ = try await manager.testConnection(saved) + } + + var session = ConnectionSession(connection: saved, driver: MockDatabaseDriver()) + session.status = .connected + manager.injectSession(session, for: saved.id) + defer { manager.removeSession(for: saved.id) } + let generation = manager.connectionAttempts.begin(for: saved.id) + await #expect(throws: lookupFailed) { + _ = try await manager.reconnectDriver(for: session, allowsCredentialPrompt: false, generation: generation) + } + manager.connectionAttempts.finish(generation, for: saved.id) + + await #expect(throws: lookupFailed) { + try await manager.reconnectSession(saved.id) + } + } +} + diff --git a/apple/TableProTests/Core/Database/DatabaseManagerTests.swift b/apple/TableProTests/Core/Database/DatabaseManagerTests.swift index 1d05998bfb..10989dcb28 100644 --- a/apple/TableProTests/Core/Database/DatabaseManagerTests.swift +++ b/apple/TableProTests/Core/Database/DatabaseManagerTests.swift @@ -412,8 +412,11 @@ struct DatabaseManagerDatabaseSwitchTests { let pluginDriver = DatabaseSwitchingDriver() let adapter = PluginDriverAdapter(connection: connection, pluginDriver: pluginDriver) - await DatabaseManager.shared.restoreSchemaAndDatabase(on: adapter, savedSchema: "idb", savedDatabase: "ice") + let warning = await DatabaseManager.shared.restoreSchemaAndDatabase( + on: adapter, savedSchema: "idb", savedDatabase: "ice" + ) #expect(pluginDriver.switches == ["database ice", "schema idb"]) + #expect(warning == nil) } } diff --git a/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift b/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift index 84bed140b7..e547d3ef92 100644 --- a/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift +++ b/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift @@ -60,6 +60,18 @@ struct DatabaseManagerTunnelTests { #expect(tunneled.passwordSource == .env(variable: "DB_PASS")) } + @Test("A tunneled connection keeps its credential profile, so it signs in with the profile's credentials") + func tunnelPreservesCredentialProfile() { + let profileId = UUID() + var connection = DatabaseConnection(name: "tunneled", host: "db.internal", port: 3_306, type: .mysql) + connection.credentialMode = .profile(id: profileId) + + let tunneled = DatabaseManager.shared.tunneledConnection(from: connection, localPort: 61_235) + + #expect(tunneled.credentialMode.profileId == profileId) + #expect(ConnectionCredentialResolver.credentialOwner(for: tunneled) == profileId) + } + @Test("A tunneled DuckDB Remote row dials the forward through its own Host and Port fields") func tunnelRewritesServerEndpointFields() { let connection = DatabaseConnection( @@ -643,3 +655,42 @@ struct DatabaseManagerTunnelTests { #expect(phase == .unavailable(.disconnected(info))) } } + +extension GlobalStateSuites { + // Writes the shared SSH profile store across suspensions. + @MainActor + struct PinnedSSHRouteTests { + @Test("A pinned route is refused once the linked profile moves, and followed while it stays") + func pinnedRouteRefusesAMovedProfile() async throws { + let profiles = SSHProfileStorage.shared + let saved = profiles.loadProfiles() + let profile = SSHProfile(name: "Jump", host: "bastion-b.example", port: 22, username: "ops", authMethod: .password) + #expect(profiles.saveProfilesWithoutSync(saved + [profile])) + defer { profiles.saveProfilesWithoutSync(saved) } + var dialed = profile.toSSHConfiguration() + dialed.host = "bastion-a.example" + var connection = DatabaseConnection( + name: "file", + type: .duckdb, + additionalFields: ["duckdbMode": "local", "duckdbFilePath": "/tmp/tablepro-tests.duckdb"] + ) + connection.sshTunnelMode = .profile(id: profile.id, snapshot: dialed) + + await #expect(throws: ConnectionTunnelError.sshProfileMoved) { + try await DatabaseManager.shared.buildEffectiveConnection( + for: connection, + pinnedSSHRoute: connection.destinationTunnel, + deadline: ConnectionDeadline(configuredSeconds: 5) + ) + } + + let current = profiles.refreshingLinkedProfile(connection) + let effective = try await DatabaseManager.shared.buildEffectiveConnection( + for: connection, + pinnedSSHRoute: current.destinationTunnel, + deadline: ConnectionDeadline(configuredSeconds: 5) + ) + #expect(effective.additionalFields["duckdbFilePath"] == "/tmp/tablepro-tests.duckdb") + } + } +} diff --git a/apple/TableProTests/Core/Database/EnvVarReferenceConsentTests.swift b/apple/TableProTests/Core/Database/EnvVarReferenceConsentTests.swift index c0c2db09cf..9f5c8b911b 100644 --- a/apple/TableProTests/Core/Database/EnvVarReferenceConsentTests.swift +++ b/apple/TableProTests/Core/Database/EnvVarReferenceConsentTests.swift @@ -145,6 +145,21 @@ extension GlobalStateSuites { }) } + @Test("A manual reconnect resolves the references as the connect does, so a declined prompt sends nothing") + func manualReconnectResolvesReferences() async throws { + let manager = DatabaseManager(envVarReferenceConsent: makeConsent()) + let connection = importedFromAttacker() + manager.setSession(ConnectionSession(connection: connection), for: connection.id) + defer { manager.removeSession(for: connection.id) } + + await withReportVariables { + await #expect(throws: EnvVarReferenceConsentError.declined) { + try await manager.reconnectSession(connection.id) + } + } + #expect(spy.questions.map(\.connectionName) == ["Staging"]) + } + @Test("MCP, the assistant and AppleScript get a declined prompt as a refusal") func bridgeReportsDeclinedAsForbidden() async throws { let connection = importedFromAttacker() diff --git a/apple/TableProTests/Core/Database/ExternalAnswerBudgetTests.swift b/apple/TableProTests/Core/Database/ExternalAnswerBudgetTests.swift new file mode 100644 index 0000000000..58115371c0 --- /dev/null +++ b/apple/TableProTests/Core/Database/ExternalAnswerBudgetTests.swift @@ -0,0 +1,174 @@ +// +// ExternalAnswerBudgetTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct ExternalAnswerBudgetTests { + private func result(_ column: String, _ rows: [[PluginCellValue]], isTruncated: Bool = false) -> QueryResult { + var result = QueryResult( + columns: [column], + columnTypes: [.text(rawType: nil)], + rows: rows, + rowsAffected: 0, + executionTime: 0, + error: nil + ) + result.isTruncated = isTruncated + return result + } + + private func textRows(_ column: String, count: Int, bytes: Int) -> QueryResult { + result(column, Array(repeating: [.text(String(repeating: "x", count: bytes))], count: count)) + } + + // What one row of a single column costs in the core's measure: the row's brackets and comma, then the name, the + // cell's quotes, colon and comma, and the value. + private func rowCost(column: String, valueBytes: Int) -> Int { + 3 + column.utf8.count + 6 + valueBytes + } + + @Test("A value past 64 KiB keeps its first 64 KiB and a mark with its full length") + func longTextIsCut() throws { + var budget = ExternalAnswerBudget.mcpAnswer + + let kept = budget.keep(result("body", [[.text(String(repeating: "a", count: 200_000))]])) + + let cell = try #require(kept.rows.first?.first) + #expect(cell == .text(String(repeating: "a", count: 65_536) + " [TablePro cut this value: 200000 bytes in all]")) + #expect(budget.valuesCut) + #expect(!budget.sizeLimited) + #expect(!kept.isTruncated) + #expect(budget.notes.count == 1) + } + + @Test("A cut never splits a character, and a value inside the limit is left as it is") + func cutKeepsWholeCharacters() { + let euros = String(repeating: "€", count: 30_000) + + let cut = ExternalAnswerBudget.cut(euros) + + let mark = " [TablePro cut this value: 90000 bytes in all]" + #expect(cut.hasSuffix(mark)) + #expect(String(cut.dropLast(mark.count)) == String(repeating: "€", count: 21_845)) + #expect(ExternalAnswerBudget.cut("short") == "short") + } + + @Test("Binary is measured as the base64 the answer carries, and a long one is cut to base64 text") + func binaryIsMeasuredAsBase64() throws { + let row = result("b", [[.bytes(Data(repeating: 7, count: 3_000))]]) + let cost = rowCost(column: "b", valueBytes: 4_000) + + var fits = ExternalAnswerBudget(bytes: cost) + var short = ExternalAnswerBudget(bytes: cost - 1) + let fitted = fits.keep(row) + let shortened = short.keep(row) + + #expect(fitted.rows.count == 1) + #expect(shortened.rows.isEmpty) + #expect(short.sizeLimited) + + let data = Data(repeating: 1, count: 100_000) + var budget = ExternalAnswerBudget.mcpAnswer + let kept = budget.keep(result("b", [[.bytes(data)]])) + let cell = try #require(kept.rows.first?.first) + #expect(cell == .text(data.prefix(49_152).base64EncodedString() + " [TablePro cut this value: 100000 bytes in all]")) + } + + @Test("Rows past the answer limit are left out, and the result says it was truncated") + func rowsPastTheLimitAreLeftOut() { + var budget = ExternalAnswerBudget.mcpAnswer + + let kept = budget.keep(textRows("body", count: 3_000, bytes: 2_000)) + + #expect(kept.rows.count == ExternalAnswerBudget.answerBytes / rowCost(column: "body", valueBytes: 2_000)) + #expect(kept.isTruncated) + #expect(budget.sizeLimited) + #expect(!budget.valuesCut) + #expect(budget.spentBytes <= ExternalAnswerBudget.answerBytes) + #expect(budget.notes.count == 1) + } + + @Test("An export keeps a long value whole, and leaves out a row that does not fit instead of cutting it") + func exportKeepsValuesWhole() { + let long = String(repeating: "y", count: 100_000) + var export = ExternalAnswerBudget.export + + let kept = export.keep(result("doc", [[.text(long)]])) + + #expect(kept.rows == [[.text(long)]]) + #expect(!export.valuesCut) + + var small = ExternalAnswerBudget(bytes: 50_000, longValues: .whole) + let leftOut = small.keep(result("doc", [[.text(long)]])) + #expect(leftOut.rows.isEmpty) + #expect(leftOut.isTruncated) + #expect(small.sizeLimited) + #expect(!small.valuesCut) + } + + @Test("A result after the limit was reached keeps no rows") + func laterResultKeepsNoRows() { + var budget = ExternalAnswerBudget(bytes: 100) + + let first = budget.keep(textRows("a", count: 1, bytes: 200)) + let second = budget.keep(textRows("b", count: 1, bytes: 1)) + + #expect(first.rows.isEmpty) + #expect(second.rows.isEmpty) + #expect(second.isTruncated) + } + + @Test("A result the answer writes twice costs twice") + func weightCountsEveryCopy() { + let rows = textRows("b", count: 6, bytes: 1_000) + var once = ExternalAnswerBudget(bytes: 10_000) + var twice = ExternalAnswerBudget(bytes: 10_000) + let keptOnce = once.keep(rows) + let keptTwice = twice.keep(rows, weight: 2) + + #expect(keptOnce.rows.count == 6) + #expect(keptTwice.rows.count == 4) + } + + @Test("A text beside the rows is cut to what the answer has left") + func textIsCutToTheRest() { + var budget = ExternalAnswerBudget(bytes: 100) + + let kept = budget.keep(text: String(repeating: "p", count: 300)) + + #expect(kept == String(repeating: "p", count: 100) + " [TablePro cut this value: 300 bytes in all]") + #expect(budget.sizeLimited) + #expect(budget.remainingBytes == 0) + } + + @Test("A part kept only whole is left out when it does not fit, and a smaller part after it still fits") + func wholePartIsLeftOut() { + var budget = ExternalAnswerBudget(bytes: 1_000) + + let large = budget.keepWhole(bytes: 1_200) + let small = budget.keepWhole(bytes: 800) + + #expect(!large) + #expect(small) + #expect(budget.spentBytes == 800) + #expect(!budget.sizeLimited) + } + + @Test("An answer that fits comes back as it was, with no notes") + func fittingAnswerIsUntouched() { + var budget = ExternalAnswerBudget.mcpAnswer + let rows = textRows("n", count: 10, bytes: 10) + + let kept = budget.keep(rows) + + #expect(kept.rows == rows.rows) + #expect(!kept.isTruncated) + #expect(budget.notes.isEmpty) + #expect(budget.spentBytes == 10 * rowCost(column: "n", valueBytes: 10)) + } +} diff --git a/apple/TableProTests/Core/Database/ForeignKeysForTablesReadTests.swift b/apple/TableProTests/Core/Database/ForeignKeysForTablesReadTests.swift new file mode 100644 index 0000000000..49c838b1a0 --- /dev/null +++ b/apple/TableProTests/Core/Database/ForeignKeysForTablesReadTests.swift @@ -0,0 +1,172 @@ +// +// ForeignKeysForTablesReadTests.swift +// TableProTests +// +// Asking for six tables' foreign keys on a driver without a bulk read fell into the schema-wide +// default, one round trip per table in the schema, and a cancelled caller kept the loop going. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CatalogReadLog: @unchecked Sendable { + private let lock = NSLock() + private var tables: [String] = [] + private var listings = 0 + private var bulk = 0 + + var perTableReads: [String] { lock.withLock { tables } } + var tableListings: Int { lock.withLock { listings } } + var bulkReads: Int { lock.withLock { bulk } } + + func recordTable(_ name: String) { lock.withLock { tables.append(name) } } + func recordListing() { lock.withLock { listings += 1 } } + func recordBulk() { lock.withLock { bulk += 1 } } +} + +private func foreignKey(on table: String) -> PluginForeignKeyInfo { + PluginForeignKeyInfo( + name: "fk_\(table)", column: "parent_id", referencedTable: "parent", referencedColumn: "id", + referencedDatabase: nil + ) +} + +private func tableInfos(_ count: Int) -> [PluginTableInfo] { + (0.. [PluginTableInfo] { + log.recordListing() + return tableInfos(tableCount) + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + log.recordTable(table) + if cancelsCallerOnRead { withUnsafeCurrentTask { $0?.cancel() } } + if let failure = failures[table] { throw failure } + return [foreignKey(on: table)] + } +} + +private final class BulkCatalogDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { true } + var supportsTransactions: Bool { false } + var currentSchema: String? { "public" } + var serverVersion: String? { nil } + var providesBulkForeignKeyFetch: Bool { true } + + let log = CatalogReadLog() + private let tableCount: Int + + init(tableCount: Int) { + self.tableCount = tableCount + } + + func ping() async throws {} + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + log.recordTable(table) + return [foreignKey(on: table)] + } + + func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { + log.recordBulk() + return Dictionary(uniqueKeysWithValues: tableInfos(tableCount).map { ($0.name, [foreignKey(on: $0.name)]) }) + } +} + +struct ForeignKeysForTablesReadTests { + private let sixTables = (0..<6).map { "t\($0)" } + + private func adapter(_ driver: any PluginDatabaseDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: .postgresql), pluginDriver: driver) + } + + @Test("Six tables on a driver without a bulk read cost six reads, not one per table in the schema") + func perTableDriverReadsOnlyTheTablesAskedFor() async throws { + let driver = PerTableCatalogDriver(tableCount: 500) + + let keys = try await adapter(driver).fetchForeignKeys(forTables: sixTables) + + #expect(driver.log.perTableReads == sixTables) + #expect(driver.log.tableListings == 0) + #expect(Set(keys.keys) == Set(sixTables)) + #expect(keys["t3"]?.first?.name == "fk_t3") + } + + @Test("A driver with a bulk read still answers more than five tables from one query") + func bulkDriverKeepsItsSingleQuery() async throws { + let driver = BulkCatalogDriver(tableCount: 500) + + let keys = try await adapter(driver).fetchForeignKeys(forTables: sixTables) + + #expect(driver.log.bulkReads == 1) + #expect(driver.log.perTableReads.isEmpty) + #expect(Set(keys.keys) == Set(sixTables)) + } + + @Test("Past five tables without a bulk read, a table that cannot be read still fails the read") + func perTableDriverPastFiveStillFails() async { + let driver = PerTableCatalogDriver(tableCount: 500, failures: ["t2": DatabaseError.queryFailed("denied")]) + + await #expect(throws: DatabaseError.self) { + _ = try await adapter(driver).fetchForeignKeys(forTables: sixTables) + } + #expect(driver.log.perTableReads == ["t0", "t1", "t2"]) + } + + @Test("Up to five tables, a table that cannot be read is left out and the rest are returned") + func smallSetSkipsAnUnreadableTable() async throws { + let driver = PerTableCatalogDriver(tableCount: 500, failures: ["t1": DatabaseError.queryFailed("denied")]) + + let keys = try await adapter(driver).fetchForeignKeys(forTables: ["t0", "t1", "t2"]) + + #expect(Set(keys.keys) == ["t0", "t2"]) + } + + @Test("A cancelled caller stops the per-table read at the next table") + func cancelledCallerStopsTheLoop() async { + let driver = PerTableCatalogDriver(tableCount: 500, cancelsCallerOnRead: true) + let reader = adapter(driver) + + let read = Task { try await reader.fetchForeignKeys(forTables: ["t0", "t1", "t2"]) } + await #expect(throws: CancellationError.self) { + _ = try await read.value + } + #expect(driver.log.perTableReads == ["t0"]) + } + + @Test("A table read that reports a cancellation ends the read instead of being skipped") + func cancellationFromTheDriverIsNotSkipped() async { + let driver = PerTableCatalogDriver(tableCount: 500, failures: ["t0": CancellationError()]) + + await #expect(throws: CancellationError.self) { + _ = try await adapter(driver).fetchForeignKeys(forTables: ["t0", "t1", "t2"]) + } + #expect(driver.log.perTableReads == ["t0"]) + } +} diff --git a/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift b/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift index 87f7872eeb..9a26a2b864 100644 --- a/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift +++ b/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift @@ -30,7 +30,7 @@ struct PostgreSQLDriverErrorClassificationTests { pluginSqlState: "28P01" ) - #expect(DatabaseManager.shared.isAuthenticationFailure(error)) + #expect(ConnectionFailureClassifier.isAuthenticationFailure(error)) } @Test("a read-only refusal lost with the connection is still classified, with the server's words") @@ -67,6 +67,6 @@ struct PostgreSQLDriverErrorClassificationTests { """, pluginSqlState: nil ) - #expect(DatabaseManager.shared.isAuthenticationFailure(error)) + #expect(ConnectionFailureClassifier.isAuthenticationFailure(error)) } } diff --git a/apple/TableProTests/Core/Database/PromptedPasswordKeyTests.swift b/apple/TableProTests/Core/Database/PromptedPasswordKeyTests.swift index ad875df4ba..179591d704 100644 --- a/apple/TableProTests/Core/Database/PromptedPasswordKeyTests.swift +++ b/apple/TableProTests/Core/Database/PromptedPasswordKeyTests.swift @@ -30,6 +30,25 @@ struct PromptedPasswordKeyTests { #expect(DatabaseManager.promptCacheKey(for: linked) != DatabaseManager.promptCacheKey(for: moved)) } + @Test("A connection on an SSH profile asks again once the profile moves to another server") + func movedProfileDoesNotReuseTheAnswer() { + let profiles = SSHProfileStorage.shared + let saved = profiles.loadProfiles() + var profile = SSHProfile(name: "Jump", host: "bastion-a.example", port: 22, username: "ops", authMethod: .password) + #expect(profiles.saveProfilesWithoutSync(saved + [profile])) + defer { profiles.saveProfilesWithoutSync(saved) } + var linked = DatabaseConnection(name: "Orders", host: "127.0.0.1", port: 5_432, type: .postgresql) + linked.sshTunnelMode = .profile(id: profile.id, snapshot: profile.toSSHConfiguration()) + let before = DatabaseManager.promptCacheKey(for: linked) + + profile.host = "bastion-b.example" + #expect(profiles.saveProfilesWithoutSync(saved + [profile])) + + #expect(DatabaseManager.promptCacheKey(for: linked) != before) + #expect(LocalCredentialConsent.question(for: linked).destination.contains("bastion-b.example")) + #expect(EnvVarReferenceConsent.question(for: linked).destination.contains("bastion-b.example")) + } + @Test("The prompt names the host the engine dials first, before a field it ignores") func summaryLeadsWithTheHost() { let connection = DatabaseConnection( diff --git a/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift b/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift index f18bfc632a..1e60929931 100644 --- a/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift +++ b/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift @@ -149,15 +149,17 @@ struct ProtectedWriteDeadlineTests { #expect(await !ran.isSet) } - @Test("A write already in the driver when the caller cancels runs to the end") + @Test("A write already in the driver when the caller cancels runs to the end, and the caller hears it may have been applied") func inDriverAtCancelFinishes() async throws { let gate = AsyncGate() let finished = Flag() + let lateCompletion = Flag() let caller = Task { try await DatabaseAccessBridge.awaitProtectedWrite( timeout: .seconds(60), timeoutSeconds: 60, + onLateCompletion: { Task { await lateCompletion.set() } }, { enterDriver -> Int in try enterDriver() await gate.wait() @@ -168,9 +170,19 @@ struct ProtectedWriteDeadlineTests { } await gate.arrived(1) caller.cancel() - await #expect(throws: CancellationError.self) { _ = try await caller.value } + let error = await #expect(throws: DatabaseAccessError.self) { _ = try await caller.value } + + guard case .stoppedWhileRunning(let detail) = error else { + Issue.record("expected the outcome to be unknown, got \(String(describing: error))") + return + } + #expect(detail == DatabaseAccessBridge.stoppedWhileRunning.message) + #expect(!detail.contains("timed out")) + #expect(await !finished.isSet) + #expect(await !lateCompletion.isSet) await gate.open() try await waitUntil { await finished.isSet } + try await waitUntil { await lateCompletion.isSet } } } diff --git a/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift b/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift index a6604f2766..3c41f37e29 100644 --- a/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift +++ b/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift @@ -13,7 +13,7 @@ struct ReconnectCredentialRecoveryTests { pluginSqlState: "28000" ) - #expect(DatabaseManager.shared.isAuthenticationFailure(error)) + #expect(ConnectionFailureClassifier.isAuthenticationFailure(error)) } @Test("An authentication failure stops an unattended reconnect with the server's message") @@ -30,6 +30,32 @@ struct ReconnectCredentialRecoveryTests { ) } + @Test("A Vault refusal stops an unattended reconnect with Vault's reason") + func vaultRefusalStopsTheReconnect() { + let error = VaultError.credentialsFailed(path: "database/creds/ro", message: "permission denied") + + #expect(DatabaseManager.shared.reconnectVerdict(for: error) == .stop(ConnectionFailureClassifier.info(for: error))) + } + + @Test("A new Vault user the database is still loading keeps the reconnect going") + func userNotLoadedKeepsRetrying() { + let error = VaultUserNotLoaded(refusal: FakePluginAuthError( + pluginErrorMessage: "Access denied for user 'v-token-ro-1'", + pluginErrorCode: 1_045, + pluginSqlState: "28000" + )) + + #expect(DatabaseManager.shared.reconnectVerdict(for: error) == .retry(ConnectionFailureClassifier.info(for: error))) + } + + @Test("An unreachable or failing Vault keeps the reconnect going and says why", arguments: [ + VaultError.unreachable("https://vault.test:8200", "The request timed out."), + VaultError.serverFailed("local node not active but active cluster node not found"), + ]) + func unreachableVaultKeepsRetrying(error: VaultError) { + #expect(DatabaseManager.shared.reconnectVerdict(for: error) == .retry(ConnectionFailureClassifier.info(for: error))) + } + @Test("A connection that is merely gone keeps the reconnect going with no reason of its own") func lostConnectionKeepsRetrying() { #expect(DatabaseManager.shared.reconnectVerdict(for: DatabaseError.notConnected) == .retry(nil)) diff --git a/apple/TableProTests/Core/Database/SchemaChangeHeldTransactionTests.swift b/apple/TableProTests/Core/Database/SchemaChangeHeldTransactionTests.swift new file mode 100644 index 0000000000..e3b682aabd --- /dev/null +++ b/apple/TableProTests/Core/Database/SchemaChangeHeldTransactionTests.swift @@ -0,0 +1,162 @@ +// +// SchemaChangeHeldTransactionTests.swift +// TableProTests +// +// On the session driver an app-owned BEGIN joins what a tab left open and its COMMIT ends it, so a +// schema change there runs inside that transaction. On a pooled connection it can wait on the +// transaction's locks, so it asks first. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class HeldDDLDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var current: PluginSessionTransactionState = .idle + private var sent: [String] = [] + + var supportsSchemas: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + var supportsTransactions: Bool { true } + var supportsTransactionalDDL: Bool { true } + + var state: PluginSessionTransactionState { + get { lock.withLock { current } } + set { lock.withLock { current = newValue } } + } + + var statements: [String] { lock.withLock { sent } } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { + sent.append(query) + switch query.uppercased() { + case "BEGIN": current = .inTransaction + case "COMMIT", "ROLLBACK": current = .idle + default: break + } + } + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { state } + + func switchDatabase(to database: String) async throws {} +} + +@Suite("Schema changes beside an open transaction", .serialized) +@MainActor +struct SchemaChangeHeldTransactionTests { + private static let first = "ALTER TABLE orders ADD COLUMN a INT;" + private static let second = "ALTER TABLE orders ADD COLUMN b INT;" + + private static func injectSession(type: DatabaseType) -> (DatabaseConnection, HeldDDLDriver, DatabaseDriver) { + let connection = TestFixtures.makeConnection(database: "orders", type: type) + let driver = HeldDDLDriver() + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + var session = ConnectionSession(connection: connection, driver: adapter) + session.browseDatabase = "orders" + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return (connection, driver, adapter) + } + + private static func tearDown(_ connection: DatabaseConnection) { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + private static func scope(_ connection: DatabaseConnection) -> DatabaseScope { + DatabaseScope(connectionId: connection.id, database: "orders", schema: nil) + } + + private static func script() -> SchemaChangeScript { + SchemaChangeScript( + tableName: "orders", + statements: [first, second].map { SchemaStatement(sql: $0, description: $0, isDestructive: false) }, + operations: [], + review: PluginSchemaChangeReview() + ) + } + + @Test("On the session driver a Structure save inside an open transaction joins it") + func sessionRouteSaveJoinsTheOpenTransaction() async throws { + let (connection, driver, _) = Self.injectSession(type: .duckdb) + defer { Self.tearDown(connection) } + driver.state = .inTransaction + + try await DatabaseManager.shared.executeSchemaChanges( + Self.script(), databaseType: .duckdb, scope: Self.scope(connection), gate: AlwaysAllowGate() + ) + + #expect(driver.statements == [Self.first, Self.second]) + #expect(driver.state == .inTransaction) + } + + @Test("On the session driver an idle Structure save keeps its own transaction") + func sessionRouteIdleSaveWraps() async throws { + let (connection, driver, _) = Self.injectSession(type: .duckdb) + defer { Self.tearDown(connection) } + + try await DatabaseManager.shared.executeSchemaChanges( + Self.script(), databaseType: .duckdb, scope: Self.scope(connection), gate: AlwaysAllowGate() + ) + + #expect(driver.statements == ["BEGIN", Self.first, Self.second, "COMMIT"]) + } + + @Test("On the session driver Create Table inside an open transaction joins it") + func sessionRouteCreateTableJoins() async throws { + let (connection, driver, _) = Self.injectSession(type: .duckdb) + defer { Self.tearDown(connection) } + driver.state = .inTransaction + + try await DatabaseManager.shared.executeCreateTable( + statements: [Self.first, Self.second], databaseType: .duckdb, scope: Self.scope(connection), + gate: AlwaysAllowGate() + ) + + #expect(driver.statements == [Self.first, Self.second]) + } + + @Test("A trigger edit that joins an open transaction sends no BEGIN or COMMIT") + func joinedTriggerEditSendsNoTransaction() async throws { + let connection = TestFixtures.makeConnection(type: .duckdb) + let driver = HeldDDLDriver() + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + + try await TriggerEditing.runInTransaction(driver: adapter, dropSQL: "DROP TRIGGER t", sql: "CREATE TRIGGER t", wraps: false) + + #expect(driver.statements == ["DROP TRIGGER t", "CREATE TRIGGER t"]) + } + + @Test("A pooled save beside an open transaction asks first, and a Cancel runs nothing") + func pooledSaveAsksBesideAHold() async throws { + let (connection, _, sessionAdapter) = Self.injectSession(type: .mysql) + defer { Self.tearDown(connection) } + let pooled = HeldDDLDriver() + let pooledAdapter = PluginDriverAdapter(connection: connection, pluginDriver: pooled) + try await pooledAdapter.connect() + MetadataConnectionPool.shared.injectEntry(pooledAdapter, scope: Self.scope(connection)) + DatabaseManager.shared.sessionTransactions.record( + .inTransaction, on: sessionAdapter, for: .init(connectionId: connection.id, lane: nil) + ) + + await #expect(throws: ExecutionGateError.self) { + try await DatabaseManager.shared.executeSchemaChanges( + Self.script(), databaseType: .mysql, scope: Self.scope(connection), gate: AlwaysAllowGate(), + holdConfirmation: { false } + ) + } + #expect(pooled.statements.isEmpty) + + try await DatabaseManager.shared.executeSchemaChanges( + Self.script(), databaseType: .mysql, scope: Self.scope(connection), gate: AlwaysAllowGate(), + holdConfirmation: { true } + ) + #expect(pooled.statements == ["BEGIN", Self.first, Self.second, "COMMIT"]) + } +} diff --git a/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift b/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift new file mode 100644 index 0000000000..5af1cd879d --- /dev/null +++ b/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift @@ -0,0 +1,86 @@ +// +// SchemaWriteGateDenialTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// A denial came back as a failed statement, so Create Database, the schema editor and Create Table +// showed the user's own Cancel at the gate as an error. +@MainActor +struct SchemaWriteGateDenialTests { + private let scope = DatabaseScope(connectionId: UUID(), database: "app", schema: nil) + + private func gateFailure(of body: () async throws -> Void) async -> GatedOperationFailure? { + do { + try await body() + return nil + } catch { + return GatedOperationFailure(error) + } + } + + private func runCreateDatabase(gate: any ExecutionGate) async -> GatedOperationFailure? { + await gateFailure { + try await DatabaseManager.shared.runContainerOperation( + description: "Create database \"sales\"", + kind: .schemaMutation, + scope: scope, + databaseType: .postgresql, + event: nil, + gate: gate + ) { _ in + Issue.record("The driver was asked after the gate said no") + } + } + } + + @Test("A Cancel at the gate ends a database operation quietly, before the driver is asked") + func containerOperationCancelIsQuiet() async { + let failure = await runCreateDatabase(gate: AlwaysDenyGate.cancelling) + + #expect(failure == .cancelledByUser) + #expect(failure?.message == nil) + } + + @Test("A refused database operation says why and is not a failed statement") + func containerOperationRefusalExplainsItself() async { + let failure = await runCreateDatabase(gate: AlwaysDenyGate(reason: "Safe Mode is read-only")) + + #expect(failure == .refused("Safe Mode is read-only")) + #expect(failure?.reportsFailure == false) + } + + @Test("A Cancel at the gate ends a schema editor apply quietly") + func containerStatementsCancelIsQuiet() async { + let failure = await gateFailure { + try await DatabaseManager.shared.runContainerStatements( + .statements([SchemaStatement(sql: "CREATE SCHEMA sales", description: "Create", isDestructive: false)]), + description: "Create Schema", + scope: scope, + databaseType: .postgresql, + event: nil, + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(failure == .cancelledByUser) + } + + @Test("A Cancel at the gate ends a Create Table quietly") + func createTableCancelIsQuiet() async { + let failure = await gateFailure { + try await DatabaseManager.shared.executeCreateTable( + statements: ["CREATE TABLE invoices (id INT)"], + databaseType: .postgresql, + scope: scope, + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(failure == .cancelledByUser) + #expect(failure?.message == nil) + } +} diff --git a/apple/TableProTests/Core/Database/SessionLanesTests.swift b/apple/TableProTests/Core/Database/SessionLanesTests.swift index 4b4e31a158..ad7404e4b5 100644 --- a/apple/TableProTests/Core/Database/SessionLanesTests.swift +++ b/apple/TableProTests/Core/Database/SessionLanesTests.swift @@ -10,6 +10,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import TableProTestSupport import Testing @@ -299,6 +300,24 @@ struct SessionLanesTests { #expect(manager.sessionLanes.parkedDatabases(for: harness.connection.id).sorted() == ["audit", "logs"]) } + @Test("An opened connection holding session state the user set up is not closed for being idle") + func openedLaneHoldingSessionStateIsNotClosedIdle() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + _ = try await run(harness, on: "logs", leaving: .idle) + let logs = try #require(harness.opener.opened["logs"]) + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "logs") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .sessionSettings), on: logs, for: lane) + + await manager.closeIdleOpenedLanes(for: connectionId, idleFor: 0) + + #expect(logs.disconnectCallCount == 0) + #expect(manager.sessionLanes.parkedDriver(for: connectionId, database: "logs") === logs) + #expect(manager.sessionLanes.isOpened(database: "logs", for: connectionId)) + } + @Test("The connection of a database the user browsed and left is not closed for being idle") func browsedConnectionIsNotClosedWhenIdle() async throws { let harness = makeHarness() @@ -634,7 +653,7 @@ struct SessionLanesTests { harness.home.sessionTransactionStateToReturn = .inTransaction let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) - await DatabaseManager.shared.sessionLanes.closeAllNotingLostTransactions(for: harness.connection.id) + await DatabaseManager.shared.sessionLanes.closeAllNotingLosses(for: harness.connection.id) #expect(harness.home.disconnectCallCount == 1) #expect(DatabaseManager.shared.executionRoute(for: scope) == .sessionDriver) @@ -645,7 +664,110 @@ struct SessionLanesTests { cancellation: .untracked ) { _ in () } } - #expect(!DatabaseManager.shared.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + #expect(!DatabaseManager.shared.sessionLanes.hasLoss(database: "app", for: harness.connection.id)) + } + + @Test("A rebuilt transport reports what the user's statements set up on a parked connection, once") + func transportRebuildReportsALanesSessionState() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: harness.home, for: lane) + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + await manager.closeTransportLanes(for: connectionId) { true } + + #expect(harness.home.disconnectCallCount == 1) + #expect(manager.executionRoute(for: scope) == .sessionDriver) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .untracked + ) { _ in () } + } + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connectionId)) + } + + @Test("A parked connection that died holding session state the user set up is replaced and the work refused once") + func deadParkedLaneReportsItsSessionState() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.pingError = DatabaseError.connectionFailed("server closed the connection") + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .userVariables), on: harness.home, for: lane) + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { _ in () } + } + + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .userVariables) + ) + #expect(error?.localizedDescription.contains(expected) == true) + let replacement = try #require(harness.opener.opened["app"]) + #expect(harness.home.disconnectCallCount == 1) + let ranOn = try await manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { driver in ObjectIdentifier(driver) } + #expect(ranOn == ObjectIdentifier(replacement)) + } + + @Test("A grid save on a database whose lane died with session state is refused once, and the next one runs") + func firstOwningTurnOnALostLaneTakesTheReport() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: harness.home, for: lane) + await manager.closeTransportLanes(for: connectionId) { true } + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .protectedWrite + ) { _ in () } + } + + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(harness.opener.opened["app"] == nil) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connectionId)) + let savedOn = try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .protectedWrite + ) { driver in ObjectIdentifier(driver) } + let reopened = try #require(harness.opener.opened["app"]) + #expect(savedOn == ObjectIdentifier(reopened)) + } + + @Test("A run notes what ran: every statement, through the one that failed, or only those a Stop let run") + func ranCountFollowsTheOutcome() { + let completed = BatchStatementOutcome.completed(results: [1, 2, 3]) + let stopped = BatchStatementOutcome.cancelled(results: [1]) + let failed = BatchStatementOutcome.failed( + results: [1], failure: .statement(sql: "SET x = 1"), errorDescription: "failed" + ) + #expect(completed.ranCount(of: 3) == 3) + #expect(stopped.ranCount(of: 3) == 1) + #expect(failed.ranCount(of: 3) == 2) } @Test("A table read leaves a lost transaction's report for the user's next statement") @@ -653,7 +775,7 @@ struct SessionLanesTests { let harness = makeHarness() defer { cleanUp(harness) } let manager = DatabaseManager.shared - manager.sessionLanes.markTransactionLost(database: "app", for: harness.connection.id) + manager.sessionLanes.markLost(database: "app", transaction: true, for: harness.connection.id) let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) let ranOn = try await manager.withTableReadDriver(scope: scope, cancellation: .untracked) { driver in @@ -661,7 +783,7 @@ struct SessionLanesTests { } #expect(ranOn == ObjectIdentifier(harness.home)) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: harness.connection.id)) await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } } @@ -764,6 +886,32 @@ struct SessionLanesTests { #expect(holding.isEmpty) } + @Test("A transaction the tracker holds is named even while its connection runs a turn") + func trackedHoldOnABusyConnectionIsNamed() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + DatabaseManager.shared.sessionTransactions.record( + .inTransaction, on: harness.home, for: .init(connectionId: harness.connection.id, lane: "app") + ) + let release = AsyncGate() + let acquired = AsyncGate() + let holder = Task { @MainActor in + try await DatabaseManager.shared.sessionDriverGate.withExclusiveAccess( + SessionDriverGate.Key(connectionId: harness.connection.id, database: "app") + ) { + await acquired.open() + await release.wait() + } + } + await acquired.wait() + + let holding = await DatabaseManager.shared.databasesHoldingTransaction(for: harness.connection.id) + + await release.open() + try await holder.value + #expect(holding == ["app"]) + } + @Test("A connection that does not answer counts as holding nothing once the bound passes") func silentConnectionIsBounded() async { let harness = makeHarness() diff --git a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift index d076adf949..8600b2c143 100644 --- a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift +++ b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift @@ -2,6 +2,7 @@ import Foundation @testable import TablePro import TableProMCP import TableProPluginKit +import TableProSQLGrammar import TableProSSHConfig import TableProTestSupport import Testing @@ -9,7 +10,7 @@ import Testing @MainActor struct SessionRecoveryLoopTests { @MainActor - private struct Fixture { + fileprivate struct Fixture { let manager: DatabaseManager let clock: ManualClock let connection: DatabaseConnection @@ -29,7 +30,7 @@ struct SessionRecoveryLoopTests { serverMessage: "self-signed certificate in certificate chain" ) - private func makeFixture( + fileprivate func makeFixture( storage: ConnectionStorage = .shared, configure: (inout DatabaseConnection) -> Void = { _ in } ) -> Fixture { @@ -48,7 +49,7 @@ struct SessionRecoveryLoopTests { return Fixture(manager: manager, clock: clock, connection: connection, driver: driver) } - private func cleanUp(_ fixture: Fixture) async { + fileprivate func cleanUp(_ fixture: Fixture) async { fixture.manager.endRecoveryLoop(for: fixture.id) await fixture.manager.stopHealthMonitor(for: fixture.id) fixture.manager.removeSession(for: fixture.id) @@ -72,11 +73,11 @@ struct SessionRecoveryLoopTests { await cleanUp(fixture) } - private func loopTask(_ fixture: Fixture) throws -> Task { + fileprivate func loopTask(_ fixture: Fixture) throws -> Task { try #require(fixture.manager.recoveryLoops[fixture.id]?.task) } - private func runNextAttempt(_ fixture: Fixture) async { + fileprivate func runNextAttempt(_ fixture: Fixture) async { await fixture.clock.waitUntilSleeping() fixture.clock.advance(by: .seconds(DatabaseManager.maxRecoveryDelay)) } @@ -314,12 +315,12 @@ struct SessionRecoveryLoopTests { #expect(fixture.session?.driver !== fixture.driver) #expect(!manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) let error = await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } } #expect(error?.localizedDescription.contains(DatabaseManager.parkedTransactionLostMessage(database: "app")) == true) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) await cleanUp(fixture) } @@ -337,7 +338,200 @@ struct SessionRecoveryLoopTests { await loop.value #expect(fixture.session?.driver !== fixture.driver) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + // Inside the user's own lease, as a query tab notes what it runs. + private func runUserStatements(_ statements: [String], on fixture: Fixture) async throws { + let manager = fixture.manager + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { driver in + await manager.noteSessionStatements(statements, ranOn: driver, scope: scope) + } + } + + private func recover(_ fixture: Fixture) async throws { + fixture.manager.beginRecovery(for: fixture.id, cause: .stoppedResponding) + let loop = try loopTask(fixture) + await runNextAttempt(fixture) + await loop.value + } + + private func runNextStatement(_ fixture: Fixture, ownsSessionState: Bool = true) async throws { + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + try await fixture.manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked, ownsSessionState: ownsSessionState + ) { _ in () } + } + + private func nextStatementRefusal(_ fixture: Fixture) async -> DatabaseError? { + do { + try await runNextStatement(fixture) + return nil + } catch { + return error as? DatabaseError + } + } + + @Test("A recovery that replaces a driver whose session set something up refuses the next statement once") + func recoveryReportsTheSessionStateItEnded() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["CREATE TABLE #orders (id int)"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id) == .temporaryObjects) + + try await recover(fixture) + + #expect(fixture.session?.driver !== fixture.driver) + let refusal = await nextStatementRefusal(fixture) + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(refusal?.localizedDescription.contains(expected) == true) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A recovery that ends a transaction and what the session set up names both in one message") + func recoveryReportsTransactionAndStateInOneMessage() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .inTransaction + try await runUserStatements(["SET XACT_ABORT ON"], on: fixture) + #expect(manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) + + try await recover(fixture) + + let refusal = await nextStatementRefusal(fixture) + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: true, state: .sessionSettings) + ) + #expect(refusal?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + // A grid save, an Explain or an MCP statement takes the report as the user's statement would, as a lost + // transaction's report always went. + @Test("The first turn that owns the session after the loss is refused once, whatever work it is, and the next runs") + func firstOwningTurnTakesTheStateReport() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["CREATE TABLE #orders (id int)"], on: fixture) + + try await recover(fixture) + + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in () } + } + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A reconnect the user asks for reports a lane's lost transaction but not what its session set up") + func manualReconnectReportsNoSessionState() async throws { + let fixture = makeFixture() + let manager = fixture.manager + let logs = MockDatabaseDriver(connection: fixture.connection) + logs.sessionTransactionStateToReturn = .idle + let audit = MockDatabaseDriver(connection: fixture.connection) + audit.sessionTransactionStateToReturn = .inTransaction + manager.sessionLanes.park(logs, database: "logs", for: fixture.id) + manager.sessionLanes.park(audit, database: "audit", for: fixture.id) + let logsLane = SessionTransactionTracker.Key(connectionId: fixture.id, lane: "logs") + let auditLane = SessionTransactionTracker.Key(connectionId: fixture.id, lane: "audit") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: logs, for: logsLane) + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .sessionSettings), on: audit, for: auditLane) + // A session that stopped answering moves the pooled transport, as a rebuilt tunnel does. + manager.updateSession(fixture.id) { $0.liveness = .unreachable(nil) } + + try? await manager.reconnectSession(fixture.id) + + #expect(logs.disconnectCallCount == 1) + #expect(audit.disconnectCallCount == 1) + #expect(!manager.sessionLanes.hasLoss(database: "logs", for: fixture.id)) + #expect( + manager.sessionLanes.takeLoss(database: "audit", for: fixture.id) + == SessionLoss(transaction: true, state: []) + ) + await cleanUp(fixture) + } + + @Test("A recovery of a session whose statements set nothing up refuses nothing") + func recoveryOfASessionThatSetNothingUpReportsNothing() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["SELECT * FROM orders", "UPDATE orders SET total = 0 WHERE id = 1"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id).isEmpty) + + try await recover(fixture) + + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A startup command the user runs again is not reported, since the reconnect runs it too") + func startupCommandRerunByTheUserIsNotReported() async throws { + let fixture = makeFixture { $0.startupCommands = "SET NOCOUNT ON" } + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["SET NOCOUNT ON;"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id).isEmpty) + + try await recover(fixture) + + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A metadata read on the session driver leaves the lost session state's report for the user's statement") + func metadataReadLeavesTheStateReport() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["DECLARE c CURSOR FOR SELECT 1"], on: fixture) + + try await recover(fixture) + try await runNextStatement(fixture, ownsSessionState: false) + + #expect(manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + let refusal = await nextStatementRefusal(fixture) + #expect(refusal != nil) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + @Test("The report names kinds only, never a statement, a name or a value it carried") + func stateMessageNamesNoStatementText() async throws { + let fixture = makeFixture() + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements( + ["SET app.key = 'secret'", "EXEC sp_set_session_context 'api_token', 'hunter2'"], on: fixture + ) + + try await recover(fixture) + + let refusal = await nextStatementRefusal(fixture) + let message = refusal?.localizedDescription ?? "" + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .sessionSettings) + ) + #expect(message.contains(expected)) + for leaked in ["secret", "app.key", "api_token", "hunter2", "sp_set_session_context"] { + #expect(!message.contains(leaked)) + } await cleanUp(fixture) } @@ -544,7 +738,7 @@ struct SessionRecoveryLoopTests { #expect(fixture.manager.sessionLanes.parkedDriver(for: fixture.id, database: "logs") == nil) #expect(lane.disconnectCallCount == 1) - #expect(fixture.manager.sessionLanes.hasTransactionLoss(database: "logs", for: fixture.id)) + #expect(fixture.manager.sessionLanes.hasLoss(database: "logs", for: fixture.id)) #expect(fixture.session?.driver === fixture.driver) await cleanUp(fixture) } @@ -587,7 +781,7 @@ struct SessionRecoveryLoopTests { #expect(manager.sessionLanes.parkedDriver(for: id, database: "logs") === replacementLane) #expect(replacementLane.disconnectCallCount == 0) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "logs", for: id)) + #expect(!manager.sessionLanes.hasLoss(database: "logs", for: id)) #expect(fixture.session?.driver !== fixture.driver) #expect(fixture.session?.liveness == .live) await cleanUp(fixture) @@ -728,3 +922,85 @@ struct SessionRecoveryLoopTests { #expect(manager.recoveryLoops.isEmpty) } } + +extension GlobalStateSuites { + // Writes the shared SSH profile store across suspensions. + @MainActor + struct SessionRecoveryMovedProfileTests { + private let loop = SessionRecoveryLoopTests() + + private func movedProfileFixture() -> (SessionRecoveryLoopTests.Fixture, restore: () -> Void) { + let profiles = SSHProfileStorage.shared + let saved = profiles.loadProfiles() + let moved = SSHProfile(name: "Jump", host: "bastion-b.example", port: 22, username: "ops", authMethod: .password) + profiles.saveProfilesWithoutSync(saved + [moved]) + var dialed = moved.toSSHConfiguration() + dialed.host = "bastion-a.example" + let fixture = loop.makeFixture { connection in + connection.sshTunnelMode = .profile(id: moved.id, snapshot: dialed) + connection.sshConfig = dialed + } + return (fixture, { profiles.saveProfilesWithoutSync(saved) }) + } + + @Test("A recovery after the linked SSH profile moved to another server stops and dials nothing") + func movedProfileEndsTheLoop() async throws { + let (fixture, restore) = movedProfileFixture() + defer { restore() } + let expected = ConnectionFailureClassifier.info(for: ConnectionTunnelError.sshProfileMoved) + + fixture.manager.beginRecovery(for: fixture.id, cause: .stoppedResponding) + let task = try loop.loopTask(fixture) + await loop.runNextAttempt(fixture) + await task.value + + #expect(fixture.session?.liveness == .unreachable(expected)) + #expect(fixture.session?.driver === fixture.driver) + #expect(fixture.manager.recoveryLoops[fixture.id] == nil) + #expect(FakeMSSQLPlugin.configurations(for: fixture.id).isEmpty) + await loop.cleanUp(fixture) + } + + @Test("A connect no window asked for keeps a dialed session off the moved profile", arguments: [ + ConnectTrigger.request, .background, + ]) + func unattendedConnectKeepsTheDialedRoute(trigger: ConnectTrigger) async throws { + let (fixture, restore) = movedProfileFixture() + defer { restore() } + var dialed = try #require(fixture.session) + dialed.effectiveConnection = fixture.connection + fixture.manager.injectSession(dialed, for: fixture.id) + fixture.manager.markSessionUnreachable(fixture.id, startedWith: fixture.driver, info: nil) + let expected = ConnectionFailureClassifier.info(for: ConnectionTunnelError.sshProfileMoved) + + for _ in 1...2 { + await #expect(throws: ConnectionTunnelError.sshProfileMoved) { + try await fixture.manager.ensureConnected(fixture.connection, trigger: trigger) + } + } + + #expect(fixture.session?.liveness == .unreachable(expected)) + #expect(fixture.session?.driver === fixture.driver) + #expect(FakeMSSQLPlugin.configurations(for: fixture.id).isEmpty) + await loop.cleanUp(fixture) + } + + @Test("A connect that failed and took the session with it still keeps the route until a disconnect") + func failedConnectKeepsTheDialedRoute() async throws { + let (fixture, restore) = movedProfileFixture() + defer { restore() } + fixture.manager.dialedConnections[fixture.id] = fixture.connection + fixture.manager.removeSession(for: fixture.id) + + await #expect(throws: ConnectionTunnelError.sshProfileMoved) { + try await fixture.manager.ensureConnected(fixture.connection) + } + #expect(fixture.session == nil) + #expect(FakeMSSQLPlugin.configurations(for: fixture.id).isEmpty) + + await fixture.manager.disconnectSession(fixture.id) + #expect(fixture.manager.dialedConnections[fixture.id] == nil) + await loop.cleanUp(fixture) + } + } +} diff --git a/apple/TableProTests/Core/Database/SessionTransactionTrackerTests.swift b/apple/TableProTests/Core/Database/SessionTransactionTrackerTests.swift index fcf531d914..7f5b437b5d 100644 --- a/apple/TableProTests/Core/Database/SessionTransactionTrackerTests.swift +++ b/apple/TableProTests/Core/Database/SessionTransactionTrackerTests.swift @@ -16,9 +16,12 @@ import Testing @MainActor struct SessionTransactionTrackerTests { private static func seed( - database: String = "app" + database: String = "app", + type: DatabaseType = .mysql, + commitMode: ConnectionCommitMode = .automatic ) -> (connection: DatabaseConnection, driver: TransactionScriptedDriver) { - let connection = TestFixtures.makeConnection(database: database, type: .mysql) + var connection = TestFixtures.makeConnection(database: database, type: type) + connection.commitMode = commitMode let driver = TransactionScriptedDriver() var session = ConnectionSession( connection: connection, @@ -41,6 +44,30 @@ struct SessionTransactionTrackerTests { } } + // The editor's single-statement path: the hold is opened inside the same turn as the write. + private static func runInMode( + _ mode: ConnectionCommitMode, + _ sql: String, + on connectionId: UUID, + type: DatabaseType = .postgresql + ) async throws { + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + _ = try await DatabaseManager.shared.withScopedDriver( + scope: scope, + route: .sessionDriver, + cancellation: .protectedWrite + ) { driver in + _ = try await ManualCommit.openIfNeeded( + on: driver, statements: [sql], databaseType: type, grammar: type.lexicalGrammar, commitMode: mode + ) + return try await driver.execute(query: sql) + } + } + + private static func scope(_ connectionId: UUID) -> DatabaseScope { + DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + } + private static var tracker: SessionTransactionTracker { DatabaseManager.shared.sessionTransactions } private static func states(_ connectionId: UUID, in tracker: SessionTransactionTracker? = nil) -> [SessionTransactionTracker.State] { @@ -239,13 +266,447 @@ struct SessionTransactionTrackerTests { #expect(tracker.holdsTransaction(connectionId: connectionId, container: "hr")) } + @Test("In Manual Commit an UPDATE sends BEGIN first and leaves the transaction open") + func manualCommitOpensTheHold() async throws { + let (connection, driver) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + try await Self.runInMode(.manual, "UPDATE t SET a = 1", on: connection.id) + + #expect(driver.statements == ["BEGIN", "UPDATE t SET a = 1"]) + #expect(Self.states(connection.id) == [.open]) + } + + @Test("Manual Commit sends no BEGIN for a read, nor a second one for a later write") + func manualCommitBeginsOnce() async throws { + let (connection, driver) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + try await Self.runInMode(.manual, "SELECT * FROM t", on: connection.id) + #expect(driver.statements == ["SELECT * FROM t"]) + #expect(Self.tracker.holds(for: connection.id).isEmpty) + + try await Self.runInMode(.manual, "UPDATE t SET a = 1", on: connection.id) + try await Self.runInMode(.manual, "UPDATE t SET a = 2", on: connection.id) + #expect(driver.statements == ["SELECT * FROM t", "BEGIN", "UPDATE t SET a = 1", "UPDATE t SET a = 2"]) + } + + @Test("Automatic commit sends no BEGIN") + func automaticCommitOpensNothing() async throws { + let (connection, driver) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + try await Self.runInMode(.automatic, "UPDATE t SET a = 1", on: connection.id) + + #expect(driver.statements == ["UPDATE t SET a = 1"]) + #expect(Self.tracker.holds(for: connection.id).isEmpty) + } + + @Test( + "A session that cannot report, or holds only a lock, refuses the write and sends nothing", + arguments: [PluginSessionTransactionState.unknown, .holdsSessionLocks] + ) + func unholdableSessionRefuses(state: PluginSessionTransactionState) async throws { + let (connection, driver) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + driver.state = state + + await #expect(throws: ManualCommitRefusal()) { + try await Self.runInMode(.manual, "UPDATE t SET a = 1", on: connection.id) + } + #expect(driver.statements.isEmpty) + } + + @Test("A driver that only answers that it cannot say is reported, and one earlier answer outlasts a later unknown") + func readerThatCannotReportIsRemembered() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + driver.state = .unknown + try await Self.run("SELECT 1", on: connection.id) + #expect(Self.tracker.cannotReportState(connectionId: connection.id, container: "app")) + + driver.state = .idle + try await Self.run("SELECT 1", on: connection.id) + driver.state = .unknown + try await Self.run("SELECT 1", on: connection.id) + #expect(!Self.tracker.cannotReportState(connectionId: connection.id, container: "app")) + } + + @Test("A table read leaves the hold's revision alone, and a write lease that owns no session state moves it") + func readOnlyTurnKeepsTheRevision() async throws { + let (connection, _) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let hold = try #require(Self.tracker.holds(for: connection.id).first) + + _ = try await DatabaseManager.shared.withTableReadDriver(scope: Self.scope(connection.id), cancellation: .untracked) { driver in + try await driver.execute(query: "SELECT * FROM t") + } + _ = try await DatabaseManager.shared.withScopedDriver( + scope: Self.scope(connection.id), route: .sessionDriver, cancellation: .untracked, + ownsSessionState: false, readsOnly: true + ) { driver in + try await driver.execute(query: "SELECT 1") + } + #expect(Self.tracker.isCurrent(hold)) + + _ = try await DatabaseManager.shared.withScopedDriver( + scope: Self.scope(connection.id), route: .sessionDriver, cancellation: .untracked, ownsSessionState: false + ) { driver in + try await driver.execute(query: "INSERT INTO t VALUES (1)") + } + #expect(!Self.tracker.isCurrent(hold)) + } + + @Test("A read-only turn that finds the transaction failed still moves the hold") + func readOnlyTurnStillReportsAFailure() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let hold = try #require(Self.tracker.holds(for: connection.id).first) + driver.failsNextStatementAborting = true + + await #expect(throws: (any Error).self) { + try await DatabaseManager.shared.withTableReadDriver(scope: Self.scope(connection.id), cancellation: .untracked) { driver in + try await driver.execute(query: "SELECT * FROM missing") + } + } + + #expect(!Self.tracker.isCurrent(hold)) + #expect(Self.states(connection.id) == [.aborted]) + } + + @Test("MCP begin is refused over a transaction already open and sends nothing") + func mcpBeginRefusedOverAnOpenTransaction() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let sent = driver.statements.count + let operation = try await ApprovedOperationFixture.approve( + [], scope: Self.scope(connection.id), databaseType: .mysql, kind: .writeQuery + ) + + await #expect(throws: DatabaseAccessError.self) { + _ = try await MCPConnectionBridge().transactionControl(operation, action: "begin") + } + #expect(driver.statements.count == sent) + } + + @Test("Manual Commit refuses every MCP transaction action", arguments: ["begin", "commit", "rollback"]) + func mcpTransactionControlRefusedInManualCommit(action: String) async throws { + let (connection, driver) = Self.seed(type: .postgresql, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let operation = try await ApprovedOperationFixture.approve( + [], scope: Self.scope(connection.id), databaseType: .postgresql, kind: .writeQuery + ) + + await #expect(throws: DatabaseAccessError.self) { + _ = try await MCPConnectionBridge().transactionControl(operation, action: action) + } + #expect(driver.statements.isEmpty) + } + + @Test("An agent's write in Manual Commit is held, and its reply says the change is pending") + func mcpWriteIsHeldAndSaysSo() async throws { + let (connection, driver) = Self.seed(type: .postgresql, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "UPDATE t SET a = 1", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + let begin = try #require(driver.statements.firstIndex(of: "BEGIN")) + let update = try #require(driver.statements.firstIndex(of: "UPDATE t SET a = 1")) + #expect(begin < update) + #expect(!driver.statements.contains("COMMIT")) + #expect(outcome.result.statusMessage?.contains(ManualCommit.pendingNotice) == true) + #expect(Self.states(connection.id) == [.open]) + } + + @Test("An agent's insert_rows in Manual Commit is held: BEGIN first, and no COMMIT") + func mcpInsertRowsIsHeld() async throws { + let (connection, driver) = Self.seed(type: .postgresql, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let rows = ["INSERT INTO t VALUES (1)", "INSERT INTO t VALUES (2)"] + let operation = try await ApprovedOperationFixture.approve( + rows.map { OperationStatement(sql: $0) }, scope: Self.scope(connection.id), databaseType: .postgresql, + kind: .writeQuery + ) + + let inserted = try await DatabaseAccessBridge().runInserts(operation) + + let begin = try #require(driver.statements.firstIndex(of: "BEGIN")) + let first = try #require(driver.statements.firstIndex(of: rows[0])) + #expect(begin < first) + #expect(driver.statements.contains(rows[1])) + #expect(!driver.statements.contains("COMMIT")) + #expect(Self.states(connection.id) == [.open]) + #expect(inserted.notice == ManualCommit.pendingNotice) + } + + @Test("An insert_rows that leaves a Teradata ANSI transaction open says so") + func mcpInsertRowsThatOpenATransactionSaySo() async throws { + let (connection, driver) = Self.seed(type: .teradata) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + driver.writesOpenATransaction = true + let operation = try await ApprovedOperationFixture.approve( + [OperationStatement(sql: "INSERT INTO t VALUES (1)")], scope: Self.scope(connection.id), databaseType: .teradata, + kind: .writeQuery + ) + + let inserted = try await DatabaseAccessBridge().runInserts(operation) + + #expect(inserted.notice == ManualCommit.statementOpenedNotice) + } + + @Test("An insert_rows of several rows joining an open SQLite transaction is taken back whole when a row fails") + func mcpInsertRowsJoiningATransactionIsOneUnit() async throws { + let (connection, driver) = Self.seed(type: .sqlite, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let rows = ["INSERT INTO t VALUES (1)", "INSERT INTO t VALUES (2)"] + driver.failingStatement = rows[1] + let operation = try await ApprovedOperationFixture.approve( + rows.map { OperationStatement(sql: $0) }, scope: Self.scope(connection.id), databaseType: .sqlite, + kind: .writeQuery + ) + + await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runInserts(operation) + } + + #expect(driver.statements.suffix(4) == [ + AgentTransactionGuard.savepointStatement, rows[0], rows[1], AgentTransactionGuard.rollbackStatement, + ]) + #expect(!driver.statements.contains("COMMIT")) + } + + @Test("An insert_rows joining an open SQL Server transaction takes SAVE TRANSACTION, which has no release") + func mcpInsertRowsOnSQLServerTakesItsSavepoint() async throws { + let (connection, driver) = Self.seed(type: .mssql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let rows = ["INSERT INTO t VALUES (1)", "INSERT INTO t VALUES (2)"] + let operation = try await ApprovedOperationFixture.approve( + rows.map { OperationStatement(sql: $0) }, scope: Self.scope(connection.id), databaseType: .mssql, + kind: .writeQuery + ) + + _ = try await DatabaseAccessBridge().runInserts(operation) + + #expect(driver.statements.suffix(3) == ["SAVE TRANSACTION tablepro_agent", rows[0], rows[1]]) + } + + @Test("An agent's COMMIT, ROLLBACK or BEGIN in Manual Commit is refused and sends nothing", arguments: ["COMMIT", "ROLLBACK", "BEGIN"]) + func mcpTransactionStatementRefusedInManualCommit(sql: String) async throws { + let (connection, driver) = Self.seed(type: .postgresql, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.runInMode(.manual, "UPDATE t SET a = 1", on: connection.id) + let sent = driver.statements.count + + let error = await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: sql, maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + } + #expect(error?.localizedDescription == ManualCommit.transactionControlRefusalMessage) + #expect(driver.statements.count == sent) + #expect(Self.states(connection.id) == [.open]) + } + + @Test( + "With Manual Commit holding a transaction, an agent statement that can end it unseen is refused", + arguments: [ + (DatabaseType.mssql, "UPDATE t SET a = 1 COMMIT"), + (DatabaseType.mssql, "EXEC dbo.p"), + (DatabaseType.postgresql, "CALL p()"), + ] + ) + func mcpUnseenTransactionControlRefused(type: DatabaseType, sql: String) async throws { + let (connection, driver) = Self.seed(type: type, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let sent = driver.statements.count + + let error = await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: sql, maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + } + #expect(error?.localizedDescription == ManualCommit.transactionControlRefusalMessage) + #expect(driver.statements.count == sent) + #expect(Self.states(connection.id) == [.open]) + } + + // Its own COMMIT would end the hold this call opens for its write. + @Test( + "On an idle Manual Commit session, an agent write that can end a transaction unseen is refused", + arguments: [ + (DatabaseType.mssql, "UPDATE t SET a = 1 COMMIT"), + (DatabaseType.postgresql, "CALL p()"), + ] + ) + func mcpUnseenWriteRefusedOnAnIdleSession(type: DatabaseType, sql: String) async throws { + let (connection, driver) = Self.seed(type: type, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let error = await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: sql, maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + } + #expect(error?.localizedDescription == ManualCommit.transactionControlRefusalMessage) + #expect(driver.statements.isEmpty) + } + + @Test("An agent statement that ended the open transaction says so in its reply") + func mcpReplySaysTheTransactionEnded() async throws { + let (connection, _) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "COMMIT", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + #expect(outcome.result.statusMessage?.contains(ManualCommit.transactionEndedNotice) == true) + #expect(Self.tracker.holds(for: connection.id).isEmpty) + } + + @Test("An agent's BEGIN over a transaction already open is refused, whatever the commit mode") + func mcpBeginStatementRefusedOverAnOpenTransaction() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let sent = driver.statements.count + + let error = await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "START TRANSACTION", maxRows: 10, timeoutSeconds: 30, + cancellation: nil + ) + } + #expect(error?.localizedDescription == ManualCommit.openTransactionRefusalMessage) + #expect(driver.statements.count == sent) + } + + @Test("An agent statement over an open PostgreSQL transaction runs inside a savepoint it releases") + func agentStatementTakesASavepoint() async throws { + let (connection, driver) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "UPDATE t SET a = 1", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + #expect(driver.statements.suffix(3) == [ + AgentTransactionGuard.savepointStatement, "UPDATE t SET a = 1", AgentTransactionGuard.releaseStatement, + ]) + } + + @Test("An agent statement that fails inside an open PostgreSQL transaction is rolled back to its savepoint") + func failedAgentStatementRollsBackToItsSavepoint() async throws { + let (connection, driver) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + driver.failingStatement = "UPDATE t SET a = 1 / 0" + + await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "UPDATE t SET a = 1 / 0", maxRows: 10, timeoutSeconds: 30, + cancellation: nil + ) + } + + #expect(driver.statements.suffix(3) == [ + AgentTransactionGuard.savepointStatement, "UPDATE t SET a = 1 / 0", AgentTransactionGuard.rollbackStatement, + ]) + } + + @Test("An agent statement over an open transaction with no savepoint is refused, since a failure would abort it", + arguments: [DatabaseType.duckdb, .teradata]) + func agentStatementRefusedWithoutSavepoints(type: DatabaseType) async throws { + let (connection, driver) = Self.seed(type: type) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let sent = driver.statements.count + + let error = await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "SELECT 1", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + } + #expect(error?.localizedDescription == AbortingTransactionRefusal().errorDescription) + #expect(driver.statements.count == sent) + } + + @Test("An agent's write that leaves a Teradata ANSI transaction open says so, and the next statement is refused") + func agentWriteThatOpensATransactionSaysSo() async throws { + let (connection, driver) = Self.seed(type: .teradata) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + driver.writesOpenATransaction = true + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "UPDATE t SET a = 1", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + #expect(outcome.result.statusMessage?.contains(ManualCommit.statementOpenedNotice) == true) + + let error = await #expect(throws: (any Error).self) { + _ = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "SELECT 1", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + } + #expect(error?.localizedDescription == AbortingTransactionRefusal().errorDescription) + } + + @Test("An agent's own BEGIN on PostgreSQL gets no notice that later statements are refused") + func agentBeginIsNotTakenForAHeldWrite() async throws { + let (connection, _) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "BEGIN", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + #expect(outcome.result.statusMessage?.contains(ManualCommit.statementOpenedNotice) != true) + } + + @Test("Turning Manual Commit on writes the session and the connection it was opened from") + func menuToggleWritesTheSession() { + let (connection, _) = Self.seed(type: .postgresql) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + DatabaseManager.shared.setCommitMode(.manual, for: connection.id) + + #expect(DatabaseManager.shared.commitMode(for: connection.id) == .manual) + #expect(DatabaseManager.shared.session(for: connection.id)?.connection.commitMode == .manual) + } + + @Test("A Manual Commit stored on a MySQL connection runs as automatic: an agent's write sends no BEGIN") + func storedManualCommitOnMySQLRunsAutomatic() async throws { + let (connection, driver) = Self.seed(type: .mysql, commitMode: .manual) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + #expect(DatabaseManager.shared.session(for: connection.id)?.connection.commitMode == .manual) + #expect(DatabaseManager.shared.commitMode(for: connection.id) == .automatic) + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: Self.scope(connection.id), query: "UPDATE t SET a = 1", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + #expect(driver.statements.contains("UPDATE t SET a = 1")) + #expect(!driver.statements.contains("BEGIN")) + #expect(outcome.result.statusMessage?.contains(ManualCommit.pendingNotice) != true) + #expect(Self.tracker.holds(for: connection.id).isEmpty) + } + @Test("Read-only hides Commit, an aborted transaction offers only Roll Back") func barContent() { let open = ConnectionTransactionBarContent( isAborted: false, connectionName: "Prod", database: "sales", - safeModeLevel: .silent + effectiveSafeModeLevel: .silent ) #expect(open.title == "Transaction Open") #expect(open.subtitle == "sales") @@ -256,7 +717,7 @@ struct SessionTransactionTrackerTests { isAborted: false, connectionName: "Prod", database: nil, - safeModeLevel: .readOnly + effectiveSafeModeLevel: .readOnly ) #expect(!readOnly.offersCommit) #expect(readOnly.subtitle == "Prod") @@ -265,7 +726,7 @@ struct SessionTransactionTrackerTests { isAborted: true, connectionName: "Prod", database: "", - safeModeLevel: .silent + effectiveSafeModeLevel: .silent ) #expect(!aborted.offersCommit) #expect(aborted.title == "Transaction Failed") @@ -280,6 +741,13 @@ private final class TransactionScriptedDriver: PluginDriverStub, @unchecked Send private var current: PluginSessionTransactionState = .idle private var sent: [String] = [] private var abortsNext = false + private var opensOnWrite = false + + // As a Teradata ANSI session does: a change opens a transaction only COMMIT ends. + var writesOpenATransaction: Bool { + get { lock.withLock { opensOnWrite } } + set { lock.withLock { opensOnWrite = newValue } } + } var state: PluginSessionTransactionState { get { lock.withLock { current } } @@ -293,10 +761,18 @@ private final class TransactionScriptedDriver: PluginDriverStub, @unchecked Send set { lock.withLock { abortsNext = newValue } } } + private var failing: String? + + // That statement fails and aborts the transaction, the way PostgreSQL answers a division by zero. + var failingStatement: String? { + get { lock.withLock { failing } } + set { lock.withLock { failing = newValue } } + } + func execute(query: String) async throws -> PluginQueryResult { let failure: Bool = lock.withLock { sent.append(query) - if abortsNext { + if abortsNext || query == failing { abortsNext = false current = .abortedTransaction return true @@ -304,6 +780,8 @@ private final class TransactionScriptedDriver: PluginDriverStub, @unchecked Send switch query.uppercased() { case "BEGIN": current = .inTransaction case "COMMIT", "ROLLBACK": current = .idle + case let write where opensOnWrite && (write.hasPrefix("UPDATE") || write.hasPrefix("INSERT")): + current = .inTransaction default: break } return false diff --git a/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift b/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift new file mode 100644 index 0000000000..11fbac5867 --- /dev/null +++ b/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift @@ -0,0 +1,135 @@ +// +// StartupCommandWarningTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct ServerRefusal: PluginDriverError { + let pluginErrorMessage: String + let pluginSqlState: String? +} + +// A startup command can be a safety rail, so a session that opened without one says so on the connection's warning. +@MainActor +struct StartupCommandWarningTests { + @Test("A startup command the server refused is named on the session's warning, the ones that ran are not") + func refusedStartupCommandIsNamed() async { + let driver = MockDatabaseDriver() + driver.failsStatementContaining = ("SET ROLE", DatabaseError.queryFailed("role \"app_ro\" does not exist")) + + let failures = await DatabaseManager.shared.applyTimeoutAndStartupCommands( + on: driver, + queryTimeoutSeconds: 0, + startupCommands: "SET ROLE app_ro\nSET search_path = app", + connectionName: "Reporting" + ) + let warning = DatabaseManager.startupWarning(for: failures) + + #expect(failures.map { $0.statement } == ["SET ROLE app_ro"]) + #expect(warning?.contains("SET ROLE app_ro") == true) + #expect(warning?.contains("SET search_path") == false) + } + + // PostgreSQL can quote the statement's values back in its message, so only the code reaches the screen. + @Test("The warning names the server's code, never its message") + func warningShowsTheCodeNotTheMessage() { + let refusal = ServerRefusal( + pluginErrorMessage: "invalid value for parameter \"app.api_key\": \"sk-live-123\"", pluginSqlState: "22023" + ) + let failure = StartupCommandFailure(statement: "SET app.api_key = 42", error: refusal) + let warning = DatabaseManager.startupWarning(for: [failure]) + + #expect(failure.code == "22023") + #expect(warning?.contains("SET app.api_key = 42") == true) + #expect(warning?.contains("22023") == true) + #expect(warning?.contains("sk-live-123") == false) + } + + @Test("Startup commands that all ran leave no warning") + func successfulStartupCommandsLeaveNoWarning() async { + let driver = MockDatabaseDriver() + + let failures = await DatabaseManager.shared.applyTimeoutAndStartupCommands( + on: driver, queryTimeoutSeconds: 0, startupCommands: "SET search_path = app", connectionName: "Reporting" + ) + + #expect(DatabaseManager.startupWarning(for: failures) == nil) + } + + @Test("Several failed startup commands are each named") + func severalFailuresAreEachNamed() { + let warning = DatabaseManager.startupWarning(for: [ + StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704"), + StartupCommandFailure(statement: "SET default_transaction_read_only = on", code: nil), + ]) + + #expect(warning?.contains("SET ROLE app_ro (42704)") == true) + #expect(warning?.contains("SET default_transaction_read_only = on") == true) + } + + // A startup command can carry a password or a key, and the banner is on screen, screen sharing included. + @Test("A failed startup command is shown only up to its first literal or credential clause") + func shownStatementStopsBeforeSecrets() { + #expect(DatabaseManager.shownStatement("SELECT set_config('app.api_key', 'sk-live-123', false)") + == "SELECT set_config(…") + #expect(DatabaseManager.shownStatement("SET ROLE reporting IDENTIFIED BY hunter2") == "SET ROLE reporting…") + #expect(DatabaseManager.shownStatement("ALTER USER app WITH PASSWORD 'x'") == "ALTER USER app WITH…") + #expect(DatabaseManager.shownStatement("SET search_path = app") == "SET search_path = app") + + let warning = DatabaseManager.startupWarning(for: [ + StartupCommandFailure(statement: "SET ROLE reporting IDENTIFIED BY hunter2", code: "28000"), + ]) + #expect(warning?.contains("hunter2") == false) + } + + @Test("A reconnect whose startup commands all ran drops the banner about the session before it") + func cleanReconnectClearsTheStartupWarning() { + let connectionId = UUID() + let manager = DatabaseManager.shared + defer { manager.reportedStartupFailures.removeValue(forKey: connectionId) } + + let failed = manager.openingWarning( + driver: MockDatabaseDriver(), + startupFailures: [StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704")], + for: connectionId + ) + let clean = manager.openingWarning(driver: MockDatabaseDriver(), startupFailures: [], for: connectionId) + + #expect(failed?.contains("SET ROLE app_ro") == true) + #expect(clean == nil) + #expect(manager.reportedStartupFailures[connectionId]?.isEmpty == true) + } + + @Test("The login's warning and the startup warning are both kept") + func warningsAreJoined() { + #expect(DatabaseManager.joinedWarning(["ORA-28002", nil, "startup"]) == "ORA-28002\n\nstartup") + #expect(DatabaseManager.joinedWarning([nil, ""]) == nil) + } + + // The pool opens entries for as long as the window is open, so a failure it was already told about stays dismissed. + @Test("A pool or lane session's failed startup command reaches the window once") + func poolFailureIsReportedOnce() { + let connection = TestFixtures.makeConnection(name: "Startup rail") + let manager = DatabaseManager.shared + manager.injectSession(ConnectionSession(connection: connection), for: connection.id) + defer { manager.removeSession(for: connection.id) } + let role = StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704") + let path = StartupCommandFailure(statement: "SET search_path = app", code: "3F000") + + manager.reportStartupFailures([role], for: connection.id) + #expect(manager.session(for: connection.id)?.connectionWarning?.contains("SET ROLE app_ro") == true) + + manager.dismissConnectionWarning(for: connection.id) + manager.reportStartupFailures([role], for: connection.id) + #expect(manager.session(for: connection.id)?.connectionWarning == nil) + + manager.reportStartupFailures([role, path], for: connection.id) + let warning = manager.session(for: connection.id)?.connectionWarning + #expect(warning?.contains("SET search_path = app") == true) + #expect(warning?.contains("SET ROLE") == false) + } +} diff --git a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift index 1d8ab421c1..2e741ec1a0 100644 --- a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift +++ b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift @@ -110,7 +110,7 @@ struct SwitchDatabasePooledConnectionTests { #expect(driver.disconnectCallCount == 0) #expect(manager.sessionTransactions.holdsTransaction(connectionId: connection.id, container: "app")) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connection.id)) #expect(manager.session(for: connection.id)?.resolvedBrowseDatabase == "app") #expect(manager.session(for: connection.id)?.status == .connected) } @@ -145,7 +145,7 @@ struct SwitchDatabasePooledConnectionTests { try? await manager.switchDatabase(to: "unreachable", for: connection.id, persist: false) #expect(driver.disconnectCallCount >= 1) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: connection.id)) } @Test("A switch that reconnects a connection that stopped answering closes its pooled connections") diff --git a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift index e67a5599bd..0b19025ea9 100644 --- a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift +++ b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift @@ -437,3 +437,89 @@ struct TriggerEditSafetyTests { #expect(requests.isEmpty) } } + +// A Cancel in the gate's confirmation reached the trigger editor as an error row under the SQL. +@MainActor +struct TriggerGateDenialTests { + private func install() async throws -> (connection: DatabaseConnection, scope: DatabaseScope, stub: StubTriggerDriver) { + let connection = TestFixtures.makeConnection(database: "app", type: .postgresql) + let stub = StubTriggerDriver() + stub.dropToReturn = "DROP TRIGGER t" + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: stub) + try await adapter.connect() + DatabaseManager.shared.injectSession(ConnectionSession(connection: connection, driver: adapter), for: connection.id) + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + MetadataConnectionPool.shared.injectEntry(adapter, scope: scope) + return (connection, scope, stub) + } + + private func uninstall(_ connection: DatabaseConnection) { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + private func failure(of body: () async throws -> Void) async -> GatedOperationFailure? { + do { + try await body() + return nil + } catch { + return GatedOperationFailure(error) + } + } + + @Test("A Cancel at the gate ends a trigger save and a trigger drop quietly, with nothing run") + func cancelIsQuiet() async throws { + let installed = try await install() + defer { uninstall(installed.connection) } + + let save = await failure { + try await TriggerEditing.apply( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + sql: "CREATE TRIGGER t", + isEdit: false, + originalName: nil, + originalDefinition: nil, + gate: AlwaysDenyGate.cancelling + ) + } + let drop = await failure { + try await TriggerEditing.drop( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + name: "t", + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(save == .cancelledByUser) + #expect(drop == .cancelledByUser) + #expect(save?.message == nil) + #expect(installed.stub.executedQueries.isEmpty) + } + + @Test("A refused trigger save says why and is not a failed statement") + func refusalExplainsItself() async throws { + let installed = try await install() + defer { uninstall(installed.connection) } + + let save = await failure { + try await TriggerEditing.apply( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + sql: "CREATE TRIGGER t", + isEdit: false, + originalName: nil, + originalDefinition: nil, + gate: AlwaysDenyGate(reason: "Safe Mode is read-only") + ) + } + + #expect(save == .refused("Safe Mode is read-only")) + #expect(save?.reportsFailure == false) + #expect(installed.stub.executedQueries.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Database/Vault/VaultAuthenticatorTests.swift b/apple/TableProTests/Core/Database/Vault/VaultAuthenticatorTests.swift index 7f97f034c6..db1320948d 100644 --- a/apple/TableProTests/Core/Database/Vault/VaultAuthenticatorTests.swift +++ b/apple/TableProTests/Core/Database/Vault/VaultAuthenticatorTests.swift @@ -153,7 +153,7 @@ struct VaultAuthenticatorTests { @Test("OIDC sign-in opens the browser, trades the callback code for a token and keeps it for the launch") func oidcSignIn() async throws { let transport = VaultFakeTransport() - transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"https://idp.test/authorize?x=1"}}"#) + transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"https://idp.test/authorize?x=1&state=st"}}"#) transport.reply("GET", "auth/oidc/oidc/callback", VaultFixtures.authReply(token: "oidc-token", ttl: 3_600)) let receiver = VaultFakeCallbackReceiver(parameters: ["state": "st", "code": "cd"]) let opened = VaultFixtures.OpenedURLs() @@ -165,8 +165,9 @@ struct VaultAuthenticatorTests { #expect(token.value == "oidc-token") #expect(again == token) - #expect(opened.all.map(\.absoluteString) == ["https://idp.test/authorize?x=1"]) + #expect(opened.all.map(\.absoluteString) == ["https://idp.test/authorize?x=1&state=st"]) #expect(receiver.wasStopped) + #expect(receiver.expectedStates == ["st"]) let authURLRequest = try #require(transport.requests(to: "auth/oidc/oidc/auth_url").first) #expect(authURLRequest.body["redirect_uri"] == "http://localhost:8250/oidc/callback") #expect(authURLRequest.body["role"] == "dba") @@ -179,7 +180,7 @@ struct VaultAuthenticatorTests { @Test("An identity provider error ends the OIDC sign-in with its description") func oidcProviderError() async { let transport = VaultFakeTransport() - transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"https://idp.test/authorize"}}"#) + transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"https://idp.test/authorize?state=st"}}"#) let receiver = VaultFakeCallbackReceiver(parameters: ["error": "access_denied", "error_description": "User cancelled"]) let authenticator = VaultFixtures.authenticator(transport: transport, receiver: receiver) @@ -213,7 +214,7 @@ struct VaultAuthenticatorTests { @Test("A local identity provider over plain http on this Mac still opens") func oidcAllowsLoopbackSignInPage() async throws { let transport = VaultFakeTransport() - transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"http://127.0.0.1:5556/auth?x=1"}}"#) + transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"http://127.0.0.1:5556/auth?x=1&state=st"}}"#) transport.reply("GET", "auth/oidc/oidc/callback", VaultFixtures.authReply(token: "oidc-token")) let receiver = VaultFakeCallbackReceiver(parameters: ["state": "st", "code": "cd"]) let opened = VaultFixtures.OpenedURLs() @@ -222,18 +223,79 @@ struct VaultAuthenticatorTests { let token = try await authenticator.token(for: .oidc(mount: "oidc", role: ""), endpoint: endpoint, interactive: true) #expect(token.value == "oidc-token") - #expect(opened.all.map { $0.absoluteString } == ["http://127.0.0.1:5556/auth?x=1"]) + #expect(opened.all.map { $0.absoluteString } == ["http://127.0.0.1:5556/auth?x=1&state=st"]) + } + + @Test("A sign-in page without a state is refused before the browser opens anything") + func oidcRefusesSignInPageWithoutState() async { + let transport = VaultFakeTransport() + transport.reply("POST", "auth/oidc/oidc/auth_url", #"{"data":{"auth_url":"https://idp.test/authorize?x=1"}}"#) + let receiver = VaultFakeCallbackReceiver(parameters: ["state": "st", "code": "cd"]) + let opened = VaultFixtures.OpenedURLs() + let authenticator = VaultFixtures.authenticator(transport: transport, receiver: receiver, opened: opened) + + await #expect(throws: VaultError.self) { + try await authenticator.token(for: .oidc(mount: "oidc", role: ""), endpoint: endpoint, interactive: true) + } + #expect(opened.all.isEmpty) + #expect(receiver.expectedStates.isEmpty) + #expect(transport.requests(to: "auth/oidc/oidc/callback").isEmpty) } - @Test("The callback listener hands back the query of GET /oidc/callback only") + @Test("The callback listener hands back the query of GET /oidc/callback carrying the expected state") func loopbackParsesCallback() { let head = Data("GET /oidc/callback?code=abc&state=xyz HTTP/1.1\r\nHost: localhost:8250\r\n\r\n".utf8) - #expect(VaultOIDCLoopback.callbackParameters(in: head) == ["code": "abc", "state": "xyz"]) + #expect(VaultOIDCLoopback.callbackParameters(in: head, expectedState: "xyz") == ["code": "abc", "state": "xyz"]) + + let denied = Data("GET /oidc/callback?error=access_denied&error_description=User+cancelled&state=xyz HTTP/1.1\r\n\r\n".utf8) + #expect( + VaultOIDCLoopback.callbackParameters(in: denied, expectedState: "xyz") + == ["error": "access_denied", "error_description": "User cancelled", "state": "xyz"] + ) let favicon = Data("GET /favicon.ico HTTP/1.1\r\nHost: localhost:8250\r\n\r\n".utf8) - #expect(VaultOIDCLoopback.callbackParameters(in: favicon) == nil) + #expect(VaultOIDCLoopback.callbackParameters(in: favicon, expectedState: "xyz") == nil) - let partial = Data("GET /oidc/callback?code=abc HTTP/1.1\r\nHost: local".utf8) - #expect(VaultOIDCLoopback.callbackParameters(in: partial) == nil) + let partial = Data("GET /oidc/callback?code=abc&state=xyz HTTP/1.1\r\nHost: local".utf8) + #expect(VaultOIDCLoopback.callbackParameters(in: partial, expectedState: "xyz") == nil) + } + + @Test("A callback without this sign-in's state is ignored", arguments: [ + "/oidc/callback?code=abc&state=forged", + "/oidc/callback?code=abc", + "/oidc/callback?code=abc&state=xyz&state=xyz", + "/oidc/callback?error=access_denied&state=forged", + "/oidc/callback?error=access_denied" + ]) + func loopbackIgnoresStrayCallback(target: String) { + let head = Data("GET \(target) HTTP/1.1\r\nHost: localhost:8250\r\n\r\n".utf8) + #expect(VaultOIDCLoopback.callbackParameters(in: head, expectedState: "xyz") == nil) + } + + @Test("A stray callback is answered and ignored, and the real one still ends the wait") + func loopbackKeepsListeningAfterAStrayCallback() async throws { + let loopback = VaultOIDCLoopback(expectedState: "xyz", port: 0) + let port = try await loopback.startListening() + defer { loopback.stop() } + let parameters = Task { try await loopback.callback(timeout: .seconds(30)) } + + let forged = await Self.status(of: "/oidc/callback?code=forged&state=wrong", port: port) + let redirect = await Self.status(of: "/oidc/callback?code=real&state=xyz", port: port) + + #expect([forged, redirect] == [400, 200]) + #expect(try await parameters.value == ["code": "real", "state": "xyz"]) + } + + private static func status(of target: String, port: UInt16) async -> Int? { + guard let url = URL(string: "http://127.0.0.1:\(port)\(target)") else { return nil } + let session = URLSession(configuration: .ephemeral) + defer { session.finishTasksAndInvalidate() } + let request = URLRequest(url: url, timeoutInterval: 5) + guard let (_, response) = try? await session.data(for: request), + let httpResponse = response as? HTTPURLResponse + else { + return nil + } + return httpResponse.statusCode } } diff --git a/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift b/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift index 7c81dd214d..30d3d516a2 100644 --- a/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift +++ b/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift @@ -21,6 +21,7 @@ private final class TokenRecorder: @unchecked Sendable { struct VaultDatabaseCredentialsTests { private let role = VaultDatabaseRole(address: "https://vault.test:8200", mount: "database", role: "readonly") + private let owner = UUID() private func credentials(_ transport: VaultFakeTransport, files: [String: String] = [:]) -> VaultDatabaseCredentials { VaultDatabaseCredentials( @@ -36,7 +37,7 @@ struct VaultDatabaseCredentialsTests { transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) - let signIn = try await credentials(transport).signIn(role, interactive: false) + let signIn = try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) #expect(signIn.username == "v-token-readonly-1") #expect(signIn.password == "s3cret") @@ -45,15 +46,33 @@ struct VaultDatabaseCredentialsTests { lease.release() } - @Test("Each sign-in reads a new lease") - func newLeasePerSignIn() async throws { + @Test("Sign-ins for one owner and role share one lease") + func signInsShareTheLease() async throws { let transport = VaultFakeTransport() transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) let credentials = credentials(transport) - let first = try await credentials.signIn(role, interactive: false) - let second = try await credentials.signIn(role, interactive: false) + let first = try await credentials.signIn(role, owner: owner, for: nil, interactive: false) + first.lease?.release() + let second = try await credentials.signIn(role, owner: owner, for: nil, interactive: false) + + #expect(transport.requests(to: "database/creds/readonly").count == 1) + #expect(second.username == first.username) + #expect(second.lease?.leaseId == first.lease?.leaseId) + second.lease?.release() + } + + @Test("Another owner signs in as a user of its own") + func otherOwnerReadsItsOwnLease() async throws { + let transport = VaultFakeTransport() + transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) + transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) + transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) + let credentials = credentials(transport) + + let first = try await credentials.signIn(role, owner: owner, for: nil, interactive: false) + let second = try await credentials.signIn(role, owner: UUID(), for: nil, interactive: false) #expect(transport.requests(to: "database/creds/readonly").count == 2) #expect(first.lease != second.lease) @@ -68,7 +87,29 @@ struct VaultDatabaseCredentialsTests { transport.reply("GET", "database/creds/readonly", status: 403, #"{"errors":["1 error occurred:\n\t* permission denied\n\n"]}"#) await #expect(throws: VaultError.credentialsFailed(path: "database/creds/readonly", message: "permission denied")) { - try await credentials(transport).signIn(role, interactive: false) + try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) + } + } + + @Test("A sealed Vault reads as unreachable, so an unattended reconnect tries again", arguments: [503, 429]) + func sealedVaultIsUnreachable(status: Int) async { + let transport = VaultFakeTransport() + transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) + transport.reply("GET", "database/creds/readonly", status: status, #"{"errors":["Vault is sealed"]}"#) + + await #expect(throws: VaultError.unreachable("https://vault.test:8200", "Vault is sealed")) { + try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) + } + } + + @Test("Vault's own failure names its reason and is worth another try") + func vaultFailureNamesItsReason() async { + let transport = VaultFakeTransport() + transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) + transport.reply("GET", "database/creds/readonly", status: 500, #"{"errors":["local node not active"]}"#) + + await #expect(throws: VaultError.serverFailed("local node not active")) { + try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) } } @@ -82,22 +123,26 @@ struct VaultDatabaseCredentialsTests { var appRole = role appRole.auth = .appRole(mount: "approle", roleId: "r", secretIdPath: "/keys/secret-id") - let signIn = try await credentials(transport, files: ["secret-id": "s"]).signIn(appRole, interactive: false) + let signIn = try await credentials(transport, files: ["secret-id": "s"]) + .signIn(appRole, owner: owner, for: nil, interactive: false) #expect(transport.requests(to: "database/creds/readonly").map(\.token) == ["old", "new"]) #expect(signIn.username == "v-token-readonly-1") signIn.lease?.release() } - @Test("Releasing the sign-in's lease revokes it with the same token") - func releaseRevokes() async throws { + @Test("Ending the last connection that used a lease revokes it, with the token that issued it") + func endUseRevokes() async throws { let transport = VaultFakeTransport() transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) transport.reply("PUT", "sys/leases/revoke", status: 204, "") + let credentials = credentials(transport) + let connection = UUID() - let signIn = try await credentials(transport).signIn(role, interactive: false) + let signIn = try await credentials.signIn(role, owner: owner, for: connection, interactive: false) signIn.lease?.release() + credentials.endUse(connection: connection, live: []) try await waitUntil { !transport.requests(to: "sys/leases/revoke").isEmpty } let revoke = try #require(transport.requests(to: "sys/leases/revoke").first) diff --git a/apple/TableProTests/Core/Database/Vault/VaultFakeTransport.swift b/apple/TableProTests/Core/Database/Vault/VaultFakeTransport.swift index 2fc57da8a5..ebe1bfcae0 100644 --- a/apple/TableProTests/Core/Database/Vault/VaultFakeTransport.swift +++ b/apple/TableProTests/Core/Database/Vault/VaultFakeTransport.swift @@ -69,6 +69,7 @@ final class VaultFakeCallbackReceiver: VaultOIDCCallbackReceiving, @unchecked Se private let parameters: [String: String] private let lock = NSLock() private var stopped = false + private var states: [String] = [] init(parameters: [String: String]) { self.parameters = parameters @@ -76,6 +77,13 @@ final class VaultFakeCallbackReceiver: VaultOIDCCallbackReceiving, @unchecked Se var wasStopped: Bool { lock.withLock { stopped } } + var expectedStates: [String] { lock.withLock { states } } + + func expect(state: String) -> VaultFakeCallbackReceiver { + lock.withLock { states.append(state) } + return self + } + func start() async throws {} func callback(timeout: Duration) async throws -> [String: String] { parameters } @@ -118,7 +126,7 @@ enum VaultFixtures { readFile: { path in files.first { path.hasSuffix($0.key) }?.value }, now: now, openURL: { opened.append($0) }, - makeCallbackReceiver: { receiver } + makeCallbackReceiver: { receiver.expect(state: $0) } ) } diff --git a/apple/TableProTests/Core/Database/Vault/VaultLeaseCacheTests.swift b/apple/TableProTests/Core/Database/Vault/VaultLeaseCacheTests.swift new file mode 100644 index 0000000000..9fb1ad33b3 --- /dev/null +++ b/apple/TableProTests/Core/Database/Vault/VaultLeaseCacheTests.swift @@ -0,0 +1,426 @@ +// +// VaultLeaseCacheTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +private final class Revocations: VaultLeaseRenewing, @unchecked Sendable { + private let lock = NSLock() + private var revoked: [String] = [] + private var renewed = 0 + // What Vault answers a renewal: the lease is still there, or it is gone. + var leaseExists = false + + var leases: [String] { lock.withLock { revoked } } + var renewals: Int { lock.withLock { renewed } } + + func renew(leaseId: String, increment: Duration, timeout: Duration) async throws -> VaultLeaseGrant { + let exists = lock.withLock { () -> Bool in + renewed += 1 + return leaseExists + } + guard exists else { throw VaultRequestError(status: 400, messages: ["lease not found"]) } + return VaultLeaseGrant(leaseDuration: increment, tokenRemaining: nil) + } + + func revoke(leaseId: String) async { + lock.withLock { revoked.append(leaseId) } + } +} + +private final class Names: @unchecked Sendable { + private let lock = NSLock() + private var recorded: [String] = [] + + var values: [String] { lock.withLock { recorded } } + + func record(_ name: String) { + lock.withLock { recorded.append(name) } + } +} + +// Vault's side: each read issues the next lease. +private final class LeaseIssuer: @unchecked Sendable { + private let lock = NSLock() + private var issued = 0 + private let clock: ManualClock + private let seconds: Int64 + let revocations = Revocations() + var gate: AsyncGate? + + private let renewable: Bool + + init(clock: ManualClock, seconds: Int64 = 3_600, renewable: Bool = false) { + self.clock = clock + self.seconds = seconds + self.renewable = renewable + } + + var reads: Int { lock.withLock { issued } } + + func issue() async throws -> VaultLeaseCache.Issued { + await gate?.wait() + let number = lock.withLock { () -> Int in + issued += 1 + return issued + } + return VaultLeaseCache.Issued( + username: "v-token-ro-\(number)", + password: "s3cret", + lease: VaultLease( + leaseId: "database/creds/ro/\(number)", + grant: VaultLeaseGrant(leaseDuration: .seconds(seconds), tokenRemaining: nil), + renewable: renewable, + renewer: revocations, + clock: clock + ), + token: "t0" + ) + } +} + +struct VaultLeaseCacheTests { + private let owner = UUID() + private let role = VaultDatabaseRole(mount: "database", role: "ro") + + private func key(owner: UUID? = nil, role: VaultDatabaseRole? = nil, token: String = "t0") throws -> VaultLeaseCache.Key { + VaultLeaseCache.Key( + owner: owner ?? self.owner, + role: role ?? self.role, + endpoint: try VaultEndpoint.resolve(address: "https://vault.test:8200", environment: [:]), + token: token + ) + } + + private func acquire( + _ cache: VaultLeaseCache, + _ issuer: LeaseIssuer, + key: VaultLeaseCache.Key, + for connection: UUID? = nil + ) async throws -> VaultLeaseHold { + try await cache.acquire(key, for: connection) { try await issuer.issue() } + } + + @Test("One owner and role read one lease") + func oneReadPerOwnerAndRole() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + let first = try await acquire(cache, issuer, key: try key()) + let second = try await acquire(cache, issuer, key: try key()) + + #expect(issuer.reads == 1) + #expect(first.leaseId == second.leaseId) + #expect(first != second) + } + + @Test("Another owner, role or token reads a lease of its own") + func otherKeysReadTheirOwn() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + let holds = [ + try await acquire(cache, issuer, key: try key()), + try await acquire(cache, issuer, key: try key(owner: UUID())), + try await acquire(cache, issuer, key: try key(role: VaultDatabaseRole(mount: "database", role: "rw"))), + try await acquire(cache, issuer, key: try key(token: "someone-else")), + ] + + #expect(issuer.reads == 4) + #expect(Set(holds.map(\.leaseId)).count == 4) + } + + @Test("Concurrent sign-ins for one key read one lease, whichever order they reach the cache in") + func concurrentSignInsReadOneLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let gate = AsyncGate() + issuer.gate = gate + let key = try key() + + async let first = acquire(cache, issuer, key: key) + async let second = acquire(cache, issuer, key: key) + await gate.arrived(1) + await gate.open() + let holds = try await [first, second] + + #expect(issuer.reads == 1) + #expect(holds[0].leaseId == holds[1].leaseId) + } + + @Test("A failed read leaves nothing behind, so the next sign-in reads again") + func failedReadIsNotKept() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let key = try key() + + await #expect(throws: VaultError.self) { + try await cache.acquire(key, for: nil) { throw VaultError.noAddress } + } + _ = try await acquire(cache, issuer, key: key) + + #expect(issuer.reads == 1) + } + + @Test("A sign-in whose caller gave up leaves its lease for the next one, unrevoked") + func abandonedSignInIsKept() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + _ = try await acquire(cache, issuer, key: try key()) + let next = try await acquire(cache, issuer, key: try key()) + + #expect(issuer.reads == 1) + #expect(next.leaseId == "database/creds/ro/1") + #expect(issuer.revocations.leases.isEmpty) + } + + @Test("A lease nothing holds is revoked after the idle retention, unless a sign-in takes it first") + func idleRetention() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + try await acquire(cache, issuer, key: try key()).release() + try await waitUntil { clock.pendingSleeperCount == 2 } + clock.advance(by: VaultLeaseCache.idleRetention - .seconds(1)) + let again = try await acquire(cache, issuer, key: try key()) + // Only the lease's own schedule is left: the idle timer was cancelled. + try await waitUntil { clock.pendingSleeperCount == 1 } + clock.advance(by: .seconds(1)) + #expect(issuer.revocations.leases.isEmpty) + + again.release() + try await waitUntil { clock.pendingSleeperCount == 2 } + clock.advance(by: VaultLeaseCache.idleRetention) + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + #expect(issuer.reads == 1) + } + + @Test("An ended lease is not handed out again, each hold hears its ending, and the last one revokes it") + func endedLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, seconds: 60) + let endings = Names() + let first = try await acquire(cache, issuer, key: try key()) + let second = try await acquire(cache, issuer, key: try key()) + first.start { _ in endings.record("first") } + second.start { _ in endings.record("second") } + + await clock.waitUntilSleeping() + clock.advance(by: .seconds(50)) + + try await waitUntil { Set(endings.values) == ["first", "second"] } + let next = try await acquire(cache, issuer, key: try key()) + #expect(next.leaseId == "database/creds/ro/2") + first.release() + #expect(issuer.revocations.leases.isEmpty) + second.release() + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("A refused sign-in on a lease Vault no longer has ends it for every hold and replaces it") + func refusalOfARevokedLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, renewable: true) + let endings = Names() + let session = try await acquire(cache, issuer, key: try key()) + session.start { _ in endings.record("session") } + let refused = try await acquire(cache, issuer, key: try key()) + + refused.signInRefused(credentials: false) + + try await waitUntil { endings.values == ["session"] && refused.hasEnded } + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/2") + session.release() + refused.release() + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test( + "A refused sign-in on a lease Vault still has keeps it, unless a server took it and then refused its password", + arguments: [(false, true), (true, false)] + ) + func refusalOfALiveLease(accepted: Bool, credentials: Bool) async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, renewable: true) + issuer.revocations.leaseExists = true + let refused = try await acquire(cache, issuer, key: try key()) + if accepted { + refused.signInAccepted() + } + + refused.signInRefused(credentials: credentials) + try await waitUntil { issuer.revocations.renewals == 1 } + + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/1") + #expect(!refused.hasEnded) + #expect(issuer.revocations.leases.isEmpty) + } + + @Test("A user a server took and then refused for its password is not handed out again, and sessions keep it") + func refusalOfAUserDroppedInTheDatabase() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, renewable: true) + issuer.revocations.leaseExists = true + let endings = Names() + let session = try await acquire(cache, issuer, key: try key()) + session.start { _ in endings.record("session") } + session.signInAccepted() + let refused = try await acquire(cache, issuer, key: try key()) + + refused.signInRefused(credentials: true) + refused.release() + try await waitUntil { issuer.revocations.renewals == 1 } + session.release() + + // A listed lease would wait out the idle retention instead. + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + #expect(endings.values.isEmpty) + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/2") + } + + @Test("Ending a connection revokes a lease only it used, including after a late release") + func endUseRevokesTheConnectionsLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let connection = UUID() + let session = try await acquire(cache, issuer, key: try key(), for: connection) + let pooledRead = try await acquire(cache, issuer, key: try key(), for: connection) + session.release() + + cache.endUse(connection: connection, live: []) + // Still there while the read holds it. + #expect(pooledRead.redialDelay(afterFailures: 1) != nil) + pooledRead.release() + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/2") + } + + @Test("A read left over from a connection ended earlier does not keep the lease past the last disconnect") + func leftoverReadDoesNotKeepTheLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let first = UUID() + let second = UUID() + let firstsRead = try await acquire(cache, issuer, key: try key(), for: first) + let seconds = try await acquire(cache, issuer, key: try key(), for: second) + + cache.endUse(connection: first, live: [second]) + seconds.release() + cache.endUse(connection: second, live: []) + firstsRead.release() + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("Ending a connection leaves a lease another live connection is reconnecting with") + func endUseKeepsALeaseBetweenDrivers() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let ended = UUID() + let reconnecting = UUID() + try await acquire(cache, issuer, key: try key(), for: ended).release() + try await acquire(cache, issuer, key: try key(), for: reconnecting).release() + + cache.endUse(connection: ended, live: [reconnecting]) + + #expect(try await acquire(cache, issuer, key: try key(), for: reconnecting).leaseId == "database/creds/ro/1") + #expect(issuer.reads == 1) + } + + @Test("Ending a hold whose password left the process revokes the lease at once") + func endRevokesAtOnce() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + try await acquire(cache, issuer, key: try key(owner: UUID())).end() + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("Ending one connection leaves a lease another connection on the profile still holds") + func endUseLeavesSharedLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let ended = UUID() + let other = UUID() + let mine = try await acquire(cache, issuer, key: try key(), for: ended) + let theirs = try await acquire(cache, issuer, key: try key(), for: other) + mine.release() + + cache.endUse(connection: ended, live: [other]) + + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/1") + #expect(issuer.revocations.leases.isEmpty) + theirs.release() + } + + @Test("Quit revokes every lease, held or not, and waits for Vault to answer") + func revokeAll() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let held = try await acquire(cache, issuer, key: try key()) + try await acquire(cache, issuer, key: try key(owner: UUID())).release() + + await cache.revokeAll(within: .seconds(3)) + + #expect(Set(issuer.revocations.leases) == ["database/creds/ro/1", "database/creds/ro/2"]) + held.release() + } + + @Test("A lease read after the quit revoke is revoked at once and the sign-in is cancelled") + func readFinishingAfterQuit() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let gate = AsyncGate() + issuer.gate = gate + + let signIn = Task { try await acquire(cache, issuer, key: try key()) } + await gate.arrived(1) + await cache.revokeAll(within: .seconds(3)) + await gate.open() + + await #expect(throws: CancellationError.self) { try await signIn.value } + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("A refused new user is dialed again after 2, 4 and 8 seconds, then only until the window closes") + func redialDelays() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let hold = try await acquire(cache, LeaseIssuer(clock: clock), key: try key()) + + #expect(hold.redialDelay(afterFailures: 1) == .seconds(2)) + #expect(hold.redialDelay(afterFailures: 2) == .seconds(4)) + #expect(hold.redialDelay(afterFailures: 3) == .seconds(8)) + clock.advance(by: .seconds(14)) + #expect(hold.redialDelay(afterFailures: 4) == .seconds(16)) + clock.advance(by: .seconds(6)) + #expect(hold.redialDelay(afterFailures: 4) == .seconds(10)) + clock.advance(by: .seconds(10)) + #expect(hold.redialDelay(afterFailures: 5) == nil) + } +} diff --git a/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift b/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift index 9c305ab03e..66798cdae1 100644 --- a/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift +++ b/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift @@ -11,14 +11,63 @@ import Testing private final class SilentPluginDriver: PluginDriverStub, @unchecked Sendable {} +private struct ServerRefusal: PluginDriverError { + let pluginErrorMessage: String + let pluginErrorCode: Int? + let pluginSqlState: String? + + static let unknownUser = ServerRefusal( + pluginErrorMessage: "Access denied for user 'v-token-ro-1'", + pluginErrorCode: 1_045, + pluginSqlState: "28000" + ) +} + +// Refuses the first `refusals` dials the way a database that has not loaded a new user does. +private final class SlowToAcceptDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var dials = 0 + private let refusals: Int + private let refusal: Error + + init(refusals: Int, refusal: Error = ServerRefusal.unknownUser) { + self.refusals = refusals + self.refusal = refusal + } + + var attempts: Int { lock.withLock { dials } } + + func connect() async throws { + let attempt = lock.withLock { () -> Int in + dials += 1 + return dials + } + if attempt <= refusals { throw refusal } + } +} + +private final class Reads: @unchecked Sendable { + private let lock = NSLock() + private var reads = 0 + + var value: Int { lock.withLock { reads } } + + func count() { + lock.withLock { reads += 1 } + } +} + private final class RevokeRecorder: VaultLeaseRenewing, @unchecked Sendable { private let lock = NSLock() private var revoked: [String] = [] + // What Vault answers a renewal: the lease is still there, or it is gone. + var leaseExists = false var revokedLeases: [String] { lock.withLock { revoked } } func renew(leaseId: String, increment: Duration, timeout: Duration) async throws -> VaultLeaseGrant { - throw VaultRequestError(status: 400, messages: ["lease not found"]) + guard lock.withLock({ leaseExists }) else { throw VaultRequestError(status: 400, messages: ["lease not found"]) } + return VaultLeaseGrant(leaseDuration: increment, tokenRemaining: nil) } func revoke(leaseId: String) async { @@ -28,28 +77,55 @@ private final class RevokeRecorder: VaultLeaseRenewing, @unchecked Sendable { @MainActor struct VaultLeaseOwnershipTests { - private func lease(_ recorder: RevokeRecorder, clock: ManualClock, seconds: Int64 = 3_600) -> VaultLease { - VaultLease( - leaseId: "database/creds/ro/1", - grant: VaultLeaseGrant(leaseDuration: .seconds(seconds), tokenRemaining: nil), - renewable: false, - renewer: recorder, - clock: clock - ) + private let owner = UUID() + private let role = VaultDatabaseRole(mount: "database", role: "ro") + + private func hold( + _ cache: VaultLeaseCache, + recorder: RevokeRecorder, + clock: ManualClock, + seconds: Int64 = 3_600, + renewable: Bool = false, + for connection: UUID? = nil, + reads: Reads = Reads() + ) async throws -> VaultLeaseHold { + let endpoint = try VaultEndpoint.resolve(address: "https://vault.test:8200", environment: [:]) + let key = VaultLeaseCache.Key(owner: owner, role: role, endpoint: endpoint, token: "t0") + return try await cache.acquire(key, for: connection) { + reads.count() + return VaultLeaseCache.Issued( + username: "v-token-ro-1", + password: "s3cret", + lease: VaultLease( + leaseId: "database/creds/ro/1", + grant: VaultLeaseGrant(leaseDuration: .seconds(seconds), tokenRemaining: nil), + renewable: renewable, + renewer: recorder, + clock: clock + ), + token: "t0" + ) + } } - @Test("A driver revokes the lease it signed in with when it disconnects") - func disconnectRevokes() async throws { + @Test("A driver's disconnect hands the lease back, and the cache revokes it once it sits unused") + func disconnectHandsTheLeaseBack() async throws { let recorder = RevokeRecorder() + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) let adapter = PluginDriverAdapter( connection: TestFixtures.makeConnection(), pluginDriver: SilentPluginDriver(), - credentialLease: lease(recorder, clock: ManualClock()) + credentialLease: try await hold(cache, recorder: recorder, clock: clock) ) adapter.disconnect() adapter.disconnect() + // The lease's own end and the idle retention. + try await waitUntil { clock.pendingSleeperCount == 2 } + #expect(recorder.revokedLeases.isEmpty) + clock.advance(by: VaultLeaseCache.idleRetention) try await waitUntil { recorder.revokedLeases == ["database/creds/ro/1"] } } @@ -57,10 +133,11 @@ struct VaultLeaseOwnershipTests { func endedLeaseIsLostConnection() async throws { let recorder = RevokeRecorder() let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) let adapter = PluginDriverAdapter( connection: TestFixtures.makeConnection(), pluginDriver: SilentPluginDriver(), - credentialLease: lease(recorder, clock: clock, seconds: 60) + credentialLease: try await hold(cache, recorder: recorder, clock: clock, seconds: 60) ) #expect(!adapter.hasLostConnection) @@ -70,4 +147,182 @@ struct VaultLeaseOwnershipTests { try await waitUntil { adapter.hasLostConnection } adapter.disconnect() } + + @Test("A refused sign-in on a new lease dials again until the database accepts the user") + func youngLeaseRedials() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let driver = SlowToAcceptDriver(refusals: 2) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: try await hold(cache, recorder: RevokeRecorder(), clock: clock) + ) + + let connect = Task { try await adapter.connect() } + try await waitUntil { driver.attempts == 1 && clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(2)) + try await waitUntil { driver.attempts == 2 && clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(4)) + try await connect.value + + #expect(driver.attempts == 3) + #expect(adapter.status == .connected) + adapter.disconnect() + } + + @Test("A user still refused when the window closes fails with the reason and is kept for the next attempt") + func refusedThroughTheWindow() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let driver = SlowToAcceptDriver(refusals: 10) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: try await hold(cache, recorder: recorder, clock: clock, reads: reads) + ) + + let connect = Task { try await adapter.connect() } + for (attempt, wait) in [(1, 2), (2, 4), (3, 8), (4, 16)] { + try await waitUntil { driver.attempts == attempt && clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(wait)) + } + + let error = await #expect(throws: VaultUserNotLoaded.self) { try await connect.value } + #expect(driver.attempts == 5) + #expect(error?.pluginErrorCode == 1_045) + #expect(error?.recoverySuggestion != nil) + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 1) + } + + @Test("A refusal of a dial that started past the window fails at once and keeps a user Vault still has") + func oldLeaseFailsAtOnce() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let driver = SlowToAcceptDriver(refusals: 1) + let lease = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + clock.advance(by: VaultLeaseCache.acceptanceWindow) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: lease + ) + + await #expect(throws: ServerRefusal.self) { try await adapter.connect() } + #expect(driver.attempts == 1) + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 1) + } + + @Test("A refused sign-in on any engine asks Vault, and a user Vault revoked is replaced, without dialing again") + func otherEngineRefusalReplacesTheUser() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let refusal = ServerRefusal( + pluginErrorMessage: "password authentication failed for user \"v-token-ro-1\"", + pluginErrorCode: nil, + pluginSqlState: nil + ) + let driver = SlowToAcceptDriver(refusals: 1, refusal: refusal) + let lease = try await hold(cache, recorder: recorder, clock: clock, renewable: true, reads: reads) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: .postgresql), + pluginDriver: driver, + credentialLease: lease + ) + + await #expect(throws: ServerRefusal.self) { try await adapter.connect() } + #expect(driver.attempts == 1) + try await waitUntil { lease.hasEnded } + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 2) + } + + @Test("A MySQL user a connection signed in with and another is refused for is not handed out again") + func acceptedUserRefusedLater() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let recorder = RevokeRecorder() + recorder.leaseExists = true + let first = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: SlowToAcceptDriver(refusals: 0), + credentialLease: try await hold(cache, recorder: recorder, clock: clock, renewable: true) + ) + try await first.connect() + clock.advance(by: VaultLeaseCache.acceptanceWindow) + let second = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: SlowToAcceptDriver(refusals: 1), + credentialLease: try await hold(cache, recorder: recorder, clock: clock, renewable: true) + ) + + await #expect(throws: ServerRefusal.self) { try await second.connect() } + second.disconnect() + first.disconnect() + + // A lease still handed out would wait out the idle retention instead. + try await waitUntil { recorder.revokedLeases == ["database/creds/ro/1"] } + } + + @Test("A refused database is not dialed again and keeps the user the profile signs in with") + func otherErrorFailsAtOnce() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let wrongDatabase = ServerRefusal( + pluginErrorMessage: "Access denied for user 'v-token-ro-1' to database 'orders'", + pluginErrorCode: 1_044, + pluginSqlState: "42000" + ) + let driver = SlowToAcceptDriver(refusals: 1, refusal: wrongDatabase) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: try await hold(cache, recorder: recorder, clock: clock, reads: reads) + ) + + await #expect(throws: ServerRefusal.self) { try await adapter.connect() } + #expect(driver.attempts == 1) + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 1) + } + + @Test("A connect cancelled while it waits for the database dials no more") + func cancelledWaitStopsDialing() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let driver = SlowToAcceptDriver(refusals: 5) + let ended = UUID() + let lease = try await hold(cache, recorder: RevokeRecorder(), clock: clock, for: ended) + // Unlisted, so letting go of it revokes it and leaves no idle timer among the sleepers counted below. + cache.endUse(connection: ended, live: []) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: lease + ) + + let connect = Task { try await adapter.connect() } + try await waitUntil { driver.attempts == 1 && clock.pendingSleeperCount == 2 } + connect.cancel() + await #expect(throws: CancellationError.self) { try await connect.value } + // Neither the redial's wait nor the lease's schedule is left. + try await waitUntil { clock.pendingSleeperCount == 0 } + clock.advance(by: VaultLeaseCache.acceptanceWindow) + + #expect(driver.attempts == 1) + } } diff --git a/apple/TableProTests/Core/Diagnostics/TableLoadTracerSinkTests.swift b/apple/TableProTests/Core/Diagnostics/TableLoadTracerSinkTests.swift index e726c64734..64cd3786e0 100644 --- a/apple/TableProTests/Core/Diagnostics/TableLoadTracerSinkTests.swift +++ b/apple/TableProTests/Core/Diagnostics/TableLoadTracerSinkTests.swift @@ -1,10 +1,6 @@ -// -// TableLoadTracerSinkTests.swift -// TableProTests -// - import Foundation @testable import TablePro +import TableProTestSupport import Testing private final class RecordingSink: TableLoadSummarySink, @unchecked Sendable { @@ -152,4 +148,23 @@ struct TableLoadTracerSinkTests { tracer.finish(token: stranger, outcome: .completed) #expect(sink.records.isEmpty) } + + @Test("The shared tracer keeps a finished load in the history file, under a test host as in the app") + func sharedTracerRecordsToTheHistoryFile() async throws { + let marker = "TracerProbe-\(UUID().uuidString)" + let tracer = TableLoadTracer.shared + let token = tracer.begin( + tabId: UUID(), + table: "users", + origin: .sidebar, + environment: TableLoadEnvironment(databaseTypeId: marker, usesSSH: false, openTabCount: 1) + ) + tracer.finish(token: token, outcome: .completed) + + let fileURL = TableLoadHistoryStore.defaultFileURL() + try await waitUntil { + let data = (try? Data(contentsOf: fileURL)) ?? Data() + return String(decoding: data, as: UTF8.self).contains(marker) + } + } } diff --git a/apple/TableProTests/Core/Execution/EditModeDataCommandsCommandTests.swift b/apple/TableProTests/Core/Execution/EditModeDataCommandsCommandTests.swift new file mode 100644 index 0000000000..fcdabc4ed7 --- /dev/null +++ b/apple/TableProTests/Core/Execution/EditModeDataCommandsCommandTests.swift @@ -0,0 +1,165 @@ +// +// EditModeDataCommandsCommandTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct EditModeDataCommandsCommandTests { + private func statement(_ sql: String, databaseType: DatabaseType) -> ExternalStatementGate.Statement { + ExternalStatementGate.Statement( + sql: sql, + connectionId: UUID(), + databaseType: databaseType, + externalAccess: .readWrite, + loadsExtensions: false, + allowsDestructive: false, + allowsOnlyDataStatements: true + ) + } + + private func refusal(_ sql: String, on databaseType: DatabaseType) -> ExternalStatementGateError? { + do { + _ = try ExternalStatementGate.classify(statement(sql, databaseType: databaseType)) + return nil + } catch let error as ExternalStatementGateError { + return error + } catch { + return nil + } + } + + @Test("Edit refuses a Redis command that changes no data", arguments: [ + "ACL SETUSER bob on >pw +@all ~*", #""\x41CL" SETUSER bob on"#, + "SELECT 3", "CLIENT SETNAME x", "CLIENT REPLY OFF", "HELLO 3", + "MULTI", "WATCH k", + "SLOWLOG RESET", "MEMORY PURGE", "HOTKEYS START METRICS 1 CPU", "CLIENT KILL ID 7", "FT.CONFIG SET TIMEOUT 1", + "PFDEBUG GETREG h", + "READWRITE", "SEARCH.CLUSTERREFRESH", "TIMESERIES.REFRESHCLUSTER", + "FT.CREATE idx ON HASH SCHEMA name TEXT", "FT.ALIASADD a idx", "FT.SYNUPDATE idx g a b", + "TS.CREATERULE a b AGGREGATION avg 60000", + "PUBLISH ch hi", "SUBSCRIBE ch", + "DB 2 ACL SETUSER bob on", "OUTSIDEBLOCK CLIENT KILL ID 7", "MUSTAPPLY", + "SOME.NEW.COMMAND x", "XGROUP", #"SET "a b"#, + ]) + func redisRefusesNonDataCommands(sql: String) { + #expect(refusal(sql, on: .redis) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit leaves Redis server code to the code rule", arguments: [#"EVAL "return 1" 0"#, "FCALL_RO f 0"]) + func redisCodeStaysRefused(sql: String) { + #expect(QueryClassifier.reachesFilesystemOrExecutesCode(sql, databaseType: .redis)) + #expect(refusal(sql, on: .redis) != nil) + } + + @Test("Edit runs Redis reads and data changes", arguments: [ + "SET k v EX 10", "EXPIRE k 60", "RENAME a b", "HSET h f v", "HEXPIRE h 60 FIELDS 1 f", "LPUSH l a", "SADD s a", + "ZADD z 1 a", "XADD s * f v", "XGROUP CREATE s g $ MKSTREAM", "SETBIT b 7 1", "GEOADD g 13.4 52.5 berlin", + "PFADD h a", "ARSET a 0 x", #"JSON.SET doc $ '{"a":1}'"#, "TS.ADD t * 1", "BF.ADD bf a", + "VADD vs VALUES 2 0.1 0.2 e1", + "DB 1 SET k v", "MUSTAPPLY SET k v NX", + "GET k", "FT.SEARCH idx *", "FT.AGGREGATE idx * LOAD 1 @name", "TS.MRANGE - + FILTER a=b", "HTTL h FIELDS 1 f", + ]) + func redisRunsDataWork(sql: String) { + #expect(refusal(sql, on: .redis) == nil) + } + + @Test("Edit refuses a Memcached command that changes the server", arguments: [ + "stats reset", "stats detail on", "stats sizes_enable", "verbosity 1", "mg k v", #""\x73tats" reset"#, + ]) + func memcachedRefusesServerCommands(sql: String) { + #expect(refusal(sql, on: .memcached) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Memcached reads and data changes", arguments: [ + "set k 0 60 v", "add k 0 0 v", "cas k 0 0 123 v", "append k 0 0 x", "incr n 1", "touch k 60", "gat 60 k", + "get k", "stats", "stats items", + ]) + func memcachedRunsDataWork(sql: String) { + #expect(refusal(sql, on: .memcached) == nil) + } + + @Test("Edit runs a Memcached delete, which takes one key, and refuses flush_all as destructive") + func memcachedDeleteRunsInEdit() { + #expect(QueryClassifier.isDataStatement("delete k", databaseType: .memcached)) + #expect(refusal("delete k", on: .memcached) == nil) + #expect(refusal("flush_all", on: .memcached) == .denied(ExternalStatementGate.destructiveRefusal)) + } + + @Test("Edit refuses an etcd command that changes no data", arguments: [ + "user add mallory pw", "role add admin", "user grant-role mallory root", "user revoke-role app reader", + "auth enable", "select 1", "frobnicate x", + ]) + func etcdRefusesNonDataCommands(sql: String) { + #expect(refusal(sql, on: .etcd) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit still refuses the etcd commands refused before", arguments: [ + "auth disable", "user delete app", "compaction 5", "lease revoke 694da1d22d63c20a", "del app/ --prefix", "defrag", + ]) + func etcdKeepsEarlierRefusals(sql: String) { + #expect(refusal(sql, on: .etcd) != nil) + } + + @Test("Edit refuses an etcd del of a prefix or a range as destructive", arguments: [ + "del app/ --prefix", "del --prefix app/", "del app/a app/z", "del --from-key app/a", "del app/ --prefix=true", + ]) + func etcdManyKeyDeleteIsDestructive(sql: String) { + #expect(refusal(sql, on: .etcd) == .denied(ExternalStatementGate.destructiveRefusal)) + } + + @Test("Edit runs etcd reads and data changes", arguments: [ + "put app/a 1", "put app/a 1 --lease=694da1d22d63c20a", "create app/b 2", "del app/a", "delete app/a", + "lease grant 60", "lease keep-alive 694da1d22d63c20a", "get app/ --prefix", "user list", + ]) + func etcdRunsDataWork(sql: String) { + #expect(refusal(sql, on: .etcd) == nil) + } + + @Test("Edit refuses text KafkaQL cannot parse") + func kafkaRefusesUnparsedText() { + #expect(refusal("ALTER TOPIC orders", on: .kafka) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit refuses DROP TOPIC as destructive") + func kafkaDropTopicStaysDestructive() { + #expect(refusal("DROP TOPIC \"orders\"", on: .kafka) == .denied(ExternalStatementGate.destructiveRefusal)) + } + + @Test("Edit runs KafkaQL reads and PRODUCE", arguments: [ + "PRODUCE INTO \"orders\" KEY \"k\" VALUE \"v\"", "CONSUME \"orders\" FROM OLDEST LIMIT 10", + "SELECT * FROM orders", "SHOW TOPICS", "DESCRIBE GROUP \"g\"", + ]) + func kafkaRunsDataWork(sql: String) { + #expect(refusal(sql, on: .kafka) == nil) + } + + @Test("Edit refuses PRODUCE into a topic another service keeps its state in", arguments: [ + "PRODUCE INTO \"_schemas\" KEY \"k\" VALUE \"v\"", + "PRODUCE INTO \"_confluent-ksql-default__command_topic\" VALUE \"v\"", + "PRODUCE INTO \"__consumer_offsets\" VALUE \"v\"", + "PRODUCE INTO \"connect-configs\" KEY \"connector-sink\" VALUE \"{}\"", + "PRODUCE INTO \"connect-offsets\" VALUE \"v\"", + "PRODUCE INTO \"connect-status\" VALUE \"v\"", + "PRODUCE INTO \"mm2-configs.east.internal\" VALUE \"v\"", + "PRODUCE INTO \"east.checkpoints.internal\" VALUE \"v\"", + "PRODUCE INTO \".hidden\" VALUE \"v\"", + "PRODUCE INTO \"orders-app-counts-store-changelog\" VALUE \"v\"", + "PRODUCE INTO \"orders-app-KSTREAM-AGGREGATE-0000000003-repartition\" VALUE \"v\"", + ]) + func kafkaRefusesServiceTopics(sql: String) { + #expect(refusal(sql, on: .kafka) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("A topic that only looks like a service's is a user's", arguments: [ + "PRODUCE INTO \"schemas\" VALUE \"v\"", + "PRODUCE INTO \"my_connect-configs\" VALUE \"v\"", + "PRODUCE INTO \"orders.internal\" VALUE \"v\"", + "CONSUME \"_schemas\" FROM OLDEST LIMIT 10", + ]) + func kafkaRunsUserTopics(sql: String) { + #expect(refusal(sql, on: .kafka) == nil) + } +} diff --git a/apple/TableProTests/Core/Execution/EditModeDataCommandsLanguageTests.swift b/apple/TableProTests/Core/Execution/EditModeDataCommandsLanguageTests.swift new file mode 100644 index 0000000000..9594b470bc --- /dev/null +++ b/apple/TableProTests/Core/Execution/EditModeDataCommandsLanguageTests.swift @@ -0,0 +1,188 @@ +// +// EditModeDataCommandsLanguageTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct EditModeDataCommandsLanguageTests { + private func statement(_ sql: String, databaseType: DatabaseType) -> ExternalStatementGate.Statement { + ExternalStatementGate.Statement( + sql: sql, + connectionId: UUID(), + databaseType: databaseType, + externalAccess: .readWrite, + loadsExtensions: false, + allowsDestructive: false, + allowsMultiStatement: false, + destructiveAlternative: nil, + allowsOnlyDataStatements: true + ) + } + + private func refusal(_ statement: ExternalStatementGate.Statement) -> ExternalStatementGateError? { + do { + _ = try ExternalStatementGate.classify(statement) + return nil + } catch let error as ExternalStatementGateError { + return error + } catch { + return nil + } + } + + // MARK: - MongoDB + + @Test("Edit refuses a MongoDB session, schema, cursor or server command, and a command it cannot name", arguments: [ + "use admin", + "db.orders.createIndex({a: 1})", + #"db.createCollection("x")"#, + #"db.createView("v", "orders", [])"#, + #"db.orders.hideIndex("a_1")"#, + #"db.runCommand({collMod: "orders", validator: {}})"#, + #"db.runCommand({create: "x"})"#, + #"db.runCommand({killCursors: "orders", cursors: []})"#, + #"db.runCommand({validate: "orders", repair: true})"#, + "db.killOp(123)", + "sleep(1000)", + #"db.orders[("insert" + "One")]({a: 1})"#, + "db.runCommand({insert: 1, documents: []})" + ]) + func mongoNonDataRefused(sql: String) { + #expect(refusal(statement(sql, databaseType: .mongodb)) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit refuses a MongoDB write into admin, config, local or a system collection", arguments: [ + #"db.getSiblingDB("admin").getCollection("system.users").insertOne({user: "x"})"#, + #"db.getSiblingDB("config").settings.updateOne({_id: "chunksize"}, {$set: {value: 1}}, {upsert: true})"#, + #"db.getCollection("system.js").insertOne({_id: "f"})"#, + #"db.getSiblingDB("local").startup_log.insertOne({a: 1})"#, + #"db.getMongo().getDB("Admin").orders.insertOne({a: 1})"#, + #"db.adminCommand({insert: "system.version", documents: [{}]})"#, + #"db.adminCommand({insert: "orders", documents: [{a: 1}]})"#, + #"db.runCommand({update: "settings", updates: [{q: {_id: 1}, u: {$set: {a: 1}}}], $db: "config"})"#, + #"db.runCommand({insert: "orders", documents: [{a: 1}], insert: "system.js"})"# + ]) + func mongoSystemWriteRefused(sql: String) { + #expect(refusal(statement(sql, databaseType: .mongodb)) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs MongoDB reads and data changes", arguments: [ + "db.orders.find({a: 1}).limit(5)", + "db.orders.aggregate([{$match: {}}])", + "show collections", + "db.orders.insertOne({a: 1})", + "db.orders.insertMany([{a: 1}])", + "db.orders.updateOne({_id: 1}, {$set: {a: 2}})", + "db.orders.updateMany({a: 1}, {$set: {b: 1}})", + "db.orders.replaceOne({_id: 1}, {a: 3})", + "db.orders.deleteOne({_id: 1})", + "db.orders.findOneAndUpdate({_id: 1}, {$inc: {n: 1}})", + "db.orders.bulkWrite([{insertOne: {document: {a: 1}}}])", + #"db.getSiblingDB("shop").orders.insertOne({a: 1})"#, + #"db.getSiblingDB("admin").getCollection("system.users").find({})"#, + #"db.runCommand({insert: "orders", documents: [{a: 1}]})"#, + #"db.runCommand({findAndModify: "orders", query: {_id: 1}, update: {$set: {a: 1}}})"#, + #"db.runCommand({find: "orders"})"#, + #"db.runCommand({listCollections: 1})"# + ]) + func mongoDataPasses(sql: String) { + #expect(refusal(statement(sql, databaseType: .mongodb)) == nil) + } + + @Test("MongoDB data calls are named in mongosh notation and commands by their command name") + func mongoNames() { + #expect(QueryClassifier.mongoDataCommands("db.orders.insertOne({a: 1})", systemDatabases: []) + == [DataCommand.named("db.collection.insertOne")]) + #expect(QueryClassifier.mongoDataCommands(#"db["orders"].updateMany({}, {$set: {a: 1}})"#, systemDatabases: []) + == [DataCommand.named("db.collection.updateMany")]) + #expect(QueryClassifier.mongoDataCommands(#"db.runCommand({insert: "orders"})"#, systemDatabases: []) + == [DataCommand.named("insert")]) + #expect(QueryClassifier.mongoDataCommands("db.orders.find({}).limit(1)", systemDatabases: []) + == [DataCommand.read, .read]) + #expect(QueryClassifier.mongoDataCommands("db.orders.find(", systemDatabases: []) == nil) + } + + // MARK: - Neo4j + + @Test("Edit refuses Cypher schema, privilege, database, transaction and server commands", arguments: [ + "CREATE INDEX person_name FOR (n:Person) ON (n.name)", + "CREATE CONSTRAINT c FOR (n:Person) REQUIRE n.id IS UNIQUE", + "CREATE USER bob SET PASSWORD 'secret' CHANGE NOT REQUIRED", + "GRANT ROLE admin TO bob", + "ALTER USER bob SET PASSWORD 'x'", + "CREATE DATABASE other", + "CREATE OR REPLACE ALIAS a FOR DATABASE neo4j", + "ALTER DATABASE neo4j SET ACCESS READ ONLY", + "START DATABASE neo4j", + "SHOW TRANSACTIONS YIELD transactionId AS id TERMINATE TRANSACTIONS id", + "ALTER CURRENT GRAPH TYPE SET {(p:Person => {name :: STRING})}", + "USE system CREATE USER x SET PASSWORD 'y'", + "CYPHER 25 CREATE INDEX i FOR (n:N) ON (n.x)", + "EXPLAIN CREATE CONSTRAINT c FOR (n:N) REQUIRE n.x IS UNIQUE", + "MERGE (n:N {id: 1}) ON CREATE INDEX i FOR (m:N) ON (m.x)", + #"CREATE INDEX i FOR (n:N) ON (n.x)"# + ]) + func cypherAdministrationRefused(sql: String) { + #expect(refusal(statement(sql, databaseType: .neo4j)) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit refuses a Cypher procedure that is neither a read nor a listed data change", arguments: [ + "CALL dbms.setConfigValue('db.logs.query.enabled', 'OFF')", + "CALL tx.setMetaData({app: 'x'})", + "CALL db.createLabel('X')", + "CALL `db`.`createLabel`('X')", + "CALL db.schema.`visualization`()", + "MATCH (n) CALL apoc.periodic.iterate('MATCH (m) RETURN m', 'SET m.x = 1', {})" + ]) + func cypherProcedureRefused(sql: String) { + #expect(refusal(statement(sql, databaseType: .neo4j)) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Cypher reads and data changes", arguments: [ + "MATCH (n:Person) RETURN n LIMIT 10", + "CREATE (n:Person {name: 'a'})", + "CREATE p = (a:A)-[:R]->(b:B) RETURN p", + "MERGE (n:Person {id: 1}) ON CREATE SET n.created = timestamp()", + "MATCH (n:Person {id: 1}) SET n.name = 'b'", + "MATCH (n:Person {id: 1}) REMOVE n:Temp", + "MATCH (n:Person {id: 1}) DETACH DELETE n", + "UNWIND [1, 2] AS i CREATE (:N {i: i})", + "MATCH (n:N) CALL (n) { SET n.x = 1 } IN TRANSACTIONS", + "CALL db.labels()", + "MATCH (d:Doc {id: 1}) CALL db.create.setNodeVectorProperty(d, 'embedding', [0.1, 0.2])", + "SHOW INDEXES", + "EXPLAIN MATCH (n) RETURN n", + "CYPHER 25 runtime=slotted MATCH (n) RETURN n", + "USE neo4j MATCH (n) RETURN count(n)", + "USE graph.byName('shop') MATCH (n) RETURN n" + ]) + func cypherDataPasses(sql: String) { + #expect(refusal(statement(sql, databaseType: .neo4j)) == nil) + } + + @Test("Cypher data clauses are named upper case and procedures lower case") + func cypherNames() { + #expect(QueryClassifier.cypherDataCommands("MATCH (n {id: 1}) DETACH DELETE n") + == [DataCommand.named("DETACH DELETE")]) + #expect(QueryClassifier.cypherDataCommands("MERGE (n:N {id: 1}) ON CREATE SET n.x = 1") + == [DataCommand.named("MERGE"), .named("SET")]) + #expect(QueryClassifier.cypherDataCommands("MATCH (d) CALL db.create.setVectorProperty(d, 'e', [1.0])") + == [DataCommand.named("db.create.setvectorproperty")]) + #expect(QueryClassifier.cypherDataCommands("MATCH (n) RETURN n") == [DataCommand.read]) + } + + // MARK: - Refused earlier + + @Test("A destructive command is refused as destructive before the data rule reads it", arguments: [ + (DatabaseType.mongodb, #"db.runCommand({createUser: "bob", pwd: "x", roles: []})"#), + (.mongodb, "db.orders.deleteMany({})"), + (.neo4j, "DROP INDEX person_name"), + (.neo4j, "REVOKE ROLE admin FROM bob") + ]) + func destructiveRefusedFirst(type: DatabaseType, sql: String) { + #expect(refusal(statement(sql, databaseType: type)) == .denied(ExternalStatementGate.destructiveRefusal)) + } +} diff --git a/apple/TableProTests/Core/Execution/EditModeDataCommandsRequestTests.swift b/apple/TableProTests/Core/Execution/EditModeDataCommandsRequestTests.swift new file mode 100644 index 0000000000..8ee124d131 --- /dev/null +++ b/apple/TableProTests/Core/Execution/EditModeDataCommandsRequestTests.swift @@ -0,0 +1,254 @@ +// +// EditModeDataCommandsRequestTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct EditModeDataCommandsRequestTests { + private func statement(_ sql: String, databaseType: DatabaseType) -> ExternalStatementGate.Statement { + ExternalStatementGate.Statement( + sql: sql, + connectionId: UUID(), + databaseType: databaseType, + externalAccess: .readWrite, + loadsExtensions: false, + allowsDestructive: false, + allowsOnlyDataStatements: true + ) + } + + private func refusal(_ sql: String, on databaseType: DatabaseType) -> ExternalStatementGateError? { + do { + try ExternalStatementGate.classify(statement(sql, databaseType: databaseType)) + return nil + } catch let error as ExternalStatementGateError { + return error + } catch { + return nil + } + } + + @Test("Edit refuses an Elasticsearch request that is not a read or a document change", arguments: [ + "PUT /orders2", + #"PUT /orders/_mapping\#n{"properties": {"note": {"type": "keyword"}}}"#, + #"POST /_aliases\#n{"actions": [{"add": {"index": "orders", "alias": "o"}}]}"#, + #"PUT /_security/user/bob\#n{"password": "secret123", "roles": ["superuser"]}"#, + #"POST /_security/api_key\#n{"name": "k"}"#, + #"PUT /_cluster/settings\#n{"persistent": {"action.destructive_requires_name": false}}"#, + #"PUT /_ilm/policy/purge\#n{"policy": {"phases": {"delete": {"min_age": "0ms", "actions": {"delete": {}}}}}}"#, + "GET /_flush", + #"PUT /_ingest/pipeline/p\#n{"processors": [{"set": {"field": "x", "value": 1}}]}"#, + #"PUT /_watcher/watch/w\#n{"trigger": {"schedule": {"interval": "1m"}}, "actions": {}}"#, + #"PUT /_inference/text_embedding/e\#n{"service": "openai", "service_settings": {"api_key": "k", "model_id": "m"}}"#, + "POST /_update_by_query/abc:1/_rethrottle?requests_per_second=-1", + #"POST /_reindex\#n{"source": {"index": "orders"}, "dest": {"index": "orders2"}}"#, + #"PUT /.watches/_doc/w1\#n{"trigger": {}}"#, + #"POST /_bulk\#n{"index": {"_index": ".security-7", "_id": "x"}}\#n{"a": 1}"#, + #"POST /orders*/_update_by_query?expand_wildcards=all\#n{"query": {"match_all": {}}}"#, + "PUT /orders/_doc/../../_cluster/settings\n{}", + #"POST /_query/async\#n{"query": "FROM orders", "keep_on_completion": true}"#, + ]) + func elasticsearchRefusesTheRest(sql: String) { + #expect(refusal(sql, on: .elasticsearch) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Elasticsearch reads and document changes", arguments: [ + #"PUT /orders/_doc/1\#n{"total": 5}"#, + #"POST /orders/_doc\#n{"total": 5}"#, + #"PUT /orders/_create/2\#n{"total": 7}"#, + #"POST /orders/_update/1\#n{"doc": {"total": 6}}"#, + #"POST /orders/_update_by_query\#n{"query": {"term": {"status": "new"}}, "script": {"source": "ctx._source.status = 'old'"}}"#, + #"POST /_bulk\#n{"index": {"_index": "orders", "_id": "3"}}\#n{"total": 1}"#, + #"GET /orders/_update_by_query\#n{"query": {"match_all": {}}, "script": {"source": "ctx._source.n = 1"}}"#, + "GET /orders/_search", + #"POST /_query\#n{"query": "FROM orders | LIMIT 5"}"#, + #"POST /orders/_mget\#n{"ids": ["1", "2"]}"#, + #"GET /_all/_search\#n{"query": {"match_all": {}}}"#, + ]) + func elasticsearchRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .elasticsearch) == nil) + } + + @Test("Edit refuses a Typesense request that is not a read or a document change", arguments: [ + #"POST /collections\#n{"name": "c", "fields": [{"name": "t", "type": "string"}]}"#, + #"PATCH /collections/books\#n{"fields": [{"name": "title", "drop": true}]}"#, + #"PUT /aliases/a\#n{"collection_name": "books"}"#, + #"PUT /collections/books/synonyms/s\#n{"synonyms": ["a", "b"]}"#, + #"POST /config\#n{"log-slow-requests-time-ms": 1}"#, + "POST /operations/cache/clear", + #"POST /conversations/models\#n{"model_name": "openai/gpt-4o", "api_key": "k", "history_collection": "h"}"#, + #"POST /proxy\#n{"url": "http://169.254.169.254/", "method": "GET"}"#, + #"POST /analytics/events\#n{"type": "click", "name": "e", "data": {}}"#, + "POST /collections/books/documents/../../../config\n{}", + ]) + func typesenseRefusesTheRest(sql: String) { + #expect(refusal(sql, on: .typesense) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Typesense reads and document changes", arguments: [ + #"POST /collections/books/documents\#n{"id": "1", "title": "x"}"#, + #"POST /collections/books/documents?action=upsert\#n{"id": "1", "title": "y"}"#, + #"POST /collections/books/documents/import?action=upsert\#n{"id": "1", "title": "a"}\#n{"id": "2", "title": "b"}"#, + #"PATCH /collections/books/documents/1\#n{"title": "z"}"#, + #"PATCH /collections/books/documents?filter_by=year:<2000\#n{"old": true}"#, + "GET /collections/books/documents/search?q=*&query_by=title", + #"POST /multi_search\#n{"searches": [{"collection": "books", "q": "*"}]}"#, + ]) + func typesenseRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .typesense) == nil) + } + + @Test("Edit refuses a Qdrant request that is not a read or a point change", arguments: [ + #"PUT /collections/docs\#n{"vectors": {"size": 4, "distance": "Cosine"}}"#, + #"PATCH /collections/docs\#n{"optimizers_config": {"indexing_threshold": 0}}"#, + #"PUT /collections/docs/index\#n{"field_name": "city", "field_schema": "keyword"}"#, + #"PUT /collections/docs/vectors/v2\#n{"size": 4, "distance": "Dot"}"#, + #"PUT /collections/docs/shards\#n{"shard_key": "k"}"#, + "PUT /quotas\n{}", + #"POST /locks\#n{"write": true, "error_message": "x"}"#, + ]) + func qdrantRefusesTheRest(sql: String) { + #expect(refusal(sql, on: .qdrant) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Qdrant reads and point changes", arguments: [ + #"PUT /collections/docs/points?wait=true\#n{"points": [{"id": 1, "vector": [0.1, 0.2, 0.3, 0.4], "payload": {"city": "Hanoi"}}]}"#, + #"POST /collections/docs/points/payload\#n{"payload": {"a": 1}, "points": [1]}"#, + #"PUT /collections/docs/points/vectors\#n{"points": [{"id": 1, "vector": [0.4, 0.3, 0.2, 0.1]}]}"#, + #"POST /collections/docs/points/query\#n{"query": [0.1, 0.2, 0.3, 0.4], "limit": 3}"#, + "GET /collections/docs", + ]) + func qdrantRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .qdrant) == nil) + } + + @Test("Edit refuses a Weaviate request that is not a read or an object change", arguments: [ + #"POST /v1/schema\#n{"class": "Article"}"#, + #"POST /v1/schema/Article/properties\#n{"name": "x", "dataType": ["text"]}"#, + #"POST /v1/schema/Article/tenants\#n[{"name": "t1"}]"#, + "POST /v1/users/db/bob", + #"POST /v1/authz/users/bob/assign\#n{"roles": ["admin"]}"#, + #"POST /v1/backups/filesystem\#n{"id": "b1"}"#, + #"POST /v1/mcp\#n{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}"#, + #"POST /v1/classifications/\#n{"class": "Article", "classifyProperties": ["cat"], "basedOnProperties": ["title"], "type": "knn"}"#, + #"PUT /v1/objects/../schema/Article\#n{"class": "Article"}"#, + ]) + func weaviateRefusesTheRest(sql: String) { + #expect(refusal(sql, on: .weaviate) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Weaviate reads and object changes", arguments: [ + #"POST /v1/objects\#n{"class": "Article", "properties": {"title": "x"}}"#, + #"PATCH /v1/objects/Article/36ddd591-2dee-4e7e-a3cc-eb86d30a4303\#n{"properties": {"title": "y"}}"#, + #"POST /v1/batch/objects\#n{"objects": [{"class": "Article", "properties": {"title": "a"}}]}"#, + #"POST /v1/batch/references\#n[{"from": "weaviate://localhost/Article/1/author", "to": "weaviate://localhost/Author/2"}]"#, + "{ Get { Article { title } } }", + #"POST /v1/graphql\#n{"query": "{ Get { Article { title } } }"}"#, + "GET /v1/objects?class=Article&limit=10", + ]) + func weaviateRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .weaviate) == nil) + } + + @Test("Edit refuses a Realtime Database write into the server's own nodes", arguments: [ + #"PUT .settings/defaultWriteSizeLimit "unlimited""#, + "PUT .settings/strictTriggerValidation false", + #"PATCH / {".settings/defaultWriteSizeLimit": "unlimited"}"#, + ]) + func realtimeDatabaseRefusesServerNodes(sql: String) { + #expect(refusal(sql, on: .firebaseRealtimeDatabase) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Realtime Database reads and node changes", arguments: [ + #"PUT users/alice {"age": 31}"#, + #"PATCH users/alice {"age": 32}"#, + #"PATCH / {"users/alice/age": 33}"#, + #"POST logs {"msg": "hi"}"#, + #"GET users?orderBy="$key"&limitToFirst=5"#, + ]) + func realtimeDatabaseRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .firebaseRealtimeDatabase) == nil) + } + + @Test("Edit refuses a Firestore write through a dot segment, and the PUT Firestore lacks", arguments: [ + #"PATCH ./../x/y {"a": 1}"#, + #"POST ./../x {"a": 1}"#, + #"PUT users/alice {"age": 31}"#, + ]) + func firestoreRefusesTheRest(sql: String) { + #expect(refusal(sql, on: .firestore) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Firestore reads and document changes", arguments: [ + #"POST users?documentId=alice {"age": 31}"#, + #"PATCH users/alice?updateMask=age {"age": 32}"#, + #"QUERY {"structuredQuery": {"from": [{"collectionId": "users"}]}}"#, + "GET users/alice", + ]) + func firestoreRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .firestore) == nil) + } + + @Test("Edit refuses a Bigtable request the driver cannot read", arguments: ["CREATE events", "DROP events"]) + func bigtableRefusesTheRest(sql: String) { + #expect(refusal(sql, on: .bigtable) == .denied(ExternalStatementGate.dataStatementRefusal)) + } + + @Test("Edit runs Bigtable reads and row changes", arguments: [ + #"INSERT events/user#42 {"profile:name": "Ann"}"#, + #"PUT events/user#42 {"profile:name": "Bo"}"#, + "GET events/user#42", + "SCAN events/user#?limit=20", + #"READ events {"rowsLimit": "10"}"#, + ]) + func bigtableRunsDataRequests(sql: String) { + #expect(refusal(sql, on: .bigtable) == nil) + } + + @Test("Edit runs a delete of one item by its id or key, which is a write", arguments: [ + (DatabaseType.elasticsearch, "DELETE /orders/_doc/1"), + (.typesense, "DELETE /collections/books/documents/1"), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [1]}"#), + (.weaviate, "DELETE /v1/objects/Article/36ddd591-2dee-4e7e-a3cc-eb86d30a4303"), + (.firestore, "DELETE users/alice"), + (.firebaseRealtimeDatabase, "DELETE users/alice"), + (.bigtable, "DELETE events/user#42"), + ]) + func oneItemDeleteRunsInEdit(type: DatabaseType, sql: String) { + #expect(QueryClassifier.isDataStatement(sql, databaseType: type)) + #expect(refusal(sql, on: type) == nil) + } + + @Test("Edit refuses a delete of a container, a batch, a filter or every item as destructive", arguments: [ + (DatabaseType.elasticsearch, "DELETE /orders"), + (.elasticsearch, "DELETE /orders,archive/_doc/1"), + (.elasticsearch, #"POST /orders/_delete_by_query\#n{"query": {"match_all": {}}}"#), + (.typesense, "DELETE /collections/books/documents?filter_by=year:<2000"), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [1, 2]}"#), + (.qdrant, #"POST /collections/docs/points/delete\#n{"filter": {"must": [{"key": "city", "match": {"value": "x"}}]}}"#), + (.weaviate, "DELETE /v1/objects/36ddd591-2dee-4e7e-a3cc-eb86d30a4303"), + (.weaviate, #"DELETE /v1/batch/objects\#n{"match": {"class": "Article", "where": {"operator": "Like", "path": ["id"], "valueText": "*"}}}"#), + (.firestore, "DELETE users"), + (.firebaseRealtimeDatabase, "DELETE users"), + (.firebaseRealtimeDatabase, #"PATCH / {"users": null}"#), + (.firebaseRealtimeDatabase, #"PUT users {"x": {"name": "a"}}"#), + (.firebaseRealtimeDatabase, #"PATCH / {"users": {"x": 1}}"#), + ]) + func manyItemDeleteIsRefusedInEdit(type: DatabaseType, sql: String) { + #expect(refusal(sql, on: type) == .denied(ExternalStatementGate.destructiveRefusal)) + } + + @Test("A write into the engine's own containers is not data, even where another rule refuses it first", arguments: [ + (DatabaseType.firebaseRealtimeDatabase, #"PUT .settings/rules {"rules": {".read": true, ".write": true}}"#), + (.firebaseRealtimeDatabase, #"PUT %2Esettings/rules {"rules": {".read": true}}"#), + (.firestore, "DELETE ./../collectionGroups/users/indexes/abc"), + (.elasticsearch, "DELETE /.security-7/_doc/x"), + ]) + func systemWriteIsNotData(type: DatabaseType, sql: String) { + #expect(!QueryClassifier.isDataStatement(sql, databaseType: type)) + #expect(refusal(sql, on: type) != nil) + } +} diff --git a/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift b/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift index d97525a317..c1ea551ec6 100644 --- a/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift +++ b/apple/TableProTests/Core/Execution/ExternalStatementGateTests.swift @@ -20,7 +20,8 @@ struct ExternalStatementGateTests { loadsExtensions: Bool = false, allowsDestructive: Bool = true, allowsMultiStatement: Bool = false, - destructiveAlternative: String? = nil + destructiveAlternative: String? = nil, + allowsOnlyDataStatements: Bool = false ) -> ExternalStatementGate.Statement { ExternalStatementGate.Statement( sql: sql, @@ -30,7 +31,8 @@ struct ExternalStatementGateTests { loadsExtensions: loadsExtensions, allowsDestructive: allowsDestructive, allowsMultiStatement: allowsMultiStatement, - destructiveAlternative: destructiveAlternative + destructiveAlternative: destructiveAlternative, + allowsOnlyDataStatements: allowsOnlyDataStatements ) } @@ -51,6 +53,55 @@ struct ExternalStatementGateTests { #expect(classification.tier == .safe) } + @Test("A caller held to data statements is refused a grant, a role change or a SET, and only that caller", arguments: [ + "GRANT ALL ON orders TO app", "ALTER ROLE app WITH SUPERUSER", "SET search_path = evil", "SELECT set_config('role', 'admin', false)", + ]) + func dataStatementsOnlyRefusesTheRest(sql: String) { + #expect(refusal(statement(sql, allowsOnlyDataStatements: true)) == .denied(ExternalStatementGate.dataStatementRefusal)) + #expect(refusal(statement(sql)) == nil) + } + + @Test("A caller held to data statements runs reads and row changes") + func dataStatementsOnlyRunsRowWork() { + for sql in ["SELECT * FROM orders", "UPDATE orders SET paid = true WHERE id = 1", "INSERT INTO orders (id) VALUES (1)"] { + #expect(refusal(statement(sql, allowsOnlyDataStatements: true)) == nil) + } + } + + // An engine with a query language of its own is read by its own reader against its curated list. + @Test("A caller held to data statements runs data and refuses a definition on an engine with its own language", arguments: [ + (DatabaseType.mongodb, "db.orders.insertOne({a: 1})", true), + (.kafka, "PRODUCE INTO \"orders\" KEY \"k\" VALUE \"v\"", true), + (.surrealdb, "CREATE person:one SET name = 'a'", true), + (.surrealdb, "DEFINE TABLE person SCHEMAFULL", false), + (.surrealdb, "SELECT fn::mk() FROM ONLY person:one", false), + ]) + func dataStatementsOnlyReadsOtherLanguages(type: DatabaseType, sql: String, runs: Bool) { + #expect(QueryClassifier.isSQLDataStatement(sql, databaseType: type) == nil) + let refused = refusal(statement(sql, databaseType: type, allowsOnlyDataStatements: true)) + #expect(refused == (runs ? nil : .denied(ExternalStatementGate.dataStatementRefusal))) + } + + @Test("Edit runs a SurrealQL DELETE of one record by id, and refuses one of a table or a range as destructive", arguments: [ + ("DELETE person:three", false), + ("DELETE person", true), + ("DELETE person:1..3", true), + ]) + func surrealRecordDeleteInEdit(sql: String, refused: Bool) { + let edit = statement(sql, databaseType: .surrealdb, allowsDestructive: false, allowsOnlyDataStatements: true) + #expect(refusal(edit) == (refused ? .denied(ExternalStatementGate.destructiveRefusal) : nil)) + } + + @Test("A write into a keyspace or database the engine keeps its own state in is a system scope") + func systemScopesOfOtherLanguages() { + #expect(QueryClassifier.writesSystemScope(database: "system_auth", schema: nil, databaseType: .cassandra)) + #expect(QueryClassifier.writesSystemScope(database: "audit", schema: nil, databaseType: .scylladb)) + #expect(!QueryClassifier.writesSystemScope(database: "audit", schema: nil, databaseType: .cassandra)) + #expect(QueryClassifier.writesSystemScope(database: "admin", schema: nil, databaseType: .mongodb)) + #expect(!QueryClassifier.writesSystemScope(database: "shop", schema: nil, databaseType: .mongodb)) + #expect(!QueryClassifier.writesSystemScope(database: "admin", schema: nil, databaseType: .postgresql)) + } + @Test("A MongoDB read passes a connection that is read only for external clients", arguments: [ #"db.c.aggregate([{"$match": {"k": {"$nin": ["x"]}}}])"#, #"db.c.find({}, {"_id": 1}).limit(1)"#, diff --git a/apple/TableProTests/Core/Execution/TabExecutionRegistryTests.swift b/apple/TableProTests/Core/Execution/TabExecutionRegistryTests.swift index 44649cef4f..8bc43894d0 100644 --- a/apple/TableProTests/Core/Execution/TabExecutionRegistryTests.swift +++ b/apple/TableProTests/Core/Execution/TabExecutionRegistryTests.swift @@ -400,6 +400,142 @@ struct TabExecutionRegistryTests { #expect(registry.isStoppable(tabId) == false) } + // MARK: - A stopped claim keeping its work + + @Test("A claim Stop ended keeps its work once, and settle still refuses it") + func stoppedClaimKeepsItsWorkOnce() { + var registry = TabExecutionRegistry() + let tabId = UUID() + let claim = registry.claim(tabId) + + _ = registry.stop(tabId) + + let settled = registry.settle(claim) + #expect(settled == false) + let kept = registry.settleStopped(claim, replacingContent: true) + #expect(kept) + let keptAgain = registry.settleStopped(claim, replacingContent: true) + #expect(keptAgain == false) + #expect(registry.isAnyExecuting == false) + } + + @Test("Keeping a stopped claim's work gives the tab new content only when it replaces the result") + func keepingStoppedWorkChangesTheContentOnlyWhenItReplaces() { + var replacing = TabExecutionRegistry() + let replacingTab = UUID() + let replacingClaim = replacing.claim(replacingTab) + _ = replacing.stop(replacingTab) + let replacedContent = replacing.contentEpoch(for: replacingTab) + + let replaced = replacing.settleStopped(replacingClaim, replacingContent: true) + + #expect(replaced) + #expect(replacing.isSameContent(replacedContent, for: replacingTab) == false) + + var keeping = TabExecutionRegistry() + let keepingTab = UUID() + let keepingClaim = keeping.claim(keepingTab) + _ = keeping.stop(keepingTab) + let keptContent = keeping.contentEpoch(for: keepingTab) + + let kept = keeping.settleStopped(keepingClaim, replacingContent: false) + + #expect(kept) + #expect(keeping.isSameContent(keptContent, for: keepingTab)) + } + + @Test("A second Stop with nothing running leaves the stopped claim its work") + func secondStopKeepsTheStoppedClaim() { + var registry = TabExecutionRegistry() + let tabId = UUID() + let claim = registry.claim(tabId) + _ = registry.stop(tabId) + + let second = registry.stop(tabId) + + #expect(second.ended.isEmpty) + let kept = registry.settleStopped(claim, replacingContent: true) + #expect(kept) + } + + @Test("A new run after Stop leaves the stopped claim nothing to write") + func newRunForgetsTheStoppedClaim() { + var registry = TabExecutionRegistry() + let tabId = UUID() + let stopped = registry.claim(tabId) + _ = registry.stop(tabId) + + let successor = registry.claim(tabId) + + let kept = registry.settleStopped(stopped, replacingContent: true) + #expect(kept == false) + #expect(registry.isCurrent(successor)) + } + + @Test( + "A retarget, a tab close or a lost session after Stop leaves the stopped claim nothing to write", + arguments: [ExecutionEndReason.supersededNavigation, .abandoned, .sessionEnded] + ) + func invalidationForgetsTheStoppedClaim(reason: ExecutionEndReason) { + var registry = TabExecutionRegistry() + let tabId = UUID() + let stopped = registry.claim(tabId) + _ = registry.stop(tabId) + + _ = registry.invalidate(tabId, reason: reason) + + let kept = registry.settleStopped(stopped, replacingContent: true) + #expect(kept == false) + + var all = TabExecutionRegistry() + let allTabId = UUID() + let allStopped = all.claim(allTabId) + _ = all.stop(allTabId) + + _ = all.invalidateAll(reason: reason) + + let keptAll = all.settleStopped(allStopped, replacingContent: true) + #expect(keptAll == false) + } + + @Test("Stop records nothing for a committing claim it keeps") + func keptCommittingClaimIsNotStopped() { + var registry = TabExecutionRegistry() + let tabId = UUID() + let claim = registry.claim(tabId) + let marked = registry.enterUninterruptiblePhase(claim) + #expect(marked) + + let outcome = registry.stop(tabId) + + #expect(outcome.keptUninterruptibleClaim) + let kept = registry.settleStopped(claim, replacingContent: true) + #expect(kept == false) + let settled = registry.settle(claim) + #expect(settled) + } + + @Test("Only the claim Stop ended, on its own tab, keeps its work") + func onlyTheStoppedClaimKeepsItsWork() { + var registry = TabExecutionRegistry() + let tabId = UUID() + let other = UUID() + let earlier = registry.claim(tabId) + _ = registry.stop(tabId) + let later = registry.claim(tabId) + let otherClaim = registry.claim(other) + _ = registry.stop(tabId) + + let keptEarlier = registry.settleStopped(earlier, replacingContent: true) + #expect(keptEarlier == false) + let keptOther = registry.settleStopped(otherClaim, replacingContent: true) + #expect(keptOther == false) + let keptLater = registry.settleStopped(later, replacingContent: true) + #expect(keptLater) + let settledOther = registry.settle(otherClaim) + #expect(settledOther) + } + @Test("A run's start is the claim's own, and work with no claim reports none") func startedAtIsTheClaimsOwn() { var registry = TabExecutionRegistry() diff --git a/apple/TableProTests/Core/Export/TableTransferServiceTests.swift b/apple/TableProTests/Core/Export/TableTransferServiceTests.swift index 127e7522a9..c268bb1847 100644 --- a/apple/TableProTests/Core/Export/TableTransferServiceTests.swift +++ b/apple/TableProTests/Core/Export/TableTransferServiceTests.swift @@ -9,7 +9,6 @@ import Testing @testable import TablePro -/// Serves one table's rows as a source, and records what reaches it as a destination. private final class TransferStubDriver: PluginDriverStub, @unchecked Sendable { let header: [String] let rows: [PluginRow] @@ -104,8 +103,7 @@ struct TableTransferServiceTests { #expect(values["email"] == .text("ada@example.com")) } - /// A driver that stops sending values once the rest of a row is null would otherwise lose the - /// whole row: the sink writes by column name, and a missing key is not a null. + // A driver can stop sending values once the rest of a row is null, and the sink writes by column name: a missing key is not a null. @Test("A row shorter than its header is padded with nulls") func shortRowIsPadded() { let values = TableTransferService.dictionary( @@ -143,8 +141,7 @@ struct TableTransferServiceTests { #expect(values["missing"] == .null) } - /// The transfer moves rows, so a request naming only definition objects has nothing to do and - /// must say so rather than reporting a successful transfer of nothing. + // Refused rather than reported as a successful transfer of nothing. @MainActor @Test("A request with no row-carrying object is refused") func requestWithoutTablesIsRefused() { let service = TableTransferService() @@ -195,8 +192,7 @@ struct TableTransferServiceTests { } } - /// The server refused the INSERT only after "Delete existing rows first" had run, and with no - /// transaction around the table the deletion stood: the destination was left empty. + // The server refused the INSERT after "Delete existing rows first" ran outside a transaction, which left the destination empty. @MainActor @Test("A contested mapping deletes nothing on the destination") func contestedMappingDeletesNothing() async { let source = TransferStubDriver(header: ["first_name", "last_name"], rows: [[.text("Ada"), .text("Lovelace")]]) @@ -226,8 +222,7 @@ struct TableTransferServiceTests { #expect(destination.executedQueries.isEmpty) } - /// `Name` and `name` both matched a lone `name`, and even with only one of them mapped the - /// sink folded the other onto it, so the INSERT named the column twice. + // `Name` and `name` both matched a lone `name`, and the sink folded the unmapped one onto it, so the INSERT named the column twice. @MainActor @Test("A source column that differs only by case from a mapped one is not written twice") func caseTwinIsNotWrittenTwice() async throws { let source = TransferStubDriver( @@ -351,7 +346,6 @@ struct TableTransferServiceTests { #expect(destination.executedParameters.last == [.text("5"), .text("1")]) } - // MySQL's header says BIT without the width, so the digits take the catalog's BIT(8). @MainActor @Test("A MySQL BIT transferred to a PostgreSQL bit varying arrives as its digits, at the source's width") func mysqlBitReachesPostgresAsItsDigits() async throws { let source = TransferStubDriver( @@ -359,7 +353,7 @@ struct TableTransferServiceTests { rows: [[.text("1"), .text("5")]], columns: ["id", "flags"], types: ["id": "int", "flags": "bit(8)"], - headerTypes: ["id": "INT", "flags": "BIT"] + headerTypes: ["id": "INT", "flags": "BIT(8)"] ) let destination = TransferStubDriver(columns: ["id", "flags"], types: ["id": "integer", "flags": "varbit"]) let request = TableTransferService.Request( diff --git a/apple/TableProTests/Core/LSP/LSPDocumentManagerTests.swift b/apple/TableProTests/Core/LSP/LSPDocumentManagerTests.swift new file mode 100644 index 0000000000..c9f6a62ef3 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPDocumentManagerTests.swift @@ -0,0 +1,41 @@ +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct LSPDocumentManagerTests { + @Test("a version starts at zero on open, rises by one per change and starts over on a reopen") + func versionsFollowOpenAndChange() { + let manager = LSPDocumentManager() + let uri = "file:///query-1.sql" + let edit = [LSPTextDocumentContentChangeEvent(text: "SELECT 1")] + + #expect(manager.changeDocument(uri: uri, changes: edit)?.textDocument.version == nil) + #expect(manager.openDocument(uri: uri, languageId: "sql", text: "").version == 0) + #expect(manager.changeDocument(uri: uri, changes: edit)?.textDocument.version == 1) + #expect(manager.changeDocument(uri: uri, changes: edit)?.textDocument.version == 2) + + _ = manager.openDocument(uri: uri, languageId: "sql", text: "") + #expect(manager.version(for: uri) == 0) + #expect(manager.closeDocument(uri: uri)?.uri == uri) + #expect(manager.version(for: uri) == nil) + #expect(manager.closeDocument(uri: uri)?.uri == nil) + } + + @Test("a full text change encodes no range, and a range change round-trips with it") + func changeEventEncodesItsRangeOnlyWhenSet() throws { + let full = try JSONSerialization.jsonObject( + with: JSONEncoder().encode(LSPTextDocumentContentChangeEvent(text: "x")) + ) as? [String: Any] + #expect(full?.keys.sorted() == ["text"]) + + let ranged = LSPTextDocumentContentChangeEvent( + range: LSPRange(start: LSPPosition(line: 3, character: 1), end: LSPPosition(line: 4, character: 0)), + text: "y" + ) + let data = try JSONEncoder().encode(ranged) + let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] + #expect(object?.keys.sorted() == ["range", "text"]) + #expect(try JSONDecoder().decode(LSPTextDocumentContentChangeEvent.self, from: data) == ranged) + } +} diff --git a/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift b/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift new file mode 100644 index 0000000000..7af8c30567 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift @@ -0,0 +1,48 @@ +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +struct LSPTransportBrokenPipeTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + // A server on its way out: it stops reading, asks the client something, then says it has stopped. + private static let closingServer = #""" + close STDIN; $| = 1; + for my $body ('{"jsonrpc":"2.0","id":7,"method":"workspace/configuration","params":{}}', + '{"jsonrpc":"2.0","method":"test/inputClosed"}') { + print "Content-Length: " . length($body) . "\r\n\r\n" . $body; + } + sleep 30; + """# + + @Test( + "A server that stopped reading fails the request instead of ending the app", + .enabled(if: LSPTransportBrokenPipeTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func closedInputFailsTheRequest() async throws { + let transport = LSPTransport() + let inputClosed = AsyncGate() + await transport.onNotification(method: "test/inputClosed") { _ in + Task { await inputClosed.open() } + } + try await transport.start(executablePath: Self.perl, arguments: ["-e", Self.closingServer]) + await inputClosed.wait() + + let error = await #expect(throws: (any Error).self) { + _ = try await transport.sendRequest(method: "test/afterClose", params: EmptyLSPParams()) + } + await transport.stop() + + let failure = error.map { $0 as NSError } + let posix = failure?.domain == NSPOSIXErrorDomain + ? failure + : failure?.underlyingErrors.first.map { $0 as NSError } + #expect(posix?.code == Int(EPIPE)) + } +} diff --git a/apple/TableProTests/Core/LSP/LSPTransportReadTests.swift b/apple/TableProTests/Core/LSP/LSPTransportReadTests.swift new file mode 100644 index 0000000000..c749a76575 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPTransportReadTests.swift @@ -0,0 +1,58 @@ +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +struct LSPTransportReadTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + // The pause lets the test's AsyncBytes read park before the second message is written. + private static let pausingServer = #""" + $| = 1; + for my $method ('test/first', 'test/second') { + my $body = '{"jsonrpc":"2.0","method":"' . $method . '"}'; + print "Content-Length: " . length($body) . "\r\n\r\n" . $body; + select(undef, undef, undef, 0.3); + } + sleep 30; + """# + + @Test( + "Server messages arrive while an AsyncBytes read elsewhere in the app waits on a quiet pipe", + .enabled(if: LSPTransportReadTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func messagesArriveBesideAParkedAsyncBytesRead() async throws { + let quiet = Pipe() + defer { withExtendedLifetime(quiet) {} } + let quietOutput = quiet.fileHandleForReading + let parked = Task { + for try await _ in quietOutput.bytes {} + } + + let transport = LSPTransport() + let first = AsyncGate() + let second = AsyncGate() + await transport.onNotification(method: "test/first") { _ in + Task { await first.open() } + } + await transport.onNotification(method: "test/second") { _ in + Task { await second.open() } + } + try await transport.start(executablePath: Self.perl, arguments: ["-e", Self.pausingServer]) + + let arrived: Void? = await HeldOpenWriter(quiet.fileHandleForWriting).finished { + await first.wait() + await second.wait() + } + await transport.stop() + + try #require(arrived != nil) + try quiet.fileHandleForWriting.close() + _ = await parked.result + } +} diff --git a/apple/TableProTests/Core/MCP/CLIPairingContractTests.swift b/apple/TableProTests/Core/MCP/CLIPairingContractTests.swift index e5bd5729df..56d632522a 100644 --- a/apple/TableProTests/Core/MCP/CLIPairingContractTests.swift +++ b/apple/TableProTests/Core/MCP/CLIPairingContractTests.swift @@ -30,7 +30,7 @@ struct CLIPairingContractTests { #expect(PairingExchangeStore.sha256Base64Url(of: request.verifier) == request.challenge) } - @Test("The app's approve and deny redirects reach the command as approved and denied") + @Test("The app's approve, deny and failure redirects reach the command as approved, denied and failed") func redirectsRoundTrip() throws { let request = CLIPairingRequest.random() let parsed = try pairingRequest(request) @@ -44,6 +44,11 @@ struct CLIPairingContractTests { base: parsed.redirectURL, state: parsed.state, mode: parsed.responseMode )) #expect(CLIPairingCallback.parse(requestTarget: Self.target(of: denied), state: request.state) == .denied) + + let failed = try #require(PairingRedirectBuilder.failed( + base: parsed.redirectURL, state: parsed.state, mode: parsed.responseMode + )) + #expect(CLIPairingCallback.parse(requestTarget: Self.target(of: failed), state: request.state) == .failed) } // What a browser puts on the request line for the URL it was handed. diff --git a/apple/TableProTests/Core/MCP/Clients/MCPClientRowPresentationTests.swift b/apple/TableProTests/Core/MCP/Clients/MCPClientRowPresentationTests.swift new file mode 100644 index 0000000000..72412c8d1c --- /dev/null +++ b/apple/TableProTests/Core/MCP/Clients/MCPClientRowPresentationTests.swift @@ -0,0 +1,179 @@ +import Foundation +@testable import TablePro +import TableProMCPClients +import Testing + +struct MCPClientRowPresentationTests { + private func row( + config: MCPClientConfigState = .notConfigured, + isInstalled: Bool = true, + installRoute: MCPClientRoute = .file, + updateRoute: MCPClientRoute = .file, + removeRoute: MCPClientRoute? = nil, + unsupported: MCPClientUnsupported? = nil, + isConfigWritable: Bool = true, + commandLine: String? = nil + ) -> MCPClientRow { + MCPClientRow( + id: "cursor", + name: "Cursor", + iconSlug: "cursor", + isDetected: isInstalled, + isInstalled: isInstalled, + config: config, + installRoute: installRoute, + updateRoute: updateRoute, + removeRoute: removeRoute, + unsupported: unsupported, + applies: .live, + configPath: "/Users/me/.cursor/mcp.json", + displayConfigPath: "~/.cursor/mcp.json", + configFileExists: true, + isConfigWritable: isConfigWritable, + snippet: "{}", + commandLine: commandLine + ) + } + + private func present( + _ row: MCPClientRow, + activity: MCPClientActivity? = nil, + connection: MCPClientConnection = .notConnected + ) -> MCPClientRowPresentation { + MCPClientRowPresentation(row: row, activity: activity, connection: connection) + } + + @Test("A client that hands off to its own app offers Install with an ellipsis") + func deeplinkInstall() { + let presentation = present(row(installRoute: .deeplink(scheme: "cursor"))) + #expect(presentation.action == .install(handsOff: true)) + #expect(presentation.action?.title == String(localized: "Install…")) + #expect(presentation.subtitle == String(format: String(localized: "Opens %@ to confirm"), "Cursor")) + #expect(presentation.status == nil) + } + + @Test("A client set up through its command or its file offers a plain Install") + func inPlaceInstall() { + let viaCommand = present(row(installRoute: .cli(binary: "claude"))) + #expect(viaCommand.action?.title == String(localized: "Install")) + #expect(viaCommand.subtitle == String(format: String(localized: "Adds through the %@ command"), "claude")) + + let viaFile = present(row(installRoute: .file)) + #expect(viaFile.action == .install(handsOff: false)) + #expect(viaFile.subtitle == String(format: String(localized: "Edits %@"), "~/.cursor/mcp.json")) + } + + @Test("A client set up for this copy offers no button and never offers Remove as one") + func currentHasNoButton() { + let presentation = present(row(config: .current, removeRoute: .file)) + #expect(presentation.action == nil) + #expect(presentation.status == String(localized: "Set up")) + } + + @Test("A client set up for a moved or other copy offers Update") + func staleOffersUpdate() { + let moved = present(row(config: .moved)) + #expect(moved.action == .update(handsOff: false)) + #expect(moved.status == String(localized: "TablePro moved")) + + let otherCopy = present(row(config: .otherCopy, updateRoute: .deeplink(scheme: "cursor"))) + #expect(otherCopy.action?.title == String(localized: "Update…")) + #expect(otherCopy.status == String(localized: "Uses another copy")) + } + + @Test("A client that is not on this Mac offers Copy") + func notFoundOffersCopy() { + let presentation = present(row(isInstalled: false, installRoute: .copy, updateRoute: .copy)) + #expect(presentation.action == .copy) + #expect(presentation.status == String(localized: "Not found")) + #expect(presentation.subtitle == String(format: String(localized: "Copy the setup into %@"), "~/.cursor/mcp.json")) + } + + @Test("A settings file TablePro cannot read is never written, so the row offers Copy") + func unreadableOffersCopy() { + let presentation = present(row(config: .unreadable, installRoute: .file)) + #expect(presentation.action == .copy) + #expect(presentation.status == String(localized: "Can't read settings")) + } + + @Test("A client that cannot reach a local server says why and offers nothing") + func unsupportedHasNoAction() { + let presentation = present(row(unsupported: .remoteOnly), connection: .connected) + #expect(presentation.action == nil) + #expect(presentation.status == nil) + #expect(presentation.subtitle == String(localized: "Reaches only servers on the internet")) + } + + @Test("A running action replaces the button and names what it does") + func busyRow() { + let presentation = present(row(), activity: .running(.install)) + #expect(presentation.isBusy) + #expect(presentation.action == nil) + #expect(presentation.status == String(localized: "Installing…")) + } + + @Test("A live session outranks the settings status, and a new one reads just now") + func connectionStatus() { + #expect(present(row(config: .current), connection: .connected).status == String(localized: "Connected")) + #expect(present(row(config: .current), connection: .justConnected).status == String(localized: "Connected just now")) + } + + @Test("A failed action stays on the row until the next one") + func failedStatus() { + let presentation = present(row(config: .moved), activity: .failed(.update, message: "boom", output: nil)) + #expect(presentation.status == String(localized: "Couldn't update")) + #expect(presentation.action == .update(handsOff: false)) + } + + @Test("A file only the client writes is copied as its command") + func commandOnlyCopy() { + let claudeCode = row(isConfigWritable: false, commandLine: "claude mcp add --scope user tablepro -- /x") + #expect(claudeCode.copyText == "claude mcp add --scope user tablepro -- /x") + #expect(row(commandLine: "codex mcp add tablepro -- /x").copyText == "{}") + } + + @Test("A token setup, a setup turned off in the client and a project override offer no button") + func userChoicesHaveNoButton() { + let token = present(row(config: .usesToken, removeRoute: .file)) + #expect(token.action == nil) + #expect(token.status == String(localized: "Set up with a token")) + #expect(MCPClientDetailText.configState(.usesToken, name: "Cursor") != nil) + + let off = present(row(config: .disabled)) + #expect(off.action == nil) + #expect(off.status == String(localized: "Turned off")) + #expect(MCPClientDetailText.configState(.disabled, name: "Cursor") != nil) + + let shadowed = present(row(config: .shadowed(project: "~/work/shop"))) + #expect(shadowed.action == nil) + #expect(shadowed.status == String(localized: "Old setup in a project")) + #expect(MCPClientDetailText.configState(.shadowed(project: "~/work/shop"), name: "Cursor")?.contains("~/work/shop") == true) + } + + @Test("The engine's statuses map to row states that keep the entry") + func statusMapping() { + #expect(MCPClientConfigState(.usesToken) == .usesToken) + #expect(MCPClientConfigState(.disabled) == .disabled) + let shadowed = MCPClientConfigState(.stale(.shadowed(scope: "/Users/me/shop"))) { $0.replacingOccurrences(of: "/Users/me", with: "~") } + #expect(shadowed == .shadowed(project: "~/shop")) + #expect(MCPClientConfigState.usesToken.hasEntry) + #expect(!MCPClientConfigState.usesToken.needsUpdate) + #expect(!MCPClientConfigState.shadowed(project: "~/shop").needsUpdate) + #expect(MCPClientConfigState.usesToken.isUserChoice) + #expect(MCPClientConfigState.disabled.isUserChoice) + #expect(!MCPClientConfigState.otherCopy.isUserChoice) + } + + @Test("Each row button's spoken label names its client") + func accessibilityLabels() { + let labels = [ + MCPClientRowAction.install(handsOff: true).accessibilityLabel(clientName: "Cursor"), + MCPClientRowAction.update(handsOff: false).accessibilityLabel(clientName: "Cursor"), + MCPClientRowAction.copy.accessibilityLabel(clientName: "Cursor"), + ] + #expect(labels.allSatisfy { $0.contains("Cursor") }) + #expect(Set(labels).count == 3) + #expect(MCPClientRowAction.install(handsOff: false).accessibilityLabel(clientName: "Zed") + != MCPClientRowAction.install(handsOff: false).accessibilityLabel(clientName: "Cursor")) + } +} diff --git a/apple/TableProTests/Core/MCP/Clients/MCPClientSetupFixture.swift b/apple/TableProTests/Core/MCP/Clients/MCPClientSetupFixture.swift new file mode 100644 index 0000000000..ed1a92e6b3 --- /dev/null +++ b/apple/TableProTests/Core/MCP/Clients/MCPClientSetupFixture.swift @@ -0,0 +1,252 @@ +import Combine +import Foundation +@testable import TablePro +import TableProMCPClients + +enum MCPClientSetupFixture { + static let catalogJSON = #""" + { + "schemaVersion": 1, + "serverName": "tablepro", + "configs": { + "editor": { "macos": { "path": "~/.editor/mcp.json", "format": "json", "keyPath": ["mcpServers"] } }, + "agent": { + "macos": { "path": "~/.agent.json", "format": "json", "keyPath": ["mcpServers"], "writable": false, "otherScopes": [["projects", "*", "mcpServers"]] } + }, + "shared": { "macos": { "path": "~/.shared/config.toml", "format": "toml", "keyPath": ["mcp_servers"] } } + }, + "clients": [ + { + "id": "editor", "name": "Editor", "icon": "editor", "config": "editor", + "clientInfo": { "names": ["editor-client"], "prefixes": [] }, + "macos": { + "detect": [{ "bundleId": "com.example.editor" }, { "path": "~/.editor" }], + "entry": { "command": "{command}", "args": [], "tools": ["*"] }, + "install": [ + { "kind": "deeplink", "scheme": "editor", "url": "editor://install?name={name}&config={entry|json|base64|uri}" }, + { "kind": "cli", "binary": "editor", "add": ["mcp", "add", "{name}", "--", "{command}"], "remove": ["mcp", "remove", "{name}"], "update": "add" }, + { "kind": "file" } + ], + "applies": "live" + }, + "linux": { + "entry": { "url": "{url}", "headers": { "Authorization": "{authorization}" } }, + "install": [{ "kind": "file" }] + } + }, + { + "id": "agent", "name": "Agent", "icon": "agent", "config": "agent", + "clientInfo": { "names": ["agent-cli"], "prefixes": [] }, + "macos": { + "detect": [{ "binary": "agent", "paths": ["~/bin/agent"] }], + "entry": { "type": "stdio", "command": "{command}", "args": [] }, + "install": [ + { + "kind": "cli", "binary": "agent", + "add": ["mcp", "add", "--scope", "user", "{name}", "--", "{command}"], + "remove": ["mcp", "remove", "--scope", "user", "{name}"], + "update": "removeThenAdd" + } + ], + "applies": "newSession" + }, + "linux": { + "entry": { "type": "http", "url": "{url}", "headers": { "Authorization": "{authorization}" } }, + "install": [ + { "kind": "cli", "binary": "agent", "add": ["mcp", "add", "--transport", "http", "{name}", "{url}", "--header", "Authorization: {authorization}"] } + ] + } + }, + { + "id": "desk", "name": "Desk", "icon": "desk", "config": "shared", + "clientInfo": { "names": ["shared-client"], "prefixes": [] }, + "macos": { + "detect": [{ "bundleId": "com.example.desk" }], + "entry": { "command": "{command}", "args": [] }, + "install": [{ "kind": "file" }], + "applies": "restartApp" + } + }, + { + "id": "term", "name": "Term", "icon": "term", "config": "shared", + "clientInfo": { "names": ["shared-client"], "prefixes": [] }, + "macos": { + "detect": [{ "path": "~/.shared" }], + "entry": { "command": "{command}", "args": [] }, + "install": [{ "kind": "file" }], + "applies": "newSession" + } + }, + { + "id": "remote", "name": "Remote", "icon": "remote", "config": null, + "clientInfo": { "names": [], "prefixes": [] }, + "macos": { "detect": [{ "bundleId": "com.example.remote" }], "unsupported": "remoteOnly" } + }, + { + "id": "penguin", "name": "Penguin", "icon": "penguin", "config": "editor", + "clientInfo": { "names": ["penguin"], "prefixes": [] }, + "linux": { "detect": [{ "path": "~/.penguin" }], "entry": { "url": "{url}" }, "install": [{ "kind": "file" }] } + } + ] + } + """# + + static func catalog() throws -> MCPClientCatalog { + try MCPClientCatalog.decode(Data(catalogJSON.utf8)) + } +} + +// A home folder of its own, so no test reads or writes the real one. +final class MCPClientTestHome { + let root: URL + let bridge: URL + + init() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-clients-\(UUID().uuidString.prefix(8))", isDirectory: true) + bridge = root.appendingPathComponent("Applications/TablePro.app/Contents/MacOS/tablepro-mcp") + try FileManager.default.createDirectory(at: bridge.deletingLastPathComponent(), withIntermediateDirectories: true) + FileManager.default.createFile(atPath: bridge.path, contents: Data()) + } + + deinit { + try? FileManager.default.removeItem(at: root) + } + + var path: String { root.path } + + func url(_ relative: String) -> URL { + root.appendingPathComponent(relative) + } + + func write(_ relative: String, _ text: String) throws { + let file = url(relative) + try FileManager.default.createDirectory(at: file.deletingLastPathComponent(), withIntermediateDirectories: true) + try Data(text.utf8).write(to: file) + } + + func read(_ relative: String) -> String? { + try? String(contentsOf: url(relative), encoding: .utf8) + } + + func makeFolder(_ relative: String) throws { + try FileManager.default.createDirectory(at: url(relative), withIntermediateDirectories: true) + } + + func makeExecutable(_ relative: String) throws { + try write(relative, "#!/bin/sh\n") + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: url(relative).path) + } + + func remove(_ relative: String) throws { + try FileManager.default.removeItem(at: url(relative)) + } + + func context(workspace: FakeMCPClientWorkspace) -> MCPClientSetupContext { + let home = path + return MCPClientSetupContext( + home: home, + environment: [:], + bridgePath: bridge.path, + workspace: workspace, + isExecutable: { $0.hasPrefix(home + "/") && FileManager.default.isExecutableFile(atPath: $0) } + ) + } +} + +final class FakeMCPClientWorkspace: MCPClientWorkspace, @unchecked Sendable { + private let lock = NSLock() + private let installedBundleIDs: Set + private let handledSchemes: Set + private var openedURLs: [URL] = [] + + init(installedBundleIDs: Set = [], handledSchemes: Set = []) { + self.installedBundleIDs = installedBundleIDs + self.handledSchemes = handledSchemes + } + + var opened: [URL] { + lock.lock() + defer { lock.unlock() } + return openedURLs + } + + func applicationURL(forBundleIdentifier identifier: String) -> URL? { + installedBundleIDs.contains(identifier) ? URL(fileURLWithPath: "/Applications/\(identifier).app") : nil + } + + func applicationURL(toOpen url: URL) -> URL? { + guard let scheme = url.scheme, handledSchemes.contains(scheme) else { return nil } + return URL(fileURLWithPath: "/Applications/\(scheme).app") + } + + func open(_ url: URL) async throws { + lock.withLock { openedURLs.append(url) } + } +} + +final class FakeMCPClientRunner: MCPClientCommandRunning, @unchecked Sendable { + struct Call: Equatable { + let executable: String + let arguments: [String] + let home: String? + } + + private let lock = NSLock() + private var replies: [MCPClientCommandResult] + private var recorded: [Call] = [] + + init(replies: [MCPClientCommandResult] = []) { + self.replies = replies + } + + var calls: [Call] { + lock.lock() + defer { lock.unlock() } + return recorded + } + + func run(executable: URL, arguments: [String], environment: [String: String]) async throws -> MCPClientCommandResult { + lock.withLock { + recorded.append(Call(executable: executable.path, arguments: arguments, home: environment["HOME"])) + return replies.isEmpty ? MCPClientCommandResult(exitCode: 0, output: "") : replies.removeFirst() + } + } +} + +@MainActor +final class MCPClientAnnouncements { + private(set) var messages: [String] = [] + + func record(_ message: String) { + messages.append(message) + } +} + +@MainActor +struct MCPClientServiceHarness { + let service: MCPClientSetupService + let sessions: PassthroughSubject<[MCPClientSessionSighting], Never> + let announcements: MCPClientAnnouncements + + init( + home: MCPClientTestHome, + workspace: FakeMCPClientWorkspace = FakeMCPClientWorkspace(), + runner: FakeMCPClientRunner = FakeMCPClientRunner(), + isAppOnReadOnlyVolume: Bool = false + ) throws { + let sessions = PassthroughSubject<[MCPClientSessionSighting], Never>() + let announcements = MCPClientAnnouncements() + self.sessions = sessions + self.announcements = announcements + service = MCPClientSetupService( + catalog: try MCPClientSetupFixture.catalog(), + context: home.context(workspace: workspace), + runner: runner, + isAppOnReadOnlyVolume: isAppOnReadOnlyVolume, + sessions: sessions.eraseToAnyPublisher(), + activations: Empty().eraseToAnyPublisher(), + announce: { announcements.record($0) } + ) + } +} diff --git a/apple/TableProTests/Core/MCP/Clients/MCPClientSetupServiceTests.swift b/apple/TableProTests/Core/MCP/Clients/MCPClientSetupServiceTests.swift new file mode 100644 index 0000000000..87e6540b5b --- /dev/null +++ b/apple/TableProTests/Core/MCP/Clients/MCPClientSetupServiceTests.swift @@ -0,0 +1,429 @@ +import Combine +import Foundation +@testable import TablePro +import TableProMCPClients +import Testing + +@MainActor +struct MCPClientSetupServiceTests { + @Test("Each client this Mac can run is listed with the state of its settings") + func rowsCarryConfigStatus() async throws { + let home = try MCPClientTestHome() + try home.write(".editor/mcp.json", #"{ "mcpServers": { "tablepro": { "command": "\#(home.bridge.path)", "args": [] } } }"#) + try home.write(".shared/config.toml", "[mcp_servers.tablepro]\ncommand = \"/Volumes/Gone/TablePro.app/Contents/MacOS/tablepro-mcp\"\nargs = []\n") + let harness = try MCPClientServiceHarness( + home: home, + workspace: FakeMCPClientWorkspace(installedBundleIDs: ["com.example.desk"]) + ) + + await harness.service.refresh() + + let service = harness.service + #expect(service.rows.map { $0.id } == ["editor", "agent", "desk", "term", "remote"]) + #expect(service.row(id: "editor")?.config == .current) + #expect(service.row(id: "agent")?.config == .notConfigured) + #expect(service.row(id: "agent")?.isInstalled == false) + #expect(service.row(id: "desk")?.config == .moved) + #expect(service.row(id: "desk")?.isInstalled == true) + #expect(service.row(id: "term")?.config == .moved) + #expect(service.row(id: "remote")?.unsupported == .remoteOnly) + } + + @Test("Install takes the first method this Mac offers, in the catalog's order") + func installRouteFollowsAvailability() async throws { + let home = try MCPClientTestHome() + try home.makeFolder(".editor") + + let withHandler = try MCPClientServiceHarness(home: home, workspace: FakeMCPClientWorkspace(handledSchemes: ["editor"])) + await withHandler.service.refresh() + #expect(withHandler.service.row(id: "editor")?.installRoute == .deeplink(scheme: "editor")) + + try home.makeExecutable(".local/bin/editor") + let withCommand = try MCPClientServiceHarness(home: home) + await withCommand.service.refresh() + #expect(withCommand.service.row(id: "editor")?.installRoute == .cli(binary: "editor")) + + try home.remove(".local/bin/editor") + let withFileOnly = try MCPClientServiceHarness(home: home) + await withFileOnly.service.refresh() + #expect(withFileOnly.service.row(id: "editor")?.installRoute == .file) + + try home.remove(".editor") + let notFound = try MCPClientServiceHarness(home: home) + await notFound.service.refresh() + let row = try #require(notFound.service.row(id: "editor")) + #expect(row.installRoute == .copy) + #expect(notFound.service.presentation(for: row).action == .copy) + } + + @Test("Updating a client whose command cannot replace removes first, and a missing entry counts as removed") + func removeThenAddUpdate() async throws { + let home = try MCPClientTestHome() + try home.makeExecutable("bin/agent") + try home.write(".agent.json", #"{ "mcpServers": { "tablepro": { "type": "stdio", "command": "/Volumes/Gone/tablepro-mcp", "args": [] } } }"#) + let runner = FakeMCPClientRunner(replies: [ + MCPClientCommandResult(exitCode: 1, output: "No MCP server named tablepro found in user config"), + MCPClientCommandResult(exitCode: 0, output: "Added stdio MCP server tablepro to user config") + ]) + let harness = try MCPClientServiceHarness(home: home, runner: runner) + await harness.service.refresh() + let row = try #require(harness.service.row(id: "agent")) + #expect(row.config == .moved) + #expect(MCPClientRowPresentation.action(row: row) == .update(handsOff: false)) + + let outcome = await harness.service.update("agent") + + #expect(outcome == .done(backupPath: nil)) + #expect(runner.calls.map { $0.arguments } == [ + ["mcp", "remove", "--scope", "user", "tablepro"], + ["mcp", "add", "--scope", "user", "tablepro", "--", home.bridge.path] + ]) + #expect(runner.calls.allSatisfy { $0.executable == home.url("bin/agent").path }) + #expect(runner.calls.allSatisfy { $0.home == home.path }) + } + + @Test("A command that fails keeps its output for the detail area") + func failedCommandKeepsOutput() async throws { + let home = try MCPClientTestHome() + try home.makeExecutable("bin/agent") + let runner = FakeMCPClientRunner(replies: [MCPClientCommandResult(exitCode: 2, output: "permission denied\n")]) + let harness = try MCPClientServiceHarness(home: home, runner: runner) + await harness.service.refresh() + + let outcome = await harness.service.install("agent") + + guard case let .failed(_, output) = outcome else { + Issue.record("Expected a failure, got \(outcome)") + return + } + #expect(output == "permission denied") + let row = try #require(harness.service.row(id: "agent")) + #expect(harness.service.presentation(for: row).status == String(localized: "Couldn't install")) + } + + @Test("A copy running from a disk image refuses to install and writes nothing") + func readOnlyVolumeRefuses() async throws { + let home = try MCPClientTestHome() + try home.makeFolder(".editor") + try home.makeExecutable("bin/agent") + let runner = FakeMCPClientRunner() + let harness = try MCPClientServiceHarness(home: home, runner: runner, isAppOnReadOnlyVolume: true) + await harness.service.refresh() + + #expect(await harness.service.install("editor") == .refusedReadOnlyVolume) + #expect(await harness.service.install("agent") == .refusedReadOnlyVolume) + #expect(home.read(".editor/mcp.json") == nil) + #expect(runner.calls.isEmpty) + } + + @Test("Install merges TablePro into the settings file and Remove takes only its entry out") + func fileInstallAndRemove() async throws { + let home = try MCPClientTestHome() + try home.write(".editor/mcp.json", #"{ "mcpServers": { "other": { "command": "other-server" } } }"#) + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + #expect(harness.service.row(id: "editor")?.installRoute == .file) + + let installed = await harness.service.install("editor") + + guard case .done(let backup) = installed else { + Issue.record("Expected the install to finish, got \(installed)") + return + } + #expect(backup?.hasSuffix("mcp.json.tablepro-backup") == true) + let servers = try Self.servers(in: try #require(home.read(".editor/mcp.json"))) + #expect((servers["tablepro"] as? [String: Any])?["command"] as? String == home.bridge.path) + #expect((servers["other"] as? [String: Any])?["command"] as? String == "other-server") + #expect(harness.service.row(id: "editor")?.config == .current) + #expect(harness.service.row(id: "editor")?.removeRoute == .file) + + let removed = await harness.service.remove("editor") + + guard case .done = removed else { + Issue.record("Expected the removal to finish, got \(removed)") + return + } + let remaining = try Self.servers(in: try #require(home.read(".editor/mcp.json"))) + #expect(remaining["tablepro"] == nil) + #expect(remaining["other"] != nil) + #expect(harness.service.row(id: "editor")?.config == .notConfigured) + } + + @Test("A deeplink install hands off to the client and waits for it to confirm") + func deeplinkHandsOff() async throws { + let home = try MCPClientTestHome() + let workspace = FakeMCPClientWorkspace(installedBundleIDs: ["com.example.editor"], handledSchemes: ["editor"]) + let harness = try MCPClientServiceHarness(home: home, workspace: workspace) + await harness.service.refresh() + + #expect(await harness.service.install("editor") == .handedOff) + + let opened = try #require(workspace.opened.first) + #expect(opened.scheme == "editor") + #expect(opened.absoluteString.contains("name=tablepro")) + let row = try #require(harness.service.row(id: "editor")) + #expect(harness.service.presentation(for: row).status == String(format: String(localized: "Finish in %@"), "Editor")) + } + + @Test("A client that connects while the sheet is open reads Connected just now and is announced once") + func newSessionIsAnnounced() async throws { + let home = try MCPClientTestHome() + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + let open = MCPClientSessionSighting(id: "open", clientInfoName: "agent-cli", tokenName: MCPTokenStore.stdioBridgeTokenName) + + harness.sessions.send([open]) + + #expect(harness.service.connection(for: "agent") == .connected) + #expect(harness.announcements.messages.isEmpty) + + harness.sessions.send([ + open, + MCPClientSessionSighting( + id: "new", clientInfoName: "shared-client", displayName: "Shared", tokenName: MCPTokenStore.stdioBridgeTokenName + ) + ]) + + #expect(harness.service.connection(for: "desk") == .justConnected) + #expect(harness.service.connection(for: "term") == .justConnected) + #expect(harness.service.connection(for: "agent") == .connected) + #expect(harness.announcements.messages == [String(format: String(localized: "%@ connected"), "Shared")]) + let desk = try #require(harness.service.row(id: "desk")) + #expect(harness.service.presentation(for: desk).status == String(localized: "Connected just now")) + } + + @Test("A session two rows share is announced by the one found on this Mac") + func sharedSessionNamesTheClientFoundHere() async throws { + let home = try MCPClientTestHome() + try home.makeFolder(".shared") + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + #expect(harness.service.row(id: "term")?.isDetected == true) + #expect(harness.service.row(id: "desk")?.isDetected == false) + + harness.sessions.send([]) + harness.sessions.send([ + MCPClientSessionSighting( + id: "new", clientInfoName: "shared-client", displayName: "Shared", tokenName: MCPTokenStore.stdioBridgeTokenName + ) + ]) + + #expect(harness.announcements.messages == [String(format: String(localized: "%@ connected"), "Term")]) + #expect(harness.service.connection(for: "term") == .justConnected) + #expect(harness.service.connection(for: "desk") == .notConnected) + } + + @Test("A session already open when the sheet opens lights only the row found here once the rows are read") + func sharedSessionSettlesOnRefresh() async throws { + let home = try MCPClientTestHome() + try home.makeFolder(".shared") + let harness = try MCPClientServiceHarness(home: home) + harness.sessions.send([ + MCPClientSessionSighting(id: "open", clientInfoName: "shared-client", tokenName: MCPTokenStore.stdioBridgeTokenName) + ]) + #expect(harness.service.connection(for: "desk") == .connected) + #expect(harness.service.connection(for: "term") == .connected) + + await harness.service.refresh() + + #expect(harness.service.connection(for: "term") == .connected) + #expect(harness.service.connection(for: "desk") == .notConnected) + } + + @Test("A clientInfo name two rows share names one of them only when it alone is on this Mac") + func sharedSessionAttribution() throws { + let catalog = try MCPClientSetupFixture.catalog() + let bridge = MCPTokenStore.stdioBridgeTokenName + #expect(catalog.client(forSessionNamed: "shared-client", tokenName: bridge, detected: ["term"])?.id == "term") + #expect(catalog.client(forSessionNamed: "shared-client", tokenName: bridge, detected: []) == nil) + #expect(catalog.client(forSessionNamed: "shared-client", tokenName: bridge, detected: ["desk", "term"]) == nil) + #expect(catalog.clientIDs(forSessionNamed: "shared-client", tokenName: bridge, detected: []) == ["desk", "term"]) + #expect(catalog.clientIDs(forSessionNamed: "shared-client", tokenName: bridge, detected: ["desk"]) == ["desk"]) + #expect(catalog.iconSlug(forSessionNamed: "shared-client", tokenName: bridge, detected: ["desk"]) == "desk") + #expect(catalog.iconSlug(forSessionNamed: "shared-client", tokenName: bridge, detected: []) == nil) + #expect(catalog.client(forSessionNamed: "editor-client", tokenName: bridge, detected: [])?.id == "editor") + let sessions = [(clientInfoName: "shared-client", tokenName: Optional(bridge)), (clientInfoName: "editor-client", tokenName: Optional(bridge))] + #expect(catalog.ambiguousClientIDs(forSessionsNamed: sessions) == ["desk", "term"]) + } + + @Test("An old setup in a project names the project and offers no Remove; a project folder that is gone is ignored") + func shadowedProject() async throws { + let home = try MCPClientTestHome() + try home.makeExecutable("bin/agent") + try home.makeFolder("work/shop") + func agentConfig(project: String) -> String { + """ + { "projects": { "\(project)": { "mcpServers": { "tablepro": { "command": "/Volumes/Gone/tablepro-mcp" } } } }, + "mcpServers": { "tablepro": { "type": "stdio", "command": "\(home.bridge.path)", "args": [] } } } + """ + } + try home.write(".agent.json", agentConfig(project: home.url("work/gone").path)) + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + #expect(harness.service.row(id: "agent")?.config == .current) + #expect(harness.service.row(id: "agent")?.removeRoute == .cli(binary: "agent")) + + try home.write(".agent.json", agentConfig(project: home.url("work/shop").path)) + await harness.service.refresh() + + let row = try #require(harness.service.row(id: "agent")) + #expect(row.config == .shadowed(project: "~/work/shop")) + #expect(row.removeRoute == nil) + #expect(await harness.service.remove("agent") == .unavailable) + #expect(MCPClientDetailText.configState(row.config, name: row.name)?.contains("~/work/shop") == true) + } + + @Test("Update rewrites how the entry reaches TablePro and keeps the tool list the user narrowed") + func updateKeepsRestrictions() async throws { + let home = try MCPClientTestHome() + try home.write(".editor/mcp.json", """ + { "mcpServers": { "tablepro": { "command": "/Volumes/Gone/TablePro.app/Contents/MacOS/tablepro-mcp", "args": [], "tools": ["list_connections"] } } } + """) + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + #expect(harness.service.row(id: "editor")?.config == .moved) + + #expect(Self.isDone(await harness.service.update("editor"))) + + let text = try #require(home.read(".editor/mcp.json")) + let server = try #require(try Self.servers(in: text)["tablepro"] as? [String: Any]) + #expect(server["command"] as? String == home.bridge.path) + #expect(server["tools"] as? [String] == ["list_connections"]) + #expect(harness.service.row(id: "editor")?.config == .current) + } + + @Test("A setup made with a token or turned off in the client is never replaced by Install or Update") + func userChoicesAreKept() async throws { + let home = try MCPClientTestHome() + let token = #"{ "mcpServers": { "tablepro": { "url": "http://127.0.0.1:23508/mcp", "headers": { "Authorization": "Bearer tp_x" } } } }"# + try home.write(".editor/mcp.json", token) + let turnedOff = "[mcp_servers.tablepro]\ncommand = \"\(home.bridge.path)\"\nargs = []\nenabled = false\n" + try home.write(".shared/config.toml", turnedOff) + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + + let editor = try #require(harness.service.row(id: "editor")) + #expect(editor.config == .usesToken) + #expect(harness.service.presentation(for: editor).action == nil) + #expect(harness.service.presentation(for: editor).status == String(localized: "Set up with a token")) + #expect(await harness.service.update("editor") == .unavailable) + #expect(await harness.service.install("editor") == .unavailable) + #expect(home.read(".editor/mcp.json") == token) + + let term = try #require(harness.service.row(id: "term")) + #expect(term.config == .disabled) + #expect(harness.service.presentation(for: term).action == nil) + #expect(harness.service.presentation(for: term).status == String(localized: "Turned off")) + #expect(await harness.service.update("term") == .unavailable) + #expect(home.read(".shared/config.toml") == turnedOff) + } + + @Test("Update merges into the settings file and keeps the keys the user added, even when the client's command exists") + func updateMergesFile() async throws { + let home = try MCPClientTestHome() + try home.makeExecutable(".local/bin/editor") + try home.write(".editor/mcp.json", """ + { + "mcpServers": { + "tablepro": { + "command": "/Volumes/Gone/TablePro.app/Contents/MacOS/tablepro-mcp", + "args": [], + "autoApprove": ["list_connections"], + "timeout": 30 + } + } + } + """) + try home.write(".shared/config.toml", """ + [mcp_servers.tablepro] + command = "/Volumes/Gone/TablePro.app/Contents/MacOS/tablepro-mcp" + args = [] + enabled_tools = ["list_connections"] + startup_timeout_sec = 30 + + [mcp_servers.tablepro.env] + TOKEN = "x" + + """) + let runner = FakeMCPClientRunner() + let harness = try MCPClientServiceHarness(home: home, runner: runner) + await harness.service.refresh() + #expect(harness.service.row(id: "editor")?.updateRoute == .file) + + #expect(Self.isDone(await harness.service.update("editor"))) + #expect(Self.isDone(await harness.service.update("term"))) + + #expect(runner.calls.isEmpty) + let editorText = try #require(home.read(".editor/mcp.json")) + let server = try #require(try Self.servers(in: editorText)["tablepro"] as? [String: Any]) + #expect(server["command"] as? String == home.bridge.path) + #expect(server["autoApprove"] as? [String] == ["list_connections"]) + #expect(server["timeout"] as? Int == 30) + #expect(home.read(".shared/config.toml") == """ + [mcp_servers.tablepro] + command = "\(home.bridge.path)" + args = [] + enabled_tools = ["list_connections"] + startup_timeout_sec = 30 + + [mcp_servers.tablepro.env] + TOKEN = "x" + + """) + #expect(harness.service.row(id: "editor")?.config == .current) + #expect(harness.service.row(id: "term")?.config == .current) + } + + @Test("A write the system refuses names the file, and keeps the system's reason as output") + func refusedWriteIsReadable() async throws { + let home = try MCPClientTestHome() + try home.write(".editor/mcp.json", #"{ "mcpServers": {} }"#) + let folder = home.url(".editor").path + try FileManager.default.setAttributes([.posixPermissions: 0o500], ofItemAtPath: folder) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: home.url(".editor").path) } + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + + let outcome = await harness.service.install("editor") + + guard case let .failed(message, output) = outcome else { + Issue.record("Expected a failure, got \(outcome)") + return + } + #expect(message == String(format: String(localized: "TablePro could not write %@."), "mcp.json")) + #expect(output?.contains("NSCocoaErrorDomain") == true) + #expect(MCPClientSetupService.message(for: MCPConfigWriteError.io("x"), fileName: "mcp.json") == message) + } + + @Test("A session from TablePro's own Claude Agent lights no client") + func agentSessionIsNotAClient() async throws { + let home = try MCPClientTestHome() + let harness = try MCPClientServiceHarness(home: home) + await harness.service.refresh() + + harness.sessions.send([]) + harness.sessions.send([ + MCPClientSessionSighting(id: "turn", clientInfoName: "agent-cli", tokenName: MCPTokenStore.claudeAgentTokenName) + ]) + + #expect(harness.service.connection(for: "agent") == .notConnected) + #expect(harness.announcements.messages.isEmpty) + } + + @Test("A removal the client reports as already gone counts as done") + func missingEntryReadsAsRemoved() { + #expect(MCPClientSetupService.reportsMissingEntry("No MCP server named tablepro found in user config")) + #expect(MCPClientSetupService.reportsMissingEntry("Error: server \"tablepro\" not found")) + #expect(!MCPClientSetupService.reportsMissingEntry("EACCES: permission denied")) + } + + private static func isDone(_ outcome: MCPClientActionOutcome) -> Bool { + if case .done = outcome { return true } + return false + } + + private static func servers(in text: String) throws -> [String: Any] { + let object = try JSONSerialization.jsonObject(with: Data(text.utf8)) as? [String: Any] + return try #require(object?["mcpServers"] as? [String: Any]) + } +} diff --git a/apple/TableProTests/Core/MCP/Clients/MCPHTTPClientSetupTests.swift b/apple/TableProTests/Core/MCP/Clients/MCPHTTPClientSetupTests.swift new file mode 100644 index 0000000000..bb5f572a14 --- /dev/null +++ b/apple/TableProTests/Core/MCP/Clients/MCPHTTPClientSetupTests.swift @@ -0,0 +1,42 @@ +import Foundation +@testable import TablePro +import TableProMCPClients +import Testing + +struct MCPHTTPClientSetupTests { + private let token = "tp_secret0123456789" + private let masked = "tp_secre\u{2022}\u{2022}\u{2022}\u{2022}" + + private func setup() throws -> MCPHTTPClientSetup { + MCPHTTPClientSetup( + catalog: try MCPClientSetupFixture.catalog(), + context: MCPClientSetupContext(home: "/Users/me", environment: [:], bridgePath: "/Applications/TablePro.app/Contents/MacOS/tablepro-mcp"), + port: 23_508 + ) + } + + @Test("The token sheet lists the HTTP clients this Mac runs, then Other Client") + func choices() throws { + #expect(try setup().choices.map { $0.id } == ["editor", "agent", MCPHTTPClientSetup.otherClientID]) + } + + @Test("The snippet on screen carries the masked token and the copied one the real token") + func maskedSnippet() throws { + let setup = try setup() + let shown = try #require(setup.render(clientID: "editor", token: masked)) + let copied = try #require(setup.render(clientID: "editor", token: token)) + + #expect(shown.snippet?.contains(token) == false) + #expect(shown.snippet?.contains("http://127.0.0.1:23508/mcp") == true) + #expect(copied.snippet?.contains("Bearer \(token)") == true) + #expect(copied.displayConfigPath == "~/.editor/mcp.json") + } + + @Test("A client whose settings file only it writes gets its command, not a snippet") + func commandForUnwritableFile() throws { + let rendered = try #require(try setup().render(clientID: "agent", token: token)) + #expect(rendered.snippet == nil) + #expect(rendered.commandLine?.contains("--header") == true) + #expect(rendered.commandLine?.contains("Bearer \(token)") == true) + } +} diff --git a/apple/TableProTests/Core/MCP/Identity/MCPIdentityLedgerTests.swift b/apple/TableProTests/Core/MCP/Identity/MCPIdentityLedgerTests.swift index ebedf0ec72..22ba168028 100644 --- a/apple/TableProTests/Core/MCP/Identity/MCPIdentityLedgerTests.swift +++ b/apple/TableProTests/Core/MCP/Identity/MCPIdentityLedgerTests.swift @@ -119,15 +119,30 @@ struct MCPIdentityLedgerTests { #expect(await ledger.isApproved(principal: second, connectionId: connectionId) == false) } - private func meta(clientName: String, version: String) -> MCPRequestMeta { + private func meta(clientName: String, version: String, title: String? = nil) -> MCPRequestMeta { MCPRequestMeta( protocolVersion: .latest, - clientInfo: MCPImplementation(name: clientName, version: version), + clientInfo: MCPImplementation(name: clientName, title: title, version: version), clientCapabilities: .none, progressToken: nil ) } + private func bridgePrincipal(tokenId: UUID, fingerprint: String) -> MCPPrincipal { + MCPPrincipal( + tokenFingerprint: fingerprint, + tokenId: tokenId, + scopes: MCPScope.readWriteSet, + connectionAccess: .all, + metadata: MCPPrincipalMetadata( + label: MCPTokenStore.stdioBridgeTokenName, + issuedAt: .distantPast, + expiresAt: nil, + isBridgeCredential: true + ) + ) + } + @Test("Activity is folded onto one entry per client identity") func activityFoldsRepeatRequests() async { let ledger = MCPClientActivityLedger(idleTimeout: .seconds(300)) @@ -169,7 +184,7 @@ struct MCPIdentityLedgerTests { #expect(await ledger.count(now: start.addingTimeInterval(61)) == 0) } - @Test("Two tokens on one machine are two clients") + @Test("Two HTTP tokens on one machine are two clients") func distinctTokensAreDistinctEntries() async { let ledger = MCPClientActivityLedger() let start = Date(timeIntervalSince1970: 1_700_000_000) @@ -191,6 +206,77 @@ struct MCPIdentityLedgerTests { #expect(await ledger.snapshot(now: start).count == 2) } + @Test("A stdio client keeps one row, and its connected time, across a bridge credential rotation") + func bridgeClientSurvivesRotation() async { + let ledger = MCPClientActivityLedger() + let start = Date(timeIntervalSince1970: 1_700_000_000) + let requestMeta = meta(clientName: "cursor-vscode", version: "1.4") + let rotated = bridgePrincipal(tokenId: UUID(), fingerprint: "b") + + await ledger.record( + meta: requestMeta, + principal: bridgePrincipal(tokenId: UUID(), fingerprint: "a"), + address: .localSocket, + at: start + ) + await ledger.record(meta: requestMeta, principal: rotated, address: .localSocket, at: start.addingTimeInterval(60)) + + let entries = await ledger.snapshot(now: start.addingTimeInterval(60)) + #expect(entries.count == 1) + #expect(entries.first?.transport == .stdioBridge) + #expect(entries.first?.firstSeenAt == start) + #expect(entries.first?.tokenId == rotated.tokenId) + } + + @Test("Two stdio clients sharing the bridge credential are two clients") + func bridgeClientsAreToldApartByClientInfo() async { + let ledger = MCPClientActivityLedger() + let start = Date(timeIntervalSince1970: 1_700_000_000) + let shared = bridgePrincipal(tokenId: UUID(), fingerprint: "a") + + await ledger.record(meta: meta(clientName: "claude-ai", version: "1"), principal: shared, address: .localSocket, at: start) + await ledger.record(meta: meta(clientName: "Zed", version: "1"), principal: shared, address: .localSocket, at: start) + + let entries = await ledger.snapshot(now: start) + #expect(Set(entries.map(\.clientName)) == ["claude-ai", "Zed"]) + #expect(Set(entries.map(\.id)).count == 2) + } + + @Test("A client's own name is kept beside the title it shows under") + func clientInfoNameIsKept() async { + let ledger = MCPClientActivityLedger() + let start = Date(timeIntervalSince1970: 1_700_000_000) + + await ledger.record( + meta: meta(clientName: "tablepro-cli", version: "1", title: "Command Line"), + principal: principal(tokenId: UUID()), + address: .localSocket, + at: start + ) + + let entry = await ledger.snapshot(now: start).first + #expect(entry?.clientName == "Command Line") + #expect(entry?.clientInfoName == "tablepro-cli") + #expect(entry?.transport == .http) + } + + @Test("The app's socket is listed under a readable address, while the audit log keeps its fixed value") + func localSocketAddressIsReadable() async { + let ledger = MCPClientActivityLedger() + let start = Date(timeIntervalSince1970: 1_700_000_000) + + await ledger.record( + meta: meta(clientName: "Claude", version: "1"), + principal: principal(tokenId: UUID()), + address: .localSocket, + at: start + ) + + #expect(await ledger.snapshot(now: start).first?.address == "Local socket") + #expect(MCPClientAddress.localSocket.displayValue == "local socket") + #expect(MCPClientAddress.loopback.localizedDisplayValue == "127.0.0.1") + } + @Test("A revoked token's activity is forgotten with it") func forgettingATokenDropsItsActivity() async { let ledger = MCPClientActivityLedger() diff --git a/apple/TableProTests/Core/MCP/Integration/MCPBridgeIntegrationTests.swift b/apple/TableProTests/Core/MCP/Integration/MCPBridgeIntegrationTests.swift index e5d9c2fabb..f9ecb07118 100644 --- a/apple/TableProTests/Core/MCP/Integration/MCPBridgeIntegrationTests.swift +++ b/apple/TableProTests/Core/MCP/Integration/MCPBridgeIntegrationTests.swift @@ -156,7 +156,7 @@ final class MCPBridgeIntegrationTests: XCTestCase { ) } - func testAModernHostRequestIsForwardedByteForByte() async throws { + func testAModernHostRequestGainsOnlyTheBridgeInstance() async throws { let bridge = try await startBridge() defer { bridge.shutdown() } await upstream.respondWithJson(try Self.successBody(id: .number(6))) @@ -172,7 +172,11 @@ final class MCPBridgeIntegrationTests: XCTestCase { let recordedForwarded = await upstream.requests.first let forwarded = try XCTUnwrap(recordedForwarded) - XCTAssertEqual(forwarded.body, Data(line.utf8), "A modern host already carries its own metadata") + XCTAssertEqual( + forwarded.body, + try Self.stampedWithInstance(line, forwarded: forwarded.body), + "A modern host's message gains only the bridge instance" + ) } func testStampingNeverRewritesTheNumbersInAMessage() async throws { @@ -198,7 +202,7 @@ final class MCPBridgeIntegrationTests: XCTestCase { XCTAssertTrue(text.contains(MCPMetaKeys.protocolVersion)) } - func testAModernMessageWithBigNumbersIsForwardedUntouched() async throws { + func testAModernMessageWithBigNumbersGainsOnlyTheBridgeInstance() async throws { let bridge = try await startBridge() defer { bridge.shutdown() } await upstream.respondWithJson(try Self.successBody(id: .number(8))) @@ -214,7 +218,33 @@ final class MCPBridgeIntegrationTests: XCTestCase { let recordedForwarded = await upstream.requests.first let forwarded = try XCTUnwrap(recordedForwarded) - XCTAssertEqual(forwarded.body, Data(line.utf8)) + XCTAssertEqual(forwarded.body, try Self.stampedWithInstance(line, forwarded: forwarded.body)) + } + + func testEveryForwardedRequestCarriesItsOwnBridgesInstance() async throws { + let first = try await startBridge() + defer { first.shutdown() } + let second = try await startBridge() + defer { second.shutdown() } + await upstream.respondWithJson(try Self.successBody(id: .number(17))) + + try first.write(#"{"jsonrpc":"2.0","id":17,"method":"tools/list","params":{}}"#) + _ = try await first.nextMessage() + try first.write(#"{"jsonrpc":"2.0","id":17,"method":"resources/list"}"#) + _ = try await first.nextMessage() + try second.write(#"{"jsonrpc":"2.0","id":17,"method":"tools/list","params":{}}"#) + _ = try await second.nextMessage() + + let forwarded = await upstream.requests + let instances = try forwarded.map { request in + let body = try JSONDecoder().decode(JsonValue.self, from: request.body) + return try XCTUnwrap(body["params"]?["_meta"]?[MCPMetaKeys.bridgeInstance]?.stringValue) + } + XCTAssertEqual(instances.count, 3) + guard instances.count == 3 else { return } + XCTAssertFalse(instances[0].isEmpty) + XCTAssertEqual(instances[0], instances[1], "One bridge process is one session") + XCTAssertNotEqual(instances[0], instances[2], "Two bridge processes are two sessions") } func testAToolCallCarriesItsNameInTheHeader() async throws { @@ -348,6 +378,17 @@ final class MCPBridgeIntegrationTests: XCTestCase { return BridgeHarness(upstream: transport, discovery: BridgeIntegrationFixtures.discovery) } + // A modern message is forwarded as sent, plus the one reserved key that names the bridge process. + private static func stampedWithInstance(_ line: String, forwarded: Data) throws -> Data { + let body = try JSONDecoder().decode(JsonValue.self, from: forwarded) + let instance = try XCTUnwrap(body["params"]?["_meta"]?[MCPMetaKeys.bridgeInstance]?.stringValue) + let stamped = line.replacingOccurrences( + of: #""_meta":{"#, + with: #""_meta":{"\#(MCPMetaKeys.bridgeInstance)":"\#(instance)","# + ) + return Data(stamped.utf8) + } + private static func successBody(id: JsonRpcId) throws -> Data { try JsonRpcCodec.encode( .successResponse(JsonRpcSuccessResponse(id: id, result: .object(["ok": .bool(true)]))) diff --git a/apple/TableProTests/Core/MCP/Legacy/InitializeHandlerTests.swift b/apple/TableProTests/Core/MCP/Legacy/InitializeHandlerTests.swift index 61cd61f061..45e12e63a9 100644 --- a/apple/TableProTests/Core/MCP/Legacy/InitializeHandlerTests.swift +++ b/apple/TableProTests/Core/MCP/Legacy/InitializeHandlerTests.swift @@ -105,7 +105,7 @@ struct LegacyInitializeHandlerTests { @Test("The legacy initialize result keeps its legacy wire shape") func theLegacyResultKeepsItsLegacyShape() { - let result = InitializeHandler.result(protocolVersion: .v20251125) + let result = InitializeHandler.result(protocolVersion: .v20251125, scopes: MCPScope.readOnlySet) let encoded = result.asJsonValue(era: .legacy, serverInfo: MCPMethodRegistry.serverInfo) @@ -120,7 +120,7 @@ struct LegacyInitializeHandlerTests { @Test("The legacy capabilities object carries no extensions field") func legacyCapabilitiesCarryNoExtensions() { - let encoded = InitializeHandler.result(protocolVersion: .v20251125) + let encoded = InitializeHandler.result(protocolVersion: .v20251125, scopes: MCPScope.readOnlySet) .asJsonValue(era: .legacy, serverInfo: MCPMethodRegistry.serverInfo) #expect(encoded["capabilities"]?["extensions"] == nil) @@ -155,14 +155,49 @@ struct LegacyInitializeHandlerTests { #expect(modern["resources"]?["subscribe"]?.boolValue == true) } + @Test("A token without resources:read is not offered resources, prompts or completions") + func toolsOnlyTokenIsOfferedToolsOnly() { + let narrow = InitializeHandler.capabilities(for: [.toolsRead]).asJsonValue(era: .legacy) + let full = InitializeHandler.capabilities(for: MCPScope.readOnlySet).asJsonValue(era: .legacy) + + #expect(narrow["tools"] != nil) + #expect(narrow["resources"] == nil) + #expect(narrow["prompts"] == nil) + #expect(narrow["completions"] == nil) + #expect(full == MCPMethodRegistry.capabilities().asJsonValue(era: .legacy)) + } + + @Test("The initialize a client receives follows the scopes of the token that sent it") + func initializeFollowsThePrincipalScopes() async throws { + let adapter = MCPLegacyTestSupport.makeAdapter() + let params = MCPLegacyTestSupport.initializeParams() + + let (narrowResult, _) = try await adapter.handleInitialize( + params: params, + principal: MCPLegacyTestSupport.principal(scopes: [.toolsRead]) + ) + let (fullResult, _) = try await adapter.handleInitialize( + params: params, + principal: MCPLegacyTestSupport.principal(scopes: MCPScope.readOnlySet) + ) + let narrow = narrowResult.asJsonValue(era: .legacy, serverInfo: MCPMethodRegistry.serverInfo)["capabilities"] + let full = fullResult.asJsonValue(era: .legacy, serverInfo: MCPMethodRegistry.serverInfo)["capabilities"] + + #expect(narrow?["tools"] != nil) + #expect(narrow?["resources"] == nil) + #expect(narrow?["prompts"] == nil) + #expect(full?["resources"] != nil) + #expect(full?["prompts"] != nil) + } + @Test("The initialize result carries no cache hint") func theResultIsNotCacheable() { - #expect(InitializeHandler.result(protocolVersion: .v20251125).cacheHint == nil) + #expect(InitializeHandler.result(protocolVersion: .v20251125, scopes: MCPScope.readOnlySet).cacheHint == nil) } @Test("The modern envelope fields the legacy shape omits are added only for the modern era") func modernEnvelopeIsAddedOnlyForModernClients() { - let encoded = InitializeHandler.result(protocolVersion: .v20251125) + let encoded = InitializeHandler.result(protocolVersion: .v20251125, scopes: MCPScope.readOnlySet) .asJsonValue(era: .modern, serverInfo: MCPMethodRegistry.serverInfo) #expect(encoded["resultType"]?.stringValue == "complete") diff --git a/apple/TableProTests/Core/MCP/Lifecycle/MCPServerClientListTests.swift b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerClientListTests.swift new file mode 100644 index 0000000000..5ffa228d2e --- /dev/null +++ b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerClientListTests.swift @@ -0,0 +1,41 @@ +import Combine +import Foundation +import Testing + +@testable import TablePro +import TableProMCP + +@MainActor +struct MCPServerClientListTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-clients-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) + } + + @Test("A client list refresh that finds nothing new publishes nothing") + func unchangedRefreshPublishesNothing() async { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + var published = 0 + let subscription = manager.objectWillChange.sink { _ in published += 1 } + defer { subscription.cancel() } + + await manager.refreshClients() + await manager.refreshClients() + + #expect(published == 0) + #expect(manager.connectedClients.isEmpty) + } + + @Test("The client list refresh ends with the task of the view that shows it") + func refreshEndsWithItsTask() async { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + let shown = Task { await manager.refreshClientsWhileShown() } + shown.cancel() + await shown.value + #expect(manager.connectedClients.isEmpty) + } +} diff --git a/apple/TableProTests/Core/MCP/Lifecycle/MCPServerManagerClientsTests.swift b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerManagerClientsTests.swift new file mode 100644 index 0000000000..41e543c3d7 --- /dev/null +++ b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerManagerClientsTests.swift @@ -0,0 +1,176 @@ +import Combine +import Foundation +import TableProMCP +import TableProTestSupport +import Testing + +@testable import TablePro + +@MainActor +@Suite(.serialized) +struct MCPServerManagerClientsTests { + private struct Running { + let manager: MCPServerManager + let composition: MCPServerComposition + let bridgeToken: String + } + + private let directory: URL + + init() throws { + // Short, so the socket fits in sun_path inside the folder itself. + directory = URL(fileURLWithPath: NSTemporaryDirectory(), isDirectory: true) + .appendingPathComponent("mcp-\(UUID().uuidString.prefix(8))", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) + } + + private func withRunningServer(_ body: (Running) async throws -> Void) async throws { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { true }) + try await manager.start(port: 0) + do { + try await waitUntil { + await MainActor.run { + if case .failed = manager.state { return true } + return manager.isRunning + } + } + try #require(manager.isRunning, "The MCP server did not start: \(manager.state)") + let composition = try #require(manager.composition) + let data = try Data(contentsOf: MCPHandshakeLocation.file(in: directory)) + let handshake = try MCPHandshakeCoding.decoder().decode(MCPHandshakePayload.self, from: data) + try await body(Running(manager: manager, composition: composition, bridgeToken: handshake.token)) + } catch { + await manager.stop() + throw error + } + await manager.stop() + } + + private func principal(presenting token: String, from address: MCPClientAddress, to running: Running) async throws -> MCPPrincipal { + let decision = await running.composition.authenticator.authenticate( + authorizationHeader: "Bearer \(token)", + clientAddress: address + ) + guard case .allow(let principal) = decision else { + Issue.record("Expected the token to be accepted, got \(decision)") + throw CancellationError() + } + return principal + } + + private func record(_ clientName: String, as principal: MCPPrincipal, from address: MCPClientAddress, in running: Running) async { + let meta = MCPRequestMeta( + protocolVersion: .latest, + clientInfo: MCPImplementation(name: clientName, version: "1"), + clientCapabilities: .none, + progressToken: nil + ) + await running.composition.activityLedger.record(meta: meta, principal: principal, address: address, at: Date()) + } + + @Test("Disconnect on a stdio client forgets that client only and leaves the shared bridge credential working") + func stdioDisconnectKeepsTheBridgeCredential() async throws { + try await withRunningServer { running in + let bridge = try await principal(presenting: running.bridgeToken, from: .localSocket, to: running) + await record("claude-ai", as: bridge, from: .localSocket, in: running) + await record("cursor-vscode", as: bridge, from: .localSocket, in: running) + await running.manager.refreshClients() + + let clients = running.manager.connectedClients + #expect(clients.count == 2) + #expect(clients.allSatisfy { $0.transport == .stdioBridge }) + let claude = try #require(clients.first { $0.clientName == "claude-ai" }) + + #expect(await running.manager.disconnectClient(claude.id)) + + #expect(running.manager.connectedClients.map { $0.clientName } == ["cursor-vscode"]) + #expect(await running.manager.tokenStore?.validate(bearerToken: running.bridgeToken) != nil) + _ = try await principal(presenting: running.bridgeToken, from: .localSocket, to: running) + } + } + + @Test("Disconnect on a client with its own token revokes that token") + func tokenDisconnectRevokesTheToken() async throws { + try await withRunningServer { running in + let store = try #require(running.manager.tokenStore) + let minted = try await store.generate( + name: "Raycast", + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + let paired = try await principal(presenting: minted.plaintext, from: .loopback, to: running) + await record("raycast", as: paired, from: .loopback, in: running) + await running.manager.refreshClients() + let row = try #require(running.manager.connectedClients.first) + #expect(row.transport == .http) + + #expect(await running.manager.disconnectClient(row.id)) + + #expect(running.manager.connectedClients.isEmpty) + #expect(await store.validate(bearerToken: minted.plaintext) == nil) + #expect(await running.manager.tokenStore?.validate(bearerToken: running.bridgeToken) != nil) + await store.delete(tokenId: minted.token.id) + } + } + + @Test("A refresh that finds nothing new does not publish the client list again") + func unchangedClientsAreNotRepublished() async throws { + try await withRunningServer { running in + let counter = PublishCounter() + let subscription = running.manager.$connectedClients.dropFirst().sink { _ in counter.count += 1 } + defer { subscription.cancel() } + + let bridge = try await principal(presenting: running.bridgeToken, from: .localSocket, to: running) + await record("Zed", as: bridge, from: .localSocket, in: running) + await running.manager.refreshClients() + let afterNewClient = counter.count + #expect(afterNewClient >= 1) + + await running.manager.refreshClients() + await running.manager.refreshClients() + + #expect(counter.count == afterNewClient) + } + } + + @Test("Revoking with no server running reports that nothing was revoked") + func revokeWithoutAStoreFails() async { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + + #expect(await manager.revokeToken(id: UUID()) == false) + } + + @Test("The window names TablePro's own clients and tokens, not their raw ids") + func ownClientsReadAsTablePro() { + #expect(IntegrationsFormatting.displayTokenName(MCPTokenStore.claudeAgentTokenName) == AIProviderType.claudeAgent.displayName) + #expect(IntegrationsFormatting.displayTokenName("Raycast") == "Raycast") + + let command = Self.snapshot(clientName: CLIClientIdentity.title, clientInfoName: CLIClientIdentity.name) + #expect(IntegrationsFormatting.displayClientName(command) == String(localized: "tablepro command")) + let cursor = Self.snapshot(clientName: "Cursor", clientInfoName: "cursor-vscode") + #expect(IntegrationsFormatting.displayClientName(cursor) == "Cursor") + } + + private static func snapshot(clientName: String, clientInfoName: String) -> MCPServerManager.SessionSnapshot { + MCPServerManager.SessionSnapshot( + id: clientInfoName, + clientName: clientName, + clientInfoName: clientInfoName, + clientVersion: "1", + connectedSince: .distantPast, + lastActivityAt: .distantPast, + tokenName: nil, + remoteAddress: nil, + transport: .http + ) + } +} + +private final class PublishCounter { + var count = 0 +} diff --git a/apple/TableProTests/Core/MCP/Lifecycle/MCPServerOffContractTests.swift b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerOffContractTests.swift index 59ce1fdab2..ad3d22fd55 100644 --- a/apple/TableProTests/Core/MCP/Lifecycle/MCPServerOffContractTests.swift +++ b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerOffContractTests.swift @@ -108,6 +108,53 @@ struct MCPServerOffContractTests { } } +@MainActor +struct MCPServerFailedStartTests { + private let root: URL + + init() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-fail-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + + @Test("A start that fails while MCP is on leaves a refusal that says so, for the bridge that asked") + func failedStartWritesARefusal() async throws { + // Too long for a socket path, so the server falls back to the temporary folder, which is a file here. + let directory = root.appendingPathComponent(String(repeating: "d", count: 110), isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let notAFolder = root.appendingPathComponent("tmp-file", isDirectory: false) + try Data().write(to: notAFolder) + let manager = MCPServerManager( + handshake: MCPHandshakeFile(directory: directory, temporaryDirectory: notAFolder), + isEnabled: { true } + ) + let startedAt = Date() + + try await manager.start(port: 0) + + #expect(manager.state == .failed(.socketFolderUnusable)) + #expect(manager.tokenStore == nil) + let data = try Data(contentsOf: MCPHandshakeLocation.refusalFile(in: directory)) + let refusal = try MCPHandshakeCoding.decoder().decode(MCPStartRefusal.self, from: data) + #expect(refusal.reason == .failedToStart) + #expect(refusal.answers(launchRequestedAt: startedAt)) + await manager.stop() + } + + @Test("A refusal while MCP is off still reads as turned off") + func turnedOffRefusalKeepsItsReason() async throws { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: root), isEnabled: { false }) + + await #expect(throws: MCPServerStartError.turnedOff) { + try await manager.start(port: 0) + } + + let data = try Data(contentsOf: MCPHandshakeLocation.refusalFile(in: root)) + #expect(try MCPHandshakeCoding.decoder().decode(MCPStartRefusal.self, from: data).reason == .turnedOff) + } +} + struct MCPBridgeRefusalTests { private let directory: URL diff --git a/apple/TableProTests/Core/MCP/MCPAggregateReadApprovalTests.swift b/apple/TableProTests/Core/MCP/MCPAggregateReadApprovalTests.swift new file mode 100644 index 0000000000..9b31282f0f --- /dev/null +++ b/apple/TableProTests/Core/MCP/MCPAggregateReadApprovalTests.swift @@ -0,0 +1,315 @@ +// +// MCPAggregateReadApprovalTests.swift +// TableProTests +// +// A read that spans connections cannot ask about each one, so it covers only the connections the +// approval gate already lets through. A call that names one connection is asked about it once. +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProMCPTestSupport +import TableProPluginKit +import Testing + +struct MCPAggregateReadApprovalTests { + private let granted = UUID() + private let withheld = UUID() + + private func ledger() -> MCPApprovalLedger { + MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()) + } + + private func makePolicy( + approval: MCPConnectionApproval, + presenter: RecordingApprovalPresenter, + connections: [UUID: MCPConnectionAuthSnapshot]? = nil + ) -> MCPAuthPolicy { + MCPToolTestHarness.authPolicy( + connections: connections ?? [ + granted: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Granted"), + withheld: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Withheld") + ], + approval: approval, + presenter: presenter, + ledger: ledger() + ) + } + + private func entry(connectionId: UUID, query: String) -> QueryHistoryEntry { + QueryHistoryEntry( + query: query, + connectionId: connectionId, + databaseName: "shop", + databaseType: .postgresql, + source: .editor, + executionTime: 0.01, + rowCount: 1, + wasSuccessful: true + ) + } + + private func historyStore() -> ToolTestQueryHistoryStore { + ToolTestQueryHistoryStore(entries: [ + entry(connectionId: granted, query: "SELECT 1"), + entry(connectionId: withheld, query: "UPDATE employees SET salary = 250000") + ]) + } + + private func tab(on connectionId: UUID, title: String) -> MCPTabSnapshot { + MCPTabSnapshot( + tabId: UUID(), + connectionId: connectionId, + connectionName: title, + tabType: "table", + tableName: title, + databaseName: "shop", + schemaName: "public", + displayTitle: title, + windowId: UUID(), + isActive: true, + window: nil + ) + } + + private func context() -> MCPRequestContext { + MCPToolTestHarness.context(principal: MCPToolTestHarness.principal(access: .all)) + } + + private func approve(_ connectionId: UUID, in policy: MCPAuthPolicy) async throws { + try await policy.resolveAndAuthorize( + principal: MCPToolTestHarness.principal(access: .all), + tool: "list_tables", + connectionId: connectionId + ) + } + + @Test("Under Ask Once the silent set holds only the connections already granted") + func askOnceKeepsOnlyGrantedConnections() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + try await approve(granted, in: policy) + + let allowed = await policy.connectionIdsReadableWithoutPrompt(principal: MCPToolTestHarness.principal()) + + #expect(allowed == [granted]) + #expect(await presenter.askedCount == 1) + } + + @Test("A connection the user just denied stays out of the silent set") + func deniedConnectionStaysOut() async throws { + let presenter = RecordingApprovalPresenter(answer: false) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter, connections: [ + granted: MCPToolTestHarness.snapshot(policy: .alwaysAllow, name: "Granted"), + withheld: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Withheld") + ]) + await #expect(throws: DatabaseAccessError.self) { + try await approve(withheld, in: policy) + } + + let allowed = await policy.connectionIdsReadableWithoutPrompt(principal: MCPToolTestHarness.principal()) + + #expect(allowed == [granted]) + #expect(await presenter.askedCount == 1) + } + + @Test("Under Ask Every Time the silent set holds only connections set to Always Allow") + func askEveryTimeKeepsOnlyAlwaysAllow() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .everyTime, presenter: presenter, connections: [ + granted: MCPToolTestHarness.snapshot(policy: .alwaysAllow, name: "Granted"), + withheld: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Withheld") + ]) + try await approve(withheld, in: policy) + + let allowed = await policy.connectionIdsReadableWithoutPrompt(principal: MCPToolTestHarness.principal()) + + #expect(allowed == [granted]) + } + + @Test("Under Never Ask the silent set is every connection the token may see") + func neverAskKeepsEveryReadableConnection() async { + let hidden = UUID() + let blocked = UUID() + let outsideToken = UUID() + let presenter = RecordingApprovalPresenter(answer: false) + let policy = makePolicy(approval: .alwaysApprove, presenter: presenter, connections: [ + granted: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Granted"), + hidden: MCPToolTestHarness.snapshot(policy: .never, name: "Hidden"), + blocked: MCPToolTestHarness.snapshot(policy: .askEachTime, externalAccess: .blocked, name: "Blocked"), + outsideToken: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Outside") + ]) + + let allowed = await policy.connectionIdsReadableWithoutPrompt( + principal: MCPToolTestHarness.principal(access: .limited([granted, hidden, blocked])) + ) + + #expect(allowed == [granted]) + #expect(await presenter.askedCount == 0) + } + + @Test("History without a connection id leaves out a connection nobody approved, and asks nothing") + func historyWithoutConnectionCoversOnlyApprovedConnections() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + try await approve(granted, in: policy) + let store = historyStore() + + let result = try await SearchQueryHistoryTool().call( + arguments: .object(["query": .string("")]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy, history: store) + ) + + let filters = await store.receivedFilters + #expect(filters.first?.allowedConnectionIds == [granted]) + let entries = try #require(result.structuredContent?["entries"]?.arrayValue) + #expect(entries.allSatisfy { $0["connection_id"]?.stringValue == granted.uuidString }) + #expect(await presenter.askedCount == 1) + } + + @Test("History without a connection id still leaves out a connection the user denied") + func historyAfterADenialLeavesTheConnectionOut() async throws { + let presenter = RecordingApprovalPresenter(answer: false) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter, connections: [ + granted: MCPToolTestHarness.snapshot(policy: .alwaysAllow, name: "Granted"), + withheld: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Withheld") + ]) + await #expect(throws: DatabaseAccessError.self) { + try await approve(withheld, in: policy) + } + let store = historyStore() + + let result = try await SearchQueryHistoryTool().call( + arguments: .object(["query": .string("")]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy, history: store) + ) + + let filters = await store.receivedFilters + #expect(filters.first?.allowedConnectionIds == [granted]) + let entries = try #require(result.structuredContent?["entries"]?.arrayValue) + for entry in entries { + #expect(entry["query"]?.stringValue?.contains("salary") != true) + } + } + + @Test("Under Ask Every Time history without a connection id reads nothing and asks nothing") + func askEveryTimeAggregateHistoryIsEmpty() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .everyTime, presenter: presenter) + let store = historyStore() + + let result = try await SearchQueryHistoryTool().call( + arguments: .object(["query": .string("")]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy, history: store) + ) + + #expect(result.structuredContent?["entries"]?.arrayValue?.isEmpty == true) + #expect(await store.receivedFilters.isEmpty) + #expect(await presenter.askedCount == 0) + } + + @Test("Under Ask Every Time a history search that names a connection returns it, without asking again") + func askEveryTimeNamedHistoryIsReturned() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .everyTime, presenter: presenter) + let store = historyStore() + + let result = try await SearchQueryHistoryTool().call( + arguments: .object(["query": .string(""), "connection_id": .string(withheld.uuidString)]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy, history: store) + ) + + let entries = try #require(result.structuredContent?["entries"]?.arrayValue) + #expect(entries.count == 1) + #expect(entries.first?["connection_id"]?.stringValue == withheld.uuidString) + #expect(await presenter.askedCount == 0) + } + + @Test("Tabs without a connection id leave out a connection nobody approved") + func tabsWithoutConnectionCoverOnlyApprovedConnections() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + try await approve(granted, in: policy) + let tabs = [tab(on: granted, title: "orders"), tab(on: withheld, title: "payroll")] + + let result = try await ListRecentTabsTool(collectSnapshots: { tabs }).call( + arguments: .object([:]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy) + ) + + let listed = try #require(result.structuredContent?["tabs"]?.arrayValue) + #expect(listed.count == 1) + #expect(listed.first?["connection_id"]?.stringValue == granted.uuidString) + #expect(await presenter.askedCount == 1) + } + + @Test("Under Ask Every Time tabs of a named connection are listed without asking again") + func askEveryTimeNamedTabsAreListed() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .everyTime, presenter: presenter) + let tabs = [tab(on: granted, title: "orders"), tab(on: withheld, title: "payroll")] + + let result = try await ListRecentTabsTool(collectSnapshots: { tabs }).call( + arguments: .object(["connection_id": .string(withheld.uuidString)]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy) + ) + + let listed = try #require(result.structuredContent?["tabs"]?.arrayValue) + #expect(listed.count == 1) + #expect(listed.first?["connection_id"]?.stringValue == withheld.uuidString) + #expect(await presenter.askedCount == 0) + } + + @Test("The snapshot filter keeps only the given connections") + func snapshotFilterKeepsTheGivenConnections() { + let tabs = [tab(on: granted, title: "orders"), tab(on: withheld, title: "payroll")] + + let kept = MCPTabSnapshotProvider.snapshots(tabs, in: [granted]) + + #expect(kept.count == 1) + #expect(kept.first?.connectionId == granted) + #expect(MCPTabSnapshotProvider.snapshots(tabs, in: []).isEmpty) + } + + @Test("Focusing a tab asks about its connection, and a denial raises nothing") + func focusHonoursADenial() async throws { + let presenter = RecordingApprovalPresenter(answer: false) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + let payroll = tab(on: withheld, title: "payroll") + + let result = try await FocusQueryTabTool(collectSnapshots: { [payroll] }).call( + arguments: .object(["tab_id": .string(payroll.tabId.uuidString)]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy) + ) + + #expect(result.isError) + let text = MCPToolTestHarness.errorText(result) ?? "" + #expect(text.hasPrefix("denied:")) + #expect(!text.contains(withheld.uuidString)) + #expect(await presenter.askedCount == 1) + } + + @Test("Focusing a tab the user allows goes on to look for its window") + func focusAfterAnAllowLooksForTheWindow() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .everyTime, presenter: presenter) + let payroll = tab(on: withheld, title: "payroll") + + let result = try await FocusQueryTabTool(collectSnapshots: { [payroll] }).call( + arguments: .object(["tab_id": .string(payroll.tabId.uuidString)]), + context: context(), + services: MCPToolTestHarness.services(authPolicy: policy) + ) + + #expect(await presenter.askedCount == 1) + #expect(MCPToolTestHarness.errorText(result)?.hasPrefix("not_found:") == true) + } +} diff --git a/apple/TableProTests/Core/MCP/MCPAnswerBudgetPayloadTests.swift b/apple/TableProTests/Core/MCP/MCPAnswerBudgetPayloadTests.swift new file mode 100644 index 0000000000..323e9d7f79 --- /dev/null +++ b/apple/TableProTests/Core/MCP/MCPAnswerBudgetPayloadTests.swift @@ -0,0 +1,160 @@ +// +// MCPAnswerBudgetPayloadTests.swift +// TableProTests +// +// An answer the size limit cut tells the client so: is_truncated stays the flag it reads, and notes say why. +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProPluginKit +import Testing + +@MainActor +struct MCPAnswerBudgetPayloadTests { + private let scope = DatabaseScope(connectionId: UUID(), database: "shop", schema: nil) + + private func textResult(_ values: [String]) -> QueryResult { + ScriptAnsweringDriver.resultSet(columns: ["body"], rows: values.map { [$0] }) + } + + @Test("An answer with a cut value and rows left out carries both notes and is_truncated") + func cutAnswerCarriesNotes() throws { + var budget = ExternalAnswerBudget(bytes: 70_000) + let kept = budget.keep(textResult([String(repeating: "a", count: 100_000), String(repeating: "b", count: 10_000)])) + + let payload = MCPConnectionBridge.encode( + script: .statement(kept, executionTimeMs: 1, budget: budget), + scope: scope + ) + + #expect(payload["row_count"]?.intValue == 1) + #expect(payload["is_truncated"]?.boolValue == true) + #expect(payload["notes"]?.arrayValue?.compactMap { $0.stringValue } == budget.notes) + #expect(budget.notes.count == 2) + } + + @Test("An answer that fit carries no notes") + func fittingAnswerHasNoNotes() { + var budget = ExternalAnswerBudget.mcpAnswer + let kept = budget.keep(textResult(["1", "2"])) + + let payload = MCPConnectionBridge.encode(script: .statement(kept, executionTimeMs: 1, budget: budget), scope: scope) + + #expect(payload["notes"] == nil) + #expect(payload["is_truncated"]?.boolValue == false) + } + + @Test("execute_query through the bridge holds a large read to 4 MiB and says so") + func executeQueryIsHeldToTheBudget() async throws { + let driver = AnsweringPluginDriver { _ in + AnsweringPluginDriver.textRows(column: "body", count: 3_000, bytes: 2_048) + } + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + + let payload = try await MCPConnectionBridge().executeQuery( + scope: scope, query: "SELECT body FROM documents", maxRows: 3_000, timeoutSeconds: 30, cancellation: nil + ) + + let rowCount = try #require(payload["row_count"]?.intValue) + #expect(rowCount < 3_000) + #expect(rowCount * 2_048 <= ExternalAnswerBudget.answerBytes) + #expect(payload["rows"]?.arrayValue?.count == rowCount) + #expect(payload["is_truncated"]?.boolValue == true) + #expect(payload["notes"]?.arrayValue?.count == 1) + } + + @Test("The assistant's execute_query holds its answer to 256 KiB") + func assistantAnswerIsSmaller() async throws { + let driver = AnsweringPluginDriver { _ in + AnsweringPluginDriver.textRows(column: "body", count: 500, bytes: 2_048) + } + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + let context = ChatToolContext( + connectionId: scope.connectionId, + bridge: MCPConnectionBridge(), + authPolicy: MCPAuthPolicy( + connectionResolver: { _ in nil }, + connectionIdsProvider: { [] }, + historyRecorder: RecordingQueryHistory(), + executionGate: AlwaysAllowGate() + ), + chatMode: .agent, + policyLookup: { _ in .alwaysAllow } + ) + + let result = try await ExecuteQueryChatTool().execute( + input: .object(["query": .string("SELECT body FROM documents"), "max_rows": .int(500)]), + context: context + ) + + #expect(!result.isError) + let payload = try JSONDecoder().decode(JsonValue.self, from: Data(result.content.utf8)) + let rowCount = try #require(payload["row_count"]?.intValue) + #expect(rowCount > 0) + #expect(rowCount * 2_048 <= ExternalAnswerBudget.assistantAnswerBytes) + #expect(payload["is_truncated"]?.boolValue == true) + #expect(driver.sent.first?.context.resultByteLimit == ExternalAnswerBudget.assistantAnswerBytes) + } + + private func explain(planFilter filter: String) async throws -> JsonValue { + let plan = #"[{"Plan": {"Node Type": "Seq Scan", "Relation Name": "orders", "Filter": "\#(filter)"}}]"# + let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: { _ in + ScriptAnsweringDriver.resultSet(columns: ["QUERY PLAN"], rows: [[plan]]) + }) + var session = ConnectionSession(connection: connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "shop", schema: nil) + let request = try MCPConnectionBridge.explainRequest( + for: "SELECT * FROM orders", + databaseType: .postgresql, + variants: [ExplainVariant(id: "json", label: "JSON", sqlPrefix: "EXPLAIN (FORMAT JSON)", format: .postgresJson)], + variantId: nil, + analyze: false + ) + let operation = try await ApprovedOperationFixture.approve( + [OperationStatement(sql: request.statement)], scope: scope, databaseType: .postgresql + ) + return try await MCPConnectionBridge().explainQuery( + operation, request: request, timeoutSeconds: 30, cancellation: nil + ) + } + + @Test("explain_query leaves out a parsed plan past the budget, so the answer stays under 4 MiB") + func explainPlanIsHeldToTheBudget() async throws { + let payload = try await explain(planFilter: String(repeating: "f", count: 3_000_000)) + + #expect(payload["plan"] == nil) + #expect((payload["plan_text"]?.stringValue?.utf8.count ?? 0) >= 3_000_000) + #expect(payload["is_truncated"]?.boolValue == true) + #expect(payload["notes"]?.arrayValue?.isEmpty == false) + #expect(payload.jsonString(prettyPrinted: false).utf8.count <= ExternalAnswerBudget.answerBytes) + } + + @Test("explain_query keeps a parsed plan that fits") + func explainPlanThatFitsIsKept() async throws { + let payload = try await explain(planFilter: "(id > 1)") + + #expect(payload["plan"]?["root"]?["operation"]?.stringValue == "Seq Scan") + #expect(payload["is_truncated"] == nil) + #expect(payload["notes"] == nil) + } + + @Test("The result schemas declare notes and keep the fields they required") + func schemasDeclareNotes() { + let executeQuery = ExecuteQueryTool.outputSchema + #expect(executeQuery?["properties"]?["notes"]?["type"]?.stringValue == "array") + #expect(executeQuery?["required"] == MCPToolSchema.resultSet["required"]) + #expect(executeQuery?["properties"]?["result_sets"]?["items"]?["properties"]?["notes"] == nil) + #expect(MCPToolSchema.resultSet["properties"]?["notes"] != nil) + #expect(ExportDataTool.outputSchema?["properties"]?["notes"] != nil) + #expect(ExplainQueryTool.outputSchema?["properties"]?["notes"] != nil) + #expect(ExplainQueryTool.outputSchema?["properties"]?["is_truncated"] != nil) + } +} diff --git a/apple/TableProTests/Core/MCP/MCPAuditChainTests.swift b/apple/TableProTests/Core/MCP/MCPAuditChainTests.swift index 732eaaecfd..7848cb76dc 100644 --- a/apple/TableProTests/Core/MCP/MCPAuditChainTests.swift +++ b/apple/TableProTests/Core/MCP/MCPAuditChainTests.swift @@ -1,12 +1,3 @@ -// -// MCPAuditChainTests.swift -// TableProTests -// -// Every row hashes its own fields, its sequence number and the hash before it, so the log is only -// evidence while that chain holds. Editing a row, reordering two rows and deleting one all break -// it, and `verify()` names the sequence where the break starts. -// - import Foundation import SQLite3 import TableProPluginKit @@ -15,6 +6,13 @@ import Testing @testable import TablePro struct MCPAuditChainTests { + private static func makeStorage() -> (storage: MCPAuditLogStorage, path: String) { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("MCPAuditChainTests-\(UUID().uuidString)", isDirectory: true) + let path = directory.appendingPathComponent(MCPAuditLogStorage.databaseFileName).path + return (MCPAuditLogStorage(directory: directory), path) + } + private func entry(action: String, details: String? = nil) -> AuditEntry { AuditEntry( timestamp: Date(timeIntervalSince1970: 1_700_000_000), @@ -30,7 +28,7 @@ struct MCPAuditChainTests { @Test("An appended log verifies intact") func chainVerifiesIntact() async { - let storage = MCPAuditLogStorage(isolatedForTesting: true) + let storage = Self.makeStorage().storage for index in 0..<5 { await storage.addEntry(entry(action: "tool.run\(index)")) } @@ -39,7 +37,7 @@ struct MCPAuditChainTests { @Test("An empty log verifies intact") func emptyChainVerifiesIntact() async { - let storage = MCPAuditLogStorage(isolatedForTesting: true) + let storage = Self.makeStorage().storage #expect(await storage.verify() == .intact(count: 0)) } @@ -65,6 +63,72 @@ struct MCPAuditChainTests { ) } + @Test("A row without a client keeps the digest builds before the client column wrote") + func rowWithoutAClientKeepsItsDigest() throws { + let row = AuditEntry( + id: try #require(UUID(uuidString: "00000000-0000-0000-0000-0000000000AA")), + timestamp: Date(timeIntervalSince1970: 1_700_000_000), + category: .tool, + tokenId: UUID(uuidString: "00000000-0000-0000-0000-000000000001"), + tokenName: "client", + connectionId: nil, + action: "tool.run", + outcome: AuditOutcome.success, + details: "sqlDigest=abc" + ) + + #expect( + MCPAuditChainLink.digest(entry: row, sequence: 0, previousHash: MCPAuditChainLink.genesisHash) + == "7ad26117f02b46c632d1367b24359f6a8ff1fc84152c20a2ca825881623c8e99" + ) + } + + @Test("The digest covers the client a row names") + func digestCoversTheClient() { + let base = entry(action: "tool.run") + let named = AuditEntry( + id: base.id, + timestamp: base.timestamp, + category: base.category, + tokenId: base.tokenId, + tokenName: base.tokenName, + connectionId: base.connectionId, + action: base.action, + outcome: base.outcome, + details: base.details, + clientName: "Cursor" + ) + let renamed = AuditEntry( + id: base.id, + timestamp: base.timestamp, + category: base.category, + tokenId: base.tokenId, + tokenName: base.tokenName, + connectionId: base.connectionId, + action: base.action, + outcome: base.outcome, + details: base.details, + clientName: "Zed" + ) + + let previous = MCPAuditChainLink.genesisHash + let digests = Set([base, named, renamed].map { + MCPAuditChainLink.digest(entry: $0, sequence: 0, previousHash: previous) + }) + #expect(digests.count == 3) + } + + @Test("A client renamed behind the app's back is detected") + func editedClientIsDetected() async { + let (storage, path) = Self.makeStorage() + await storage.addEntry(AuditEntry(category: .tool, action: "tool.first", outcome: AuditOutcome.success, clientName: "Cursor")) + await storage.addEntry(AuditEntry(category: .tool, action: "tool.second", outcome: AuditOutcome.success, clientName: "Zed")) + + #expect(Self.execute("UPDATE audit_entries SET client_name = 'Cursor' WHERE sequence = 1;", at: path)) + + #expect(await storage.verify() == .broken(atSequence: 1)) + } + @Test("Reordering two entries breaks the chain") func reorderingBreaksTheChain() { let first = entry(action: "tool.first") @@ -87,7 +151,7 @@ struct MCPAuditChainTests { @Test("Clearing the log restarts the chain at genesis") func deleteAllResetsTheChain() async { - let storage = MCPAuditLogStorage(isolatedForTesting: true) + let storage = Self.makeStorage().storage await storage.addEntry(entry(action: "tool.first")) #expect(await storage.deleteAll() == true) await storage.addEntry(entry(action: "tool.second")) @@ -107,11 +171,10 @@ struct MCPAuditChainTests { @Test("A row edited behind the app's back is detected") func editedRowIsDetected() async throws { let marker = "tool.edited.\(UUID().uuidString)" - let storage = MCPAuditLogStorage(isolatedForTesting: true) + let (storage, path) = Self.makeStorage() for index in 0..<3 { await storage.addEntry(entry(action: index == 1 ? marker : "tool.run\(index)")) } - let path = try #require(Self.databasePath(containing: marker)) #expect( Self.execute( @@ -126,11 +189,10 @@ struct MCPAuditChainTests { @Test("A row removed behind the app's back is detected") func removedRowIsDetected() async throws { let marker = "tool.removed.\(UUID().uuidString)" - let storage = MCPAuditLogStorage(isolatedForTesting: true) + let (storage, path) = Self.makeStorage() for index in 0..<3 { await storage.addEntry(entry(action: index == 1 ? marker : "tool.gone\(index)")) } - let path = try #require(Self.databasePath(containing: marker)) #expect(Self.execute("DELETE FROM audit_entries WHERE action = '\(marker)';", at: path)) @@ -140,41 +202,15 @@ struct MCPAuditChainTests { @Test("A row reordered behind the app's back is detected") func reorderedRowIsDetected() async throws { let marker = "tool.reordered.\(UUID().uuidString)" - let storage = MCPAuditLogStorage(isolatedForTesting: true) + let (storage, path) = Self.makeStorage() await storage.addEntry(entry(action: marker)) await storage.addEntry(entry(action: "tool.second")) - let path = try #require(Self.databasePath(containing: marker)) #expect(Self.execute("UPDATE audit_entries SET sequence = 9 WHERE sequence = 1;", at: path)) #expect(await storage.verify() == .broken(atSequence: 9)) } - private static func databasePath(containing action: String) -> String? { - let directory = AppStorageEnvironment.shared.applicationSupportRoot.appendingPathComponent("TablePro") - let prefix = "mcp-audit-test_\(ProcessInfo.processInfo.processIdentifier)" - let names = ((try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? []) - .filter { $0.hasPrefix(prefix) && $0.hasSuffix(".db") } - return names - .map { directory.appendingPathComponent($0).path } - .first { contains(action: action, at: $0) } - } - - private static func contains(action: String, at path: String) -> Bool { - var db: OpaquePointer? - guard sqlite3_open(path, &db) == SQLITE_OK else { return false } - defer { sqlite3_close(db) } - - var statement: OpaquePointer? - let sql = "SELECT COUNT(*) FROM audit_entries WHERE action = ?;" - guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } - defer { sqlite3_finalize(statement) } - - sqlite3_bind_text(statement, 1, action, -1, unsafeBitCast(-1, to: sqlite3_destructor_type.self)) - guard sqlite3_step(statement) == SQLITE_ROW else { return false } - return sqlite3_column_int(statement, 0) > 0 - } - private static func execute(_ sql: String, at path: String) -> Bool { var db: OpaquePointer? guard sqlite3_open(path, &db) == SQLITE_OK else { return false } diff --git a/apple/TableProTests/Core/MCP/MCPAuditLogStorageTests.swift b/apple/TableProTests/Core/MCP/MCPAuditLogStorageTests.swift index b9d2151684..2380830626 100644 --- a/apple/TableProTests/Core/MCP/MCPAuditLogStorageTests.swift +++ b/apple/TableProTests/Core/MCP/MCPAuditLogStorageTests.swift @@ -1,14 +1,5 @@ -// -// MCPAuditLogStorageTests.swift -// TableProTests -// -// The audit log is evidence, so it has to survive being read by the wrong person and being edited -// by one. Every row carries the token id that caused it, so a token can be traced; the statement -// is reduced to a digest, so a CREATE USER with an inline password never lands on disk in -// cleartext; and the store and its journal files are owner-only. -// - import Foundation +import SQLite3 import TableProMCP import TableProPluginKit import Testing @@ -18,7 +9,12 @@ import Testing @Suite("MCP Audit Log Storage", .serialized) struct MCPAuditLogStorageTests { private func makeStorage() -> MCPAuditLogStorage { - MCPAuditLogStorage(isolatedForTesting: true) + MCPAuditLogStorage(directory: Self.makeDirectory()) + } + + private static func makeDirectory() -> URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("MCPAuditLogStorageTests-\(UUID().uuidString)", isDirectory: true) } private func makeEntry( @@ -180,16 +176,16 @@ struct MCPAuditLogStorageTests { @Test("The store and its journal files stay readable by their owner alone") func databaseFilesAreNotWorldReadable() async throws { - let storage = makeStorage() + let directory = Self.makeDirectory() + let storage = MCPAuditLogStorage(directory: directory) await storage.addEntry(makeEntry(action: "tool.permissions")) - let directory = Self.auditDirectory() let directoryPermissions = try #require(Self.permissions(ofItemAt: directory.path)) #expect(directoryPermissions & 0o077 == 0) - let paths = await storage.databaseFilePaths - #expect(paths.isEmpty == false) - for path in paths { + let databasePath = directory.appendingPathComponent(MCPAuditLogStorage.databaseFileName).path + #expect(FileManager.default.fileExists(atPath: databasePath)) + for path in [databasePath, databasePath + "-wal", databasePath + "-shm"] { guard let permissions = Self.permissions(ofItemAt: path) else { continue } #expect( permissions & 0o077 == 0, @@ -261,8 +257,213 @@ struct MCPAuditLogStorageTests { #expect(row.tokenName == "client") } - private static func auditDirectory() -> URL { - AppStorageEnvironment.shared.applicationSupportRoot.appendingPathComponent("TablePro") + @Test("A damaged log is kept beside a fresh one, which starts by recording the reset") + func damagedLogIsSetAsideAndAFreshOneStarts() async throws { + let directory = Self.makeDirectory() + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let databaseURL = directory.appendingPathComponent(MCPAuditLogStorage.databaseFileName) + let damaged = Data(repeating: 0x2A, count: 4_096) + try damaged.write(to: databaseURL) + + let storage = MCPAuditLogStorage(directory: directory) + #expect(await storage.addEntry(makeEntry(action: "tool.afterReset"))) + + let names = try FileManager.default.contentsOfDirectory(atPath: directory.path) + let copyName = try #require(names.first { $0.hasPrefix("mcp-audit.damaged-") && $0.hasSuffix(".db") }) + let copyURL = directory.appendingPathComponent(copyName) + #expect(try Data(contentsOf: copyURL) == damaged) + let copyPermissions = try #require(Self.permissions(ofItemAt: copyURL.path)) + #expect(copyPermissions & 0o077 == 0) + let freshPermissions = try #require(Self.permissions(ofItemAt: databaseURL.path)) + #expect(freshPermissions & 0o077 == 0) + + let entries = await storage.query() + let actions = Set(entries.map { $0.action }) + #expect(actions == ["auditLog.setAside", "tool.afterReset"]) + let notice = try #require(entries.first { $0.action == "auditLog.setAside" }) + #expect(notice.details == "setAside=\(copyName)") + #expect(await storage.verify() == .intact(count: 2)) + } + + @Test("The first entry after the log is reopened continues its chain") + func reopenedLogContinuesItsChain() async { + let directory = Self.makeDirectory() + do { + let first = MCPAuditLogStorage(directory: directory) + #expect(await first.addEntry(makeEntry(action: "tool.beforeRelaunch"))) + #expect(await first.addEntry(makeEntry(action: "tool.beforeRelaunch2"))) + } + + let reopened = MCPAuditLogStorage(directory: directory) + #expect(await reopened.addEntry(makeEntry(action: "tool.afterRelaunch"))) + #expect(await reopened.count() == 3) + #expect(await reopened.verify() == .intact(count: 3)) + } + + @Test("The shared log writes the storage folder's mcp-audit.db, under a test host as in the app") + func sharedLogUsesTheStorageFolder() async throws { + MCPAuditLogger.logTokenCreated(tokenId: UUID(), tokenName: "paired client") + await MCPAuditLogger.flush() + + let directory = AppStorageEnvironment.shared.supportDirectory + let names = try FileManager.default.contentsOfDirectory(atPath: directory.path) + #expect(names.contains(MCPAuditLogStorage.databaseFileName)) + #expect(names.allSatisfy { !$0.hasPrefix("mcp-audit-test_") }) + } + + @Test("A row keeps the client that caused it, and the log filters by that client") + func filterByClient() async { + let storage = makeStorage() + await storage.addEntry(AuditEntry(category: .tool, action: "tool.a", outcome: AuditOutcome.success, clientName: "Cursor")) + await storage.addEntry(AuditEntry(category: .tool, action: "tool.b", outcome: AuditOutcome.success, clientName: "Zed")) + await storage.addEntry(AuditEntry(category: .admin, action: "server.started", outcome: AuditOutcome.success)) + + let cursor = await storage.query(clientName: "Cursor") + #expect(cursor.map { $0.action } == ["tool.a"]) + #expect(cursor.first?.clientName == "Cursor") + #expect(await storage.query(category: .admin).first?.clientName == nil) + #expect(await storage.clientNames() == ["Cursor", "Zed"]) + #expect(await storage.verify() == .intact(count: 3)) + } + + @Test("Every bridge credential reads as one token, whichever rotation wrote the row") + func filterByTokenName() async { + let storage = makeStorage() + await storage.addEntry(makeEntry(tokenId: UUID(), tokenName: MCPTokenStore.stdioBridgeTokenName, action: "tool.before")) + await storage.addEntry(makeEntry(tokenId: UUID(), tokenName: MCPTokenStore.stdioBridgeTokenName, action: "tool.after")) + await storage.addEntry(makeEntry(tokenName: "Raycast", action: "tool.raycast")) + + let bridge = await storage.query(tokenName: MCPTokenStore.stdioBridgeTokenName) + #expect(Set(bridge.map { $0.action }) == ["tool.before", "tool.after"]) + } + + @Test("A log written before the client column opens with no client on its rows and keeps its chain") + func logWithoutTheClientColumnMigrates() async throws { + let directory = Self.makeDirectory() + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let path = directory.appendingPathComponent(MCPAuditLogStorage.databaseFileName).path + // Whole seconds, so the timestamp reads back from SQL text to the same value the digest used. + let old = makeEntry(tokenName: "Raycast", timestamp: Date(timeIntervalSince1970: Date().timeIntervalSince1970.rounded(.down)), action: "tool.beforeUpgrade") + let hash = MCPAuditChainLink.digest(entry: old, sequence: 0, previousHash: MCPAuditChainLink.genesisHash) + let created = Self.execute([ + """ + CREATE TABLE audit_entries ( + id TEXT PRIMARY KEY, timestamp REAL NOT NULL, category TEXT NOT NULL, token_id TEXT, + token_name TEXT, connection_id TEXT, action TEXT NOT NULL, outcome TEXT NOT NULL, details TEXT, + sequence INTEGER, previous_hash TEXT, entry_hash TEXT + ); + """, + """ + INSERT INTO audit_entries VALUES ('\(old.id.uuidString)', \(old.timestamp.timeIntervalSince1970), 'tool', + '\(old.tokenId?.uuidString ?? "")', 'Raycast', NULL, 'tool.beforeUpgrade', 'success', NULL, + 0, '\(MCPAuditChainLink.genesisHash)', '\(hash)'); + """ + ], at: path) + #expect(created) + + let storage = MCPAuditLogStorage(directory: directory) + let migrated = await storage.query() + #expect(migrated.count == 1) + #expect(migrated.first?.clientName == nil) + #expect(migrated.first?.tokenName == "Raycast") + #expect(await storage.verify() == .intact(count: 1)) + + #expect(await storage.addEntry(AuditEntry(category: .tool, action: "tool.afterUpgrade", outcome: AuditOutcome.success, clientName: "Cursor"))) + #expect(await storage.verify() == .intact(count: 2)) + #expect(await storage.query(clientName: "Cursor").map { $0.action } == ["tool.afterUpgrade"]) + + // What a build without the column runs: it names its columns, so the log stays its own. + #expect(Self.execute([ + "SELECT id, timestamp, category, token_id, token_name, connection_id, action, outcome, details FROM audit_entries;", + """ + INSERT INTO audit_entries + (id, timestamp, category, token_id, token_name, connection_id, action, outcome, details, + sequence, previous_hash, entry_hash) + VALUES ('\(UUID().uuidString)', 0, 'tool', NULL, NULL, NULL, 'tool.olderBuild', 'success', NULL, 99, '', ''); + """ + ], at: path)) + } + + @Test("A tool call names the client that made it, never its token") + func toolCallCarriesTheClient() async throws { + let tokenId = UUID() + + MCPAuditLogger.logToolCalled( + principal: makePrincipal(tokenId: tokenId, label: MCPTokenStore.stdioBridgeTokenName), + client: "Cursor", + toolName: "list_tables", + outcome: .success + ) + await MCPAuditLogger.flush() + + let row = try #require(await MCPAuditLogStorage.shared.query(tokenId: tokenId).first) + #expect(row.clientName == "Cursor") + #expect(row.tokenName == MCPTokenStore.stdioBridgeTokenName) + } + + @Test("The client is named as Connected Clients names it, before any catalog lookup") + func clientNameFromClientInfo() { + func meta(_ info: MCPImplementation?) -> MCPRequestMeta { + MCPRequestMeta(protocolVersion: .latest, clientInfo: info, clientCapabilities: .none, progressToken: nil) + } + + #expect(MCPAuditLogger.clientName(for: meta(MCPImplementation(name: "claude-ai", title: "Claude", version: "1"))) == "Claude") + #expect(MCPAuditLogger.clientName(for: meta(MCPImplementation(name: "cursor-vscode", title: " ", version: "1"))) == "cursor-vscode") + #expect(MCPAuditLogger.clientName(for: meta(MCPImplementation(name: CLIClientIdentity.name, title: CLIClientIdentity.title, version: "1"))) + == CLIClientIdentity.pairingName) + #expect(MCPAuditLogger.clientName(for: meta(nil)) == nil) + } + + @Test("The Activity Log finds a row by its client and exports the client beside the token") + @MainActor + func activityLogSearchesAndExportsTheClient() throws { + let row = AuditEntry( + timestamp: Date(timeIntervalSince1970: 1_700_000_000), + category: .tool, + tokenName: MCPTokenStore.stdioBridgeTokenName, + action: "tool.list_tables", + outcome: AuditOutcome.success, + clientName: "Cursor" + ) + let noConnection: (UUID?) -> String? = { _ in nil } + + #expect(IntegrationsActivityLogPane.searchMatches(row, needle: "cursor", connectionName: noConnection)) + #expect(!IntegrationsActivityLogPane.searchMatches(row, needle: "zed", connectionName: noConnection)) + + let lines = IntegrationsActivityLogPane.csvString(for: [row], connectionName: noConnection) + .split(separator: "\n") + .map { $0.split(separator: ",", omittingEmptySubsequences: false).map(String.init) } + let header = try #require(lines.first) + let values = try #require(lines.last) + let client = try #require(header.firstIndex(of: "Client")) + let token = try #require(header.firstIndex(of: "Token")) + #expect(values[client] == "Cursor") + #expect(values[token] == MCPTokenStore.stdioBridgeTokenName) + } + + @Test("A client name that reads as a spreadsheet formula is exported as text") + @MainActor + func exportedClientNameNeverRunsAsAFormula() throws { + let row = AuditEntry( + timestamp: Date(timeIntervalSince1970: 1_700_000_000), + category: .tool, + tokenName: MCPTokenStore.stdioBridgeTokenName, + action: "tool.list_tables", + outcome: AuditOutcome.success, + clientName: "=HYPERLINK(\"https://example.com\")" + ) + let csv = IntegrationsActivityLogPane.csvString(for: [row], connectionName: { _ in nil }) + let values = try #require(csv.split(separator: "\n").last) + + #expect(!values.contains(",=HYPERLINK")) + #expect(values.contains("'=HYPERLINK")) + } + + private static func execute(_ statements: [String], at path: String) -> Bool { + var db: OpaquePointer? + guard sqlite3_open(path, &db) == SQLITE_OK else { return false } + defer { sqlite3_close(db) } + return statements.allSatisfy { sqlite3_exec(db, $0, nil, nil, nil) == SQLITE_OK } } private static func permissions(ofItemAt path: String) -> Int? { diff --git a/apple/TableProTests/Core/MCP/MCPHistoryAnswerBudgetTests.swift b/apple/TableProTests/Core/MCP/MCPHistoryAnswerBudgetTests.swift new file mode 100644 index 0000000000..a2350bbb93 --- /dev/null +++ b/apple/TableProTests/Core/MCP/MCPHistoryAnswerBudgetTests.swift @@ -0,0 +1,99 @@ +// +// MCPHistoryAnswerBudgetTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProPluginKit +import Testing + +struct MCPHistoryAnswerBudgetTests { + private let connectionId = UUID() + + private func entry(query: String, errorMessage: String? = nil) -> QueryHistoryEntry { + QueryHistoryEntry( + query: query, + connectionId: connectionId, + databaseName: "shop", + databaseType: .postgresql, + source: .editor, + executionTime: 0.01, + rowCount: 1, + wasSuccessful: errorMessage == nil, + errorMessage: errorMessage + ) + } + + @Test("A long statement or error in history is cut before it leaves the Mac") + func longTextIsCut() { + let statement = "INSERT INTO t VALUES " + String(repeating: "(1),", count: 50_000) + let failure = "syntax error near " + String(repeating: "y", count: 200_000) + + let query = MCPConnectionBridge.externalQueryText(of: entry(query: statement), masksValues: false) + let error = MCPConnectionBridge.externalErrorText(failure, masksValues: false) + + #expect(query.utf8.count < statement.utf8.count) + #expect(query.hasPrefix(String(statement.prefix(1_000)))) + #expect(error.utf8.count < failure.utf8.count) + // The redactor drops a token too long to read whole, since part of one can hold half a secret. + #expect(error.hasPrefix("syntax error near")) + #expect(!error.contains("yyy")) + } + + @Test("Short history text passes through whole") + func shortTextIsWhole() { + #expect(MCPConnectionBridge.externalQueryText(of: entry(query: "SELECT 1"), masksValues: false) == "SELECT 1") + } + + @Test("Entries stop at the answer budget, in order, and the answer says so") + func entriesStopAtTheBudget() { + let megabyte = String(repeating: "x", count: 1 << 20) + let entries = (0..<6).map { index in JsonValue.object(["id": .int(index), "query": .string(megabyte)]) } + + let answer = MCPConnectionBridge.withinAnswerBudget(entries) + let used = answer.entries.reduce(0) { $0 + $1.jsonString().utf8.count } + + #expect(answer.isTruncated) + #expect(!answer.entries.isEmpty) + #expect(answer.entries.count < entries.count) + #expect(used <= ExternalAnswerBudget.answerBytes) + #expect(answer.entries == Array(entries.prefix(answer.entries.count))) + } + + @Test("An answer under the budget keeps every entry") + func smallAnswerIsWhole() { + let entries = (0..<3).map { index in JsonValue.object(["id": .int(index)]) } + + let answer = MCPConnectionBridge.withinAnswerBudget(entries) + + #expect(!answer.isTruncated) + #expect(answer.entries == entries) + } + + @Test("search_query_history reports is_truncated and cuts each statement") + func searchReportsTruncation() async throws { + let statement = String(repeating: "z", count: 200_000) + let store = ToolTestQueryHistoryStore(entries: (0..<80).map { _ in entry(query: statement) }) + + let result = try await SearchQueryHistoryTool().call( + arguments: .object(["query": .string(""), "limit": .int(500)]), + context: MCPToolTestHarness.context(principal: MCPToolTestHarness.principal(access: .all)), + services: MCPToolTestHarness.services( + authPolicy: MCPToolTestHarness.authPolicy(connections: [ + connectionId: MCPToolTestHarness.snapshot(policy: .alwaysAllow) + ]), + history: store + ) + ) + + let payload = try #require(result.structuredContent) + let entries = try #require(payload["entries"]?.arrayValue) + #expect(payload["is_truncated"]?.boolValue == true) + #expect(entries.count < 80) + for entry in entries { + #expect((entry["query"]?.stringValue?.utf8.count ?? 0) < statement.utf8.count) + } + } +} diff --git a/apple/TableProTests/Core/MCP/MCPPairingServiceTests.swift b/apple/TableProTests/Core/MCP/MCPPairingServiceTests.swift index ce15993444..f486e1e671 100644 --- a/apple/TableProTests/Core/MCP/MCPPairingServiceTests.swift +++ b/apple/TableProTests/Core/MCP/MCPPairingServiceTests.swift @@ -1,12 +1,4 @@ -// -// MCPPairingServiceTests.swift -// TableProTests -// -// A pairing code is a bearer credential for the whole grant, so each code allows exactly one -// attempt: a verifier that does not match burns the code instead of leaving it alive for the rest -// of the five-minute window, which is what turned PKCE into an offline guessing game. The record -// carries the token id so a caller that fails the exchange can delete the token it was holding. -// +// A pairing code is a bearer credential for the whole grant, so each code allows exactly one attempt. import CryptoKit import Foundation @@ -17,6 +9,14 @@ import Testing @testable import TablePro +private struct RefusingCredentialStore: MCPTokenCredentialStoring { + func read() -> MCPTokenBlob { .found(Data("[]".utf8)) } + + func write(_ data: Data) -> Bool { false } + + func delete() {} +} + struct MCPPairingServiceTests { private func challenge(for verifier: String) -> String { PairingExchangeStore.sha256Base64Url(of: verifier) @@ -200,6 +200,42 @@ struct MCPPairingServiceTests { #expect(await store.contains(code: "code-overflow") == false) } + @Test("An approved pairing whose token the Keychain refused fails in words and leaves no code behind") + @MainActor + func unsavedTokenFailsThePairing() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-pairing-\(UUID().uuidString)", isDirectory: true) + let store = PairingExchangeStore() + let service = MCPPairingService( + store: store, + serverManager: MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + ) + let redirect = try #require(URL(string: "http://127.0.0.1:7391/callback")) + let request = PairingRequest( + clientName: "Launcher", + challenge: challenge(for: makeVerifier()), + redirectURL: redirect, + requestedScopes: nil, + requestedConnectionIds: nil, + state: "s1", + responseMode: .query + ) + let target = try PairingRedirectValidator.validate(redirect) + let tokens = MCPTokenStore(credentialStore: RefusingCredentialStore()) + + await #expect(throws: MCPPairingError.tokenNotSaved) { + _ = try await service.issueCode( + for: request, + target: target, + approval: PairingApproval(grantedPermissions: .readOnly, allowedConnectionIds: nil, expiresAt: nil, extraScopes: []), + tokenStore: tokens + ) + } + #expect(await store.count() == 0) + #expect(await tokens.list().isEmpty) + #expect(MCPPairingError.tokenNotSaved.localizedDescription.contains("Keychain")) + } + @Test("The exchange window is the five minutes the client is told about") func exchangeWindowIsFiveMinutes() { #expect(PairingExchangeStore.exchangeWindow == 300) diff --git a/apple/TableProTests/Core/MCP/MCPScriptResultTests.swift b/apple/TableProTests/Core/MCP/MCPScriptResultTests.swift index 221b7a6283..31d439be41 100644 --- a/apple/TableProTests/Core/MCP/MCPScriptResultTests.swift +++ b/apple/TableProTests/Core/MCP/MCPScriptResultTests.swift @@ -191,7 +191,8 @@ struct MCPScriptResultTests { let context = ChatToolContext( connectionId: driver.connection.id, bridge: MCPConnectionBridge(), - authPolicy: authPolicy() + authPolicy: authPolicy(), + chatMode: .agent ) let result = try await ExecuteQueryChatTool().execute(input: .object(["query": .string(script)]), context: context) @@ -229,6 +230,7 @@ struct MCPScriptResultTests { historyRecorder: DiscardingHistory(), executionGate: gate ), + chatMode: .agent, sessionId: sessionId, approvalWasExplicit: true, policyLookup: { _ in .alwaysAllow } diff --git a/apple/TableProTests/Core/MCP/MCPSettingsTests.swift b/apple/TableProTests/Core/MCP/MCPSettingsTests.swift index 3ccb9899fd..826a3409df 100644 --- a/apple/TableProTests/Core/MCP/MCPSettingsTests.swift +++ b/apple/TableProTests/Core/MCP/MCPSettingsTests.swift @@ -106,6 +106,19 @@ struct MCPSettingsTests { #expect(decoded.port == 5_000) } + @Test("A stored port below 1024 falls back to the default, as on Linux") + func decodesAPrivilegedPort() throws { + let decoded = try JSONDecoder().decode(MCPSettings.self, from: Data(#"{"port": 80}"#.utf8)) + #expect(decoded.port == 23_508) + } + + @Test("A port written outside 1024 to 65535 is clamped into it") + func validatedPortClamps() { + #expect(MCPSettings(port: 80).validatedPort == 1_024) + #expect(MCPSettings(port: 70_000).validatedPort == 65_535) + #expect(MCPSettings(port: 23_508).validatedPort == 23_508) + } + @Test("Row limits decode to the documented defaults") func decodesRowLimitDefaults() throws { let decoded = try JSONDecoder().decode(MCPSettings.self, from: Data("{}".utf8)) diff --git a/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift b/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift index 7bea3598b0..1ed522539d 100644 --- a/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift +++ b/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift @@ -1,13 +1,8 @@ -// -// MCPTokenStoreTests.swift -// TableProTests -// // Minting a token is a grant, so the store makes the caller state the grant: `generate` takes the // connection scope and the expiry with no defaults, because the defaults it used to carry handed // every caller that forgot them a token over every connection that never expired. Two tokens may // share a name; pairing no longer revokes an existing token because a new client claimed the same // one, which let any caller name an installed client and take its access away. -// import Foundation import TableProMCP @@ -76,10 +71,15 @@ struct MCPTokenStoreTests { MCPTokenStore(credentialStore: credentialStore) } - private func makeToken(isActive: Bool = true, expiresAt: Date? = nil) -> MCPAuthToken { + private func makeToken( + name: String = "test-token", + isActive: Bool = true, + expiresAt: Date? = nil, + isClaudeAgentCredential: Bool = false + ) -> MCPAuthToken { MCPAuthToken( id: UUID(), - name: "test-token", + name: name, prefix: "tp_abc12", tokenHash: "fakehash", salt: "fakesalt", @@ -88,7 +88,8 @@ struct MCPTokenStoreTests { createdAt: Date.now, lastUsedAt: nil, expiresAt: expiresAt, - isActive: isActive + isActive: isActive, + isClaudeAgentCredential: isClaudeAgentCredential ) } @@ -490,6 +491,170 @@ struct MCPTokenStoreTests { #expect(await store.validate(bearerToken: result.plaintext) == nil) } + @Test("A token the Keychain did not save is never handed out") + func unsavedTokenIsRefused() async throws { + let credentials = InMemoryCredentialStore() + let store = makeStore(credentials) + credentials.failsWrites = true + + await #expect(throws: MCPTokenStoreError.notSaved) { + _ = try await store.generate( + name: "Raycast", + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + } + + #expect(await store.list().isEmpty) + } + + @Test("The bridge credential is still minted when the Keychain refuses the write") + func bridgeCredentialLivesInMemory() async throws { + let credentials = InMemoryCredentialStore() + let store = makeStore(credentials) + credentials.failsWrites = true + + let bridge = try await store.generate( + name: MCPTokenStore.stdioBridgeTokenName, + permissions: MCPTokenStore.bridgeTokenPermissions, + connectionAccess: .all, + expiresAt: Date.now.addingTimeInterval(3_600), + isBridgeCredential: true, + isClaudeAgentCredential: false, + extraScopes: [] + ) + + #expect(await store.validate(bearerToken: bridge.plaintext)?.id == bridge.token.id) + } + + @Test("Deleting reports whether the Keychain took the change") + func deleteReportsPersistence() async throws { + let credentials = InMemoryCredentialStore() + let store = makeStore(credentials) + let result = try await store.generate( + name: "settings", + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + + credentials.failsWrites = true + #expect(await store.delete(tokenId: result.token.id) == false) + #expect(await store.validate(bearerToken: result.plaintext) == nil) + + let reloaded = makeStore(credentials) + await reloaded.loadFromDisk() + #expect(await reloaded.token(id: result.token.id) != nil) + + credentials.failsWrites = false + let second = try await store.generate( + name: "second", + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + #expect(await store.delete(tokenId: second.token.id)) + let afterSave = makeStore(credentials) + await afterSave.loadFromDisk() + #expect(await afterSave.list().isEmpty) + } + + @Test("Only the Claude Agent's own mint may take its token name") + func agentNameIsReserved() async throws { + let store = makeStore() + let paired = try await store.generate( + name: MCPTokenStore.claudeAgentTokenName, + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + let agent = try await store.generate( + name: MCPTokenStore.claudeAgentTokenName, + permissions: .readOnly, + connectionAccess: .all, + expiresAt: Date.now.addingTimeInterval(900), + isBridgeCredential: false, + isClaudeAgentCredential: true, + extraScopes: [] + ) + + #expect(paired.token.name != MCPTokenStore.claudeAgentTokenName) + #expect(agent.token.name == MCPTokenStore.claudeAgentTokenName) + } + + @Test("A superseded bridge credential stops working at once and no revocation observer hears of it") + func supersededBridgeCredentialIsDroppedWithoutARevocation() async throws { + let store = makeStore() + let superseded = try await store.generate( + name: MCPTokenStore.stdioBridgeTokenName, + permissions: MCPTokenStore.bridgeTokenPermissions, + connectionAccess: .all, + expiresAt: Date.now.addingTimeInterval(3_600), + isBridgeCredential: true, + isClaudeAgentCredential: false, + extraScopes: [] + ) + let paired = try await store.generate( + name: "paired", + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + let recorder = RevocationRecorder() + await store.addRevocationObserver { key, _ in + await recorder.append(key) + } + + await store.dropSupersededBridgeCredential(tokenId: superseded.token.id) + await store.delete(tokenId: paired.token.id) + let pairedKey = paired.token.id.uuidString + try await waitUntil { await recorder.keys().contains(pairedKey) } + + #expect(await store.validate(bearerToken: superseded.plaintext) == nil) + #expect(await store.list().contains(where: { $0.id == superseded.token.id }) == false) + #expect(await recorder.keys() == [pairedKey]) + } + + @Test("Only a bridge credential is dropped quietly: any other token passed in is deleted as a revocation") + func supersedingAPairedTokenStillRevokesIt() async throws { + let store = makeStore() + let paired = try await store.generate( + name: "paired", + permissions: .readWrite, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: false, + isClaudeAgentCredential: false, + extraScopes: [] + ) + let recorder = RevocationRecorder() + await store.addRevocationObserver { key, _ in + await recorder.append(key) + } + + await store.dropSupersededBridgeCredential(tokenId: paired.token.id) + let key = paired.token.id.uuidString + try await waitUntil { await recorder.keys().contains(key) } + + #expect(await store.validate(bearerToken: paired.plaintext) == nil) + } + @Test("A second token claiming the same client name leaves the first one working") func sameClientNameDoesNotRevokeTheStandingToken() async throws { let store = makeStore() @@ -574,6 +739,28 @@ struct MCPTokenStoreTests { #expect(await reader.token(id: survivor.token.id) != nil) } + @Test("A token an older build let a client name like the Claude Agent is renamed and saved at load") + func claimedAgentNameIsReleasedAtLoad() async throws { + let claimed = makeToken(name: MCPTokenStore.claudeAgentTokenName) + let agent = makeToken(name: MCPTokenStore.claudeAgentTokenName, isClaudeAgentCredential: true) + let credentials = InMemoryCredentialStore( + seed: try MCPTokenStore.encodeStored(tokens: [claimed, agent], undecoded: []) + ) + + let store = makeStore(credentials) + await store.loadFromDisk() + + let renamed = try #require(await store.token(id: claimed.id)) + #expect(renamed.name == MCPTokenStore.claudeAgentTokenName + " (client)") + #expect(renamed.tokenHash == claimed.tokenHash) + #expect(renamed.isClaudeAgentCredential == false) + #expect(await store.token(id: agent.id)?.name == MCPTokenStore.claudeAgentTokenName) + + let saved = MCPTokenStore.decodeStored(Data(credentials.storedText.utf8)).tokens + #expect(saved.first { $0.id == claimed.id }?.name == MCPTokenStore.claudeAgentTokenName + " (client)") + #expect(saved.first { $0.id == agent.id }?.name == MCPTokenStore.claudeAgentTokenName) + } + @Test("An optional grant survives a reload and adds to the tier's scopes") func extraScopesRoundTrip() async throws { let credentials = InMemoryCredentialStore() diff --git a/apple/TableProTests/Core/MCP/PairingRedirectBuilderTests.swift b/apple/TableProTests/Core/MCP/PairingRedirectBuilderTests.swift index 51cbd87e52..2076c5f5a5 100644 --- a/apple/TableProTests/Core/MCP/PairingRedirectBuilderTests.swift +++ b/apple/TableProTests/Core/MCP/PairingRedirectBuilderTests.swift @@ -1,8 +1,3 @@ -// -// PairingRedirectBuilderTests.swift -// TableProTests -// - import Foundation @testable import TablePro import Testing @@ -126,6 +121,19 @@ struct PairingRedirectBuilderTests { #expect(try success(launch, mode: .legacy) == "\(launch)&context=%7B%22code%22:%22\(Self.code)%22%7D") } + @Test("A pairing that failed after approval answers with server_error in every mode, state kept") + func failureAnswersWithServerError() throws { + for base in [Self.raycast, Self.loopback] { + for mode in Self.modes { + let redirect = try #require(PairingRedirectBuilder.failed(base: try url(base), state: "s1", mode: mode)) + let fields = try parameters(of: redirect.absoluteString) + #expect(fields["error"] == "server_error") + #expect(fields["state"] == "s1") + #expect(fields["code"] == nil) + } + } + } + @Test("No state on the link means no state on the redirect") func absentStateIsNotInvented() throws { for base in [Self.raycast, Self.loopback] { diff --git a/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsCallHandlerTests.swift b/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsCallHandlerTests.swift index 019de074d0..5b01d6e98b 100644 --- a/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsCallHandlerTests.swift +++ b/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsCallHandlerTests.swift @@ -1,5 +1,6 @@ import Foundation import TableProMCP +import TableProMCPTestSupport import TableProPluginKit @testable import TablePro import Testing @@ -124,6 +125,103 @@ struct ToolsCallHandlerTests { #expect(result.payload["content"] != nil) } + @Test("A tool outside the token's allowlist is refused before it runs, with no scope to step up to") + func toolOutsideTheAllowlistIsRefused() async throws { + let history = ToolTestQueryHistoryStore() + let handler = ToolsCallHandler(services: MCPToolTestHarness.services(history: history)) + let principal = askPrincipal() + + for name in ["execute_query", "browse_table", "export_data", "search_query_history", "explain_query"] { + let params = JsonValue.object([ + "name": .string(name), + "arguments": .object(["query": .string("SELECT 1")]) + ]) + let context = await MCPProtocolHandlerTestSupport.makeContext( + method: ToolsCallHandler.method, + params: params, + principal: principal + ) + do { + _ = try await handler.handle(params: params, context: context) + Issue.record("\(name) ran for a token whose allowlist leaves it out") + } catch let error as MCPProtocolError { + #expect(error.code == JsonRpcErrorCode.forbidden) + #expect(header(named: "WWW-Authenticate", in: error) == nil) + #expect(error.data?["requiredScopes"] == nil) + } + } + #expect(await history.receivedFilters.isEmpty) + } + + @Test("A tool inside the token's allowlist still runs") + func toolInsideTheAllowlistRuns() async throws { + let params = JsonValue.object(["name": .string("list_connections"), "arguments": .object([:])]) + let context = await MCPProtocolHandlerTestSupport.makeContext( + method: ToolsCallHandler.method, + params: params, + principal: askPrincipal() + ) + + let result = try await makeHandler().handle(params: params, context: context) + + #expect(result.payload["isError"]?.boolValue == false) + } + + @Test("Under Ask Every Time, one search_query_history call that names a connection asks once") + func namedHistorySearchAsksOnce() async throws { + let connectionId = UUID() + let presenter = RecordingApprovalPresenter(answer: true) + let history = ToolTestQueryHistoryStore(entries: [ + QueryHistoryEntry( + query: "SELECT 1", + connectionId: connectionId, + databaseName: "shop", + databaseType: .postgresql, + source: .editor, + executionTime: 0.01, + rowCount: 1, + wasSuccessful: true + ) + ]) + let services = MCPToolTestHarness.services( + authPolicy: MCPToolTestHarness.authPolicy( + connections: [connectionId: MCPToolTestHarness.snapshot(policy: .askEachTime, name: "Payroll")], + approval: .everyTime, + presenter: presenter, + ledger: MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()) + ), + history: history + ) + let params = JsonValue.object([ + "name": .string("search_query_history"), + "arguments": .object(["query": .string(""), "connection_id": .string(connectionId.uuidString)]) + ]) + let context = await MCPProtocolHandlerTestSupport.makeContext( + method: ToolsCallHandler.method, + params: params, + principalScopes: [.toolsRead] + ) + + let result = try await ToolsCallHandler(services: services).handle(params: params, context: context) + + #expect(await presenter.askedCount == 1) + #expect(result.payload["isError"]?.boolValue == false) + let entries = result.payload["structuredContent"]?["entries"]?.arrayValue ?? [] + #expect(entries.count == 1) + #expect(entries.first?["connection_id"]?.stringValue == connectionId.uuidString) + } + + private func askPrincipal() -> MCPPrincipal { + MCPPrincipal( + tokenFingerprint: "ask", + tokenId: UUID(), + scopes: [.toolsRead], + connectionAccess: .limited([]), + metadata: MCPPrincipalMetadata(label: "ask", issuedAt: .distantPast, expiresAt: nil), + toolAllowlist: MCPToolProfile.schemaLookup.toolNames + ) + } + private func makeHandler() -> ToolsCallHandler { ToolsCallHandler(services: MCPProtocolHandlerTestSupport.makeToolServices()) } diff --git a/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsListHandlerTests.swift b/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsListHandlerTests.swift index 94ccf79144..dd515d5320 100644 --- a/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsListHandlerTests.swift +++ b/apple/TableProTests/Core/MCP/Protocol/Handlers/ToolsListHandlerTests.swift @@ -35,6 +35,30 @@ struct ToolsListHandlerTests { #expect(Set(readOnly).isSubset(of: Set(readWrite))) } + @Test("A principal held to a tool allowlist sees only those tools, even with wider scopes") + func allowlistNarrowsTheListing() async throws { + let allowlist = MCPToolProfile.schemaLookup.toolNames + let principal = MCPPrincipal( + tokenFingerprint: "ask", + tokenId: UUID(), + scopes: [.toolsRead, .toolsWrite], + metadata: MCPPrincipalMetadata(label: "ask", issuedAt: .distantPast, expiresAt: nil), + toolAllowlist: allowlist + ) + let context = await MCPProtocolHandlerTestSupport.makeContext( + method: ToolsListHandler.method, + principal: principal + ) + + let result = try await ToolsListHandler().handle(params: nil, context: context) + let names = try #require(result.payload["tools"]?.arrayValue).compactMap { $0["name"]?.stringValue } + + #expect(Set(names) == allowlist) + #expect(names.count == allowlist.count) + #expect(!names.contains("execute_query")) + #expect(!names.contains("connect")) + } + @Test("A principal with no scopes sees no tools at all") func noScopesListsNothing() async throws { let payload = try await runToolsList(scopes: []) diff --git a/apple/TableProTests/Core/MCP/Protocol/MCPProtocolDispatcherTests.swift b/apple/TableProTests/Core/MCP/Protocol/MCPProtocolDispatcherTests.swift index a719153081..e5045330c7 100644 --- a/apple/TableProTests/Core/MCP/Protocol/MCPProtocolDispatcherTests.swift +++ b/apple/TableProTests/Core/MCP/Protocol/MCPProtocolDispatcherTests.swift @@ -3,6 +3,7 @@ import Foundation import TableProMCP import TableProMCPTestSupport import TableProPluginKit +import TableProTestSupport import Testing struct MCPProtocolDispatcherTests { @@ -564,6 +565,41 @@ struct MCPProtocolDispatcherTests { _ = await run } + @Test + func aSupersededTokenKeepsItsInFlightRequestsUntilItsExpiry() async throws { + let clock = MCPTestClock() + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher( + handlers: [handler], + clock: clock, + handlerTimeout: .seconds(86_400) + ) + let request = MCPProtocolTestSupport.makeModernRequest(id: .number(12), method: StubTestHandler.method) + let (exchange, sink) = MCPProtocolTestSupport.makeExchange(message: request, principal: alice) + + async let run: Void = dispatcher.dispatch(exchange) + _ = await waitForInflight(dispatcher, count: 1) + + let tokenId = try #require(alice.tokenId) + let expiry = await clock.now().addingTimeInterval(900) + await dispatcher.cancelInflight(matchingTokenId: tokenId, at: expiry) + + #expect(await dispatcher.inflightCount() == 1) + #expect(await sink.jsonWrites.isEmpty) + + try await waitUntil { + await clock.advance(by: .seconds(60)) + return await dispatcher.inflightCount() == 0 + } + #expect(await clock.now() >= expiry) + + await sink.waitForCompletion() + let errorValue = try await sink.errorEnvelope() + let error = try #require(errorValue) + #expect(error.code == JsonRpcErrorCode.requestCancelled) + _ = await run + } + @Test func anInputRequiredSignalReachesTheClientAsAnInputRequiredResult() async throws { let signal = MCPInputRequired( @@ -591,6 +627,298 @@ struct MCPProtocolDispatcherTests { #expect(result["inputRequests"]?["approve_connection"] != nil) } + @Test("The bridge's discover probe is not listed as a client, the client it carries is") + func discoverProbeIsNotAClient() async throws { + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [DiscoverHandler(), StubToolsListHandler()]) + let bridge = Self.bridgePrincipal() + + let probe = Self.request(id: .string("tablepro-bridge-discover"), method: DiscoverHandler.method, clientName: "tablepro-mcp") + let (probeExchange, probeSink) = MCPProtocolTestSupport.makeExchange(message: probe, principal: bridge, clientAddress: .localSocket) + await dispatcher.dispatch(probeExchange) + await probeSink.waitForCompletion() + #expect(try await probeSink.successResult() != nil) + #expect(await dispatcher.activityLedger.snapshot(now: MCPProtocolTestSupport.referenceDate).isEmpty) + + let list = Self.request(id: .number(2), method: StubToolsListHandler.method, clientName: "claude-ai") + let (listExchange, listSink) = MCPProtocolTestSupport.makeExchange(message: list, principal: bridge, clientAddress: .localSocket) + await dispatcher.dispatch(listExchange) + await listSink.waitForCompletion() + + let entries = await dispatcher.activityLedger.snapshot(now: MCPProtocolTestSupport.referenceDate) + #expect(entries.map(\.clientName) == ["claude-ai"]) + #expect(entries.first?.transport == .stdioBridge) + } + + @Test("Ending one stdio client's requests leaves another stdio client's request on the same credential running") + func cancellingOneClientSparesTheOther() async throws { + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [handler]) + let bridge = Self.bridgePrincipal() + let (cursorExchange, cursorSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(1), method: StubTestHandler.method, clientName: "cursor-vscode"), + principal: bridge, + clientAddress: .localSocket + ) + let (zedExchange, zedSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(2), method: StubTestHandler.method, clientName: "Zed"), + principal: bridge, + clientAddress: .localSocket + ) + + async let cursorRun: Void = dispatcher.dispatch(cursorExchange) + async let zedRun: Void = dispatcher.dispatch(zedExchange) + #expect(await waitForInflight(dispatcher, count: 2) == 2) + + let entries = await dispatcher.activityLedger.snapshot(now: MCPProtocolTestSupport.referenceDate) + let cursor = try #require(entries.first { $0.clientName == "cursor-vscode" }) + #expect(await dispatcher.cancelInflight(matchingClientId: cursor.id) == 1) + + await cursorSink.waitForCompletion() + let cursorError = try #require(try await cursorSink.errorEnvelope()) + #expect(cursorError.code == JsonRpcErrorCode.requestCancelled) + #expect(await dispatcher.inflightCount() == 1) + #expect(await zedSink.jsonWrites.isEmpty) + + _ = await dispatcher.cancelAllInflight() + await zedSink.waitForCompletion() + _ = await cursorRun + _ = await zedRun + } + + @Test("Two stdio clients may use the same request id, and one's cancellation stops only its own request") + func stdioClientsShareARequestIdWithoutCollapsing() async throws { + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [handler]) + let bridge = Self.bridgePrincipal() + let (cursorExchange, cursorSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(1), method: StubTestHandler.method, clientName: "cursor-vscode"), + principal: bridge, + clientAddress: .localSocket + ) + let (zedExchange, zedSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(1), method: StubTestHandler.method, clientName: "Zed"), + principal: bridge, + clientAddress: .localSocket + ) + + async let cursorRun: Void = dispatcher.dispatch(cursorExchange) + async let zedRun: Void = dispatcher.dispatch(zedExchange) + #expect(await waitForInflight(dispatcher, count: 2) == 2) + + let cancel = JsonRpcMessage.notification(JsonRpcNotification( + method: "notifications/cancelled", + params: .object([ + "requestId": .int(1), + "_meta": .object(MCPProtocolTestSupport.metaFields(clientName: "Zed")) + ]) + )) + let (cancelExchange, cancelSink) = MCPProtocolTestSupport.makeExchange( + message: cancel, + principal: bridge, + clientAddress: .localSocket + ) + await dispatcher.dispatch(cancelExchange) + #expect(await cancelSink.acceptedCount == 1) + + await zedSink.waitForCompletion() + let zedError = try #require(try await zedSink.errorEnvelope()) + #expect(zedError.code == JsonRpcErrorCode.requestCancelled) + #expect(await dispatcher.inflightCount() == 1) + #expect(await cursorSink.jsonWrites.isEmpty) + + _ = await dispatcher.cancelAllInflight() + await cursorSink.waitForCompletion() + _ = await cursorRun + _ = await zedRun + } + + @Test("A stdio cancellation that names no client stops nothing") + func stdioCancellationWithoutAClientIsDropped() async throws { + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [handler]) + let bridge = Self.bridgePrincipal() + let (exchange, sink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(4), method: StubTestHandler.method, clientName: "Zed"), + principal: bridge, + clientAddress: .localSocket + ) + + async let run: Void = dispatcher.dispatch(exchange) + #expect(await waitForInflight(dispatcher, count: 1) == 1) + + let (cancelExchange, cancelSink) = MCPProtocolTestSupport.makeExchange( + message: MCPProtocolTestSupport.makeNotification( + method: "notifications/cancelled", + params: .object(["requestId": .int(4)]) + ), + principal: bridge, + clientAddress: .localSocket + ) + await dispatcher.dispatch(cancelExchange) + #expect(await cancelSink.acceptedCount == 1) + #expect(await dispatcher.inflightCount() == 1) + #expect(await sink.jsonWrites.isEmpty) + + _ = await dispatcher.cancelAllInflight() + await sink.waitForCompletion() + _ = await run + } + + @Test("A stdio cancellation sent with the renewed bridge credential stops the request the old one started") + func stdioCancellationCrossesACredentialRotation() async throws { + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [handler], handlerTimeout: .seconds(10)) + let (exchange, sink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(3), method: StubTestHandler.method, clientName: "claude-code", bridgeInstance: "one"), + principal: Self.bridgePrincipal(fingerprint: "a"), + clientAddress: .localSocket + ) + + async let run: Void = dispatcher.dispatch(exchange) + #expect(await waitForInflight(dispatcher, count: 1) == 1) + + let (cancelExchange, cancelSink) = MCPProtocolTestSupport.makeExchange( + message: Self.cancellation(requestId: 3, clientName: "claude-code", bridgeInstance: "one"), + principal: Self.bridgePrincipal(fingerprint: "b"), + clientAddress: .localSocket + ) + await dispatcher.dispatch(cancelExchange) + #expect(await cancelSink.acceptedCount == 1) + #expect(await dispatcher.inflightCount() == 0) + + _ = await dispatcher.cancelAllInflight() + await sink.waitForCompletion() + let error = try #require(try await sink.errorEnvelope()) + #expect(error.code == JsonRpcErrorCode.requestCancelled) + _ = await run + } + + @Test("A stdio cancellation without clientInfo stops the request that sent it, matched by bridge instance") + func stdioCancellationWithoutClientInfo() async throws { + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [handler], handlerTimeout: .seconds(10)) + let bridge = Self.bridgePrincipal() + let (exchange, sink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(3), method: StubTestHandler.method, clientName: "claude-code", bridgeInstance: "one"), + principal: bridge, + clientAddress: .localSocket + ) + + async let run: Void = dispatcher.dispatch(exchange) + #expect(await waitForInflight(dispatcher, count: 1) == 1) + + let (otherSession, _) = MCPProtocolTestSupport.makeExchange( + message: Self.cancellation(requestId: 3, clientName: nil, bridgeInstance: "two"), + principal: bridge, + clientAddress: .localSocket + ) + await dispatcher.dispatch(otherSession) + #expect(await dispatcher.inflightCount() == 1) + + let (cancelExchange, cancelSink) = MCPProtocolTestSupport.makeExchange( + message: Self.cancellation(requestId: 3, clientName: nil, bridgeInstance: "one"), + principal: bridge, + clientAddress: .localSocket + ) + await dispatcher.dispatch(cancelExchange) + #expect(await cancelSink.acceptedCount == 1) + #expect(await dispatcher.inflightCount() == 0) + + _ = await dispatcher.cancelAllInflight() + await sink.waitForCompletion() + let error = try #require(try await sink.errorEnvelope()) + #expect(error.code == JsonRpcErrorCode.requestCancelled) + _ = await run + } + + @Test("Sessions of one stdio client share a request id: a cancellation stops its own session, Disconnect ends them all") + func sessionsOfOneStdioClientKeepTheirRequestsApart() async throws { + let handler = StubTestHandler(behavior: .waitForCancellation) + let dispatcher = MCPProtocolTestSupport.makeDispatcher(handlers: [handler], handlerTimeout: .seconds(10)) + let bridge = Self.bridgePrincipal() + let (oneExchange, oneSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(1), method: StubTestHandler.method, clientName: "claude-code", bridgeInstance: "one"), + principal: bridge, + clientAddress: .localSocket + ) + let (twoExchange, twoSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(1), method: StubTestHandler.method, clientName: "claude-code", bridgeInstance: "two"), + principal: bridge, + clientAddress: .localSocket + ) + let (threeExchange, threeSink) = MCPProtocolTestSupport.makeExchange( + message: Self.request(id: .number(1), method: StubTestHandler.method, clientName: "claude-code", bridgeInstance: "three"), + principal: bridge, + clientAddress: .localSocket + ) + + async let oneRun: Void = dispatcher.dispatch(oneExchange) + async let twoRun: Void = dispatcher.dispatch(twoExchange) + async let threeRun: Void = dispatcher.dispatch(threeExchange) + #expect(await waitForInflight(dispatcher, count: 3) == 3) + + let entries = await dispatcher.activityLedger.snapshot(now: MCPProtocolTestSupport.referenceDate) + #expect(entries.count == 1) + let row = try #require(entries.first) + + let (cancelExchange, _) = MCPProtocolTestSupport.makeExchange( + message: Self.cancellation(requestId: 1, clientName: "claude-code", bridgeInstance: "three"), + principal: bridge, + clientAddress: .localSocket + ) + await dispatcher.dispatch(cancelExchange) + #expect(await dispatcher.inflightCount() == 2) + #expect(await oneSink.jsonWrites.isEmpty) + #expect(await twoSink.jsonWrites.isEmpty) + + #expect(await dispatcher.cancelInflight(matchingClientId: row.id) == 2) + #expect(await dispatcher.inflightCount() == 0) + + _ = await dispatcher.cancelAllInflight() + for sink in [oneSink, twoSink, threeSink] { + await sink.waitForCompletion() + let error = try #require(try await sink.errorEnvelope()) + #expect(error.code == JsonRpcErrorCode.requestCancelled) + } + _ = await oneRun + _ = await twoRun + _ = await threeRun + } + + private static func bridgePrincipal(fingerprint: String = "bridge") -> MCPPrincipal { + MCPPrincipal( + tokenFingerprint: fingerprint, + tokenId: UUID(), + scopes: MCPScope.readWriteSet, + connectionAccess: .all, + metadata: MCPPrincipalMetadata( + label: MCPTokenStore.stdioBridgeTokenName, + issuedAt: MCPProtocolTestSupport.referenceDate, + expiresAt: nil, + isBridgeCredential: true + ) + ) + } + + private static func request(id: JsonRpcId, method: String, clientName: String, bridgeInstance: String? = nil) -> JsonRpcMessage { + .request(JsonRpcRequest(id: id, method: method, params: .object(["_meta": .object(meta(clientName, bridgeInstance))]))) + } + + private static func cancellation(requestId: Int, clientName: String?, bridgeInstance: String?) -> JsonRpcMessage { + .notification(JsonRpcNotification( + method: "notifications/cancelled", + params: .object(["requestId": .int(requestId), "_meta": .object(meta(clientName, bridgeInstance))]) + )) + } + + private static func meta(_ clientName: String?, _ bridgeInstance: String?) -> [String: JsonValue] { + var fields = MCPProtocolTestSupport.metaFields(clientName: clientName) + if let bridgeInstance { + fields[MCPMetaKeys.bridgeInstance] = .string(bridgeInstance) + } + return fields + } + private func dispatch( _ message: JsonRpcMessage, handlers: [any MCPMethodHandler], diff --git a/apple/TableProTests/Core/MCP/Protocol/Tools/ExportDataBudgetTests.swift b/apple/TableProTests/Core/MCP/Protocol/Tools/ExportDataBudgetTests.swift new file mode 100644 index 0000000000..4538de797b --- /dev/null +++ b/apple/TableProTests/Core/MCP/Protocol/Tools/ExportDataBudgetTests.swift @@ -0,0 +1,70 @@ +// +// ExportDataBudgetTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProPluginKit +import Testing + +@MainActor +struct ExportDataBudgetTests { + nonisolated private static let wideValue = String(repeating: "w", count: 100_000) + + // `wide` holds values past 64 KiB, `big` alone passes 4 MiB, and `after` comes once the answer is spent. + nonisolated private static func answer(_ query: String) -> QueryResult { + if query.contains("\"wide\"") { + return ScriptAnsweringDriver.resultSet(columns: ["doc"], rows: Array(repeating: [wideValue], count: 3)) + } + if query.contains("\"big\"") { + return ScriptAnsweringDriver.resultSet( + columns: ["body"], + rows: Array(repeating: [String(repeating: "x", count: 1_000)], count: 4_500) + ) + } + return ScriptAnsweringDriver.resultSet(columns: ["n"], rows: Array(repeating: ["1"], count: 5)) + } + + @Test("The tables of one export share one budget, and every value stays whole") + func tablesShareOneBudget() async throws { + let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: Self.answer) + var session = ConnectionSession(connection: connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let result = try await ExportDataTool().perform( + arguments: .object([ + "connection_id": .string(connection.id.uuidString), + "format": .string("csv"), + "tables": .array([.string("wide"), .string("big"), .string("after")]), + "max_rows": .int(5_000) + ]), + context: MCPToolTestHarness.context(), + services: MCPToolTestHarness.services( + authPolicy: MCPToolTestHarness.authPolicy(executionGate: AlwaysAllowGate()), + settings: MCPSettings(defaultRowLimit: 500, maxRowLimit: 10_000) + ) + ) + + let payload = try #require(result.structuredContent) + let exports = try #require(payload["exports"]?.arrayValue) + try #require(exports.count == 3) + #expect(exports[0]["row_count"]?.intValue == 3) + #expect(exports[0]["is_truncated"]?.boolValue == false) + let wide = try #require(exports[0]["data"]?.stringValue) + #expect(wide.contains(Self.wideValue)) + #expect(!wide.contains("TablePro cut")) + let bigRows = try #require(exports[1]["row_count"]?.intValue) + #expect(bigRows > 0 && bigRows < 4_500) + #expect((bigRows * 1_000) + (3 * 100_000) <= ExternalAnswerBudget.answerBytes) + #expect(exports[1]["is_truncated"]?.boolValue == true) + #expect(exports[2]["row_count"]?.intValue == 0) + #expect(exports[2]["is_truncated"]?.boolValue == true) + #expect(payload["is_truncated"]?.boolValue == true) + #expect(payload["notes"]?.arrayValue?.count == 1) + } +} diff --git a/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolErrorSurfaceTests.swift b/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolErrorSurfaceTests.swift index e07fa7792d..1366d0d896 100644 --- a/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolErrorSurfaceTests.swift +++ b/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolErrorSurfaceTests.swift @@ -48,6 +48,24 @@ struct MCPToolErrorSurfaceTests { #expect(MCPToolTestHarness.errorText(result) == "query_failed: syntax error at or near \"SELEC\"") } + @Test("A long error of URLs with no @ is redacted at once, and a token cut at the scan's end is dropped", .timeLimit(.minutes(1))) + func longErrorIsRedactedQuickly() { + let urls = String(repeating: "a://", count: 50_000) + #expect(MCPErrorRedactor.redact(urls) == MCPErrorRedactor.placeholder) + + let head = String(repeating: "x ", count: MCPErrorRedactor.scannedLength / 2 - 4) + let secret = "postgres://ada:hunter2@db.internal.example:5432/app" + let redacted = MCPErrorRedactor.redact(head + secret, secrets: []) + #expect(!redacted.contains("hunter2")) + #expect(!redacted.contains("db.internal")) + } + + @Test("The assistant's transcript reads a tool failure's own message, not Foundation's generic text") + func localizedDescriptionCarriesTheMessage() { + let error: Error = MCPToolExecutionError.queryFailed("relation \"ordrs\" does not exist") + #expect(error.localizedDescription == "query_failed: relation \"ordrs\" does not exist") + } + @Test("A timeout and a denied write also come back as results, not protocol errors") func timeoutAndDenialAreToolResults() async throws { let timedOut = try await result(for: MCPToolExecutionError.timedOut("Query timed out after 30 seconds")) diff --git a/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolTestHarness.swift b/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolTestHarness.swift index 9c988fb871..2b926da0b7 100644 --- a/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolTestHarness.swift +++ b/apple/TableProTests/Core/MCP/Protocol/Tools/MCPToolTestHarness.swift @@ -166,13 +166,26 @@ enum MCPToolTestHarness { } // A test that reaches a confirmation passes a stub gate: the default is the real alert gate. + // A test whose connection asks passes an approval and a presenter: the defaults read Settings and raise an alert. static func authPolicy( connections: [UUID: MCPConnectionAuthSnapshot] = [:], - executionGate: any ExecutionGate = ExecutionGateProvider.shared + executionGate: any ExecutionGate = ExecutionGateProvider.shared, + approval: MCPConnectionApproval? = nil, + presenter: (any MCPApprovalPresenting)? = nil, + ledger: MCPApprovalLedger? = nil ) -> MCPAuthPolicy { - MCPAuthPolicy( + let approvalSetting: any MCPConnectionApprovalReading + if let approval { + approvalSetting = MCPFixedApprovalReader(approval) + } else { + approvalSetting = MCPSettingsApprovalReader() + } + return MCPAuthPolicy( connectionResolver: { id in connections[id] }, connectionIdsProvider: { Set(connections.keys) }, + approvalLedger: ledger ?? MCPApprovalLedger(), + approvalSetting: approvalSetting, + presenter: presenter ?? MCPApprovalAlertPresenter(), executionGate: executionGate ) } diff --git a/apple/TableProTests/Core/MCP/Protocol/Tools/StoppedWriteOutcomeTests.swift b/apple/TableProTests/Core/MCP/Protocol/Tools/StoppedWriteOutcomeTests.swift new file mode 100644 index 0000000000..ca1b8802a0 --- /dev/null +++ b/apple/TableProTests/Core/MCP/Protocol/Tools/StoppedWriteOutcomeTests.swift @@ -0,0 +1,218 @@ +// +// StoppedWriteOutcomeTests.swift +// TableProTests +// +// A write already in the driver runs to its end when the caller stops waiting. The caller used to hear a plain +// CancellationError, history kept no row, and the write committed anyway. +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProPluginKit +import TableProTestSupport +import Testing + +private struct StoppedTool: MCPToolImplementation { + static let name = "stopped_test_tool" + static let description = "A tool whose write was stopped in the driver." + static let requiredScopes: Set = [.toolsRead] + static let inputSchema = MCPToolSchema.empty + + let cancelsItsTask: Bool + + func perform( + arguments: JsonValue, + context: MCPRequestContext, + services: MCPToolServices + ) async throws -> MCPToolCallResult { + if cancelsItsTask { + withUnsafeCurrentTask { $0?.cancel() } + } + throw DatabaseAccessBridge.stoppedWhileRunning + } +} + +@MainActor +struct StoppedWriteOutcomeTests { + private static let write = "UPDATE orders SET n = n + 1 WHERE id = 1" + + private func makeServices(history: RecordingQueryHistory) -> MCPToolServices { + MCPToolServices( + connectionBridge: MCPConnectionBridge(), + authPolicy: MCPAuthPolicy( + connectionResolver: { _ in nil }, + connectionIdsProvider: { [] }, + historyRecorder: history, + executionGate: AlwaysAllowGate() + ) + ) + } + + @Test("A write stopped in the driver is recorded in history as failed, with the outcome unknown") + func stoppedWriteIsRecorded() async throws { + let gate = AsyncGate() + let driver = AnsweringPluginDriver(hold: gate) + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + let history = RecordingQueryHistory() + let services = makeServices(history: history) + let operation = try await ApprovedOperationFixture.approve( + [OperationStatement(sql: Self.write)], scope: scope, databaseType: .postgresql + ) + + let caller = Task { + try await ToolQueryExecutor.executeAndLog( + services: services, + operation: operation, + maxRows: 500, + timeoutSeconds: 30, + principal: MCPToolTestHarness.principal() + ) + } + await gate.arrived(1) + caller.cancel() + let error = await #expect(throws: DatabaseAccessError.self) { _ = try await caller.value } + + guard case .stoppedWhileRunning? = error else { + Issue.record("expected the outcome to be unknown, got \(String(describing: error))") + return + } + let rows = history.requests + #expect(rows.count == 1) + #expect(rows.first?.query == Self.write) + #expect(rows.first?.wasSuccessful == false) + #expect(rows.first?.errorMessage == MCPErrorRedactor.message(for: DatabaseAccessBridge.stoppedWhileRunning)) + #expect(driver.finishedCount == 0) + + await gate.open() + try await waitUntil { driver.finishedCount == 1 } + } + + @Test("A cancelled MCP request whose write was in the driver keeps -33002 and says the outcome is unknown") + func cancelledRequestKeepsItsCode() async throws { + do { + _ = try await Task { + try await StoppedTool(cancelsItsTask: true).call( + arguments: .object([:]), + context: MCPToolTestHarness.context(), + services: MCPToolTestHarness.services() + ) + }.value + Issue.record("Expected a cancellation protocol error") + } catch let error as MCPProtocolError { + #expect(error.code == JsonRpcErrorCode.requestCancelled) + #expect(error.data == .object(["outcome": .string("unknown")])) + } + } + + @Test("Outside a cancelled request the stopped write is a timeout result that says it may have been applied") + func uncancelledStopIsATimeoutResult() async throws { + let result = try await StoppedTool(cancelsItsTask: false).call( + arguments: .object([:]), + context: MCPToolTestHarness.context(), + services: MCPToolTestHarness.services() + ) + + #expect(result.isError) + #expect(MCPToolTestHarness.errorText(result) == "timeout: \(DatabaseAccessBridge.stoppedWhileRunning.message)") + } + + @Test("An analyzed write explain_query runs is recorded in history, and a plan of a read is not") + func analyzedWriteIsRecorded() async throws { + let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: { _ in + ScriptAnsweringDriver.resultSet(columns: ["QUERY PLAN"], rows: [["Update on orders"]]) + }) + var session = ConnectionSession(connection: connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "shop", schema: nil) + let variants = [ + ExplainVariant(id: "plan", label: "Plan", sqlPrefix: "EXPLAIN"), + ExplainVariant(id: "analyze", label: "Analyze", sqlPrefix: "EXPLAIN ANALYZE") + ] + let analyzedWrite = try MCPConnectionBridge.explainRequest( + for: Self.write, databaseType: .postgresql, variants: variants, variantId: nil, analyze: true + ) + let plannedRead = try MCPConnectionBridge.explainRequest( + for: "SELECT 1", databaseType: .postgresql, variants: variants, variantId: nil, analyze: false + ) + let history = RecordingQueryHistory() + let services = makeServices(history: history) + + for (request, kind) in [(analyzedWrite, OperationKind.writeQuery), (plannedRead, .readQuery)] { + let operation = try await ApprovedOperationFixture.approve( + [OperationStatement(sql: request.statement)], scope: scope, databaseType: .postgresql, kind: kind + ) + _ = try await ToolQueryExecutor.explainAndLog( + services: services, + operation: operation, + request: request, + timeoutSeconds: 30, + context: MCPToolTestHarness.context(), + secrets: [] + ) + } + + #expect(driver.sentStatements == [analyzedWrite.statement, plannedRead.statement]) + #expect(history.requests.map { $0.query } == [analyzedWrite.statement]) + #expect(history.requests.first?.wasSuccessful == true) + } +} + +// A request that reaches a write needs a saved connection open to external writes. +extension GlobalStateSuites { + @MainActor + struct StoppedWriteRequestTests { + @Test("execute_query cancelled during a write in the driver answers -33002, and history keeps the write") + func executeQueryCancelledInTheDriver() async throws { + let gate = AsyncGate() + let driver = AnsweringPluginDriver(hold: gate) + var connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + connection.externalAccess = .readWrite + let savedBefore = ConnectionStorage.shared.loadConnections() + ConnectionStorage.shared.saveConnections(savedBefore + [connection]) + let scope = AnsweringPluginDriver.install(driver, connection: connection) + defer { + DatabaseManager.shared.removeSession(for: scope.connectionId) + ConnectionStorage.shared.saveConnections(savedBefore) + } + let history = RecordingQueryHistory() + let services = MCPToolServices( + connectionBridge: MCPConnectionBridge(), + authPolicy: MCPAuthPolicy( + connectionResolver: { _ in nil }, + connectionIdsProvider: { [] }, + historyRecorder: history, + executionGate: AlwaysAllowGate() + ) + ) + + let caller = Task { + try await ExecuteQueryTool().call( + arguments: .object([ + "connection_id": .string(connection.id.uuidString), + "query": .string("UPDATE orders SET n = n + 1 WHERE id = 1") + ]), + context: MCPToolTestHarness.context(), + services: services + ) + } + await gate.arrived(1) + caller.cancel() + do { + _ = try await caller.value + Issue.record("Expected the cancelled request to end as a cancellation") + } catch let error as MCPProtocolError { + #expect(error.code == JsonRpcErrorCode.requestCancelled) + #expect(error.data == .object(["outcome": .string("unknown")])) + } + #expect(history.requests.map { $0.wasSuccessful } == [false]) + + await gate.open() + try await waitUntil { driver.finishedCount == 1 } + } + } +} diff --git a/apple/TableProTests/Core/Menu/ManualCommitMenuTests.swift b/apple/TableProTests/Core/Menu/ManualCommitMenuTests.swift new file mode 100644 index 0000000000..244921f413 --- /dev/null +++ b/apple/TableProTests/Core/Menu/ManualCommitMenuTests.swift @@ -0,0 +1,119 @@ +// +// ManualCommitMenuTests.swift +// TableProTests +// + +import AppKit +@testable import TablePro +import Testing + +@MainActor +struct ManualCommitMenuTests { + private func databaseMenu() throws -> NSMenu { + let bar = MainMenuBuilder.build(keyboard: KeyboardSettings()).bar + return try #require(bar.items.first { $0.title == String(localized: "Database") }?.submenu) + } + + private func enabled(_ selector: Selector, _ context: MenuValidationContext) -> Bool { + MainSplitViewController.isEnabled(selector, context: context) + } + + @Test("Manual Commit, its footnotes, Commit Transaction and Roll Back Transaction follow Safe Mode Level") + func itemsFollowSafeModeLevel() throws { + let items = try databaseMenu().items + let safeMode = try #require(items.firstIndex { $0.title == String(localized: "Safe Mode Level") }) + try #require(safeMode + 5 < items.count) + + #expect(items[safeMode + 1].title == String(localized: "Manual Commit")) + #expect(items[safeMode + 1].action == #selector(MainSplitViewController.toggleManualCommit(_:))) + #expect(items[safeMode + 2].identifier == DatabaseMenuBuilder.manualCommitFootnoteIdentifier) + #expect(items[safeMode + 2].isHidden) + #expect(items[safeMode + 3].identifier == DatabaseMenuBuilder.manualCommitUnavailableFootnoteIdentifier) + #expect(items[safeMode + 3].attributedTitle?.string.replacingOccurrences(of: "\n", with: " ") + == ManualCommit.unavailableExplanation) + #expect(items[safeMode + 3].isHidden) + #expect(items[safeMode + 4].title == String(localized: "Commit Transaction")) + #expect(items[safeMode + 4].action == #selector(MainSplitViewController.commitOpenTransaction(_:))) + #expect(items[safeMode + 5].title == String(localized: "Roll Back Transaction")) + #expect(items[safeMode + 5].action == #selector(MainSplitViewController.rollBackOpenTransaction(_:))) + } + + @Test("Commit and Roll Back Transaction are Keyboard rows with no default chord") + func transactionCommandsShipUnbound() throws { + for action in [ShortcutAction.commitTransaction, .rollbackTransaction] { + #expect(KeyboardSettings.defaultShortcuts[action] == nil) + let item = try #require(try databaseMenu().items.first { $0.identifier == MenuItemFactory.identifier(for: action) }) + #expect(item.keyEquivalent.isEmpty) + } + } + + @Test("Manual Commit needs a connection that can hold a transaction, and stays enabled while on") + func manualCommitEnablement() { + let toggle = #selector(MainSplitViewController.toggleManualCommit(_:)) + var context = MenuValidationContext() + context.transaction.isAvailable = true + context.transaction.canHoldTransactions = true + #expect(!enabled(toggle, context)) + + context.isConnected = true + #expect(enabled(toggle, context)) + + context.transaction.canHoldTransactions = false + #expect(!enabled(toggle, context)) + + context.transaction.isManualCommitOn = true + #expect(enabled(toggle, context)) + } + + @Test("Where DDL commits on its own, Manual Commit is disabled with its footnote, even when stored on") + func manualCommitUnavailable() { + let toggle = #selector(MainSplitViewController.toggleManualCommit(_:)) + var context = MenuValidationContext() + context.isConnected = true + context.transaction.canHoldTransactions = true + context.transaction.isManualCommitOn = true + #expect(!enabled(toggle, context)) + #expect(MainSplitViewController.shownManualCommitFootnote(context.transaction, isConnected: true) + == DatabaseMenuBuilder.manualCommitUnavailableFootnoteIdentifier) + #expect(MainSplitViewController.shownManualCommitFootnote(context.transaction, isConnected: false) == nil) + } + + @Test("The reporting footnote shows only where the engine can hold but its driver can't report, while off") + func reportingFootnote() { + var state = TransactionMenuState() + state.isAvailable = true + #expect(MainSplitViewController.shownManualCommitFootnote(state, isConnected: true) + == DatabaseMenuBuilder.manualCommitFootnoteIdentifier) + state.canHoldTransactions = true + #expect(MainSplitViewController.shownManualCommitFootnote(state, isConnected: true) == nil) + state.canHoldTransactions = false + state.isManualCommitOn = true + #expect(MainSplitViewController.shownManualCommitFootnote(state, isConnected: true) == nil) + } + + @Test("Commit needs an open transaction it may commit; Roll Back needs any held transaction") + func commitAndRollBackEnablement() { + let commit = #selector(MainSplitViewController.commitOpenTransaction(_:)) + let rollBack = #selector(MainSplitViewController.rollBackOpenTransaction(_:)) + var context = MenuValidationContext() + context.isConnected = true + #expect(!enabled(commit, context)) + #expect(!enabled(rollBack, context)) + + context.transaction.holdsTransaction = true + #expect(!enabled(commit, context)) + #expect(enabled(rollBack, context)) + + context.transaction.canCommit = true + #expect(enabled(commit, context)) + } + + // The level is the gate's, managed floor included, so Commit is never offered for the gate to refuse. + @Test("Commit is offered for an open hold below the effective Read-Only, never for an aborted one") + func commitFollowsSafeModeAndState() { + #expect(ConnectionTransactionActions.offersCommit(.open, effectiveLevel: .silent)) + #expect(ConnectionTransactionActions.offersCommit(.open, effectiveLevel: .safeModeFull)) + #expect(!ConnectionTransactionActions.offersCommit(.open, effectiveLevel: .readOnly)) + #expect(!ConnectionTransactionActions.offersCommit(.aborted, effectiveLevel: .silent)) + } +} diff --git a/apple/TableProTests/Core/Menu/QueryTabRevertMenuTests.swift b/apple/TableProTests/Core/Menu/QueryTabRevertMenuTests.swift new file mode 100644 index 0000000000..2152832a19 --- /dev/null +++ b/apple/TableProTests/Core/Menu/QueryTabRevertMenuTests.swift @@ -0,0 +1,89 @@ +// +// QueryTabRevertMenuTests.swift +// TableProTests +// + +import AppKit +@testable import TablePro +import Testing + +@MainActor +struct QueryTabRevertMenuTests { + private static let revert = #selector(MainSplitViewController.revertToQueryTabVersion(_:)) + private static let browse = #selector(MainSplitViewController.browseQueryTabVersions(_:)) + + private func fileMenu() throws -> NSMenu { + let bar = MainMenuBuilder.build(keyboard: KeyboardSettings()).bar + return try #require(bar.items.first { $0.title == String(localized: "File") }?.submenu) + } + + private func summary(_ id: Int64, _ reason: QueryTabVersionReason) -> QueryTabVersionSummary { + QueryTabVersionSummary(id: id, reason: reason, savedAt: Date(timeIntervalSince1970: 1_800_000_000 + Double(id))) + } + + @Test("Revert To follows Save As in File, fills when it opens and carries no shortcut") + func revertToFollowsSaveAs() throws { + let file = try fileMenu() + let saveAs = try #require(file.items.firstIndex { + $0.action == #selector(MainSplitViewController.saveDocumentAs(_:)) + }) + let revertTo = file.items[saveAs + 1] + + #expect(revertTo.title == String(localized: "Revert To")) + #expect(revertTo.keyEquivalent.isEmpty) + #expect(revertTo.submenu?.delegate is QueryTabRevertMenuDelegate) + #expect(revertTo.submenu?.items.isEmpty == true) + } + + @Test("The list is the tab's versions, then Browse All Versions…") + func listsVersionsThenBrowse() throws { + let menu = NSMenu() + let tabId = UUID() + + QueryTabRevertMenuDelegate.fill(menu, tabId: tabId, summaries: [summary(2, .run), summary(1, .idle)]) + + #expect(menu.items.count == 4) + #expect(menu.items[0].action == Self.revert) + #expect(menu.items[0].target == nil) + #expect(menu.items[0].representedObject as? QueryTabRevertChoice == QueryTabRevertChoice(tabId: tabId, versionId: 2)) + #expect(menu.items[1].representedObject as? QueryTabRevertChoice == QueryTabRevertChoice(tabId: tabId, versionId: 1)) + #expect(menu.items[0].title.contains(String(localized: "Run"))) + #expect(menu.items[2].isSeparatorItem) + #expect(menu.items[3].title == String(localized: "Browse All Versions…")) + #expect(menu.items[3].action == Self.browse) + #expect(menu.items[3].target == nil) + } + + @Test("With no versions the list says so and still offers Browse All Versions…") + func emptyListShowsThePlaceholder() { + let menu = NSMenu() + menu.addItem(withTitle: "stale", action: nil, keyEquivalent: "") + + QueryTabRevertMenuDelegate.fill(menu, tabId: UUID(), summaries: []) + + #expect(menu.items.map { $0.title } == [ + String(localized: "None Available"), "", String(localized: "Browse All Versions…"), + ]) + #expect(menu.items[0].isEnabled == false) + } + + @Test("Revert To and Browse All Versions need a query tab, and no connection") + func commandsNeedAQueryTab() { + for selector in [Self.revert, Self.browse] { + var context = MenuValidationContext() + #expect(MainSplitViewController.resolvedEnablement(selector, context: context) == false) + + context.isQueryTab = true + #expect(MainSplitViewController.resolvedEnablement(selector, context: context) == true) + + context.isAgentMode = true + #expect(MainSplitViewController.resolvedEnablement(selector, context: context) == false) + } + } + + @Test("The window answers both commands") + func windowImplementsBoth() { + #expect(MainSplitViewController.instancesRespond(to: Self.revert)) + #expect(MainSplitViewController.instancesRespond(to: Self.browse)) + } +} diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift index 772fdf6ba5..554f16d353 100644 --- a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift @@ -17,6 +17,7 @@ private final class CopyDriver: PluginDriverStub, @unchecked Sendable { var batchSize = 2 var executedQueries: [String] = [] var executedParameters: [[PluginCellValue]] = [] + var executedTypeNames: [[String?]?] = [] var quote: (String) -> String = { "`\($0)`" } var declaredParameterStyle: ParameterStyle = .questionMark var transactionEvents: [String] = [] @@ -34,6 +35,15 @@ private final class CopyDriver: PluginDriverStub, @unchecked Sendable { return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) } + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + executedTypeNames.append(context.parameterTypeNames) + return try await executeParameterized(query: query, parameters: parameters) + } + func streamRows(query: String) -> AsyncThrowingStream { let rows = streamed let size = batchSize @@ -49,6 +59,13 @@ private final class CopyDriver: PluginDriverStub, @unchecked Sendable { } } + var streamContexts: [PluginStatementContext] = [] + + func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { + streamContexts.append(context) + return streamRows(query: query) + } + func quoteIdentifier(_ name: String) -> String { quote(name) } func beginTransaction() async throws { transactionEvents.append("begin") } @@ -104,6 +121,19 @@ struct ObjectCopyRowCopierTests { #expect(target.executedParameters.flatMap { $0 }.count == 10) } + // A source session holding a transaction runs the copy's SELECT in it at Read-Only, which needs the call check. + @Test + func aReadOnlySourceReadSaysItCallsNoUnlistedFunction() async throws { + let source = CopyDriver() + source.streamed = rows(1) + + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + .copy(from: source, to: CopyDriver(), readContext: PluginStatementContext(enforcesReadOnly: true)) { _ in } + + #expect(source.streamContexts.map { $0.enforcesReadOnly } == [true]) + #expect(source.streamContexts.map { $0.callsUnlistedFunction } == [false]) + } + /// One multi-row INSERT per batch, not one per row: that is the whole reason the copy is /// faster than exporting to a file and importing it back. @Test @@ -361,6 +391,38 @@ struct ObjectCopyRowCopierTests { #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "00000101"]) } + // MARK: - Column types + + // Oracle stored a copied `00123` in a VARCHAR2 as 123: the driver was never told the column was text. + @Test + func eachValueTellsTheTargetItsColumnType() async throws { + let source = CopyDriver() + source.streamed = [[.text("1"), .text("00123")], [.text("2"), .text("0.50")]] + let target = CopyDriver() + + _ = try await ObjectCopyRowCopier( + step: step(columnTypeNames: ["id": "NUMBER", "name": "VARCHAR2(20 BYTE)"]), + sourceDatabaseType: .postgresql, + targetDatabaseType: .oracle + ).copy(from: source, to: target) { _ in } + + // Oracle takes one row per INSERT. + #expect(target.executedTypeNames == [["NUMBER", "VARCHAR2(20 BYTE)"], ["NUMBER", "VARCHAR2(20 BYTE)"]]) + #expect(target.executedParameters.map { row in row.map { $0.sortKey } } == [["1", "00123"], ["2", "0.50"]]) + } + + @Test + func aTargetOfNoKnownColumnTypesIsToldNone() async throws { + let source = CopyDriver() + source.streamed = rows(1) + let target = CopyDriver() + + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + .copy(from: source, to: target) { _ in } + + #expect(target.executedTypeNames == [nil]) + } + private static func draft( columns: [PluginColumnInfo], from source: DatabaseType, diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift new file mode 100644 index 0000000000..c9e9c4e532 --- /dev/null +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift @@ -0,0 +1,107 @@ +// +// ObjectCopyRowCutTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CappedSourceDriver: PluginDriverStub, @unchecked Sendable { + let declaredLimit: Int? + let rowCount: Int + + init(declaredLimit: Int?, rowCount: Int) { + self.declaredLimit = declaredLimit + self.rowCount = rowCount + } + + func streamRows(query: String) -> AsyncThrowingStream { + var header = PluginStreamHeader(columns: ["id", "name"], columnTypeNames: []) + header.rowLimit = declaredLimit + let rows: [PluginRow] = (0 ..< rowCount).map { [.text(String($0)), .text("row\($0)")] } + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } +} + +private final class WritingTargetDriver: PluginDriverStub, @unchecked Sendable { + var parameterStyle: ParameterStyle { .questionMark } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ name: String) -> String { "`\(name)`" } +} + +struct ObjectCopyRowCutTests { + private func step() -> ObjectCopyTableStep { + ObjectCopyTableStep( + selection: ObjectCopySelection(kind: .table, name: "orders", schema: "public"), + dropStatements: [], + sequenceStatements: [], + createStatements: [], + truncateStatements: [], + columns: ["id", "name"], + targetColumnTypeNames: [:], + primaryKeyColumns: ["id"], + sourceQuery: "SELECT `id`, `name` FROM `public`.`orders`", + targetTable: "orders", + targetSchema: "public", + estimatedRows: nil, + copiesData: true, + copiesIdentityColumn: false, + coercer: nil, + serverSideInsert: nil, + note: nil + ) + } + + private func copy(from source: CappedSourceDriver) async throws -> ObjectCopyRowCopier.Outcome { + try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + .copy(from: source, to: WritingTargetDriver()) { _ in } + } + + // A 6,000,000-row SAP HANA table arrived as 5,000,000 rows and the copy reported success. + @Test("A copy whose source stream stopped at the driver's limit says where") + func cutStreamIsReported() async throws { + let outcome = try await copy(from: CappedSourceDriver(declaredLimit: 3, rowCount: 3)) + + #expect(outcome.inserted == 3) + #expect(outcome.rowCut == 3) + } + + @Test("A copy that read fewer rows than the driver's limit is whole") + func shortStreamIsWhole() async throws { + let outcome = try await copy(from: CappedSourceDriver(declaredLimit: 10, rowCount: 3)) + + #expect(outcome.inserted == 3) + #expect(outcome.rowCut == nil) + } + + @Test("A cut table's outcome carries the export's wording") + func cutOutcomeCarriesTheWarning() { + let selection = ObjectCopySelection(kind: .table, name: "orders", schema: "public") + let result = ObjectCopyRunResult( + outcomes: [ + ObjectCopyObjectOutcome( + selection: selection, + rowsCopied: 3, + error: nil, + warning: StreamRowCutWarning.text(table: selection.qualifiedName, limit: 3) + ) + ], + rowsCopied: 3, + cancelled: false, + createdDatabase: nil + ) + + #expect(result.warnings == [StreamRowCutWarning.text(table: selection.qualifiedName, limit: 3)]) + #expect(result.failedCount == 0) + } +} diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyTransactionTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyTransactionTests.swift index ab8b8205a5..77179ab158 100644 --- a/apple/TableProTests/Core/ObjectCopy/ObjectCopyTransactionTests.swift +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyTransactionTests.swift @@ -286,6 +286,24 @@ struct ObjectCopyTransactionTests { #expect(result.pendingInSessionTransaction == false) } + // A held copy on a pooled lease would be rolled back when the pool closes it, after the sheet reported success. + @Test("Copy To commits its own transaction even on a connection in Manual Commit") + func manualCommitConnectionStillCommitsTheCopy() async throws { + // MySQL runs a stored Manual Commit as automatic, so the session is on an engine that holds. + var connection = TestFixtures.makeConnection(id: targetScope.connectionId, type: .postgresql) + connection.commitMode = .manual + DatabaseManager.shared.injectSession(ConnectionSession(connection: connection), for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let log = CopyEventLog() + + _ = try await run( + replacementPlan(), source: RecordingCopyDriver("source", log: log), target: RecordingCopyDriver("target", log: log) + ) + + #expect(log.events.first == "target begin") + #expect(log.events.last == "target commit") + } + /// The statements still run: they join the transaction the user opened, and only the user can /// end it. Opening one over it is what aborted it on DuckDB. @Test("A session holding a transaction is joined rather than wrapped, with its keys as they are") diff --git a/apple/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift b/apple/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift index 29f8559502..4c191de08a 100644 --- a/apple/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift +++ b/apple/TableProTests/Core/Operations/AppSettingsCategoryParityTests.swift @@ -23,4 +23,32 @@ struct AppSettingsCategoryParityTests { func notificationsSync() { #expect(AppSettingsCategory.synced.contains(AppSettingsCategory.notifications)) } + + @MainActor + @Test("Value colors sync as their own category, and a rewrite of the Data Grid blob keeps them") + func valueColorsSyncAsTheirOwnCategory() throws { + #expect(AppSettingsCategory.synced.contains(AppSettingsCategory.valueColors)) + let pool = EphemeralDefaultsPool() + let storage = AppSettingsStorage(userDefaults: try pool.make().defaults) + storage.saveValueColors(ValueColorSettings(colorsValuesByType: true)) + + storage.saveDataGrid(.default) + + #expect(storage.loadValueColors().colorsValuesByType) + } + + @MainActor + @Test("Value colors persist under their own key, read off when never saved, and reset with the rest") + func valueColorsStorageRoundTrip() throws { + let pool = EphemeralDefaultsPool() + let storage = AppSettingsStorage(userDefaults: try pool.make().defaults) + #expect(storage.loadValueColors() == .default) + + storage.saveValueColors(ValueColorSettings(colorsValuesByType: true)) + #expect(storage.loadValueColors().colorsValuesByType) + #expect(storage.loadDataGrid() == .default) + + storage.resetToDefaults() + #expect(storage.loadValueColors() == .default) + } } diff --git a/apple/TableProTests/Core/Operations/RunningWorkTests.swift b/apple/TableProTests/Core/Operations/RunningWorkTests.swift index 2b223d8bb4..4486651f38 100644 --- a/apple/TableProTests/Core/Operations/RunningWorkTests.swift +++ b/apple/TableProTests/Core/Operations/RunningWorkTests.swift @@ -121,13 +121,13 @@ struct RunningWorkTests { let prompt = try #require(RunningWorkPrompt.make( action: .close, work: [work(.restore, subject: "sales"), work(.query, subject: "Query 2")], - otherLosses: RunningWorkPrompt.transactionLines(for: ["app"]) + otherLosses: ["Unsaved changes"] )) #expect(prompt.message.hasPrefix(""" Restoring “sales” on Production Running query in “Query 2” on Production - Open transaction in “app” + Unsaved changes Closing stops this work. """)) @@ -151,13 +151,18 @@ struct RunningWorkTests { """) } - @Test("The quit a relaunch starts stops the approved work without asking again") - func relaunchApprovedQuitStopsWork() { + // The work is stopped by the quit once its commits landed, never by the approval. + @Test("The quit a relaunch starts asks nothing and plans the approved work's stop") + func relaunchApprovedQuitStopsWork() throws { var stopped = 0 let registry = RunningWorkRegistry(providers: []) registry.begin(.restore, subject: "sales", connectionName: "", connectionIds: []) { stopped += 1 } - #expect(QuitApproval.approve(registry: registry, relaunchApproved: true)) + let plan = try #require(QuitApproval.approve(registry: registry, relaunchApproved: true)) + #expect(plan.commits.isEmpty) + #expect(stopped == 0) + + registry.cancel(plan.work) #expect(stopped == 1) } diff --git a/apple/TableProTests/Core/Plugins/ConnectionFieldTrustCoverageTests.swift b/apple/TableProTests/Core/Plugins/ConnectionFieldTrustCoverageTests.swift index 8a84c3f491..e3143eda6a 100644 --- a/apple/TableProTests/Core/Plugins/ConnectionFieldTrustCoverageTests.swift +++ b/apple/TableProTests/Core/Plugins/ConnectionFieldTrustCoverageTests.swift @@ -10,6 +10,7 @@ private let hostOwnedKeys: [String] = [ DatabaseConnection.postDisconnectScriptKey, DatabaseConnection.scriptTimeoutSecondsKey, DatabaseConnection.completionIdentifierCaseKey, + DatabaseConnection.commitModeKey, DatabaseConnection.binaryInlineLimitKilobytesKey, DatabaseConnection.maskingRulesKey, DatabaseConnection.sshForwardUnixSocketPathKey, diff --git a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift index 0e380bac45..6704212a34 100644 --- a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift +++ b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRowLimitTests.swift @@ -19,6 +19,99 @@ private final class StreamRecordingDriver: PluginDriverStub, @unchecked Sendable func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } } +// No stream of its own, so its rows reach the export through PluginKit's default, the way SAP HANA's do. +private final class CuttingQueryDriver: PluginDriverStub, @unchecked Sendable { + private let rowCount: Int + private let isCut: Bool + + init(rowCount: Int, isCut: Bool) { + self.rowCount = rowCount + self.isCut = isCut + } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult( + columns: ["id"], + columnTypeNames: ["INTEGER"], + rows: (0.. String { "\"\(name)\"" } +} + +// A SAP HANA export over 5,000,000 rows wrote the first 5,000,000 and reported success: the default +// stream dropped the result's cut, so nothing named the file as partial. +@MainActor +struct ExportDriverRowCutTests { + private let orders = PluginExportTable(name: "orders", databaseName: "SALES", tableType: "table", schema: "SALES", kind: .table) + + private func exportAdapter(_ driver: CuttingQueryDriver, type: DatabaseType) -> ExportDataSourceAdapter { + ExportDataSourceAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver), + databaseType: type + ) + } + + private func rowCount(_ stream: AsyncThrowingStream) async throws -> Int { + var count = 0 + for try await element in stream { + if case .rows(let rows) = element { count += rows.count } + } + return count + } + + @Test("A table export the driver cut at its row limit is named as partial") + func cutTableExportIsReported() async throws { + let adapter = exportAdapter(CuttingQueryDriver(rowCount: 3, isCut: true), type: .sapHana) + + let exported = try await rowCount(adapter.streamRows(for: orders)) + + #expect(exported == 3) + #expect(adapter.cappedTableWarnings.count == 1) + #expect(adapter.cappedTableWarnings.first?.contains("orders") == true) + #expect(adapter.cappedTableWarnings.first?.contains("3") == true) + } + + @Test("A table export the driver read whole carries no warning") + func wholeTableExportIsNotReported() async throws { + let adapter = exportAdapter(CuttingQueryDriver(rowCount: 3, isCut: false), type: .sapHana) + + let exported = try await rowCount(adapter.streamRows(for: orders)) + + #expect(exported == 3) + #expect(adapter.cappedTableWarnings.isEmpty) + } + + @Test("A table export that reaches the engine's own ceiling is still named as partial") + func engineCeilingIsStillReported() async throws { + let adapter = exportAdapter(CuttingQueryDriver(rowCount: 10_000, isCut: false), type: .cloudflareR2SQL) + + _ = try await rowCount(adapter.streamRows(for: orders)) + + #expect(adapter.cappedTableWarnings.count == 1) + } + + @Test("A query export the driver cut is named as partial at the limit it declared") + func cutQueryExportIsReported() async throws { + let driver = PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: .sapHana), + pluginDriver: CuttingQueryDriver(rowCount: 3, isCut: true) + ) + let source = StreamingQueryExportDataSource(query: "SELECT id FROM orders", driver: driver, databaseType: .sapHana) + + let exported = try await rowCount(source.streamRows(table: "result", databaseName: "")) + + #expect(exported == 3) + #expect(source.driverRowCut == 3) + #expect(ExportService.leadingRowsCapWarning(exportedRows: 3, pagination: .offset, driverCut: source.driverRowCut) != nil) + #expect(ExportService.leadingRowsCapWarning(exportedRows: 3, pagination: .offset, driverCut: nil) == nil) + } +} + struct ExportDataSourceAdapterRowLimitTests { private func streamedQuery( type: DatabaseType, diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift index 6af9834a0a..a30117c995 100644 --- a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift @@ -101,13 +101,16 @@ struct ImportDataSinkAdapterBitColumnTests { #expect(sql.last == "UPDATE `devices` SET `name` = ? WHERE `flags` = CAST(? AS UNSIGNED)") } + // Every engine is asked once, for the types its driver writes each value by and its binary and boolean columns. @Test - func anEngineThatReadsBitsAsDigitsIsNotAsked() async throws { + func anEngineThatReadsBitsAsDigitsGetsNoCast() async throws { let driver = BitTableDriver() + let target = sink(on: driver, type: .postgresql) - try await sink(on: driver, type: .postgresql).insertRows([["id": .text("1"), "flags": .text("101")]]) + try await target.insertRows([["id": .text("1"), "flags": .text("101")]]) + try await target.insertRows(columns: ["id", "flags"], values: [[.text("2"), .text("110")]]) - #expect(driver.columnFetches == 0) - #expect(driver.statements.first?.sql.contains("CAST") == false) + #expect(driver.columnFetches == 1) + #expect(driver.statements.allSatisfy { !$0.sql.contains("CAST") }) } } diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift new file mode 100644 index 0000000000..2df5b46f5f --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift @@ -0,0 +1,134 @@ +// +// ImportDataSinkAdapterCopyTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +private final class CopyingDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private let declaresCopy: Bool + private var copies: [(statement: String, data: Data)] = [] + + init(declaresCopy: Bool) { + self.declaresCopy = declaresCopy + } + + var capabilities: PluginCapabilities { + declaresCopy ? [.transactions, .copyFromStandardInput] : [.transactions] + } + + var received: [(statement: String, data: Data)] { + lock.withLock { copies } + } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + var data = Data() + while let piece = try rows.nextPiece() { + data.append(piece) + } + lock.withLock { copies.append((statement, data)) } + return .empty + } +} + +// Reads the first piece, then holds the copy until a Stop cancels the query, as a copy waiting on its server does. +private final class StoppableCopyDriver: PluginDriverStub, @unchecked Sendable { + let started = AsyncGate() + private let stopped = AsyncGate() + private let lock = NSLock() + private var cancels = 0 + + var capabilities: PluginCapabilities { [.transactions, .copyFromStandardInput] } + + var cancelCount: Int { lock.withLock { cancels } } + + func cancelQuery() throws { + lock.withLock { cancels += 1 } + Task { await stopped.open() } + } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + _ = try rows.nextPiece() + await started.open() + await stopped.wait() + throw CancellationError() + } +} + +struct ImportDataSinkAdapterCopyTests { + private static func rows(_ text: String) -> PluginCopyData { + let remaining = OSAllocatedUnfairLock(initialState: Data(text.utf8)) + return PluginCopyData { + remaining.withLock { data in + defer { data = nil } + return data + } + } + } + + private static func sink(on driver: CopyingDriver) -> ImportDataSinkAdapter { + let connection = TestFixtures.makeConnection(type: .postgresql) + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + return ImportDataSinkAdapter(driver: adapter, databaseType: .postgresql) + } + + @Test("A COPY reaches a driver that declares copy-in with its header and every row byte") + func copyReachesTheDriver() async throws { + let driver = CopyingDriver(declaresCopy: true) + let sink = Self.sink(on: driver) + + try await sink.copyFromStandardInput(statement: "COPY t (a) FROM stdin", line: 4, rows: Self.rows("1\n2\n")) + + let received = driver.received + #expect(received.map { $0.statement } == ["COPY t (a) FROM stdin"]) + #expect(received.map { $0.data } == [Data("1\n2\n".utf8)]) + } + + @Test("A driver without copy-in refuses the COPY at its line before anything is sent") + func driverWithoutCopyRefuses() async throws { + let driver = CopyingDriver(declaresCopy: false) + let sink = Self.sink(on: driver) + + await #expect(throws: SQLFileParseError.copyFromStandardInput(line: 4)) { + try await sink.copyFromStandardInput(statement: "COPY t (a) FROM stdin", line: 4, rows: Self.rows("1\n")) + } + #expect(driver.received.isEmpty) + } + + @Test("Stop during an import's COPY cancels the driver's copy at once and stops the run with rollback") + func stopCancelsTheCopy() async throws { + let driver = StoppableCopyDriver() + let run = ImportRun() + let connection = TestFixtures.makeConnection(type: .postgresql) + let sink = ImportDataSinkAdapter( + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver), + databaseType: .postgresql, + importRun: run + ) + let copy = Task { + try await sink.copyFromStandardInput(statement: "COPY t (a) FROM stdin", line: 1, rows: Self.rows("1\n")) + } + await driver.started.wait() + + run.requestPause() + + #expect(driver.cancelCount == 1) + #expect(run.stopChoice == .rollBack) + await #expect(throws: CancellationError.self) { try await copy.value } + #expect(run.questionIfPaused() == nil) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift index c7661521e8..4875bdacd6 100644 --- a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift @@ -132,9 +132,8 @@ struct ImportDataSinkAdapterMappingTests { #expect(columns.isEmpty) } - /// A row carrying nothing has nothing to lose, so it passes through. Only a row holding values - /// that reach no column is worth stopping for, and conflating the two would turn an empty - /// object in an NDJSON file into a failed import. + // An empty object is how JSON export writes an all-NULL row without its NULLs, so it is written as a row of + // defaults. Only a row holding values that reach no column is an error. @Test("A row with no values at all is not an error") func emptyRowIsNotAnError() async throws { let sink = adapter(mapping: ["name": "name"]) diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterParameterTypeTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterParameterTypeTests.swift new file mode 100644 index 0000000000..8f34cb63b3 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterParameterTypeTests.swift @@ -0,0 +1,119 @@ +// +// ImportDataSinkAdapterParameterTypeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct SentStatement { + let sql: String + let typeNames: [String?]? +} + +private struct CatalogRefused: Error {} + +private final class TypedTableDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var sent: [SentStatement] = [] + private let refusesCatalog: Bool + + init(refusesCatalog: Bool = false) { + self.refusesCatalog = refusesCatalog + } + + var statements: [SentStatement] { lock.withLock { sent } } + + func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + if refusesCatalog { throw CatalogRefused() } + return [ + PluginColumnInfo(name: "ID", dataType: "NUMBER(10)", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "CODE", dataType: "VARCHAR2(20 BYTE)") + ] + } + + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + lock.withLock { sent.append(SentStatement(sql: query, typeNames: context.parameterTypeNames)) } + guard query.hasPrefix("SELECT") else { return .empty } + return PluginQueryResult( + columns: ["position", "count"], columnTypeNames: [], rows: [[.text("0"), .text("1")]], + rowsAffected: 0, executionTime: 0 + ) + } +} + +// Oracle stored an imported `00123` in a VARCHAR2 as 123: the driver was never told the column was text. +struct ImportDataSinkAdapterParameterTypeTests { + private static let mapping = ["ID": "ID", "CODE": "CODE"] + + private func sink(on driver: TypedTableDriver, rowWriting: ImportRowWriting = .insert) -> ImportDataSinkAdapter { + ImportDataSinkAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: driver), + databaseType: .postgresql, + targetTable: "PEOPLE", + columnMapping: Self.mapping, + sourceFields: Set(Self.mapping.keys), + rowWriting: rowWriting + ) + } + + @Test + func importedRowsTellTheDriverEachValuesColumnType() async throws { + let driver = TypedTableDriver() + + try await sink(on: driver).insertRows([["ID": .text("1"), "CODE": .text("00123")]]) + + let insert = try #require(driver.statements.first) + #expect(insert.sql.hasPrefix("INSERT INTO \"PEOPLE\" (\"CODE\", \"ID\")")) + #expect(insert.typeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aSingleRowAndOrderedRowsTellTheDriverTheTypesToo() async throws { + let driver = TypedTableDriver() + let target = sink(on: driver) + + try await target.insertRow(["ID": .text("1"), "CODE": .text("00123")]) + try await target.insertRows(columns: ["ID", "CODE"], values: [[.text("2"), .text("00456")]]) + + let typeNames = driver.statements.map { $0.typeNames } + #expect(typeNames == [ + ["VARCHAR2(20 BYTE)", "NUMBER(10)"], + ["NUMBER(10)", "VARCHAR2(20 BYTE)"] + ]) + } + + @Test + func matchingTellsTheDriverTheKeyAndAssignmentTypes() async throws { + let driver = TypedTableDriver() + let target = sink(on: driver, rowWriting: ImportRowWriting(mode: .update, keyColumns: ["ID"])) + + try await target.insertRows([["ID": .text("7"), "CODE": .text("00123")]]) + + let match = try #require(driver.statements.first) + let update = try #require(driver.statements.last) + #expect(match.sql.hasPrefix("SELECT 0, COUNT(*)")) + #expect(match.typeNames == ["NUMBER(10)"]) + #expect(update.sql.hasPrefix("UPDATE \"PEOPLE\" SET \"CODE\"")) + #expect(update.typeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aTargetWhoseColumnsCannotBeReadStillImportsUntyped() async throws { + let driver = TypedTableDriver(refusesCatalog: true) + + try await sink(on: driver).insertRows([["ID": .text("1"), "CODE": .text("00123")]]) + + let insert = try #require(driver.statements.first) + #expect(insert.sql.hasPrefix("INSERT INTO \"PEOPLE\"")) + #expect(insert.typeNames == nil) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift new file mode 100644 index 0000000000..348e6cc792 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift @@ -0,0 +1,204 @@ +// +// ImportDataSinkAdapterTargetValueTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct WrittenStatement { + let sql: String + let parameters: [PluginCellValue] +} + +private final class TypedTableDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private let columns: [PluginColumnInfo] + private var written: [WrittenStatement] = [] + + init(columns: [(String, String)], generated: Set = []) { + self.columns = columns.map { PluginColumnInfo(name: $0.0, dataType: $0.1, isGenerated: generated.contains($0.0)) } + } + + var statements: [WrittenStatement] { lock.withLock { written } } + + func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { columns } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + lock.withLock { written.append(WrittenStatement(sql: query, parameters: parameters)) } + return .empty + } +} + +// A file holds text; what it stands for depends on the column it lands in. +struct ImportDataSinkAdapterTargetValueTests { + private func sink( + _ driver: TypedTableDriver, + type: DatabaseType, + mapping: [String], + binaryText: ImportBinaryText? = nil, + rowWriting: ImportRowWriting = .insert + ) -> ImportDataSinkAdapter { + ImportDataSinkAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver), + databaseType: type, + targetTable: "t", + columnMapping: Dictionary(uniqueKeysWithValues: mapping.map { ($0, $0) }), + sourceFields: Set(mapping), + rowWriting: rowWriting, + binaryText: binaryText + ) + } + + @Test("CSV and XLSX hex lands in a binary column as its bytes, and only there") + func prefixedHexBecomesBytes() async throws { + let driver = TypedTableDriver(columns: [("payload", "bytea"), ("note", "text")]) + let target = sink(driver, type: .postgresql, mapping: ["payload", "note"], binaryText: .prefixedHex) + + try await target.insertRows([ + ["payload": .text("0x89504E47"), "note": .text("0x41")], + ["payload": .text("0x"), "note": .text("x")], + ["payload": .text("plain"), "note": .text("y")], + ]) + + let parameters = try #require(driver.statements.first).parameters + #expect(parameters == [ + .text("0x41"), .bytes(Data([0x89, 0x50, 0x4E, 0x47])), + .text("x"), .bytes(Data()), + .text("y"), .text("plain"), + ]) + } + + @Test("Hex cut short fails the row and names the column") + func brokenHexFailsTheRow() async throws { + let driver = TypedTableDriver(columns: [("payload", "blob")]) + let target = sink(driver, type: .mysql, mapping: ["payload"], binaryText: .prefixedHex) + + let error = await #expect(throws: PluginImportError.self) { + try await target.insertRow(["payload": .text("0x41…")]) + } + #expect(error?.localizedDescription.contains("payload") == true) + #expect(driver.statements.isEmpty) + } + + @Test("JSON's base64 lands in a binary column as its bytes, and other text stays text") + func base64BecomesBytes() async throws { + let driver = TypedTableDriver(columns: [("payload", "bytea")]) + let target = sink(driver, type: .postgresql, mapping: ["payload"], binaryText: .base64) + + try await target.insertRow(["payload": .text(Data([0x00, 0xFF]).base64EncodedString())]) + try await target.insertRow(["payload": .text("hello world")]) + try await target.insertRow(["payload": .text("\\x4142")]) + + #expect(driver.statements.map { $0.parameters } == [ + [.bytes(Data([0x00, 0xFF]))], [.text("hello world")], [.text("\\x4142")], + ]) + } + + @Test("A format with no binary spelling, and rows in column order, keep the text") + func noBinaryTextKeepsText() async throws { + let driver = TypedTableDriver(columns: [("payload", "bytea")]) + + try await sink(driver, type: .postgresql, mapping: ["payload"]).insertRow(["payload": .text("0x41")]) + try await sink(driver, type: .postgresql, mapping: ["payload"], binaryText: .prefixedHex) + .insertRows(columns: ["payload"], values: [[.text("0x41")]]) + + #expect(driver.statements.map { $0.parameters } == [[.text("0x41")], [.text("0x41")]]) + } + + @Test( + "A boolean word becomes 1 or 0 where the engine writes booleans as numbers", + arguments: [ + (DatabaseType.mysql, "tinyint(1)"), + (DatabaseType.mssql, "bit"), + (DatabaseType.sqlite, "BOOLEAN"), + ] + ) + func booleanWordsBecomeDigits(type: DatabaseType, column: String) async throws { + let driver = TypedTableDriver(columns: [("active", column), ("label", "varchar(10)")]) + let target = sink(driver, type: type, mapping: ["active", "label"]) + + try await target.insertRows(["true", "No", "T", "y", "1", " off ", "maybe"].map { word in + ["active": .text(word), "label": .text(word)] + }) + + let parameters = try #require(driver.statements.first).parameters + let active = stride(from: 0, to: parameters.count, by: 2).map { parameters[$0] } + let labels = stride(from: 1, to: parameters.count, by: 2).map { parameters[$0] } + #expect(active == ["1", "0", "1", "1", "1", "0", "maybe"]) + #expect(labels == ["true", "No", "T", "y", "1", " off ", "maybe"]) + } + + @Test("An engine that reads TRUE and FALSE gets the file's own words") + func trueFalseEngineKeepsWords() async throws { + let driver = TypedTableDriver(columns: [("active", "boolean")]) + + try await sink(driver, type: .postgresql, mapping: ["active"]).insertRow(["active": .text("yes")]) + + #expect(driver.statements.first?.parameters == [.text("yes")]) + } + + @Test("A row with no field is written as a row of defaults and counted") + func emptyRowIsARowOfDefaults() async throws { + let driver = TypedTableDriver(columns: [("id", "serial"), ("note", "text")]) + let target = sink(driver, type: .postgresql, mapping: ["note"]) + + try await target.insertRows([["note": .text("a")], [:], [:]]) + try await target.insertRow([:]) + + #expect(driver.statements.map { $0.sql } == [ + "INSERT INTO \"t\" (\"note\") VALUES ($1)", + "INSERT INTO \"t\" DEFAULT VALUES", + "INSERT INTO \"t\" DEFAULT VALUES", + "INSERT INTO \"t\" DEFAULT VALUES", + ]) + #expect(target.rowTally.inserted == 4) + } + + @Test("Oracle's row of defaults names the first column that is not virtual") + func oracleDefaultsNameAColumn() async throws { + let driver = TypedTableDriver(columns: [("FULL_NAME", "VARCHAR2(41)"), ("ID", "NUMBER(19)")], generated: ["FULL_NAME"]) + + try await sink(driver, type: .oracle, mapping: ["ID"]).insertRow([:]) + + #expect(driver.statements.map { $0.sql } == ["INSERT INTO \"t\" (\"ID\") VALUES (DEFAULT)"]) + } + + @Test("An engine with no INSERT for a row of defaults fails the row and says why") + func noDefaultsInsertSaysWhy() async throws { + let driver = TypedTableDriver(columns: [("id", "UInt64")]) + + let error = await #expect(throws: PluginImportError.self) { + try await sink(driver, type: .clickhouse, mapping: ["id"]).insertRow([:]) + } + #expect(error?.localizedDescription.contains("row of only defaults") == true) + } + + @Test("A row with no field cannot be matched to an existing row") + func emptyRowHasNoKeyToMatch() async throws { + let driver = TypedTableDriver(columns: [("id", "int"), ("note", "text")]) + let target = sink( + driver, type: .postgresql, mapping: ["id", "note"], + rowWriting: ImportRowWriting(mode: .upsert, keyColumns: ["id"]) + ) + + await #expect(throws: PluginImportError.self) { + try await target.insertRows([[:]]) + } + #expect(driver.statements.isEmpty) + } +} + +struct ImportBinaryTextTests { + @Test("Each format reads bytes back the way its export wrote them") + func formatIds() { + #expect(ImportBinaryText(formatId: "csv") == .prefixedHex) + #expect(ImportBinaryText(formatId: "xlsx") == .prefixedHex) + #expect(ImportBinaryText(formatId: "json") == .base64) + #expect(ImportBinaryText(formatId: "sql") == nil) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportManualCommitTests.swift b/apple/TableProTests/Core/Plugins/ImportManualCommitTests.swift new file mode 100644 index 0000000000..0faf907b45 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportManualCommitTests.swift @@ -0,0 +1,154 @@ +// +// ImportManualCommitTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class ImportHoldDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var current: PluginSessionTransactionState + private var sent: [String] = [] + + init(state: PluginSessionTransactionState = .idle) { + current = state + } + + var supportsSchemas: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + var statements: [String] { lock.withLock { sent } } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { + sent.append(query) + switch query.uppercased() { + case "BEGIN": current = .inTransaction + case "COMMIT", "ROLLBACK": current = .idle + default: break + } + } + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { lock.withLock { current } } + + func foreignKeyDisableStatements() -> [String]? { ["PRAGMA foreign_keys = OFF"] } + func foreignKeyEnableStatements() -> [String]? { ["PRAGMA foreign_keys = ON"] } +} + +struct ImportManualCommitTests { + private func sink( + on driver: ImportHoldDriver, + type: DatabaseType = .sqlite, + commitMode: ConnectionCommitMode = .manual + ) -> ImportDataSinkAdapter { + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver) + return ImportDataSinkAdapter(driver: adapter, databaseType: type, commitMode: commitMode, importRun: ImportRun()) + } + + @Test("A Manual Commit import holds its transaction: BEGIN, and no COMMIT at the end") + func importIsHeld() async throws { + let driver = ImportHoldDriver() + let sink = sink(on: driver) + + try await sink.prepareImportRun() + try await sink.beginTransaction() + try await sink.execute(statement: "INSERT INTO t VALUES (1)", line: 1) + try await sink.commitTransaction() + + #expect(driver.statements == ["BEGIN", "INSERT INTO t VALUES (1)"]) + } + + @Test( + "A Manual Commit import the session can't hold is refused before anything runs", + arguments: [PluginSessionTransactionState.unknown, .holdsSessionLocks] + ) + func unholdableImportIsRefused(state: PluginSessionTransactionState) async { + let driver = ImportHoldDriver(state: state) + let sink = sink(on: driver) + + await #expect(throws: ManualCommitRefusal()) { + try await sink.prepareImportRun() + } + #expect(driver.statements.isEmpty) + } + + // SQLite ignores the re-enable inside a held transaction, so the checks would stay off for the session. + @Test("A Manual Commit import that turned foreign key checks off is refused before its first write") + func foreignKeyToggledImportIsRefused() async throws { + let driver = ImportHoldDriver() + let sink = sink(on: driver) + + try await sink.prepareImportRun() + try await sink.disableForeignKeyChecks() + await #expect(throws: ManualCommitForeignKeyRefusal()) { + try await sink.beginTransaction() + } + try await sink.enableForeignKeyChecks() + + #expect(driver.statements == ["PRAGMA foreign_keys = OFF", "PRAGMA foreign_keys = ON"]) + } + + // Skip and Continue goes on after a refusal, so every later write is refused as well. + @Test("A Manual Commit import that never begins a transaction is refused at each write, before it runs") + func unwrappedImportIsRefused() async throws { + let driver = ImportHoldDriver() + let sink = sink(on: driver) + + try await sink.prepareImportRun() + await #expect(throws: ManualCommitUnwrappedImportRefusal()) { + try await sink.execute(statement: "INSERT INTO t VALUES (1)", line: 1) + } + await #expect(throws: ManualCommitUnwrappedImportRefusal()) { + try await sink.execute(statement: "INSERT INTO t VALUES (2)", line: 2) + } + + #expect(driver.statements.isEmpty) + } + + @Test("A held import runs a dump's DDL inside its transaction, with no COMMIT") + func heldImportRunsDDL() async throws { + let driver = ImportHoldDriver() + let sink = sink(on: driver) + try await sink.prepareImportRun() + try await sink.beginTransaction() + + try await sink.execute(statement: "DROP TABLE IF EXISTS t", line: 1) + try await sink.execute(statement: "INSERT INTO t VALUES (1)", line: 2) + try await sink.commitTransaction() + + #expect(driver.statements == ["BEGIN", "DROP TABLE IF EXISTS t", "INSERT INTO t VALUES (1)"]) + } + + @Test("A held import marks the turn's hold at its BEGIN") + func heldImportMarksTheTurn() async throws { + let driver = ImportHoldDriver() + let sink = sink(on: driver) + let held = ManualCommitTurnHold() + + try await ManualCommit.$turnHold.withValue(held) { + try await sink.prepareImportRun() + try await sink.beginTransaction() + } + + #expect(held.isHeld) + } + + @Test("An automatic import with foreign key checks off still wraps and commits") + func automaticImportWithToggleCommits() async throws { + let driver = ImportHoldDriver() + let sink = sink(on: driver, commitMode: .automatic) + + try await sink.prepareImportRun() + try await sink.disableForeignKeyChecks() + try await sink.beginTransaction() + try await sink.commitTransaction() + + #expect(driver.statements == ["PRAGMA foreign_keys = OFF", "BEGIN", "COMMIT"]) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift b/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift index 5e78070475..a556b4c497 100644 --- a/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift @@ -53,6 +53,7 @@ struct ImportTypeMapperTests { #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .sqlite) == "INTEGER") #expect(ImportTypeMapper.sqlType(for: .real, databaseType: .sqlite) == "REAL") #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .sqlite) == "TEXT") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .sqlite) == "BOOLEAN") } @Test("Oracle gets types it has, never TEXT") diff --git a/apple/TableProTests/Core/Plugins/MySQLProtocolVariantParityTests.swift b/apple/TableProTests/Core/Plugins/MySQLProtocolVariantParityTests.swift index 4ccdd7da4d..6e7313f9f0 100644 --- a/apple/TableProTests/Core/Plugins/MySQLProtocolVariantParityTests.swift +++ b/apple/TableProTests/Core/Plugins/MySQLProtocolVariantParityTests.swift @@ -47,7 +47,7 @@ struct MySQLProtocolVariantParityTests { @Test("A variant whose driver reads mysql_affected_rows holds a keyless save to its row count", arguments: variants) func rowCountsAgree(type: DatabaseType) { - #expect(DataWriteRowCounts.areMeaningful(for: type) == !Self.outsideMySQLDialect.contains(type)) + #expect(DataWriteRowCounts.countsMatchedRows(for: type) == !Self.outsideMySQLDialect.contains(type)) } /// The app classifies a database list by the connection type's registry entry and the driver flags metadata by diff --git a/apple/TableProTests/Core/Plugins/PluginCodeSignatureVerifierTests.swift b/apple/TableProTests/Core/Plugins/PluginCodeSignatureVerifierTests.swift index a25965d1c6..f1e316d423 100644 --- a/apple/TableProTests/Core/Plugins/PluginCodeSignatureVerifierTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginCodeSignatureVerifierTests.swift @@ -1,8 +1,3 @@ -// -// PluginCodeSignatureVerifierTests.swift -// TableProTests -// - import Foundation @testable import TablePro import Testing @@ -14,10 +9,14 @@ struct PluginCodeSignatureVerifierTests { // Calculator carries a valid Apple signature that is not TablePro's, the same shape as a // bundle signed with another developer's Developer ID. + private static func calculator() throws -> Bundle { + try #require(Bundle(path: "/System/Applications/Calculator.app")) + } + @Test("A validly signed bundle from another signer is refused") func refusesAnotherSigner() throws { guard !Self.verifierIsBypassed else { return } - let bundle = try #require(Bundle(path: "/System/Applications/Calculator.app")) + let bundle = try Self.calculator() let error = #expect(throws: PluginError.self) { try PluginCodeSignatureVerifier.verify(bundle: bundle) @@ -50,4 +49,60 @@ struct PluginCodeSignatureVerifierTests { return } } + + @Test("The app's own Team ID builds a signing requirement") + func resolvedTeamBuildsRequirement() throws { + _ = try PluginCodeSignatureVerifier.firstPartyRequirement( + signingTeam: PluginCodeSignatureVerifier.resolvedSigningTeamId + ) + } + + // A lone quote used to leave no requirement, and the `or anchor apple` team one every Apple-signed bundle meets. + @Test( + "A validly signed bundle is refused when the Team ID would break TablePro's requirement", + arguments: [ + "\"", + "ABCDEFGHIJ\" or anchor apple or certificate leaf[subject.OU] = \"X" + ] + ) + func refusesWhenTeamBreaksRequirement(team: String) throws { + let bundle = try Self.calculator() + + let error = #expect(throws: PluginError.self) { + try PluginCodeSignatureVerifier.verify(bundle: bundle, signingTeam: team) + } + guard case .signatureInvalid(let detail) = error else { + Issue.record("Expected signatureInvalid, got \(String(describing: error))") + return + } + #expect(detail == "TablePro's signing Team ID is not valid") + } + + @Test( + "A Team ID that is not ten capital letters or digits builds no requirement", + arguments: ["", "d7hj5tfycu", "D7HJ5TFYC", "D7HJ5TFYCU1", "D7HJ5-FYCU"] + ) + func refusesMalformedTeam(team: String) { + let error = #expect(throws: PluginError.self) { + _ = try PluginCodeSignatureVerifier.firstPartyRequirement(signingTeam: team) + } + guard case .signatureInvalid = error else { + Issue.record("Expected signatureInvalid, got \(String(describing: error))") + return + } + } + + @Test("Requirement text that does not compile is an error, never an empty requirement") + func uncompilableRequirementThrows() { + let error = #expect(throws: PluginError.self) { + _ = try PluginCodeSignatureVerifier.compiledRequirement( + "anchor apple generic and certificate leaf[subject.OU] = \"\"\"" + ) + } + guard case .signatureInvalid(let detail) = error else { + Issue.record("Expected signatureInvalid, got \(String(describing: error))") + return + } + #expect(detail.hasPrefix("TablePro's signing requirement could not be built")) + } } diff --git a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift index 775152ce39..61394989e7 100644 --- a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift @@ -24,13 +24,21 @@ private final class ContextRecordingDriver: PluginDriverStub, @unchecked Sendabl private let lock = NSLock() private var recorded: [PluginStatementContext] = [] + private var caps: [Int?] = [] var contexts: [PluginStatementContext] { lock.withLock { recorded } } - private func record(_ context: PluginStatementContext) { - lock.withLock { recorded.append(context) } + var rowCaps: [Int?] { + lock.withLock { caps } + } + + private func record(_ context: PluginStatementContext, rowCap: Int? = nil) { + lock.withLock { + recorded.append(context) + caps.append(rowCap) + } } private static let empty = PluginQueryResult( @@ -59,7 +67,7 @@ private final class ContextRecordingDriver: PluginDriverStub, @unchecked Sendabl parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - record(context) + record(context, rowCap: rowCap) return Self.empty } @@ -69,6 +77,32 @@ private final class ContextRecordingDriver: PluginDriverStub, @unchecked Sendabl } } +// A driver that bounds its read under a context, as one declaring `.enforcesReadOnlyMode` does. +private final class BoundingContextDriver: PluginDriverStub, @unchecked Sendable { + var capabilities: PluginCapabilities { [.readOnlyStatements, .enforcesReadOnlyMode] } + + private let lock = NSLock() + private var recorded: [PluginStatementContext?] = [] + + var boundedContexts: [PluginStatementContext?] { + lock.withLock { recorded } + } + + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + lock.withLock { recorded.append(nil) } + return .empty + } + + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + lock.withLock { recorded.append(context) } + return .empty + } +} + struct PluginDriverAdapterStatementContextTests { private func makeAdapter(_ type: DatabaseType, driver: ContextRecordingDriver) -> PluginDriverAdapter { PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: type), pluginDriver: driver) @@ -320,3 +354,160 @@ struct UnenforcedStatementPolicyTests { #expect(adapter(ContextRecordingDriver(capabilities: .readOnlyStatements)).runsReadOnlyStatements) } } + +struct PluginDriverAdapterParameterKindTests { + private func adapter(_ driver: ContextRecordingDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: .postgresql), pluginDriver: driver) + } + + @Test("A panel value reaches the driver with the type the user gave it") + func panelTypesReachTheDriver() async throws { + let driver = ContextRecordingDriver() + let values: [Any?] = [ + TypedParameterValue(text: "00123", kind: .text), nil, TypedParameterValue(text: "7", kind: .integer), + ] + + _ = try await adapter(driver).executeUserQuery( + query: "SELECT $1, $2, $3", rowCap: nil, parameters: values, context: PluginStatementContext() + ) + _ = try await adapter(driver).executeParameterized( + query: "SELECT $1, $2, $3", parameters: values, context: PluginStatementContext() + ) + + #expect(driver.contexts.map { $0.parameterKinds } == [[.text, nil, .integer], [.text, nil, .integer]]) + } + + @Test("Values with no type, such as a grid save's, carry none") + func untypedValuesCarryNoKinds() async throws { + let driver = ContextRecordingDriver() + + _ = try await adapter(driver).executeParameterized( + query: "UPDATE t SET a = $1", parameters: ["42"], context: PluginStatementContext() + ) + + #expect(driver.contexts.map { $0.parameterKinds } == [nil]) + } + + @Test("A typed value is written to the driver as its text") + func typedValueIsItsText() { + #expect(PluginDriverAdapter.cellValue(for: TypedParameterValue(text: "1E5", kind: .text)) == .text("1E5")) + } +} + +// Read-Only Safe Mode reaches the driver as `enforcesReadOnly`, only where the driver declares it can hold the engine to +// it, and never on the app's own catalog reads. +struct PluginDriverAdapterReadOnlyModeTests { + private static let readOnly = PluginStatementContext(enforcesReadOnly: true) + + private func adapter(_ driver: any PluginDatabaseDriver, type: DatabaseType = .postgresql) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: type), pluginDriver: driver) + } + + @Test("Only a driver that declares the mode reports that it enforces Read-Only") + func adapterReportsTheMode() { + #expect(!adapter(ContextRecordingDriver(capabilities: .readOnlyStatements)).enforcesReadOnlyMode) + #expect(adapter(ContextRecordingDriver(capabilities: .enforcesReadOnlyMode)).enforcesReadOnlyMode) + } + + @Test("A run carries the field only at Read-Only and only on a driver that declares the mode") + func runContextNeedsTheLevelAndTheMode() { + let enforcing = adapter(ContextRecordingDriver(capabilities: .enforcesReadOnlyMode)) + let older = adapter(ContextRecordingDriver(capabilities: .readOnlyStatements)) + + #expect(enforcing.readOnlyRunContext(safeModeIsReadOnly: true)?.enforcesReadOnly == true) + #expect(enforcing.readOnlyRunContext(safeModeIsReadOnly: false) == nil) + #expect(older.readOnlyRunContext(safeModeIsReadOnly: true) == nil) + } + + @Test("Every entry point that takes a context forwards the field to the driver") + func everyEntryPointForwardsTheField() async throws { + let driver = ContextRecordingDriver(capabilities: .enforcesReadOnlyMode) + let adapter = adapter(driver) + let read = "SELECT purge_orders()" + + _ = try await adapter.executeUserQuery(query: read, rowCap: 100, parameters: nil, context: Self.readOnly) + _ = try await adapter.executeParameterized(query: "SELECT $1", parameters: [1], context: Self.readOnly) + _ = try await adapter.execute(query: read, context: Self.readOnly) + for try await _ in adapter.streamExportRows(query: read, safeModeIsReadOnly: true) {} + for try await _ in adapter.streamRows(query: read, safeModeIsReadOnly: true) {} + + #expect(driver.contexts.count == 5) + #expect(driver.contexts.allSatisfy { $0.enforcesReadOnly }) + #expect(driver.contexts.allSatisfy { $0.origin == .app && !$0.readOnly }) + } + + @Test("At Read-Only every entry point says whether the statement calls a function off the built-in list") + func everyEntryPointReadsTheFunctionCalls() async throws { + let driver = ContextRecordingDriver(capabilities: .enforcesReadOnlyMode) + let adapter = adapter(driver) + let bounding = BoundingContextDriver() + let boundingAdapter = self.adapter(bounding) + + for query in ["SELECT purge_orders()", "SELECT count(*) FROM orders"] { + _ = try await adapter.executeUserQuery(query: query, rowCap: 100, parameters: nil, context: Self.readOnly) + _ = try await adapter.execute(query: query, context: Self.readOnly) + for try await _ in adapter.streamRows(query: query, safeModeIsReadOnly: true) {} + _ = try await boundingAdapter.executeBoundedQuery(query: query, rowCap: 10, context: Self.readOnly) + } + _ = try await adapter.executeUserQuery(query: "SELECT count(*) FROM orders", rowCap: nil, parameters: nil) + + #expect(driver.contexts.map { $0.callsUnlistedFunction } == [true, true, true, false, false, false, true]) + #expect(bounding.boundedContexts.map { $0?.callsUnlistedFunction } == [true, false]) + } + + @Test("The bounded read takes the context") + func boundedReadForwardsTheField() async throws { + let driver = BoundingContextDriver() + let adapter = adapter(driver) + + _ = try await adapter.executeBoundedQuery(query: "SELECT 1", rowCap: 10, context: Self.readOnly) + _ = try await adapter.executeBoundedQuery(query: "SELECT 1", rowCap: 10) + + #expect(driver.boundedContexts.count == 2) + #expect(driver.boundedContexts.first??.enforcesReadOnly == true) + #expect(driver.boundedContexts.last == .some(nil)) + } + + @Test("A driver that cannot bound under the context falls back to the user query with the same context and cap") + func boundedFallbackKeepsTheContext() async throws { + let driver = ContextRecordingDriver(capabilities: .enforcesReadOnlyMode) + let adapter = adapter(driver) + + #expect(try await adapter.executeBoundedQuery(query: "SELECT 1", rowCap: 10, context: Self.readOnly) == nil) + _ = try await QueryExecutor.fetchQueryData(driver: adapter, sql: "SELECT 1", rowCap: 10, context: Self.readOnly) + + #expect(driver.contexts.map { $0.enforcesReadOnly } == [true]) + #expect(driver.rowCaps == [10]) + } + + @Test("At Silent nothing changes: the export stream keeps the default context") + func silentKeepsTheDefault() async throws { + let driver = ContextRecordingDriver(capabilities: .enforcesReadOnlyMode) + let adapter = adapter(driver) + + for try await _ in adapter.streamExportRows(query: "SELECT 1") {} + _ = try await adapter.executeUserQuery(query: "SELECT 1", rowCap: nil, parameters: nil) + + #expect(driver.contexts.count == 2) + #expect(driver.contexts.allSatisfy { !$0.enforcesReadOnly }) + } + + @Test("A driver without the mode is never told to enforce it, even at Read-Only") + func olderDriverIsNotTold() async throws { + let driver = ContextRecordingDriver(capabilities: .readOnlyStatements) + + for try await _ in adapter(driver).streamExportRows(query: "SELECT 1", safeModeIsReadOnly: true) {} + + #expect(driver.contexts.map { $0.enforcesReadOnly } == [false]) + } + + @Test("A catalog read never carries the field") + func catalogReadIsNotMarked() async throws { + let driver = ContextRecordingDriver(capabilities: .enforcesReadOnlyMode) + + _ = try await adapter(driver).executeCatalogRead(query: "SELECT 1 FROM pg_catalog.pg_class") + + #expect(driver.contexts.map { $0.enforcesReadOnly } == [false]) + #expect(driver.contexts.first?.readsCatalog == true) + } +} diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift index e8b611a013..4a5284387e 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift @@ -158,6 +158,22 @@ private final class MockTrinoPlugin: NSObject, TableProPlugin, DriverPlugin { } } +private final class MockPostgreSQLPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Mock PostgreSQL" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Stands in for the bundled PostgreSQL plugin" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "PostgreSQL" + static let databaseDisplayName = "PostgreSQL" + static let iconName = "postgresql-icon" + static let defaultPort = 5_432 + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + fatalError("Not used in tests") + } +} + private final class MockClickHousePlugin: NSObject, TableProPlugin, DriverPlugin { static let pluginName = "Mock ClickHouse" static let pluginVersion = "1.0.0" @@ -369,8 +385,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(snapshot.capabilities.supportsRawFilter) } - /// None of these has a `DriverPlugin` static, so a plugin that registers would reset them to the struct - /// defaults: counts on every open that read every partition, a header sort CQL refuses, and Match Any. + // None of these has a `DriverPlugin` static, so a plugin that registers would reset them to the struct + // defaults: counts on every open that read every partition, a header sort CQL refuses, and Match Any. @Test("Cassandra keeps its count, sort and filter limits when its plugin registers") func cassandraKeepsItsCuratedQueryLimits() { let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockCassandraPlugin.self) @@ -433,8 +449,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { } } - /// Measured: `ADD … NOT NULL` and `COMMENT ON COLUMN` are syntax errors, `ALTER … TYPE` is refused, and only a - /// primary key column can be renamed, so the Structure tab offers adding and dropping a column and nothing else. + // Measured: `ADD … NOT NULL` and `COMMENT ON COLUMN` are syntax errors, `ALTER … TYPE` is refused, and only a + // primary key column can be renamed, so the Structure tab offers adding and dropping a column and nothing else. @Test("Cassandra and ScyllaDB offer only the column edits CQL can run", arguments: ["Cassandra", "ScyllaDB"]) func cassandraStructureEdits(typeId: String) throws { let snapshot = try #require(PluginMetadataRegistry.shared.snapshot(forRegisteredTypeId: typeId)) @@ -480,8 +496,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.connection.defaultLocalSocketPath == MySQLLocalSocket.defaultPath) } - /// A MySQL-protocol session opened with no database has none at all, so it lists nothing until - /// one is chosen. Databend's session falls back to its `default` database instead. + // A MySQL-protocol session opened with no database has none at all, so it lists nothing until + // one is chosen. Databend's session falls back to its `default` database instead. @Test("Only the MySQL engines with no default database require a selected database to browse") func browsingRequiresSelectedDatabasePerEngine() { let registry = PluginMetadataRegistry.shared @@ -591,6 +607,49 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.verifiesServerWithSystemTrust == true) } + @Test("PostgreSQL keeps system trust, and TLS asked for without a mode stays Required, when its plugin registers") + func postgreSQLKeepsItsTLSCapabilities() { + let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockPostgreSQLPlugin.self) + + #expect(built.capabilities.verifiesServerWithSystemTrust == true) + #expect(built.capabilities.requestedTLSVerifiesIdentity == false) + } + + @Test("Trino and ClickHouse keep Verify Identity for TLS asked for without a mode when their plugins register") + func requestedTLSSurvivesRegistration() { + let registry = PluginMetadataRegistry.shared + + #expect(registry.buildMetadataSnapshot(from: MockTrinoPlugin.self).capabilities.requestedTLSVerifiesIdentity) + #expect(registry.buildMetadataSnapshot(from: MockClickHousePlugin.self).capabilities.requestedTLSVerifiesIdentity) + #expect(registry.buildMetadataSnapshot(from: MockHanaPlugin.self).capabilities.requestedTLSVerifiesIdentity) + } + + @Test( + "The libpq engines check the system trust store, while TLS asked for without a mode stays Required", + arguments: ["PostgreSQL", "Redshift", "CockroachDB", "KingbaseES"] + ) + func libpqEnginesSplitTheTwoTLSFlags(typeId: String) throws { + let curated = try #require( + PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == typeId }?.snapshot + ) + + #expect(curated.capabilities.verifiesServerWithSystemTrust == true) + #expect(curated.capabilities.requestedTLSVerifiesIdentity == false) + } + + @Test( + "The engines that already verified the system trust store keep both TLS flags", + arguments: ["SAP HANA", "Kafka", "Trino", "Presto", "Qdrant", "DuckDB", "ClickHouse"] + ) + func systemTrustEnginesKeepBothTLSFlags(typeId: String) throws { + let curated = try #require( + PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == typeId }?.snapshot + ) + + #expect(curated.capabilities.verifiesServerWithSystemTrust == true) + #expect(curated.capabilities.requestedTLSVerifiesIdentity == true) + } + @Test("etcd offers no generic SSL Mode, since its driver reads only its own TLS Mode field") @MainActor func etcdDeclaresNoGenericSSL() { @@ -624,6 +683,7 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.filtersTypedByReadRows == false) #expect(built.capabilities.tlsImpliedPorts.isEmpty) #expect(built.capabilities.verifiesServerWithSystemTrust == false) + #expect(built.capabilities.requestedTLSVerifiesIdentity == false) #expect(built.capabilities.supportsPerConnectionCertificatePaths == true) #expect(built.capabilities.newDatabaseFileExtensions.isEmpty) #expect(built.schema.implicitSchemaName == nil) diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryKingbaseESTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryKingbaseESTests.swift index 88182458fc..6cf10aa5c6 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryKingbaseESTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryKingbaseESTests.swift @@ -24,6 +24,15 @@ struct PluginMetadataRegistryKingbaseESTests { #expect(snapshot.connection.additionalConnectionFields.map(\.id) == ["usePgpass", "connectionOptions"]) } + // Its driver is PostgreSQL's, which runs an external read in START TRANSACTION READ ONLY, so the Options pane says so. + @Test("KingbaseES describes Read Only for external clients as PostgreSQL does") + func readOnlyNoteMatchesItsDriver() { + #expect(ReadOnlyEnforcement(for: .kingbasees) == .sessionDefault) + #expect(ReadOnlyEnforcement(for: .kingbasees) == ReadOnlyEnforcement(for: .postgresql)) + #expect(ReadOnlyEnforcement(for: .kingbasees, liveDriverRunsReadOnly: true) == .sessionDefault) + #expect(ReadOnlyEnforcement(for: .kingbasees, liveDriverRunsReadOnly: false) == .loginPrivileges) + } + @Test("Both KingbaseES URL schemes keep credentials and route to the dedicated engine", arguments: ["kingbase", "kingbase8"]) func connectionURLs(scheme: String) throws { let parsed = try ConnectionURLParser.parse("\(scheme)://system:pass@localhost:55432/test").get() diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryStarRocksTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryStarRocksTests.swift index c82b0b9db1..79d0c2de92 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryStarRocksTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryStarRocksTests.swift @@ -39,7 +39,7 @@ struct PluginMetadataRegistryStarRocksTests { @Test("A DELETE on a duplicate key table reports 0 rows, so a keyless save is not held to its count") func rowCountsAreNotMeaningful() { - #expect(!DataWriteRowCounts.areMeaningful(for: .starrocks)) + #expect(!DataWriteRowCounts.countsMatchedRows(for: .starrocks)) } @Test("A StarRocks URL keeps the catalog.database path and routes to StarRocks") diff --git a/apple/TableProTests/Core/Plugins/RejectedPluginRemovalTests.swift b/apple/TableProTests/Core/Plugins/RejectedPluginRemovalTests.swift new file mode 100644 index 0000000000..8d9ecce183 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/RejectedPluginRemovalTests.swift @@ -0,0 +1,87 @@ +import Foundation +@testable import TablePro +import Testing + +@MainActor +private final class TrashRecorder { + var trashed: [URL] = [] +} + +@MainActor +struct RejectedPluginRemovalTests { + private let root = FileManager.default.temporaryDirectory + .appendingPathComponent("RejectedPluginRemovalTests-\(UUID().uuidString)", isDirectory: true) + + private func makeBundle(in directory: URL) throws -> RejectedPlugin { + let url = directory.appendingPathComponent("Broken.tableplugin", isDirectory: true) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return RejectedPlugin( + url: url, + bundleId: "com.example.broken", + registryId: nil, + name: "Broken", + reason: "Could not be loaded", + isOutdated: false, + providedDatabaseTypeIds: [] + ) + } + + @Test("A rejected plugin in the user's folder moves to the Trash and leaves the list") + func userPluginMovesToTheTrash() throws { + defer { try? FileManager.default.removeItem(at: root) } + let userPlugins = root.appendingPathComponent("Plugins", isDirectory: true) + let plugin = try makeBundle(in: userPlugins) + let manager = PluginManager(builtInPluginsURL: nil, userPluginsDir: userPlugins) + manager.rejectedPlugins = [plugin] + let recorder = TrashRecorder() + + #expect(manager.canRemoveRejectedPlugin(plugin)) + try manager.removeRejectedPlugin(plugin) { recorder.trashed.append($0) } + + #expect(recorder.trashed == [plugin.url]) + #expect(manager.rejectedPlugins.isEmpty) + } + + @Test("A rejected plugin outside the user's folder, such as one inside the app, is refused and kept") + func bundledPluginIsRefused() throws { + defer { try? FileManager.default.removeItem(at: root) } + let userPlugins = root.appendingPathComponent("Plugins", isDirectory: true) + try FileManager.default.createDirectory(at: userPlugins, withIntermediateDirectories: true) + let plugin = try makeBundle(in: root.appendingPathComponent("TablePro.app/Contents/PlugIns", isDirectory: true)) + let manager = PluginManager(builtInPluginsURL: nil, userPluginsDir: userPlugins) + manager.rejectedPlugins = [plugin] + let recorder = TrashRecorder() + + #expect(!manager.canRemoveRejectedPlugin(plugin)) + let error = #expect(throws: PluginError.self) { + try manager.removeRejectedPlugin(plugin) { recorder.trashed.append($0) } + } + guard case .cannotUninstallBuiltIn = error else { + Issue.record("expected cannotUninstallBuiltIn, got \(String(describing: error))") + return + } + + #expect(recorder.trashed.isEmpty) + #expect(FileManager.default.fileExists(atPath: plugin.url.path)) + #expect(manager.rejectedPlugins.map(\.url) == [plugin.url]) + } + + @Test("A path that climbs out of the user's folder is refused") + func climbingPathIsRefused() throws { + defer { try? FileManager.default.removeItem(at: root) } + let userPlugins = root.appendingPathComponent("Plugins", isDirectory: true) + try FileManager.default.createDirectory(at: userPlugins, withIntermediateDirectories: true) + let manager = PluginManager(builtInPluginsURL: nil, userPluginsDir: userPlugins) + let escaping = RejectedPlugin( + url: userPlugins.appendingPathComponent("../Other.tableplugin"), + bundleId: nil, + registryId: nil, + name: "Other", + reason: "", + isOutdated: false, + providedDatabaseTypeIds: [] + ) + + #expect(!manager.canRemoveRejectedPlugin(escaping)) + } +} diff --git a/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift b/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift index 1e8662832e..dd89bb8dab 100644 --- a/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift +++ b/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift @@ -44,6 +44,36 @@ struct SqlFileImportSourceEncodingTests { #expect(result.first == "SET client_encoding = 'UTF8'") } + @Test("A Latin-1 pg_dump's COPY rows arrive as UTF-8, and its statements still run with a UTF-8 client encoding") + func latin1PgDumpCopyRows() async throws { + let dump = "SET client_encoding = 'LATIN1';\nCOPY t (a) FROM stdin;\nCafé\nZürich\n\\.\nSELECT 'é';\n" + let bytes = try #require(dump.data(using: .isoLatin1, allowLossyConversion: false)) + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("SqlFileImportSourceEncodingTests-\(UUID().uuidString).sql") + try bytes.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let source = SqlFileImportSource(url: url, encoding: .isoLatin1, grammar: { TestGrammar.postgres }, family: .postgres) + + var statements: [String] = [] + var rows = Data() + for try await element in try await source.elements() { + switch element { + case let .statement(statement, _): + statements.append(statement) + case let .copyFromStandardInput(statement, _, copy): + statements.append(statement) + while let piece = try copy.nextPiece() { + rows.append(piece) + } + @unknown default: + Issue.record("An element this test does not read: \(element)") + } + } + + #expect(statements == ["SET client_encoding = 'UTF8'", "COPY t (a) FROM stdin", "SELECT 'é'"]) + #expect(rows == Data("Café\nZürich\n".utf8)) + } + @Test("A UTF-8 file keeps its own character set statement, since its bytes go to the server unchanged") func utf8FileKeepsItsDeclaration() async throws { let dump = "/*!40101 SET NAMES latin1 */;\nINSERT INTO t VALUES ('é');\n" diff --git a/apple/TableProTests/Core/Process/PipeLineSplitterTests.swift b/apple/TableProTests/Core/Process/PipeLineSplitterTests.swift new file mode 100644 index 0000000000..9505dee992 --- /dev/null +++ b/apple/TableProTests/Core/Process/PipeLineSplitterTests.swift @@ -0,0 +1,30 @@ +import Foundation +@testable import TablePro +import Testing + +struct PipeLineSplitterTests { + @Test("A character split across two chunks arrives whole") + func multibyteCharacterAcrossChunks() { + var splitter = PipeLineSplitter() + let bytes = Data("café\n".utf8) + let cut = bytes.firstIndex(of: 0xC3).map { bytes.index(after: $0) } ?? bytes.endIndex + + #expect(splitter.lines(appending: bytes[.. 0 {} + } + + func closePeer() { + Darwin.close(peer) + peer = -1 + } + + func close() { + libssh2_session_free(session) + Darwin.close(local) + if peer >= 0 { Darwin.close(peer) } + } +} diff --git a/apple/TableProTests/Core/SSH/SFTPDownloadChunkSizeTests.swift b/apple/TableProTests/Core/SSH/SFTPDownloadChunkSizeTests.swift new file mode 100644 index 0000000000..e15517ad52 --- /dev/null +++ b/apple/TableProTests/Core/SSH/SFTPDownloadChunkSizeTests.swift @@ -0,0 +1,31 @@ +// +// SFTPDownloadChunkSizeTests.swift +// TableProTests +// + +@testable import TablePro +import Testing + +struct SFTPDownloadChunkSizeTests { + private static let minimum = 32 * 1_024 + private static let maximum = 2 * 1_024 * 1_024 + + @Test("A file under 32 KiB still reads into a 32 KiB buffer") + func smallFileGetsTheFloor() { + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 0) == Self.minimum) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 4_096) == Self.minimum) + } + + @Test("A file over 2 MiB reads into a 2 MiB buffer, which fills libssh2's 8 MiB read-ahead") + func largeFileGetsTheCap() { + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 500 * 1_024 * 1_024) == Self.maximum) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: .max) == Self.maximum) + } + + @Test("A file between the floor and the cap reads into a buffer of its own size") + func midSizeFileGetsItsOwnSize() { + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: UInt64(Self.minimum)) == Self.minimum) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 300_000) == 300_000) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: UInt64(Self.maximum)) == Self.maximum) + } +} diff --git a/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift b/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift index 1c250164d7..62a6a3dada 100644 --- a/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift +++ b/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift @@ -27,10 +27,4 @@ struct SSHTunnelDeadlineMarginTests { #expect(second.deadline.configuredSeconds == 60) #expect(second.deadline.instant == secondStart.advanced(by: .seconds(60))) } - - @Test("The accept loop notices a waiting client well inside the margin") - func acceptPollIsShort() { - #expect(LibSSH2Tunnel.acceptPollTimeoutMs <= 200) - #expect(LibSSH2Tunnel.acceptPollTimeoutMs > 0) - } } diff --git a/apple/TableProTests/Core/SSH/SSHTunnelErrorTests.swift b/apple/TableProTests/Core/SSH/SSHTunnelErrorTests.swift index 1b655b854a..b3474eb573 100644 --- a/apple/TableProTests/Core/SSH/SSHTunnelErrorTests.swift +++ b/apple/TableProTests/Core/SSH/SSHTunnelErrorTests.swift @@ -7,26 +7,6 @@ import TableProTestSupport import Testing struct SSHTunnelErrorTests { - // MARK: - Port Bind Failure Classification - - @Test("isLocalPortBindFailure detects 'already in use' pattern") - func bindFailureAlreadyInUse() { - #expect(SSHTunnelManager.isLocalPortBindFailure("Address already in use")) - } - - @Test("isLocalPortBindFailure is case-insensitive") - func bindFailureCaseInsensitive() { - #expect(SSHTunnelManager.isLocalPortBindFailure("ADDRESS ALREADY IN USE")) - } - - @Test("isLocalPortBindFailure returns false for unrelated SSH errors") - func nonBindFailures() { - #expect(!SSHTunnelManager.isLocalPortBindFailure("Permission denied")) - #expect(!SSHTunnelManager.isLocalPortBindFailure("Connection refused")) - #expect(!SSHTunnelManager.isLocalPortBindFailure("Host key verification failed")) - #expect(!SSHTunnelManager.isLocalPortBindFailure("")) - } - // MARK: - Error Descriptions @Test("SSHTunnelError.noAvailablePort has a localized description") diff --git a/apple/TableProTests/Core/SSH/SSHTunnelLocalPortTests.swift b/apple/TableProTests/Core/SSH/SSHTunnelLocalPortTests.swift new file mode 100644 index 0000000000..72da006f95 --- /dev/null +++ b/apple/TableProTests/Core/SSH/SSHTunnelLocalPortTests.swift @@ -0,0 +1,104 @@ +import Foundation +import TableProSSHConfig +import TableProSSHTransport +import Testing + +@testable import TablePro + +private final class LoopbackSocket { + let descriptor: Int32 + let port: Int + + init?(listening: Bool) { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { return nil } + + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_port = 0 + address.sin_addr.s_addr = inet_addr("127.0.0.1") + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(fd, $0, socklen_t(MemoryLayout.size)) + } + } + guard bound == 0, !listening || listen(fd, 4) == 0 else { + Darwin.close(fd) + return nil + } + + var assigned = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &assigned) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(fd, $0, &length) } + } + guard named == 0 else { + Darwin.close(fd) + return nil + } + descriptor = fd + port = Int(UInt16(bigEndian: assigned.sin_port)) + } + + var hasPendingConnection: Bool { + var pollDescriptor = pollfd(fd: descriptor, events: Int16(POLLIN), revents: 0) + return poll(&pollDescriptor, 1, 0) > 0 + } + + deinit { + Darwin.close(descriptor) + } +} + +struct SSHTunnelLocalPortTests { + private func freeLoopbackPort() throws -> Int { + try #require(LoopbackSocket(listening: false)).port + } + + @Test("A taken local port is skipped for the next free one") + func takenPortIsSkipped() throws { + let taken = try #require(LoopbackSocket(listening: true)) + let freePort = try freeLoopbackPort() + + let listener = try LibSSH2TunnelFactory.bindLocalListener(candidates: [taken.port, freePort]) + defer { Darwin.close(listener.descriptor) } + + #expect(listener.port == freePort) + withExtendedLifetime(taken) {} + } + + @Test("Every candidate port taken reports that no local port is available") + func everyPortTakenReportsNoPort() throws { + let taken = try #require(LoopbackSocket(listening: true)) + + #expect(throws: SSHTunnelError.noAvailablePort) { + try LibSSH2TunnelFactory.bindLocalListener(candidates: [taken.port]) + } + withExtendedLifetime(taken) {} + } + + @Test("A tunnel with no free local port fails before it dials or signs in to the SSH server") + func noLocalPortFailsBeforeDialing() async throws { + let taken = try #require(LoopbackSocket(listening: true)) + let sshServer = try #require(LoopbackSocket(listening: true)) + let takenPort = taken.port + let sshPort = sshServer.port + let manager = SSHTunnelManager(localPortCandidates: { [takenPort] }) + + await #expect(throws: SSHTunnelError.noAvailablePort) { + try await manager.createTunnel( + connectionId: UUID(), + sshHost: "127.0.0.1", + sshPort: sshPort, + sshUsername: "tester", + authMethod: .password, + sshPassword: "secret", + destination: .tcp(host: "127.0.0.1", port: 5_432), + deadline: ConnectionDeadline(configuredSeconds: 2) + ) + } + + #expect(!sshServer.hasPendingConnection) + withExtendedLifetime(taken) {} + } +} diff --git a/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift b/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift index c847c5ec7d..4d35111448 100644 --- a/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift +++ b/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift @@ -124,6 +124,139 @@ struct StructureChangeRebaseTests { #expect(shown == ["id", "a"]) } + private func stageDropAndReAdd(in manager: StructureChangeManager) throws { + try manager.deleteColumn(id: workingColumn("a", in: manager).id) + var readded = EditableColumnDefinition.from(textColumn("a")) + readded.id = UUID() + manager.addColumn(readded) + } + + private func textColumn(_ name: String) -> ColumnInfo { + ColumnInfo( + name: name, dataType: "TEXT", isNullable: true, isPrimaryKey: false, + defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil + ) + } + + private func stagedColumnChanges(_ manager: StructureChangeManager) -> [String] { + manager.getChangesArray().compactMap { change -> String? in + switch change { + case .deleteColumn(let column): return "drop \(column.name) \(column.dataType)" + case .addColumn(let column): return "add \(column.name) \(column.dataType)" + case .modifyColumn(let old, let new): return "\(old.name)>\(new.name)" + default: return nil + } + }.sorted() + } + + @Test("A drop and an add of one name both stay staged when the save wrote nothing") + func dropAndReAddSurviveAnUnchangedTable() throws { + let manager = manager() + try stageDropAndReAdd(in: manager) + let before = manager.getChangesArray() + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a", "x"].map { column($0) }, indexes: [], foreignKeys: [], primaryKey: ["id"] + ) + ) + + #expect(!moved) + #expect(manager.getChangesArray() == before) + } + + @Test("A drop that failed after a rename ran keeps the drop and the add of the same name") + func dropAndReAddSurviveAPartlyAppliedSave() throws { + let manager = manager() + var renamed = try workingColumn("x", in: manager) + renamed.name = "y" + manager.updateColumn(id: renamed.id, with: renamed) + try stageDropAndReAdd(in: manager) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a", "y"].map { column($0) }, indexes: [], foreignKeys: [], primaryKey: ["id"] + ) + ) + + #expect(moved) + #expect(stagedColumnChanges(manager) == ["add a TEXT", "drop a INTEGER"]) + } + + @Test("A drop and an add of one name that both ran leave nothing staged") + func dropAndReAddThatRanAreDone() throws { + let manager = manager() + var renamed = try workingColumn("x", in: manager) + renamed.name = "y" + manager.updateColumn(id: renamed.id, with: renamed) + try stageDropAndReAdd(in: manager) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: [column("id"), column("y"), textColumn("a")], indexes: [], foreignKeys: [], primaryKey: ["id"] + ) + ) + + #expect(moved) + #expect(stagedColumnChanges(manager).isEmpty) + } + + private func stageDropAndRenameInto(in manager: StructureChangeManager) throws { + try manager.deleteColumn(id: workingColumn("a", in: manager).id) + var renamed = try workingColumn("x", in: manager) + renamed.name = "a" + manager.updateColumn(id: renamed.id, with: renamed) + } + + @Test("A drop and a rename into the same name that both ran leave nothing staged, even with equal definitions") + func dropAndRenameIntoThatRanAreDone() throws { + let manager = manager() + try stageDropAndRenameInto(in: manager) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a"].map { column($0) }, indexes: [], foreignKeys: [], primaryKey: ["id"] + ) + ) + + #expect(moved) + #expect(stagedColumnChanges(manager).isEmpty) + } + + @Test("A drop and a rename into the same name keep both when the save wrote nothing") + func dropAndRenameIntoSurviveAnUnchangedTable() throws { + let manager = manager() + try stageDropAndRenameInto(in: manager) + let before = manager.getChangesArray() + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a", "x"].map { column($0) }, indexes: [], foreignKeys: [], primaryKey: ["id"] + ) + ) + + #expect(!moved) + #expect(manager.getChangesArray() == before) + } + + @Test("A chain of renames through a dropped name that all ran leaves no drop on the renamed column") + func renameChainThatRanStagesNoDrop() throws { + let manager = manager() + try stageDropAndRenameInto(in: manager) + var added = EditableColumnDefinition.from(textColumn("x")) + added.id = UUID() + manager.addColumn(added) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: [column("id"), column("a"), textColumn("x")], indexes: [], foreignKeys: [], primaryKey: ["id"] + ) + ) + + #expect(moved) + #expect(!stagedColumnChanges(manager).contains { $0.hasPrefix("drop") }) + } + @Test("A rename that did not run is staged against the table as it is") func unappliedRenameStaysStaged() throws { let manager = manager() diff --git a/apple/TableProTests/Core/SchemaTracking/StructureChangeValidationTests.swift b/apple/TableProTests/Core/SchemaTracking/StructureChangeValidationTests.swift index e05e11ea1f..a325c60432 100644 --- a/apple/TableProTests/Core/SchemaTracking/StructureChangeValidationTests.swift +++ b/apple/TableProTests/Core/SchemaTracking/StructureChangeValidationTests.swift @@ -83,6 +83,25 @@ struct StructureChangeValidationTests { #expect(manager.validationSummary.isEmpty) } + @Test("A key with more columns than referenced columns blocks the save, with Create Table's reason") + func unequalColumnCountsBlockTheSave() throws { + let manager = loadedManager() + let foreignKey = validForeignKey(columns: ["id", "customer_id"], referencedColumns: ["id"]) + manager.addForeignKey(foreignKey) + + #expect(!manager.canCommit) + let staged = try #require(manager.workingForeignKeys.last) + #expect(manager.validationErrors[.foreignKey(staged.id)] + == "A foreign key needs as many referenced columns as it has columns.") + } + + @Test("A key with as many referenced columns as columns can be saved") + func equalColumnCountsAllowTheSave() { + let manager = loadedManager() + manager.addForeignKey(validForeignKey(columns: ["id", "customer_id"], referencedColumns: ["tenant_id", "id"])) + #expect(manager.canCommit) + } + @Test("A key on a column the table does not have blocks the save") func foreignKeyOnMissingColumnBlocksCommit() { let manager = loadedManager() diff --git a/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift b/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift index 011bea2ec7..62a28411b1 100644 --- a/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift +++ b/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift @@ -57,6 +57,32 @@ struct BlobFormattingServiceCompactHexTests { } } +struct BlobFormattingServiceCopyTests { + @Test("A copy holds every byte, where the grid shows 64") + func copyIsTheWholeValue() { + let value = String(data: Data(repeating: 0xAB, count: 100), encoding: .isoLatin1) ?? "" + let copied = BlobFormattingService.shared.format(value, for: .copy) + #expect(copied == "0x" + String(repeating: "AB", count: 100)) + #expect(BlobFormattingService.shared.format(value, for: .grid)?.hasSuffix("…") == true) + } + + @Test("The copied form reads back as the bytes it spells") + func copiedHexReadsBack() { + let service = BlobFormattingService.shared + #expect(service.bytes(fromPrefixedHex: "0x4142") == Data([0x41, 0x42])) + #expect(service.bytes(fromPrefixedHex: "0X00ff") == Data([0x00, 0xFF])) + #expect(service.bytes(fromPrefixedHex: "0x") == Data()) + let bytes = Data(0...255) + let copied = service.format(bytes, for: .copy) ?? "" + #expect(service.bytes(fromPrefixedHex: copied) == bytes) + } + + @Test("Text that only starts like hex is not bytes", arguments: ["4142", "0x414", "0x41…", "0xZZ", "0x41 42", ""]) + func notCopiedHex(_ text: String) { + #expect(BlobFormattingService.shared.bytes(fromPrefixedHex: text) == nil) + } +} + @MainActor struct BlobFormattingServiceByteCountTests { @Test("Issue #1188 exact value reports 48 bytes (not 98)") diff --git a/apple/TableProTests/Core/Services/BulkReadTimeoutTests.swift b/apple/TableProTests/Core/Services/BulkReadTimeoutTests.swift new file mode 100644 index 0000000000..2109f9ee73 --- /dev/null +++ b/apple/TableProTests/Core/Services/BulkReadTimeoutTests.swift @@ -0,0 +1,40 @@ +// +// BulkReadTimeoutTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct BulkReadTimeoutTests { + private struct Failure: Error {} + + @Test("A bulk read runs with the timeout off and puts the connection's own back") + func liftsAndRestores() async throws { + var connection = TestFixtures.makeConnection() + connection.queryTimeoutSeconds = 17 + let source = MockDatabaseDriver(connection: connection) + let target = MockDatabaseDriver() + let configured = AppSettingsManager.shared.general.queryTimeoutSeconds + + let value = try await BulkReadTimeout.lifted(on: [source, target]) { 42 } + + #expect(value == 42) + #expect(source.applyQueryTimeoutValues == [0, 17]) + #expect(target.applyQueryTimeoutValues == [0, configured]) + } + + @Test("A bulk read that fails still puts the timeout back") + func restoresAfterFailure() async { + var connection = TestFixtures.makeConnection() + connection.queryTimeoutSeconds = 9 + let source = MockDatabaseDriver(connection: connection) + + await #expect(throws: Failure.self) { + try await BulkReadTimeout.lifted(on: [source]) { () async throws -> Int in throw Failure() } + } + #expect(source.applyQueryTimeoutValues == [0, 9]) + } +} diff --git a/apple/TableProTests/Core/Services/ClipboardServiceTests.swift b/apple/TableProTests/Core/Services/ClipboardServiceTests.swift index a0877b345b..2a2beb2255 100644 --- a/apple/TableProTests/Core/Services/ClipboardServiceTests.swift +++ b/apple/TableProTests/Core/Services/ClipboardServiceTests.swift @@ -101,3 +101,19 @@ struct ClipboardServiceTests { #expect(provider.readGridCells() == nil) } } + +extension GlobalStateSuites { + @Suite(.recordingClipboard) + @MainActor + struct ClipboardSecretWriteTests { + @Test("A secret copy through the shared service reaches the provider's concealed write") + func secretWriteReachesTheProvider() throws { + let clipboard = try #require(RecordingClipboard.current) + + ClipboardService.shared.writeSecretText("postgresql://app:hunter2@db.example.com/app") + + #expect(clipboard.secretWrites == ["postgresql://app:hunter2@db.example.com/app"]) + #expect(clipboard.text == "postgresql://app:hunter2@db.example.com/app") + } + } +} diff --git a/apple/TableProTests/Core/Services/ColumnTypeClassifierTests.swift b/apple/TableProTests/Core/Services/ColumnTypeClassifierTests.swift index 29832b00ab..1d9304d6a8 100644 --- a/apple/TableProTests/Core/Services/ColumnTypeClassifierTests.swift +++ b/apple/TableProTests/Core/Services/ColumnTypeClassifierTests.swift @@ -50,6 +50,20 @@ struct ColumnTypeClassifierTests { #expect(ColumnTypeClassifier(family: .postgres).classify(rawTypeName: "boolean").isBooleanType) } + // The MySQL driver names a BIT with its width and a BOOLEAN as TINYINT(1). + @Test("A MySQL BIT wider than one bit is a number; BIT, BIT(1) and TINYINT(1) are booleans") + func mysqlBitWidthDecidesTheKind() { + let mysql = ColumnTypeClassifier(family: .mysql) + #expect(mysql.classify(rawTypeName: "BIT(8)") == .integer(rawType: "BIT(8)")) + #expect(mysql.classify(rawTypeName: "bit(64)") == .integer(rawType: "bit(64)")) + #expect(mysql.classify(rawTypeName: "BIT(1)").isBooleanType) + #expect(mysql.classify(rawTypeName: "BIT").isBooleanType) + #expect(mysql.classify(rawTypeName: "TINYINT(1)").isBooleanType) + #expect(isInteger(mysql.classify(rawTypeName: "TINYINT"))) + #expect(ColumnTypeClassifier(family: .postgres).classify(rawTypeName: "bit(8)") == .text(rawType: "bit(8)")) + #expect(ColumnTypeClassifier().classify(rawTypeName: "BIT(8)").isBooleanType) + } + @Test("A compared column reads BIT by each side's family") func comparedColumnReadsBitBySide() { let column = CompareColumn( @@ -787,8 +801,7 @@ struct ColumnTypeClassifierTests { #expect(isTimestamp(classifier.classify(rawTypeName: rawTypeName))) } - /// Every other engine's DATE is a calendar date, so Oracle's own DATE reaches the classifier as a hint - /// rather than through this name. + // Oracle's DATE reaches the classifier as a hint, so the bare name stays the calendar date every other engine means. @Test("DATE stays a calendar date") func dateStaysADate() { #expect(classifier.classify(rawTypeName: "DATE") == .date(rawType: "DATE")) @@ -959,8 +972,7 @@ struct ColumnTypeClassifierTests { #expect(classifier.classify(rawTypeName: "jsonb").arrayElementEditor == nil) } - /// The badge vocabulary is semantic rather than the SQL spelling, so `jsonb[]` badges as - /// `json[]` exactly as scalar `jsonb` badges as `json`. #2897 read that as a lost `b`. + // The badge is semantic, so `jsonb[]` badges as `json[]` as scalar `jsonb` badges as `json`. #2897 read that as a lost `b`. @Test("A jsonb array keeps its raw type name behind the semantic badge") func keepsRawTypeBehindBadge() { let type = classifier.classify(rawTypeName: "jsonb[]") @@ -976,10 +988,7 @@ struct ColumnTypeClassifierTests { #expect(classifier.classify(rawTypeName: "text[]").displayName == "Text Array") } - /// PostgreSQL spells a domain's array through `format_type`, which puts the parameters - /// before the brackets. The base/params split dropped everything after the last `)`, so the - /// `[]` was lost and the column classified as its element: a `bit(8)[]` got the boolean - /// dropdown, which writes a scalar into an array column. + // PostgreSQL's format_type puts the parameters before the brackets, and the old split dropped the `[]`: `bit(8)[]` got the boolean dropdown. @Test("A parameterized array keeps its brackets") func classifiesParameterizedArrays() { let numeric = classifier.classify(rawTypeName: "numeric(10,2)[]") @@ -996,8 +1005,7 @@ struct ColumnTypeClassifierTests { #expect(classifier.classify(rawTypeName: "timestamp(3) with time zone[]").arrayElement != nil) } - /// The app's own spelling for an enum array puts the `[]` on the base and the labels in the - /// parentheses, which is the one array form the suffix test cannot see. + // The app's own `ENUM[](labels)` spelling is the one array form the suffix test cannot see. @Test("The enum array spelling still resolves its labels") func classifiesEnumArraySpelling() { let type = classifier.classify(rawTypeName: "ENUM[](mood)") @@ -1069,10 +1077,7 @@ struct ColumnTypeClassifierTests { return false } - /// `GEO_POINT` ends in `INT`, and `classifyByPattern` tests `hasSuffix("INT")` before any - /// spatial arm, so an Elasticsearch `geo_point` column classified as an integer. That drove - /// its cell alignment, its sort comparator and its filter operators, all wrong, with or - /// without a map. + // `GEO_POINT` ends in `INT` and the `hasSuffix("INT")` arm ran before any spatial one, so `geo_point` classified as an integer. @Test("geo_point is spatial, not an integer") func geoPointIsNotAnInteger() { for raw in ["geo_point", "GEO_POINT", "Geo_Point"] { @@ -1081,8 +1086,7 @@ struct ColumnTypeClassifierTests { } } - /// MySQL 8.0.11 renamed GEOMETRYCOLLECTION, and its catalog reports `geomcollection` for a - /// column declared with either spelling. Measured on MySQL 8.4.11. + // MySQL 8.0.11 renamed GEOMETRYCOLLECTION, and its catalog reports `geomcollection` for either spelling. Measured on MySQL 8.4.11. @Test("MySQL's own GEOMCOLLECTION spelling is spatial") func geomCollectionSpelling() { for raw in ["geomcollection", "GEOMCOLLECTION", "GEOMETRYCOLLECTION"] { @@ -1105,7 +1109,6 @@ struct ColumnTypeClassifierTests { #expect(isSpatial(classifier.classify(rawTypeName: "Nullable(Point)"))) } - /// The new entries must not drag neighbouring names into the spatial family. @Test("Neighbouring names keep their classification") func neighboursUnchanged() { #expect(classifier.classify(rawTypeName: "INT") == .integer(rawType: "INT")) diff --git a/apple/TableProTests/Core/Services/ContentFoundedRestoreTests.swift b/apple/TableProTests/Core/Services/ContentFoundedRestoreTests.swift new file mode 100644 index 0000000000..fb057f2811 --- /dev/null +++ b/apple/TableProTests/Core/Services/ContentFoundedRestoreTests.swift @@ -0,0 +1,51 @@ +// +// ContentFoundedRestoreTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct ContentFoundedRestoreTests { + @Test("A workspace founded on one table reads the saved tabs while the connection has not read them") + func contentFoundingReadsSavedTabsWhileTheGateIsClosed() { + let persistence = TabPersistenceCoordinator(connectionId: UUID()) + + #expect(persistence.readsSavedTabs(foundedWith: .openContent)) + #expect(persistence.readsSavedTabs(foundedWith: .newEmptyTab)) + #expect(persistence.readsSavedTabs(foundedWith: .restoreOrDefault)) + #expect(persistence.readsSavedTabs(foundedWith: nil)) + } + + @Test("A tab torn off into a second window does not pull the saved set in after it") + func contentFoundingSkipsTheRestoreOnceTheConnectionReadIt() async { + let persistence = TabPersistenceCoordinator(connectionId: UUID()) + _ = await persistence.restoreFromDisk() + + #expect(!persistence.readsSavedTabs(foundedWith: .openContent)) + #expect(persistence.readsSavedTabs(foundedWith: .restoreOrDefault)) + } + + @Test("A content-founded workspace restores its saved tabs after a reconnect instead of reopening the founding tab") + func releasedWorkspaceRestoresOnReconnect() { + let connectionId = UUID() + let workspace = ConnectionWorkspace( + connectionId: connectionId, + payload: EditorTabPayload(connectionId: connectionId, tabType: .table, tableName: "users"), + autoConnect: false, + payloadConnection: nil, + session: nil, + sessionState: nil, + trailingPaneState: nil, + phase: .connected + ) + + workspace.retireFoundingPayload() + + #expect(workspace.payload?.intent == .restoreOrDefault) + #expect(workspace.payload?.connectionId == connectionId) + #expect(workspace.payload?.tableName == nil) + } +} diff --git a/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift b/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift index ecec6574f7..1fc6accabd 100644 --- a/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift +++ b/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift @@ -369,3 +369,62 @@ struct DateFormattingServiceTimeZoneTests { #expect(written == wire) } } + +// Work off the main actor builds its own renderer, so it has to print exactly what the service prints. +@MainActor +struct DateDisplayRendererTests { + private struct Sample: Sendable { + let text: String + let columnType: ColumnType? + } + + private static var samples: [Sample] { + let shapes: [ColumnType?] = [ + .datetime(rawType: "DATETIME"), + .datetime(rawType: "DATETIME(6)"), + .date(rawType: "DATE"), + .timestamp(rawType: "TIME"), + .timestamp(rawType: "TIMETZ"), + .timestamp(rawType: "TIMESTAMPTZ"), + nil + ] + let texts = DatabaseDateParserTests.wireSpellings + [ + "2024-12-31 23:59:59-03:30", + "2024-12-31 23:59:59+05:45", + "2024-12-31 23:59:59Z", + "2024-03-15 09:30:00.000000", + "2024-03-15 09:30:00.999999999", + "09:30:00.123456+05:30", + "not-a-date" + ] + return texts.flatMap { text in shapes.map { Sample(text: text, columnType: $0) } } + } + + private static let instants: [Date] = [0, 1_700_000_000, 1_709_294_400.5, -86_400].map(Date.init(timeIntervalSince1970:)) + + // The service hands its calls to a renderer, so this pins the same text on and off the main actor; the literal tests above pin the text. + @Test("A renderer off the main actor prints the same text as the service on the main actor, under every format") + func rendererMatchesTheService() async { + let samples = Self.samples + let original = DateFormattingService.shared.currentFormat + var printed: [DateFormatOption: [String?]] = [:] + for format in DateFormatOption.allCases { + DateFormattingService.shared.updateFormat(format) + printed[format] = samples.map { DateFormattingService.shared.format(dateString: $0.text, columnType: $0.columnType) } + + Self.instants.map { Optional(DateFormattingService.shared.format($0)) } + } + DateFormattingService.shared.updateFormat(original) + + for format in DateFormatOption.allCases { + let rendered = await Self.render(samples, instants: Self.instants, format: format) + #expect(rendered == printed[format], "\(format)") + } + } + + @concurrent + nonisolated private static func render(_ samples: [Sample], instants: [Date], format: DateFormatOption) async -> [String?] { + let renderer = DateDisplayRenderer(format: format) + return samples.map { renderer.format(dateString: $0.text, columnType: $0.columnType) } + + instants.map { Optional(renderer.format($0)) } + } +} diff --git a/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift b/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift index 0cd0f79c01..3f3173c46e 100644 --- a/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift +++ b/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift @@ -28,6 +28,8 @@ private enum AutocommitOnlyFixture { return sqlServer case .oracle: return TestGrammar.oracle + case .teradata: + return DatabaseType.teradata.lexicalGrammar case .redis, .other: return TestGrammar.standard } @@ -392,3 +394,107 @@ struct AutocommitOnlyStatementSQLServerTests { #expect(!AutocommitOnlyFixture.matches(statement, .oracle)) } } + +struct AutocommitOnlyStatementTeradataTests { + @Test( + "Teradata takes nothing but the end of a transaction after these", + arguments: [ + "CREATE MULTISET TABLE sales.t (id INTEGER) PRIMARY INDEX (id)", + "create volatile table v (id integer) on commit preserve rows", + "CT sales.t (id INTEGER)", + "ALTER TABLE sales.t ADD note VARCHAR(10)", + "DROP TABLE sales.t", + "RENAME TABLE sales.t TO t2", + "REPLACE VIEW sales.v AS SELECT 1 AS a", + "COMMENT ON TABLE sales.t AS 'orders'", + "COLLECT STATISTICS COLUMN (id) ON sales.t", + "COLLECT STATS ON sales.t COLUMN id", + "DATABASE sales", + "SET SESSION DATABASE sales", + "SET SESSION DATEFORM = ANSIDATE", + "SET QUERY_BAND = 'app=tp;' FOR SESSION", + "SET QUERY_BAND = 'app=tp;' UPDATE FOR SESSION VOLATILE", + "SET QUERY_BAND = NONE FOR SESSION", + "SET ROLE analyst", + "SET TIME ZONE LOCAL", + "MODIFY USER u AS PERM = 1e9", + "DELETE DATABASE sales", + "DELETE USER u", + "GRANT SELECT ON sales TO u", + "REVOKE SELECT ON sales FROM u", + "BEGIN QUERY LOGGING ON ALL", + "END QUERY LOGGING ON ALL", + "BEGIN LOGGING ON EACH ALL ON TABLE sales.t", + "FLUSH QUERY LOGGING WITH ALL", + "-- nightly\nDROP TABLE sales.t", + "/* rebuild */ CREATE TABLE t (a INTEGER)", + ] + ) + func teradataRefusals(statement: String) { + #expect(AutocommitOnlyFixture.matches(statement, .teradata)) + } + + @Test( + "Data and transaction statements keep the wrap", + arguments: [ + "SELECT * FROM sales.t", + "SEL * FROM sales.t", + "INSERT INTO sales.t VALUES (1)", + "UPDATE sales.t SET id = 2", + "DELETE FROM sales.t", + "DELETE sales.t ALL", + "MERGE INTO sales.t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE SET id = s.id", + "HELP SESSION", + "SHOW TABLE sales.t", + "BT", + "ET", + "BEGIN TRANSACTION", + "END TRANSACTION", + "COMMIT WORK", + "ROLLBACK", + "CHECKPOINT sales.j", + "LOCKING ROW FOR ACCESS SELECT * FROM sales.t", + "SELECT 'CREATE TABLE t (a INTEGER)'", + "SET QUERY_BAND = 'a=1;' FOR TRANSACTION", + "set query_band = 'job=nightly;' for transaction;", + "SET QUERY_BAND = NONE FOR TRANSACTION", + ] + ) + func teradataAcceptances(statement: String) { + #expect(!AutocommitOnlyFixture.matches(statement, .teradata)) + } + + private static func movesDatabase(_ statement: String, _ family: TransactionEngineFamily = .teradata) -> Bool { + AutocommitOnlyStatement.movesDatabase( + statement, family: family, grammar: AutocommitOnlyFixture.grammar(for: family) + ) + } + + @Test( + "These move the session's default database", + arguments: [ + "DATABASE sales", "database sales;", "SET SESSION DATABASE sales", "SS DATABASE sales", + "-- switch\nDATABASE sales", + ] + ) + func teradataDatabaseMoves(statement: String) { + #expect(Self.movesDatabase(statement)) + } + + @Test( + "These leave the default database where it is", + arguments: [ + "SET SESSION DATEFORM = ANSIDATE", "DELETE DATABASE sales", "CREATE DATABASE d FROM sales AS PERM = 1e6", + "SELECT DATABASE", "SS", "SET SESSION", "MODIFY DATABASE sales AS PERM = 1e9", + ] + ) + func teradataNonMoves(statement: String) { + #expect(!Self.movesDatabase(statement)) + } + + @Test("A database move is read only on Teradata") + func databaseMoveIsTeradataOnly() { + #expect(!Self.movesDatabase("DATABASE sales", .postgres)) + #expect(!Self.movesDatabase("SET SESSION DATABASE sales", .mysql)) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/BareFunctionCatalogReadTests.swift b/apple/TableProTests/Core/Services/Execution/BareFunctionCatalogReadTests.swift new file mode 100644 index 0000000000..3bc5e9cd23 --- /dev/null +++ b/apple/TableProTests/Core/Services/Execution/BareFunctionCatalogReadTests.swift @@ -0,0 +1,130 @@ +// +// BareFunctionCatalogReadTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +// Answers the catalog the way Oracle 23ai answered for a schema with function PURGE, table ORDERS(ID, PURGE) and DUAL. +private final class CatalogDriver: PluginDriverStub, @unchecked Sendable { + private let fails: Bool + + init(fails: Bool) { + self.fails = fails + } + + func execute(query: String) async throws -> PluginQueryResult { + if fails { throw DatabaseError.notConnected } + let rows: [[PluginCellValue]] + if query == SQLCallableCatalog.query { + rows = [[.text("F"), .text("PURGE")]] + } else if query.contains("'DUAL'") { + rows = [[.text("0"), .text("3"), .null, .text("DUMMY")]] + } else { + rows = [[.text("0"), .text("1"), .null, .text("ID")], [.text("0"), .text("1"), .null, .text("PURGE")]] + } + return PluginQueryResult( + columns: rows.first.map { $0.indices.map { "C\($0)" } } ?? [], + columnTypeNames: rows.first.map { $0.map { _ in "VARCHAR2" } } ?? [], + rows: rows, + rowsAffected: 0, + executionTime: 0 + ) + } +} + +@MainActor +private final class CatalogProvider: ScopedMetadataProviding { + private let driver: DatabaseDriver + private let browse: DatabaseScope? + private(set) var scopes: [DatabaseScope] = [] + + init(fails: Bool = false, browse: DatabaseScope?) { + let connection = DatabaseConnection(name: "Probe", database: "FREEPDB1", type: .oracle) + driver = PluginDriverAdapter(connection: connection, pluginDriver: CatalogDriver(fails: fails)) + self.browse = browse + } + + func withMetadataDriver( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + scopes.append(scope) + return try await body(driver) + } + + func browseScope(for connectionId: UUID) -> DatabaseScope? { browse } +} + +@MainActor +struct BareFunctionCatalogReadTests { + private static func reading(_ sql: String) -> SQLBareNameReading { + SQLBareNameCalls.read(sql, databaseTypeId: "Oracle", readings: DatabaseType.oracle.lexicalReadings) + } + + private static func request(_ sql: String, connectionId: UUID, runsIn: DatabaseScope? = nil) -> OperationRequest { + OperationRequest( + connectionId: connectionId, + databaseType: .oracle, + sql: sql, + kind: .readQuery, + caller: .userInterface, + capabilities: .interactiveUser, + operationDescription: "Execute Query", + runsIn: runsIn + ) + } + + @Test("The session's catalog tells a function named bare from a column, in the schema the tab runs in") + func readsTheCatalogInTheTabScope() async { + let connectionId = UUID() + let browse = DatabaseScope(connectionId: connectionId, database: "FREEPDB1", schema: "APP") + let tab = DatabaseScope(connectionId: connectionId, database: "FREEPDB1", schema: "HR") + let provider = CatalogProvider(browse: browse) + + let call = await ExecutionGateProvider.callsBareFunction( + Self.request("SELECT purge FROM dual", connectionId: connectionId, runsIn: tab), + reading: Self.reading("SELECT purge FROM dual"), + catalog: SQLCallableCatalogCache(), + metadata: provider + ) + let column = await ExecutionGateProvider.callsBareFunction( + Self.request("SELECT purge FROM orders", connectionId: connectionId), + reading: Self.reading("SELECT purge FROM orders"), + catalog: SQLCallableCatalogCache(), + metadata: provider + ) + + #expect(call) + #expect(!column) + #expect(provider.scopes.first == tab) + #expect(provider.scopes.last == browse) + } + + @Test("A catalog that cannot be read, or no scope to read it in, counts as a call") + func unreadableCatalogRefuses() async { + let connectionId = UUID() + let browse = DatabaseScope(connectionId: connectionId, database: "FREEPDB1", schema: nil) + + let failed = await ExecutionGateProvider.callsBareFunction( + Self.request("SELECT id FROM orders", connectionId: connectionId), + reading: Self.reading("SELECT id FROM orders"), + catalog: SQLCallableCatalogCache(), + metadata: CatalogProvider(fails: true, browse: browse) + ) + let unscoped = await ExecutionGateProvider.callsBareFunction( + Self.request("SELECT id FROM orders", connectionId: connectionId), + reading: Self.reading("SELECT id FROM orders"), + catalog: SQLCallableCatalogCache(), + metadata: CatalogProvider(browse: nil) + ) + + #expect(failed) + #expect(unscoped) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/BatchCommitStatementTests.swift b/apple/TableProTests/Core/Services/Execution/BatchCommitStatementTests.swift index 687173ca76..5c29a89a4a 100644 --- a/apple/TableProTests/Core/Services/Execution/BatchCommitStatementTests.swift +++ b/apple/TableProTests/Core/Services/Execution/BatchCommitStatementTests.swift @@ -38,6 +38,18 @@ struct BatchCommitStatementTests { #expect(Self.matches(sql)) } + @Test("Teradata's ET is a commit point", arguments: ["ET", "et", "ET;", "-- done\nET"]) + func teradataETIsACommitPoint(sql: String) { + #expect(Self.matches(sql, type: .teradata)) + } + + @Test("ET followed by anything is not ET, and ET is only Teradata's") + func etWithAFollowerIsNotACommitPoint() { + #expect(Self.matches("ET AL", type: .teradata) == false) + #expect(Self.matches("ET") == false) + #expect(Self.matches("ET", type: .mysql) == false) + } + /// A block terminator is never a transaction's end, whichever engine wrote it. @Test( "END that closes a block is not a commit point", @@ -83,3 +95,31 @@ struct BatchCommitStatementTests { #expect(Self.matches("/*!40101 COMMIT */", type: .mysql)) } } + +struct TransactionEndStatementTests { + private static func matches(_ sql: String) -> Bool { + TransactionEndStatement.matches(sql, grammar: DatabaseType.teradata.lexicalGrammar) + } + + @Test( + "A commit or a rollback ends the transaction", + arguments: [ + "COMMIT", "COMMIT WORK", "ET", "END TRANSACTION", "ROLLBACK", "ROLLBACK WORK", "rollback;", "ABORT", + "ABORT 'bad load' WHERE 1 = 1", + ] + ) + func endsTheTransaction(sql: String) { + #expect(Self.matches(sql)) + } + + @Test( + "Anything else leaves it open", + arguments: [ + "ROLLBACK TO SAVEPOINT s", "BT", "BEGIN TRANSACTION", "INSERT INTO t VALUES (1)", "SELECT 'ROLLBACK'", + "CREATE TABLE t (a INTEGER)", "", + ] + ) + func leavesTheTransactionOpen(sql: String) { + #expect(Self.matches(sql) == false) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/BatchStatementRunTests.swift b/apple/TableProTests/Core/Services/Execution/BatchStatementRunTests.swift index b20e731cd0..c9fb0067c7 100644 --- a/apple/TableProTests/Core/Services/Execution/BatchStatementRunTests.swift +++ b/apple/TableProTests/Core/Services/Execution/BatchStatementRunTests.swift @@ -20,7 +20,10 @@ struct BatchStatementRunTests { commitPoints: Set = [], failing: String? = nil, stopsAfter: Int = .max, - commitOutput: ServerOutputBox = ServerOutputBox() + commitOutput: ServerOutputBox = ServerOutputBox(), + endsItsTransaction: (String) -> Bool = { _ in false }, + closesTransaction: (String) -> Bool = { _ in false }, + movesDatabase: (String) -> Bool = { _ in false } ) async -> BatchStatementOutcome { let claims = probe ?? ClaimProbe() return await BatchStatementRun.run( @@ -32,6 +35,9 @@ struct BatchStatementRunTests { gate: claims.gate, failureSQL: { $0 }, isCommitPoint: { commitPoints.contains($0) }, + endsItsTransaction: endsItsTransaction, + closesTransaction: closesTransaction, + movesDatabase: movesDatabase, commitOutput: commitOutput ) { statement in if claims.recordExecution() == stopsAfter { claims.stop() } @@ -117,6 +123,53 @@ struct BatchStatementRunTests { } } + @Test("A held run sends BEGIN and leaves the transaction open for the user to commit") + func heldRunBeginsAndNeverCommits() async { + let driver = TransactionRecordingDriver() + let outcome = await Self.run(plan: .heldTransaction, driver: driver) + #expect(driver.events == [.begin(mode: .readWrite)]) + guard case .completed(let results) = outcome else { + Issue.record("expected a completed run, got \(outcome)") + return + } + #expect(results.count == 3) + } + + @Test("A held run marks the turn's hold, and one rolled back clears it") + func heldRunMarksTheTurn() async { + let held = ManualCommitTurnHold() + _ = await ManualCommit.$turnHold.withValue(held) { + await Self.run(plan: .heldTransaction, driver: TransactionRecordingDriver()) + } + #expect(held.isHeld) + + let failed = ManualCommitTurnHold() + _ = await ManualCommit.$turnHold.withValue(failed) { + await Self.run(plan: .heldTransaction, driver: TransactionRecordingDriver(), failing: "VACUUM") + } + #expect(!failed.isHeld) + } + + @Test("A held run rolls back what it opened on a failure and on a Stop") + func heldRunRollsBackAFailureAndAStop() async { + let failing = TransactionRecordingDriver() + let failed = await Self.run(plan: .heldTransaction, driver: failing, failing: "VACUUM") + #expect(failing.events == [.begin(mode: .readWrite), .rollback]) + guard case .failed = failed else { + Issue.record("expected a failed run, got \(failed)") + return + } + + let stopped = TransactionRecordingDriver() + let outcome = await Self.run(plan: .heldTransaction, driver: stopped, stopsAfter: 1) + #expect(stopped.events == [.begin(mode: .readWrite), .rollback]) + guard case .cancelled(let results) = outcome else { + Issue.record("expected a cancelled run, got \(outcome)") + return + } + #expect(results.isEmpty) + } + @Test("The app's own transaction is rolled back on a failure, and its results are dropped") func appTransactionRollsBackAFailure() async { let driver = TransactionRecordingDriver() @@ -228,7 +281,7 @@ struct BatchStatementRunTests { @Test( "A driver without transactions is never asked to begin, commit or roll back", - arguments: [BatchTransactionPlan.appTransaction, .scriptTransaction, .autocommit, .sessionTransaction] + arguments: [BatchTransactionPlan.appTransaction, .heldTransaction, .scriptTransaction, .autocommit, .sessionTransaction] ) func driversWithoutTransactionsAreLeftAlone(plan: BatchTransactionPlan) async { let driver = TransactionRecordingDriver(supportsTransactions: false) @@ -552,6 +605,318 @@ struct BatchStatementRunTests { return } } + + // MARK: - A statement the session takes nothing after but the end of its transaction + + private static let create = "CREATE MULTISET TABLE sales.t (id INTEGER) PRIMARY INDEX (id)" + private static let insert = "INSERT INTO sales.t VALUES (1)" + + nonisolated private static func teradataDefinition(_ sql: String) -> Bool { + AutocommitOnlyStatement.matches(sql, family: .teradata, grammar: DatabaseType.teradata.lexicalGrammar) + } + + nonisolated private static func teradataEnd(_ sql: String) -> Bool { + TransactionEndStatement.matches(sql, grammar: DatabaseType.teradata.lexicalGrammar) + } + + nonisolated private static func teradataMove(_ sql: String) -> Bool { + AutocommitOnlyStatement.movesDatabase(sql, family: .teradata, grammar: DatabaseType.teradata.lexicalGrammar) + } + + private static func runTeradata( + _ statements: [String], + plan: BatchTransactionPlan = .autocommit, + driver: TransactionRecordingDriver, + probe: ClaimProbe? = nil, + stopsAfter: Int = .max + ) async -> BatchStatementOutcome { + await Self.run( + plan: plan, + driver: driver, + probe: probe, + statements: statements, + stopsAfter: stopsAfter, + endsItsTransaction: teradataDefinition, + closesTransaction: teradataEnd, + movesDatabase: teradataMove + ) + } + + // Error 3722 in ANSI mode; inside BT, Failure 3932 would roll back the whole block. + @Test("A run stops after DDL the session holds open, before the next statement, and keeps the DDL's result") + func stopsAfterADefinitionTheSessionHolds() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata([Self.create, Self.insert], driver: driver) + + #expect(driver.events.isEmpty) + #expect(driver.executedQueries == [Self.create]) + guard case .failed(let results, let failure, let description) = outcome else { + Issue.record("expected a stopped run, got \(outcome)") + return + } + #expect(results.count == 1) + #expect(failure == .awaitsCommit(sql: Self.create)) + #expect(description.isEmpty) + #expect(outcome.succeededCount == 1) + #expect(outcome.ranCount(of: 2) == 1) + } + + @Test( + "The stop commits nothing and rolls nothing back, whatever the plan", + arguments: [BatchTransactionPlan.autocommit, .scriptTransaction, .sessionTransaction, .heldTransaction, .appTransaction] + ) + func theStopLeavesTheTransactionOpen(plan: BatchTransactionPlan) async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata([Self.create, Self.insert], plan: plan, driver: driver) + + #expect(!driver.events.contains(.commit)) + #expect(!driver.events.contains(.rollback)) + guard case .failed(_, .awaitsCommit, _) = outcome else { + Issue.record("expected a stopped run, got \(outcome)") + return + } + } + + @Test( + "A run carries on when the next statement ends the transaction", + arguments: ["COMMIT WORK", "COMMIT", "ROLLBACK", "ROLLBACK WORK", "ABORT", "ET", "END TRANSACTION"] + ) + func carriesOnIntoTheTransactionEnd(end: String) async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata([Self.create, end, Self.insert], driver: driver) + + #expect(driver.executedQueries == [Self.create, end, Self.insert]) + #expect(outcome.isCompleted) + } + + // Teradata mode outside BT commits each request itself, and an unresolved mode cannot say. + @Test( + "A run carries on when the session holds no transaction after the DDL", + arguments: [PluginSessionTransactionState.idle, .unknown] + ) + func carriesOnWhenNothingIsHeld(state: PluginSessionTransactionState) async { + let driver = TransactionRecordingDriver() + driver.sessionState = state + + let outcome = await Self.runTeradata([Self.create, Self.insert], driver: driver) + + #expect(driver.executedQueries == [Self.create, Self.insert]) + #expect(outcome.isCompleted) + } + + @Test("A run whose engine has no such rule carries on past DDL") + func carriesOnWithoutTheRule() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.run(plan: .autocommit, driver: driver, statements: [Self.create, Self.insert]) + + #expect(driver.executedQueries == [Self.create, Self.insert]) + #expect(outcome.isCompleted) + } + + @Test("DDL as the last statement completes the run") + func definitionLastCompletes() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata([Self.insert, Self.create], driver: driver) + + #expect(driver.executedQueries == [Self.insert, Self.create]) + #expect(outcome.isCompleted) + } + + @Test("The stop reads the statement a replan put next") + func theStopReadsTheReplannedStatement() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + let claims = ClaimProbe() + + let outcome = await BatchStatementRun.run( + [Self.create, Self.insert], + plan: .autocommit, + mode: .readWrite, + driver: driver, + connectionId: claims.connectionId, + gate: claims.gate, + failureSQL: { $0 }, + isCommitPoint: { _ in false }, + endsItsTransaction: Self.teradataDefinition, + closesTransaction: Self.teradataEnd, + replanned: { ran in ran == 1 ? ["COMMIT WORK", Self.insert] : nil } + ) { statement in + try await driver.execute(query: statement) + } + + #expect(driver.executedQueries == [Self.create, "COMMIT WORK", Self.insert]) + #expect(outcome.isCompleted) + } + + // A Stop that lands on the DDL is a Stop: the plan takes back what it opened. + @Test("A Stop during the DDL is reported as a Stop, not as a wait for the commit") + func aStopDuringTheDefinitionWins() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata( + ["BT", Self.create, Self.insert], plan: .scriptTransaction, driver: driver, stopsAfter: 2 + ) + + #expect(driver.events == [.rollback]) + #expect(driver.executedQueries == ["BT", Self.create]) + guard case .cancelled = outcome else { + Issue.record("expected a cancelled run, got \(outcome)") + return + } + } + + // MARK: - A statement that moves the session's database + + private static let move = "DATABASE sales_dev" + private static let delete = "DELETE FROM orders" + + // A new run is pinned back to the tab's database, so the DELETE would land in the wrong one. + @Test( + "A run stops before a database move while the session holds a transaction, and sends nothing of it", + arguments: [BatchTransactionPlan.autocommit, .scriptTransaction, .sessionTransaction] + ) + func stopsBeforeAMoveInsideATransaction(plan: BatchTransactionPlan) async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata([Self.insert, Self.move, Self.delete], plan: plan, driver: driver) + + #expect(driver.executedQueries == [Self.insert]) + #expect(driver.events.isEmpty) + guard case .failed(let results, let failure, let description) = outcome else { + Issue.record("expected a stopped run, got \(outcome)") + return + } + #expect(results.count == 1) + #expect(failure == .databaseMoveAwaitsCommit(sql: Self.move)) + #expect(description.isEmpty) + #expect(outcome.ranCount(of: 3) == 1) + } + + @Test( + "Every spelling of a move stops inside a transaction", + arguments: ["DATABASE sales_dev", "SET SESSION DATABASE sales_dev", "SS DATABASE sales_dev"] + ) + func everyMoveSpellingStops(move: String) async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.runTeradata([move, Self.delete], driver: driver) + + #expect(driver.executedQueries.isEmpty) + guard case .failed(_, .databaseMoveAwaitsCommit(let sql), _) = outcome else { + Issue.record("expected a stopped run, got \(outcome)") + return + } + #expect(sql == move) + } + + // ANSI opens a transaction for the move alone; the app's own switch commits it the same way. + @Test("A move on an idle session is committed at once, and the run carries on in the same run") + func moveOnAnIdleSessionIsCommitted() async { + let probe = ClaimProbe() + let driver = TransactionRecordingDriver() + driver.sessionState = .idle + driver.stateAfterStatement = [Self.move: .inTransaction] + driver.stateAfterCommit = .idle + + let outcome = await Self.runTeradata([Self.move, Self.delete], driver: driver, probe: probe) + + #expect(driver.executedQueries == [Self.move, Self.delete]) + #expect(driver.events == [.commit]) + #expect(probe.commitPhaseEntries == 1) + #expect(probe.commitPhaseExits == 1) + #expect(probe.isStoppable) + guard case .completed(let results) = outcome else { + Issue.record("expected a completed run, got \(outcome)") + return + } + #expect(results.count == 2) + } + + @Test("A move that leaves the session idle is not committed") + func moveThatOpensNothingIsLeftAlone() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .idle + + let outcome = await Self.runTeradata([Self.move, Self.delete], driver: driver) + + #expect(driver.executedQueries == [Self.move, Self.delete]) + #expect(driver.events.isEmpty) + #expect(outcome.isCompleted) + } + + @Test("A move's commit the server refused fails the run at the move") + func refusedMoveCommitFailsTheRun() async { + let driver = TransactionRecordingDriver(failsCommit: true) + driver.sessionState = .idle + driver.stateAfterStatement = [Self.move: .inTransaction] + + let outcome = await Self.runTeradata([Self.move, Self.delete], driver: driver) + + #expect(driver.executedQueries == [Self.move]) + #expect(driver.events == [.commit]) + guard case .failed(let results, let failure, _) = outcome else { + Issue.record("expected a failed run, got \(outcome)") + return + } + #expect(results.isEmpty) + #expect(failure == .statement(sql: Self.move)) + } + + @Test("A move on a session that cannot say what it holds runs as before") + func moveOnAnUnknownSessionRunsAsBefore() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .unknown + + let outcome = await Self.runTeradata([Self.move, Self.delete], driver: driver) + + #expect(driver.executedQueries == [Self.move, Self.delete]) + #expect(driver.events.isEmpty) + #expect(outcome.isCompleted) + } + + @Test("A run whose engine has no such rule runs a move inside a transaction") + func moveWithoutTheRuleRuns() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .inTransaction + + let outcome = await Self.run(plan: .autocommit, driver: driver, statements: [Self.insert, Self.move, Self.delete]) + + #expect(driver.executedQueries == [Self.insert, Self.move, Self.delete]) + #expect(driver.events.isEmpty) + #expect(outcome.isCompleted) + } + + @Test("A stop after DDL that follows a move names the move, which a new run has to repeat") + func stopAfterAMoveNamesTheMove() async { + let driver = TransactionRecordingDriver() + driver.sessionState = .idle + driver.stateAfterStatement = [Self.move: .inTransaction, Self.create: .inTransaction] + driver.stateAfterCommit = .idle + + let outcome = await Self.runTeradata([Self.move, Self.create, Self.delete], driver: driver) + + #expect(driver.executedQueries == [Self.move, Self.create]) + #expect(driver.events == [.commit]) + guard case .failed(let results, let failure, _) = outcome else { + Issue.record("expected a stopped run, got \(outcome)") + return + } + #expect(results.count == 2) + #expect(failure == .awaitsCommit(sql: Self.create, databaseMove: Self.move)) + } } private enum TestError: Error { @@ -670,6 +1035,11 @@ private final class TransactionRecordingDriver: DatabaseDriver, @unchecked Senda private(set) var cancelCount = 0 private(set) var commitSawCancellation = false private(set) var rollbackSawCancellation = false + private(set) var executedQueries: [String] = [] + var sessionState: PluginSessionTransactionState = .unknown + // What the session holds once a statement of this text ran, and once a commit went through. + var stateAfterStatement: [String: PluginSessionTransactionState] = [:] + var stateAfterCommit: PluginSessionTransactionState? /// What each statement the run executes through this driver should throw, by its text. Only the /// commit-point cases use it: everything else fails through the run's own closure. @@ -706,6 +1076,7 @@ private final class TransactionRecordingDriver: DatabaseDriver, @unchecked Senda if let hook = whileCommitting { await MainActor.run { hook() } } commitSawCancellation = commitSawCancellation || Task.isCancelled if failsCommit { throw commitError } + if let stateAfterCommit { sessionState = stateAfterCommit } } func rollbackTransaction() async throws { @@ -722,10 +1093,16 @@ private final class TransactionRecordingDriver: DatabaseDriver, @unchecked Senda return serverOutputs.isEmpty ? .none : serverOutputs.removeFirst() } + func sessionTransactionState() async -> PluginSessionTransactionState { + sessionState + } + func execute(query: String) async throws -> QueryResult { executedStatements += 1 + executedQueries.append(query) if executedStatements == 1, let hook = whileExecuting { await MainActor.run { hook() } } if let error = failingStatements[query] { throw error } + if let next = stateAfterStatement[query] { sessionState = next } return .empty } diff --git a/apple/TableProTests/Core/Services/Execution/BatchTransactionPlanTests.swift b/apple/TableProTests/Core/Services/Execution/BatchTransactionPlanTests.swift index 185bd773ad..49eb84b51e 100644 --- a/apple/TableProTests/Core/Services/Execution/BatchTransactionPlanTests.swift +++ b/apple/TableProTests/Core/Services/Execution/BatchTransactionPlanTests.swift @@ -11,12 +11,16 @@ import Testing struct BatchTransactionPlanTests { private static let textPlans: [BatchTransactionPlan] = [.appTransaction, .scriptTransaction, .autocommit] - @Test("Only the app's own plan opens a transaction") - func onlyTheAppPlanOpensOne() { - #expect(BatchTransactionPlan.appTransaction.opensTransaction) - #expect(BatchTransactionPlan.scriptTransaction.opensTransaction == false) - #expect(BatchTransactionPlan.autocommit.opensTransaction == false) - #expect(BatchTransactionPlan.sessionTransaction.opensTransaction == false) + @Test("The app's own plan and a held one send BEGIN, and only the app's own sends COMMIT") + func onlyTheAppPlanCommits() { + #expect(BatchTransactionPlan.appTransaction.sendsBegin) + #expect(BatchTransactionPlan.appTransaction.commitsAtEnd) + #expect(BatchTransactionPlan.heldTransaction.sendsBegin) + #expect(BatchTransactionPlan.heldTransaction.commitsAtEnd == false) + for plan in [BatchTransactionPlan.scriptTransaction, .autocommit, .sessionTransaction] { + #expect(plan.sendsBegin == false) + #expect(plan.commitsAtEnd == false) + } } @Test("A joined run rolls nothing back and keeps what ran, exactly as autocommit does") @@ -25,31 +29,48 @@ struct BatchTransactionPlanTests { #expect(BatchTransactionPlan.sessionTransaction.keepsExecutedStatements) } + @Test("A held run rolls back on a failure or a Stop and reads what it left open") + func heldRunRollsBackAndReadsItsEnd() { + #expect(BatchTransactionPlan.heldTransaction.rollsBackAfterStop) + #expect(BatchTransactionPlan.heldTransaction.keepsExecutedStatements == false) + #expect(BatchTransactionPlan.heldTransaction.readsSessionEndState) + #expect(BatchTransactionPlan.appTransaction.readsSessionEndState == false) + } + @Test( "A session holding a transaction or a lock takes the plan over, whatever the text said", arguments: [PluginSessionTransactionState.inTransaction, .abortedTransaction, .holdsSessionLocks] ) func heldSessionTakesOver(state: PluginSessionTransactionState) { for plan in Self.textPlans { - #expect(plan.joining(state) == .sessionTransaction) + #expect(plan.joining(state, commitMode: .automatic) == .sessionTransaction) + #expect(plan.joining(state, commitMode: .manual) == .sessionTransaction) } } @Test("A session with nothing open leaves the plan the text decided") func idleSessionChangesNothing() { for plan in Self.textPlans { - #expect(plan.joining(.idle) == plan) + #expect(plan.joining(.idle, commitMode: .automatic) == plan) } } + @Test("Manual Commit holds a wrapped batch on an idle session, and nothing else") + func manualCommitHoldsOnlyAWrappedBatchOnAnIdleSession() { + #expect(BatchTransactionPlan.appTransaction.joining(.idle, commitMode: .manual) == .heldTransaction) + #expect(BatchTransactionPlan.appTransaction.joining(.unknown, commitMode: .manual) == .appTransaction) + #expect(BatchTransactionPlan.scriptTransaction.joining(.idle, commitMode: .manual) == .scriptTransaction) + #expect(BatchTransactionPlan.autocommit.joining(.idle, commitMode: .manual) == .autocommit) + } + /// The wrap stays for a plain batch, because its atomicity is worth more than a transaction that /// may not be there. A self-managed script stops being rolled back, because the transaction its /// text left open may predate the run. @Test("A session that cannot say keeps the wrap but stops the rollback of a self-managed script") func unknownSessionKeepsTheWrapButNotTheRollback() { - #expect(BatchTransactionPlan.appTransaction.joining(.unknown) == .appTransaction) - #expect(BatchTransactionPlan.autocommit.joining(.unknown) == .autocommit) - #expect(BatchTransactionPlan.scriptTransaction.joining(.unknown) == .sessionTransaction) + #expect(BatchTransactionPlan.appTransaction.joining(.unknown, commitMode: .automatic) == .appTransaction) + #expect(BatchTransactionPlan.autocommit.joining(.unknown, commitMode: .automatic) == .autocommit) + #expect(BatchTransactionPlan.scriptTransaction.joining(.unknown, commitMode: .automatic) == .sessionTransaction) } @Test("Joining is settled once: a joined plan stays joined") @@ -57,9 +78,21 @@ struct BatchTransactionPlanTests { for state in [ PluginSessionTransactionState.idle, .inTransaction, .abortedTransaction, .holdsSessionLocks, .unknown, ] { - #expect(BatchTransactionPlan.sessionTransaction.joining(state) == .sessionTransaction) + #expect(BatchTransactionPlan.sessionTransaction.joining(state, commitMode: .manual) == .sessionTransaction) } } + + @Test("A held run is never recorded as committed") + func heldRunIsNotCommitted() { + let completed = StatementCommitEvidence.run( + startedIn: .idle, endedIn: .inTransaction, plan: .heldTransaction, completed: true + ) + let failed = StatementCommitEvidence.run( + startedIn: .idle, endedIn: .idle, plan: .heldTransaction, completed: false + ) + #expect(completed == .runStartedIn(.idle, endedIn: .inTransaction, appTransaction: .none)) + #expect(failed == .runStartedIn(.idle, endedIn: .idle, appTransaction: .rolledBack)) + } } struct SessionTransactionStateTests { diff --git a/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift b/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift index f3e3df23bf..177584a048 100644 --- a/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift +++ b/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift @@ -238,6 +238,48 @@ struct BatchTransactionPolicyTests { #expect(Self.plan(["INSERT INTO t VALUES (1)", "CHECKPOINT"], .postgresql) == .appTransaction) } + // Inside BT a statement after DDL fails with 3932 and rolls the batch back, so a Teradata batch holding DDL runs + // each statement as its own request. + @Test( + "A Teradata batch holding DDL runs in autocommit", + arguments: [ + ["CREATE MULTISET TABLE t (id INTEGER) PRIMARY INDEX (id)", "INSERT INTO t VALUES (1)"], + ["DATABASE sales", "SELECT * FROM t"], + ["INSERT INTO t VALUES (1)", "COLLECT STATISTICS COLUMN (id) ON t"], + ["SET SESSION DATEFORM = ANSIDATE", "SELECT CURRENT_DATE"], + ["GRANT SELECT ON sales TO u", "UPDATE t SET id = 2"], + ] + ) + func teradataDefinitionBatchIsNotWrapped(statements: [String]) { + #expect(Self.plan(statements, .teradata) == .autocommit) + } + + @Test("A Teradata batch of data statements stays wrapped") + func teradataDataBatchIsWrapped() { + #expect(Self.plan(["INSERT INTO t VALUES (1)", "UPDATE t SET id = 2", "SEL * FROM t"], .teradata) == .appTransaction) + #expect(Self.plan(["DELETE FROM t", "DELETE t ALL"], .teradata) == .appTransaction) + } + + @Test("A Teradata script that begins its own transaction runs as written") + func teradataBeginTransactionIsNotWrapped() { + #expect(Self.plan(["BEGIN TRANSACTION", "INSERT INTO t VALUES (1)", "END TRANSACTION"], .teradata) == .scriptTransaction) + } + + // A wrap around BT would nest it, so its ET would close only the inner level. + @Test("BT opens a Teradata script's own transaction, and nothing on another engine") + func teradataBTIsTransactionControl() { + #expect(Self.plan(["BT", "INSERT INTO t VALUES (1)", "ET"], .teradata) == .scriptTransaction) + #expect(Self.plan(["bt;", "CREATE TABLE t (a INTEGER)", "ET"], .teradata) == .scriptTransaction) + #expect(Self.plan(["BT", "INSERT INTO t VALUES (1)"], .postgresql) == .appTransaction) + } + + // SET QUERY_BAND ... FOR TRANSACTION is DML, legal inside the wrap. + @Test("A transaction query band keeps a Teradata batch wrapped, and a session one does not") + func teradataQueryBandScope() { + #expect(Self.plan(["SET QUERY_BAND = 'job=x;' FOR TRANSACTION", "UPDATE t SET a = 1"], .teradata) == .appTransaction) + #expect(Self.plan(["SET QUERY_BAND = 'job=x;' FOR SESSION", "UPDATE t SET a = 1"], .teradata) == .autocommit) + } + @Test("A SQLite savepoint opens a transaction, so the script owns it after all") func savepointAfterAnUnwrappingStatement() { #expect(Self.plan(["VACUUM", "SAVEPOINT a", "INSERT INTO t VALUES (1)"], .sqlite) == .scriptTransaction) @@ -339,4 +381,16 @@ struct BatchTransactionPolicyTests { func redisBlockEndersDoNotOpenOne(statement: String) { #expect(Self.plan([statement], .redis) == .autocommit) } + + // Each statement then runs in a read-only transaction of its own, which an app transaction around them would refuse. + @Test("At Read-Only a batch of reads runs in autocommit", arguments: [DatabaseType.postgresql, .mysql, .oracle]) + func readOnlyBatchRunsInAutocommit(type: DatabaseType) { + let reads = ["SELECT purge_orders()", "SELECT 1"] + let plan = BatchTransactionPolicy.plan( + for: reads, databaseType: type, grammar: type.lexicalGrammar, safeModeIsReadOnly: true + ) + + #expect(plan == .autocommit) + #expect(Self.plan(reads, type) == .appTransaction) + } } diff --git a/apple/TableProTests/Core/Services/Execution/CopyDataRouteTests.swift b/apple/TableProTests/Core/Services/Execution/CopyDataRouteTests.swift new file mode 100644 index 0000000000..61dbc7f5b4 --- /dev/null +++ b/apple/TableProTests/Core/Services/Execution/CopyDataRouteTests.swift @@ -0,0 +1,77 @@ +// +// CopyDataRouteTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@MainActor +struct CopyDataRouteTests { + private static func route(_ text: String, sendsBatchesWhole: Bool) -> QueryExecutionRoute? { + QueryExecutionRoute.resolve( + QueryBatchPlanner.batches(in: text, model: .sql, grammar: TestGrammar.postgres), + sendsBatchesWhole: sendsBatchesWhole, + isPlainQuery: { _ in true } + ) + } + + @Test("A COPY carrying rows runs statement by statement even alone, where a lone statement takes the single path", arguments: [ + false, true, + ]) + func copyTakesTheStatementPath(sendsBatchesWhole: Bool) { + let route = Self.route("COPY t (a, b) FROM stdin;\n1\tone\n\\.\n", sendsBatchesWhole: sendsBatchesWhole) + guard case .statements(let statements) = route else { + Issue.record("expected statements, got \(String(describing: route))") + return + } + #expect(statements.map { $0.sql } == ["COPY t (a, b) FROM stdin"]) + #expect(statements.map { $0.copyData } == ["1\tone\n"]) + + guard case .single = Self.route("SELECT 1;", sendsBatchesWhole: sendsBatchesWhole) else { + Issue.record("expected a lone query on the single path") + return + } + } + + @Test("A COPY with no rows still takes the statement path, the one that runs a copy") + func copyWithoutRowsTakesTheStatementPath() { + guard case .statements(let statements) = Self.route("COPY t FROM stdin;", sendsBatchesWhole: false) else { + Issue.record("expected statements") + return + } + #expect(statements.map { $0.copyData } == [""]) + } +} + +struct CopyDataPiecesTests { + private static func pieces(of text: String) throws -> [Data] { + let rows = CopyDataPieces.reading(text) + var pieces: [Data] = [] + while let piece = try rows.nextPiece() { + pieces.append(piece) + } + #expect(try rows.nextPiece() == nil) + return pieces + } + + @Test("No rows make no piece") + func noRows() throws { + #expect(try Self.pieces(of: "").isEmpty) + } + + @Test("Rows go as UTF-8 in pieces of at most 64 KiB, cut wherever the limit falls, even inside a character") + func piecesCutAnywhere() throws { + let text = "12\t" + String(repeating: "é", count: 70_000) + "\n" + + let pieces = try Self.pieces(of: text) + + #expect(pieces.map { $0.count } == [65_536, 65_536, 8_932]) + #expect(Data(pieces.joined()) == Data(text.utf8)) + let first = try #require(pieces.first) + #expect(String(data: first, encoding: .utf8) == nil) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift b/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift index 0e304c6e92..35d8e896e3 100644 --- a/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift +++ b/apple/TableProTests/Core/Services/Execution/ExecutionGateTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing @MainActor @@ -859,6 +860,456 @@ struct ExecutionGateTests { try await gate.authorizing(makeRequest(sql: "DELETE FROM t", kind: .writeQuery)) {} } } + + // MARK: - Read-Only function calls on a driver without the engine's read-only mode + + private func makeReadOnlyGate( + level: SafeModeLevel = .readOnly, + policy: ReadOnlyFunctionPolicy, + confirm: StubConfirming + ) -> DefaultExecutionGate { + DefaultExecutionGate( + confirming: confirm, + authenticating: StubAuthenticating(answer: true), + safeModeLevelResolver: { _ in level }, + forcesWriteResolver: { _ in false }, + readOnlyFunctionPolicyResolver: { _, _ in policy } + ) + } + + private static let writableFunctions = ReadOnlyFunctionPolicy(functionsCanWrite: true) + + @Test("Read-Only refuses a read that calls a function the engine could let write, and names the level to switch to") + func readOnlyRefusesAFunctionCall() async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: "SELECT purge_orders()", kind: .readQuery)) + + guard case .denied(let reason, let cause) = decision else { + Issue.record("Read-Only must refuse a call it cannot tell from a write") + return + } + #expect(reason.contains("Alert")) + #expect(cause == .policy) + #expect(confirm.callCount == 0) + } + + @Test( + "A caller cleared before the gate is refused the call all the same", + arguments: [CallerCapabilities.preCleared, .confirmationPreCleared] + ) + func clearedCallerIsRefusedTheCall(cleared: CallerCapabilities) async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest( + sql: "SELECT purge_orders()", + kind: .readQuery, + capabilities: CallerCapabilities.interactiveUser.union(cleared) + )) + + #expect(!decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test("A managed policy or an engine floor holding Read-Only refuses the call without asking") + func heldReadOnlyRefusesAFunctionCall() async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate( + policy: ReadOnlyFunctionPolicy(functionsCanWrite: true, floor: .readOnly), + confirm: confirm + ) + + let decision = await gate.authorize(makeRequest(sql: "SELECT purge_orders()", kind: .readQuery)) + + guard case .denied(let reason, let cause) = decision else { + Issue.record("A held Read-Only must refuse") + return + } + #expect(reason.contains("held at Read-Only")) + #expect(!reason.contains("Alert")) + #expect(cause == .policy) + #expect(confirm.callCount == 0) + } + + @Test( + "The refusal names the weakest level the menu offers that runs the call", + arguments: [ + (SafeModeLevel?.none, SafeModeLevel?.some(.alert)), (.silent, .alert), (.alert, .alert), + (.alertFull, .alertFull), (.safeModeFull, .safeModeFull), (.readOnly, nil), + ] + ) + func refusalNamesTheLevelThatRunsIt(floor: SafeModeLevel?, named: SafeModeLevel?) async { + let policy = ReadOnlyFunctionPolicy(functionsCanWrite: true, floor: floor) + let gate = makeReadOnlyGate(policy: policy, confirm: StubConfirming(answer: true)) + + let reason = await gate.authorize(makeRequest(sql: "SELECT purge_orders()", kind: .readQuery)).deniedReason + + #expect(policy.levelThatRunsIt == named) + if let named { + #expect(reason?.hasSuffix("Switch Safe Mode to \(named.displayName) to run it.") == true) + } else { + #expect(reason?.contains("held at Read-Only") == true) + } + } + + // Oracle and Dameng call a function named bare; the session's catalog tells it from a column. + private actor ResolverCalls { + private(set) var count = 0 + func record() { count += 1 } + } + + private func makeBareNameGate(calls: ResolverCalls, answer: Bool) -> DefaultExecutionGate { + let policy = Self.writableFunctions + return DefaultExecutionGate( + confirming: StubConfirming(answer: true), + authenticating: StubAuthenticating(answer: true), + safeModeLevelResolver: { _ in .readOnly }, + forcesWriteResolver: { _ in false }, + readOnlyFunctionPolicyResolver: { _, _ in policy }, + bareNameCallResolver: { _, _ in + await calls.record() + return answer + } + ) + } + + @Test("Read-Only on Oracle and Dameng refuses a name the catalog resolves to a function", arguments: [ + DatabaseType.oracle, .dameng, + ]) + func readOnlyRefusesABareFunctionName(type: DatabaseType) async { + let calls = ResolverCalls() + let gate = makeBareNameGate(calls: calls, answer: true) + + let decision = await gate.authorize(makeRequest(sql: "SELECT purge_orders FROM dual", kind: .readQuery, databaseType: type)) + + #expect(decision.deniedReason?.contains("calls a function that can change data") == true) + #expect(await calls.count == 1) + } + + @Test("A name the catalog reads as a column runs, and a query naming nothing never asks the catalog") + func bareNameAsksOnlyWhenItMust() async { + let calls = ResolverCalls() + let gate = makeBareNameGate(calls: calls, answer: false) + + #expect(await gate.authorize(makeRequest(sql: "SELECT id FROM orders", kind: .readQuery, databaseType: .oracle)).isAuthorized) + #expect(await calls.count == 1) + #expect(await gate.authorize(makeRequest(sql: "SELECT 1 FROM dual", kind: .readQuery, databaseType: .oracle)).isAuthorized) + #expect(await gate.authorize(makeRequest(sql: "SELECT id FROM orders", kind: .readQuery, databaseType: .mysql)).isAuthorized) + #expect(await calls.count == 1) + } + + @Test("A package function named bare is refused without the catalog") + func dottedBareCallRefusedByDefault() async { + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: StubConfirming(answer: true)) + + let decision = await gate.authorize(makeRequest(sql: "SELECT pkg.purge FROM dual", kind: .readQuery, databaseType: .oracle)) + + #expect(!decision.isAuthorized) + } + + @Test("A caller that cannot prompt gets the same refusal") + func readOnlyFunctionCallDeniedWithoutPrompt() async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize( + makeRequest(sql: "SELECT purge_orders()", kind: .readQuery, capabilities: [.cannotPrompt]) + ) + + #expect(decision.deniedReason?.contains("Alert") == true) + #expect(confirm.callCount == 0) + } + + @Test( + "Read-Only runs built-ins, keywords before a parenthesis and a plain EXPLAIN", + arguments: [ + "SELECT COUNT(*), MAX(total), LOWER(name) FROM orders WHERE id IN (1, 2) AND EXISTS (SELECT 1)", + "SELECT ROW_NUMBER() OVER (PARTITION BY region ORDER BY total) FROM orders", + "SELECT 'purge_orders()' FROM orders -- purge_orders()", + "EXPLAIN SELECT purge_orders()", + ] + ) + func readOnlyRunsBuiltins(sql: String) async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: sql, kind: .readQuery)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test( + "A qualified name, a quoted name and a call in a later statement are refused", + arguments: [ + "SELECT app.count(id) FROM orders", + "SELECT `purge_orders`() FROM orders", + "SELECT 1;\nSELECT purge_orders()", + ] + ) + func readOnlyRefusesAnyOtherCall(sql: String) async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize( + makeRequest(sql: sql, kind: .readQuery, capabilities: CallerCapabilities.interactiveUser) + ) + + #expect(!decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test( + "A user function named like a keyword another engine leaves unreserved is refused", + arguments: [ + "SELECT filter(x) FROM orders", + "SELECT `within`(1) FROM orders", + "SELECT keep(1), sets(2) FROM orders", + "SELECT pivot(total) FROM orders", + "SELECT array(1) FROM orders", + "SELECT id FROM orders WHERE total = offset(1)", + "SELECT join(id) FROM orders", + "SELECT id FROM orders WHERE status IN (purge_orders())", + "SELECT 10 DIV pivot(1)", + "SELECT 10 MOD pivot(1)", + "SELECT paid XOR offset(1) FROM orders", + "SELECT id FROM orders WHERE /* don't */ offset(1) = 1", + ] + ) + func keywordNamedFunctionIsRefused(sql: String) async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: sql, kind: .readQuery)) + + #expect(!decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test( + "A keyword in the syntax that uses it before a parenthesis runs", + arguments: [ + "SELECT MAX(sal) KEEP (DENSE_RANK FIRST ORDER BY hired) FROM emp", + "SELECT LISTAGG(name, ',') WITHIN GROUP (ORDER BY name) FROM emp", + "SELECT * FROM sales PIVOT (SUM(amount) FOR quarter IN ('Q1', 'Q2'))", + "SELECT region, SUM(total) FROM orders GROUP BY ROLLUP (region)", + "SELECT region, SUM(total) FROM orders GROUP BY GROUPING SETS ((region), ())", + "SELECT * FROM orders PARTITION (p1)", + "SELECT * FROM orders WHERE total = ANY (SELECT total FROM refunds)", + "SELECT * FROM orders o JOIN (SELECT id FROM refunds) r ON o.id = r.id", + "SELECT CASE WHEN (total > 0) THEN 1 ELSE 0 END FROM orders", + ] + ) + func keywordSyntaxRuns(sql: String) async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: sql, kind: .readQuery, databaseType: .oracle)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test( + "A word operator before a parenthesis runs", + arguments: [ + "SELECT 10 DIV (2), paid XOR (refunded) FROM orders", + "SELECT id FROM orders WHERE name REGEXP ('^a') OR name RLIKE ('b$')", + "SELECT id FROM orders WHERE note LIKE 'a!%' ESCAPE ('!')", + ] + ) + func wordOperatorsRun(sql: String) async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: sql, kind: .readQuery, databaseType: .mysql)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test( + "The fallback engines' everyday built-ins run", + arguments: [ + (DatabaseType.oracle, "SELECT DECODE(status, 1, 'open', 'closed'), NVL2(a, b, c), INSTR(name, 'x') FROM orders"), + (.oracle, "SELECT ADD_MONTHS(created, 1), MONTHS_BETWEEN(a, b), TO_TIMESTAMP(t), SYS_GUID() FROM orders"), + (.oracle, "SELECT * FROM orders WHERE REGEXP_LIKE(name, '^a') AND TRUNC(created) = LAST_DAY(created)"), + (.dameng, "SELECT INITCAP(name), LPAD(code, 5, '0'), CHR(65) FROM orders"), + (.mysql, "SELECT DATE_FORMAT(created_at, '%Y-%m') AS m, COUNT(*) FROM orders GROUP BY m"), + (.mysql, "SELECT IF(paid, 1, 0), GROUP_CONCAT(name), JSON_EXTRACT(doc, '$.a') FROM orders USE INDEX (idx)"), + (.mariadb, "SELECT * FROM orders WHERE MATCH (title) AGAINST ('x') AND DATEDIFF(NOW(), created) < 7"), + (.oceanbase, "SELECT DATE_ADD(created, INTERVAL (1) DAY), UUID(), MD5(name) FROM orders"), + ] + ) + func everydayBuiltinsRun(type: DatabaseType, sql: String) async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: sql, kind: .readQuery, databaseType: type)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + // GET_LOCK is refused as a write; SLEEP and BENCHMARK are off the built-in list. + @Test( + "A built-in that locks, sleeps or waits never runs on its own", + arguments: ["SELECT GET_LOCK('a', 10)", "SELECT SLEEP(5)", "SELECT BENCHMARK(1000, MD5('a'))"] + ) + func lockingOrSleepingBuiltinDoesNotRun(sql: String) async { + let confirm = StubConfirming(answer: true) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: sql, kind: .readQuery)) + + #expect(!decision.isAuthorized) + } + + @Test("The built-in lists hold nothing that writes, locks, sleeps or reaches a file") + func builtinListsHoldOnlyReads() { + let forbidden: Set = [ + "GET_LOCK", "RELEASE_LOCK", "IS_FREE_LOCK", "SLEEP", "PG_SLEEP", "BENCHMARK", "LOAD_FILE", "NEXTVAL", + "SETVAL", "LAST_INSERT_ID", "SET_CONFIG", "MASTER_POS_WAIT", "SOURCE_POS_WAIT", "TXID_CURRENT", + ] + for type in [DatabaseType.oracle, .dameng, .mysql, .mariadb, .oceanbase, .postgresql, .sqlite] { + let builtins = ReadOnlyFunctionCalls.builtins(for: type) + #expect(builtins.isDisjoint(with: forbidden)) + #expect(!builtins.contains { $0.hasPrefix("DBMS_") || $0.hasPrefix("UTL_") || $0.hasPrefix("PG_") }) + } + } + + @Test("A driver that runs the statement in the engine's read-only mode runs the call") + func enforcingDriverRunsTheCall() async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(policy: ReadOnlyFunctionPolicy(), confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: "SELECT purge_orders()", kind: .readQuery)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test("Only Read-Only refuses: Silent and Alert run the same call", arguments: [SafeModeLevel.silent, .alert]) + func otherLevelsDoNotAsk(level: SafeModeLevel) async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(level: level, policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: "SELECT purge_orders()", kind: .readQuery)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } + + @Test("A metadata read is never refused for its calls") + func metadataReadSkipsTheFunctionCheck() async { + let confirm = StubConfirming(answer: false) + let gate = makeReadOnlyGate(policy: Self.writableFunctions, confirm: confirm) + + let decision = await gate.authorize(makeRequest(sql: "SELECT pg_get_viewdef(1)", kind: .metadataRead)) + + #expect(decision.isAuthorized) + #expect(confirm.callCount == 0) + } +} + +// Which engines let a function a query calls write, and what the live driver adds to that. +@MainActor +struct ReadOnlyFunctionPolicyTests { + @Test( + "Oracle, Dameng, MySQL, MariaDB, OceanBase and the PostgreSQL family count as engines whose functions can write", + arguments: [DatabaseType.oracle, .dameng, .mysql, .mariadb, .oceanbase, .postgresql, .kingbasees] + ) + func enginesWhoseFunctionsWrite(type: DatabaseType) { + #expect(PluginMetadataRegistry.queryFunctionsCanWrite.contains(type)) + } + + @Test( + "Engines that refuse a write from a function in a query, or have no such function, do not", + arguments: [ + DatabaseType.mssql, .snowflake, .bigQuery, .trino, .redshift, .clickhouse, .sqlite, .libsql, .duckdb, + .cassandra, .mongodb, .redis, .tidb, .starrocks, .databend, + ] + ) + func enginesWhoseFunctionsDoNotWrite(type: DatabaseType) { + #expect(!PluginMetadataRegistry.queryFunctionsCanWrite.contains(type)) + } + + @Test("A live MySQL driver without the mode is refused the call, and one with it is not") + func liveDriverDecides() { + let connection = DatabaseConnection(name: "Shop", database: "shop", type: .mysql, safeModeLevel: .readOnly) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + install(connection, capabilities: [.readOnlyStatements]) + let older = ExecutionGateProvider.readOnlyFunctionPolicy(connectionId: connection.id, databaseType: .mysql) + install(connection, capabilities: [.readOnlyStatements, .enforcesReadOnlyMode]) + let enforcing = ExecutionGateProvider.readOnlyFunctionPolicy(connectionId: connection.id, databaseType: .mysql) + + #expect(older.functionsCanWrite) + #expect(older.levelThatRunsIt == .alert) + #expect(!enforcing.functionsCanWrite) + #expect(ExecutionGateProvider.runsAtReadOnly(connection.id)) + } + + // The read-only mode would end what a session holds, so the read runs outside it and its calls are checked. + @Test( + "A live MySQL driver with the mode is refused the call while its session holds a transaction or a lock", + arguments: [ + (PluginSessionTransactionState.idle, false), + (.inTransaction, true), + (.abortedTransaction, true), + (.holdsSessionLocks, true), + (.unknown, true), + ] + ) + func liveSessionStateDecides(state: PluginSessionTransactionState, refused: Bool) async { + let connection = DatabaseConnection(name: "Shop", database: "shop", type: .mysql, safeModeLevel: .readOnly) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + install(connection, driver: SessionStateDriver(state)) + + let policy = await ExecutionGateProvider.liveReadOnlyFunctionPolicy( + connectionId: connection.id, databaseType: .mysql + ) + + #expect(policy.functionsCanWrite == refused) + } + + private func install(_ connection: DatabaseConnection, capabilities: PluginCapabilities) { + install(connection, driver: CapabilityOnlyDriver(capabilities)) + } + + private func install(_ connection: DatabaseConnection, driver: any PluginDatabaseDriver) { + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + } +} + +private final class CapabilityOnlyDriver: PluginDriverStub, @unchecked Sendable { + private let declared: PluginCapabilities + + init(_ capabilities: PluginCapabilities) { + declared = capabilities + } + + var capabilities: PluginCapabilities { declared } +} + +private final class SessionStateDriver: PluginDriverStub, @unchecked Sendable { + private let state: PluginSessionTransactionState + + init(_ state: PluginSessionTransactionState) { + self.state = state + } + + var capabilities: PluginCapabilities { [.readOnlyStatements, .enforcesReadOnlyMode] } + var supportsTransactions: Bool { true } + + func sessionTransactionState() async -> PluginSessionTransactionState { state } } // The editor's request for a text the gate must tier by what the server runs, behind each executable comment form. diff --git a/apple/TableProTests/Core/Services/Execution/ManualCommitTests.swift b/apple/TableProTests/Core/Services/Execution/ManualCommitTests.swift new file mode 100644 index 0000000000..c891b8e502 --- /dev/null +++ b/apple/TableProTests/Core/Services/Execution/ManualCommitTests.swift @@ -0,0 +1,322 @@ +// +// ManualCommitTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSSHConfig +import Testing + +struct ManualCommitTests { + private func decision( + _ sql: String, + mode: ConnectionCommitMode = .manual, + type: DatabaseType = .postgresql, + supportsTransactions: Bool = true, + state: PluginSessionTransactionState = .idle + ) -> ManualCommit.Decision { + decision([sql], mode: mode, type: type, supportsTransactions: supportsTransactions, state: state) + } + + private func decision( + _ statements: [String], + mode: ConnectionCommitMode = .manual, + type: DatabaseType = .postgresql, + supportsTransactions: Bool = true, + state: PluginSessionTransactionState = .idle + ) -> ManualCommit.Decision { + ManualCommit.decision( + commitMode: mode, + statements: statements, + databaseType: type, + grammar: type.lexicalGrammar, + supportsTransactions: supportsTransactions, + sessionState: state + ) + } + + @Test("A write on an idle session is held in Manual Commit and runs as typed otherwise") + func writeOnAnIdleSessionIsHeld() { + #expect(decision("UPDATE t SET a = 1") == .hold) + #expect(decision("UPDATE t SET a = 1", mode: .automatic) == .runAsIs) + } + + @Test("A read opens nothing") + func readOpensNothing() { + #expect(decision("SELECT * FROM t") == .runAsIs) + } + + @Test("A write joins a transaction already open, aborted or not") + func writeJoinsAnOpenTransaction() { + #expect(decision("UPDATE t SET a = 1", state: .inTransaction) == .runAsIs) + #expect(decision("UPDATE t SET a = 1", state: .abortedTransaction) == .runAsIs) + } + + @Test( + "A write the session cannot hold is refused rather than committed", + arguments: [PluginSessionTransactionState.unknown, .holdsSessionLocks] + ) + func unholdableStateRefuses(state: PluginSessionTransactionState) { + #expect(decision("UPDATE t SET a = 1", state: state) == .refuse) + } + + @Test("An engine without transactions refuses a write in Manual Commit") + func engineWithoutTransactionsRefuses() { + #expect(decision("UPDATE t SET a = 1", supportsTransactions: false) == .refuse) + #expect(decision("SELECT 1", supportsTransactions: false) == .runAsIs) + } + + @Test("Transaction control and statements refused inside a block run as typed") + func transactionControlAndAutocommitOnlyRunAsTyped() { + #expect(decision("BEGIN") == .runAsIs) + #expect(decision("VACUUM") == .runAsIs) + } + + @Test("Manual Commit is offered where DDL waits for Commit", arguments: [ + DatabaseType.postgresql, .sqlite, .mssql, .duckdb, + ]) + func availableWhereDDLWaits(type: DatabaseType) { + #expect(ManualCommit.isAvailable(for: type)) + #expect(ManualCommit.effectiveMode(.manual, databaseType: type) == .manual) + } + + @Test("Manual Commit is not offered where DDL commits on its own, and a stored one runs as automatic", arguments: [ + DatabaseType.mysql, .mariadb, .oracle, .clickhouse, .redshift, .cockroachdb, + ]) + func unavailableWhereDDLCommits(type: DatabaseType) { + #expect(!ManualCommit.isAvailable(for: type)) + #expect(ManualCommit.effectiveMode(.manual, databaseType: type) == .automatic) + } + + @Test("A run that mixes a write with a statement that can't run inside a transaction is refused") + func autocommitOnlyBesideAWriteRefuses() { + #expect(decision(["UPDATE t SET a = 1", "VACUUM t"]) == .refuseMixed) + #expect(decision(["DELETE FROM t", "CREATE INDEX CONCURRENTLY i ON t (a)"]) == .refuseMixed) + #expect(decision(["VACUUM t"]) == .runAsIs) + } + + @Test("DDL beside a write is held like the write where DDL waits for Commit", arguments: [ + DatabaseType.postgresql, .sqlite, .mssql, + ]) + func ddlBesideAWriteIsHeld(type: DatabaseType) { + #expect(decision(["UPDATE t SET a = 1", "ALTER TABLE u ADD c INT"], type: type) == .hold) + #expect(decision(["ALTER TABLE u ADD c INT"], type: type) == .hold) + } + + @Test("Two reads in Manual Commit are not held: the run keeps the plan an automatic connection uses") + func readOnlyRunIsNotHeld() { + let reads = ["SELECT 1", "SELECT 2"] + let decided = decision(reads) + #expect(decided == .runAsIs) + #expect(ManualCommit.runPlan(text: .appTransaction, decision: decided, sessionState: .idle) == .appTransaction) + #expect(ManualCommit.runPlan(text: .appTransaction, decision: .hold, sessionState: .idle) == .heldTransaction) + #expect(ManualCommit.runPlan(text: .autocommit, decision: .runAsIs, sessionState: .inTransaction) == .sessionTransaction) + } + + @Test("An EXPLAIN ANALYZE of a write is held like the write") + func explainAnalyzeOfAWriteIsHeld() { + #expect(decision("EXPLAIN ANALYZE DELETE FROM orders WHERE id = 1") == .hold) + #expect(decision("EXPLAIN SELECT * FROM orders") == .runAsIs) + } + + @Test("A read that timed out after the turn's own BEGIN records the hold it opened") + func timedOutReadAfterTheTurnsBeginIsOpen() { + #expect(ManualCommit.recordedState(read: .unknown, turnOpenedHold: true) == .inTransaction) + #expect(ManualCommit.recordedState(read: .unknown, turnOpenedHold: false) == .unknown) + #expect(ManualCommit.recordedState(read: .idle, turnOpenedHold: true) == .idle) + } + + @Test("COMMIT, ROLLBACK, BEGIN, SAVEPOINT and END control the transaction; a PL/SQL block does not") + func transactionControlIsRecognised() { + let postgres = DatabaseType.postgresql + for sql in ["COMMIT", "ROLLBACK", "ROLLBACK TO SAVEPOINT a", "BEGIN", "START TRANSACTION", "SAVEPOINT a", "END"] { + #expect(ManualCommit.controlsTransaction([sql], databaseType: postgres, grammar: postgres.lexicalGrammar), "\(sql)") + } + #expect(!ManualCommit.controlsTransaction(["UPDATE t SET a = 1"], databaseType: postgres, grammar: postgres.lexicalGrammar)) + let oracle = DatabaseType.oracle + #expect(!ManualCommit.controlsTransaction( + ["BEGIN UPDATE t SET a = 1; END;"], databaseType: oracle, grammar: oracle.lexicalGrammar + )) + } + + @Test("Over an open transaction, an agent statement takes a savepoint where one exists and is refused where none does") + func agentProtectionFollowsTheFamily() { + #expect(AgentTransactionGuard.protection(family: .postgres, sessionState: .inTransaction, controlsTransaction: false) == .savepoint) + #expect(AgentTransactionGuard.protection(family: .cockroach, sessionState: .inTransaction, controlsTransaction: false) == .savepoint) + #expect(AgentTransactionGuard.protection(family: .duckdb, sessionState: .inTransaction, controlsTransaction: false) == .refuse) + #expect(AgentTransactionGuard.protection(family: .redshift, sessionState: .inTransaction, controlsTransaction: false) == .refuse) + #expect(AgentTransactionGuard.protection(family: .mysql, sessionState: .inTransaction, controlsTransaction: false) == .none) + #expect(AgentTransactionGuard.protection(family: .teradata, sessionState: .inTransaction, controlsTransaction: false) == .refuse) + #expect(AgentTransactionGuard.protection(family: .postgres, sessionState: .idle, controlsTransaction: false) == .none) + #expect(AgentTransactionGuard.protection(family: .postgres, sessionState: .inTransaction, controlsTransaction: true) == .none) + } + + @Test("A call of several writes joining an open transaction takes the engine's savepoint, and nothing else does") + func oneUnitSavepointFollowsTheFamily() { + func savepoint(_ family: TransactionEngineFamily, _ state: PluginSessionTransactionState, unit: Bool) + -> AgentTransactionGuard.Savepoint? { + let protection = AgentTransactionGuard.protection(family: family, sessionState: state, controlsTransaction: false) + return AgentTransactionGuard.savepoint(family: family, protection: protection, sessionState: state, asOneUnit: unit) + } + #expect(savepoint(.sqlite, .inTransaction, unit: true) == .standard) + #expect(savepoint(.mysql, .inTransaction, unit: true) == .standard) + #expect(savepoint(.sqlServer, .inTransaction, unit: true)?.open == "SAVE TRANSACTION tablepro_agent") + #expect(savepoint(.sqlServer, .inTransaction, unit: true)?.release == nil) + #expect(savepoint(.oracle, .inTransaction, unit: true)?.release == nil) + #expect(savepoint(.sqlite, .inTransaction, unit: false) == nil) + #expect(savepoint(.sqlite, .idle, unit: true) == nil) + #expect(savepoint(.other, .inTransaction, unit: true) == nil) + #expect(savepoint(.postgres, .inTransaction, unit: false) == .standard) + #expect(savepoint(.duckdb, .inTransaction, unit: true) == nil) + } + + @Test("Unseen transaction control: a T-SQL COMMIT inside a batch, an IF, a procedure call") + func unseenTransactionControlIsRecognised() { + let mssql = DatabaseType.mssql + for sql in ["UPDATE t SET a = 1 COMMIT", "IF @@TRANCOUNT > 0 ROLLBACK", "EXEC dbo.p", "SAVE TRAN s"] { + #expect(ManualCommit.mayEndTransactionUnseen([sql], databaseType: mssql, grammar: mssql.lexicalGrammar), "\(sql)") + } + let mysql = DatabaseType.mysql + #expect(ManualCommit.mayEndTransactionUnseen(["CALL p()"], databaseType: mysql, grammar: mysql.lexicalGrammar)) + for sql in ["UPDATE t SET a = CASE WHEN b = 1 THEN 1 ELSE 2 END", "SELECT 'commit'", "UPDATE t SET a = 1"] { + #expect(!ManualCommit.mayEndTransactionUnseen([sql], databaseType: mysql, grammar: mysql.lexicalGrammar), "\(sql)") + } + } + + @Test("Away from SQL Server, a column named like a transaction word is not transaction control", arguments: [ + DatabaseType.postgresql, .mysql, + ]) + func transactionWordsAsColumnsPass(type: DatabaseType) { + let sql = ["SELECT release, commit FROM t"] + #expect(!ManualCommit.mayEndTransactionUnseen(sql, databaseType: type, grammar: type.lexicalGrammar)) + #expect(!ManualCommit.controlsTransaction(sql, databaseType: type, grammar: type.lexicalGrammar)) + } + + @Test("A reply says the transaction ended when it did, and that a held change is pending otherwise") + func agentNoticeFollowsTheEndState() { + #expect(AgentTransactionGuard.notice(startState: .inTransaction, endState: .idle, pendingMatters: true, openedHold: false) + == ManualCommit.transactionEndedNotice) + #expect(AgentTransactionGuard.notice(startState: .inTransaction, endState: .idle, pendingMatters: false, openedHold: false) + == ManualCommit.transactionEndedNotice) + #expect(AgentTransactionGuard.notice(startState: .idle, endState: .idle, pendingMatters: true, openedHold: true) + == ManualCommit.transactionEndedNotice) + #expect(AgentTransactionGuard.notice(startState: .idle, endState: .inTransaction, pendingMatters: true, openedHold: true) + == ManualCommit.pendingNotice) + #expect(AgentTransactionGuard.notice( + startState: .inTransaction, endState: .inTransaction, pendingMatters: false, openedHold: false + ) == nil) + #expect(AgentTransactionGuard.notice(startState: .idle, endState: .inTransaction, pendingMatters: false, openedHold: false) + == ManualCommit.statementOpenedNotice) + #expect(AgentTransactionGuard.notice(startState: .idle, endState: .idle, pendingMatters: false, openedHold: false) == nil) + } + + @Test("Teradata's BT and ET are transaction control, and only there") + func teradataShortFormsControlTheTransaction() { + let teradata = DatabaseType.teradata + for sql in ["BT", "ET", "bt;"] { + #expect(ManualCommit.controlsTransaction([sql], databaseType: teradata, grammar: teradata.lexicalGrammar), "\(sql)") + } + #expect(ManualCommit.startsTransaction(["BT"], databaseType: teradata, grammar: teradata.lexicalGrammar)) + #expect(!ManualCommit.startsTransaction(["ET"], databaseType: teradata, grammar: teradata.lexicalGrammar)) + let postgres = DatabaseType.postgresql + #expect(!ManualCommit.controlsTransaction(["ET"], databaseType: postgres, grammar: postgres.lexicalGrammar)) + #expect(!ManualCommit.controlsTransaction(["BT"], databaseType: postgres, grammar: postgres.lexicalGrammar)) + } + + @Test("The refusal says how to run the change") + func refusalNamesTheWayOut() { + #expect( + ManualCommitRefusal().errorDescription + == "This connection can't hold this change until you commit, so it did not run. Turn off Manual Commit in the Database menu to run it." + ) + #expect(ManualCommit.Decision.refuseMixed.refusal?.localizedDescription == ManualCommitMixedRefusal().errorDescription) + } +} + +struct ConnectionCommitModeStorageTests { + @Test("Manual Commit round-trips through the connection's fields, and Automatic removes the key") + func commitModeRoundTrips() { + var connection = TestFixtures.makeConnection(type: .postgresql) + #expect(connection.commitMode == .automatic) + #expect(connection.additionalFields[DatabaseConnection.commitModeKey] == nil) + + connection.commitMode = .manual + #expect(connection.additionalFields[DatabaseConnection.commitModeKey] == "manual") + #expect(connection.commitMode == .manual) + + connection.commitMode = .automatic + #expect(connection.additionalFields[DatabaseConnection.commitModeKey] == nil) + } + + @Test("A session starts in the connection's commit mode") + func sessionSeedsFromTheConnection() { + var connection = TestFixtures.makeConnection(type: .postgresql) + connection.commitMode = .manual + #expect(ConnectionSession(connection: connection).commitMode == .manual) + } + + @MainActor + @Test("The form's Manual Commit toggle is off and refuses a change where the engine can't hold") + func formToggleIsDisabledWhereUnavailable() { + let coordinator = ConnectionFormCoordinator(connectionId: nil) + coordinator.network.type = .mysql + coordinator.customization.commitMode = .manual + #expect(!coordinator.isManualCommitAvailable) + #expect(!coordinator.effectiveManualCommit) + coordinator.effectiveManualCommit = false + #expect(coordinator.customization.commitMode == .manual) + + coordinator.network.type = .postgresql + #expect(coordinator.isManualCommitAvailable) + #expect(coordinator.effectiveManualCommit) + coordinator.effectiveManualCommit = false + #expect(coordinator.customization.commitMode == .automatic) + } + + @Test("The form writes Manual Commit into the saved fields, and turning it off removes the key") + func formEditsCarryTheMode() { + var base = TestFixtures.makeConnection(type: .postgresql) + base.commitMode = .manual + #expect(ConnectionFormEditsSample.edits(commitMode: .automatic).applied(to: base).commitMode == .automatic) + let saved = ConnectionFormEditsSample.edits(commitMode: .manual).applied(to: TestFixtures.makeConnection()) + #expect(saved.additionalFields[DatabaseConnection.commitModeKey] == "manual") + } +} + +private enum ConnectionFormEditsSample { + static func edits(commitMode: ConnectionCommitMode) -> ConnectionFormEdits { + ConnectionFormEdits( + name: "Edited", + host: "db.example.com", + port: 6_000, + database: "edited_db", + username: "edited_user", + type: .postgresql, + sshConfig: SSHConfiguration(), + sslConfig: SSLConfiguration(), + color: .blue, + iconName: nil, + tagIds: [], + groupId: nil, + sshProfileId: nil, + sshTunnelMode: .disabled, + credentialMode: .inline, + cloudflareTunnelMode: .disabled, + cloudSQLProxyMode: .disabled, + socksProxyMode: .disabled, + tunnelCommandMode: .disabled, + safeModeLevel: .silent, + aiPolicy: nil, + aiRules: nil, + externalAccess: .readOnly, + commitMode: commitMode, + redisDatabase: nil, + startupCommands: nil, + localOnly: false, + additionalFields: [:], + ownedAdditionalFieldIDs: ConnectionFormEdits.appManagedAdditionalFieldIDs + ) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift b/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift index 6b31814b93..98bf4eca5a 100644 --- a/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift +++ b/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift @@ -107,7 +107,7 @@ struct MultiStatementFailureTests { ) #expect(firstStatement.message == "Statement 1/3 failed: database is locked") - for plan in [BatchTransactionPlan.appTransaction, .scriptTransaction] { + for plan in [BatchTransactionPlan.appTransaction, .heldTransaction, .scriptTransaction] { let rolledBack = Self.report( .statement(sql: "VACUUM"), executed: 2, @@ -228,7 +228,7 @@ struct MultiStatementFailureTests { /// `.appTransaction` and a script's own commit under `.scriptTransaction` are both unanswered. @Test( "The unknown-outcome wording does not change with the plan", - arguments: [BatchTransactionPlan.appTransaction, .scriptTransaction, .sessionTransaction, .autocommit] + arguments: [BatchTransactionPlan.appTransaction, .heldTransaction, .scriptTransaction, .sessionTransaction, .autocommit] ) func unknownCommitOutcomeReadsTheSameUnderEveryPlan(plan: BatchTransactionPlan) { let report = Self.report( @@ -308,6 +308,22 @@ struct MultiStatementFailureTests { #expect(!report.message.contains("stays applied")) } + // A Teradata ANSI session holds an unwrapped DDL statement in a transaction it leaves open. + @Test("An unwrapped statement failure over a transaction left open says so") + func statementFailureReportsOpenTransaction() { + let report = Self.report( + .statement(sql: "INSERT INTO t VALUES (1)"), + executed: 1, + total: 2, + error: "Only a COMMIT WORK or null statement is legal after a DDL Statement.", + plan: .autocommit, + sessionState: .inTransaction, + unit: .statement + ) + #expect(report.message.contains("still open")) + #expect(!report.message.contains("stays applied")) + } + /// Work done inside a transaction the user already had open is pending in it, or went with it when the error /// ended it, and the session cannot say which, so the note claims neither. @Test("A batch failure inside the user's own transaction never says the work stays applied") @@ -354,4 +370,134 @@ struct MultiStatementFailureTests { #expect(MultiStatementFailure.batch(sql: "x").ranStatementCount(executedCount: 2, totalCount: 3) == 2) #expect(MultiStatementFailure.statement(sql: "x").ranStatementCount(executedCount: 2, totalCount: 3) == 3) } + + // MARK: - A run stopped to wait for the commit + + @Test( + "A run stopped after DDL tells the user to commit, and names the statement it stopped before", + arguments: [BatchTransactionPlan.autocommit, .sessionTransaction] + ) + func awaitsCommitTellsTheUserToCommit(plan: BatchTransactionPlan) { + let report = Self.report( + .awaitsCommit(sql: "CREATE TABLE t (a INTEGER)"), + executed: 2, + total: 4, + error: "", + plan: plan, + sessionState: .inTransaction + ) + #expect(report.message == """ + Statement 2/4 is a DDL statement, and the session takes nothing after it but COMMIT or ROLLBACK. \ + The run stopped before statement 3. Click Commit on the Transaction Open bar, then run the rest. + """) + #expect(!report.message.contains("still open")) + #expect(!report.message.localizedCaseInsensitiveContains("failed")) + #expect(report.resultLabel == "Stopped") + #expect(report.failedStatementIndex == 2) + #expect(report.failedSQL == nil) + } + + @Test("A run stopped to wait for the commit counts only the statements that ran") + func awaitsCommitCountsWhatRan() { + #expect(MultiStatementFailure.awaitsCommit(sql: "x").ranStatementCount(executedCount: 2, totalCount: 4) == 2) + #expect( + MultiStatementFailure.databaseMoveAwaitsCommit(sql: "DATABASE y").ranStatementCount(executedCount: 1, totalCount: 3) + == 1 + ) + } + + // A new run is pinned back to the tab's database, so the rest needs the move again. + @Test("A stop after DDL that followed a database move says to begin the rest with the move") + func awaitsCommitAfterAMoveNamesTheMove() { + let report = Self.report( + .awaitsCommit(sql: "CREATE TABLE t (a INTEGER)", databaseMove: " DATABASE sales_dev\n"), + executed: 3, + total: 5, + error: "", + plan: .autocommit, + sessionState: .inTransaction + ) + #expect(report.message == """ + Statement 3/5 is a DDL statement, and the session takes nothing after it but COMMIT or ROLLBACK. \ + The run stopped before statement 4. Click Commit on the Transaction Open bar. A new run starts in \ + the tab's database, so begin the rest with “DATABASE sales_dev”. + """) + #expect(report.resultLabel == "Stopped") + #expect(report.failedStatementIndex == 3) + #expect(report.failedSQL == nil) + } + + @Test( + "A run stopped before a database move says to end the transaction and run again from the move", + arguments: [BatchTransactionPlan.autocommit, .sessionTransaction] + ) + func databaseMoveTellsTheUserToEndTheTransaction(plan: BatchTransactionPlan) { + let report = Self.report( + .databaseMoveAwaitsCommit(sql: "DATABASE sales_dev"), + executed: 1, + total: 3, + error: "", + plan: plan, + sessionState: .inTransaction + ) + #expect(report.message == """ + Statement 2/3 moves the session to another database, which this session takes only outside a \ + transaction. The run stopped before it. Click Commit or Roll Back on the Transaction Open bar, then run \ + the script again from statement 2. + """) + #expect(!report.message.contains("still open")) + #expect(!report.message.localizedCaseInsensitiveContains("failed")) + #expect(report.resultLabel == "Stopped") + #expect(report.failedStatementIndex == 1) + #expect(report.failedSQL == nil) + } + + // The script's own BT would be left behind by a rerun from the move, and its ET would then fail. + @Test("A stop before a database move inside the script's BT says to roll back and move it above the BT") + func databaseMoveInsideTheScriptsBlock() { + let report = Self.report( + .databaseMoveAwaitsCommit(sql: "DATABASE sales_dev"), + executed: 2, + total: 4, + error: "", + plan: .scriptTransaction, + sessionState: .inTransaction + ) + #expect(report.message == """ + Statement 3/4 moves the session to another database, which this session takes only outside a \ + transaction. The run stopped before it. Click Roll Back on the Transaction Open bar, move the statement \ + above the script's BT, then run the script again from there. + """) + #expect(report.resultLabel == "Stopped") + #expect(report.failedStatementIndex == 2) + #expect(report.failedSQL == nil) + } + + // The rest has to open the block again, or its ET would end one the app opened around it. + @Test("A stop after DDL inside the script's BT says to begin the rest with BT, after the move if there was one") + func awaitsCommitInsideTheScriptsBlock() { + let plain = Self.report( + .awaitsCommit(sql: "CREATE TABLE t (a INTEGER)"), + executed: 2, + total: 4, + error: "", + plan: .scriptTransaction, + sessionState: .inTransaction + ) + #expect(plain.message == """ + Statement 2/4 is a DDL statement, and the session takes nothing after it but COMMIT or ROLLBACK. \ + The run stopped before statement 3. Click Commit on the Transaction Open bar, then begin the rest with \ + “BT”. + """) + let moved = Self.report( + .awaitsCommit(sql: "CREATE TABLE t (a INTEGER)", databaseMove: "DATABASE sales_dev;"), + executed: 3, + total: 5, + error: "", + plan: .scriptTransaction, + sessionState: .inTransaction + ) + #expect(moved.message.hasSuffix("then begin the rest with “DATABASE sales_dev; BT”.")) + #expect(moved.failedStatementIndex == 3) + } } diff --git a/apple/TableProTests/Core/Services/Execution/QueryBatchResultTests.swift b/apple/TableProTests/Core/Services/Execution/QueryBatchResultTests.swift index f52a2d2e5d..4722b88946 100644 --- a/apple/TableProTests/Core/Services/Execution/QueryBatchResultTests.swift +++ b/apple/TableProTests/Core/Services/Execution/QueryBatchResultTests.swift @@ -2,8 +2,8 @@ // QueryBatchResultTests.swift // TableProTests // -// `GO n` sends a batch n times, and the editor and every external caller fold the answers into one with -// `followed(by:)`, so the ceiling, the counts and the place of each error are decided in one place. +// `GO n` sends a batch n times, and every caller folds the answers into one here, so the ceiling, the counts and +// the place of each error are decided in one place. The editor keeps the newest result sets past the ceiling. // import Foundation @@ -51,4 +51,17 @@ struct QueryBatchResultTests { #expect(combined.errors.map(\.precedingResultSetCount) == [3]) } + + @Test("Keeping the latest, result sets past the ceiling push out the oldest and are counted as superseded") + func latestPastTheCeiling() { + let limit = QueryBatchResult.keptResultSetLimit + let combined = answer(resultSets: limit - 1).followedKeepingLatest(by: answer(resultSets: 3, errorAfter: 3)) + + #expect(combined.resultSets.count == limit) + #expect(combined.resultSets.last?.columns == ["c2"]) + #expect(combined.resultSets.dropLast(3).last?.columns == ["c\(limit - 2)"]) + #expect(combined.supersededResultSetCount == 2) + #expect(combined.discardedResultSetCount == 0) + #expect(combined.errors.map(\.precedingResultSetCount) == [limit]) + } } diff --git a/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift b/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift index b1184108b9..45c3e2c438 100644 --- a/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift +++ b/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift @@ -74,11 +74,26 @@ struct TransactionEngineFamilyTests { } } + @Test("Only Teradata ends a transaction at a definition statement and keeps unwrapped work open") + func teradataAloneHoldsDefinitionsAndUnwrappedWork() { + for family in TransactionEngineFamily.allCases { + #expect(family.definitionEndsTransaction == (family == .teradata)) + #expect(family.unwrappedWorkStaysOpen == (family == .teradata)) + } + } + @Test("Redis has rules of its own, so it is not the fallback") func redisIsItsOwnFamily() { #expect(TransactionEngineFamily.of(.redis) == .redis) } + @Test("Teradata, keyed by the id its plugin registers, reads its own rules") + func teradataIsItsOwnFamily() { + #expect(TransactionEngineFamily.of(.teradata) == .teradata) + #expect(TransactionEngineFamily.of(DatabaseType(rawValue: "Teradata")) == .teradata) + #expect(TransactionEngineFamily.teradata.wrapsBatchInTransaction) + } + /// Redis is the only engine the app cannot open a transaction on: `MULTI` queues commands until /// `EXEC` and answers `+QUEUED` in place of every reply, and `DISCARD` can only drop a queue /// nothing has applied. diff --git a/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift b/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift index 50470ab3dd..f2913a8481 100644 --- a/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift +++ b/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift @@ -195,4 +195,339 @@ struct ExecutionAuditLogTests { #expect(url.deletingLastPathComponent() == AppStorageEnvironment.shared.supportDirectory) #expect(url.path.hasPrefix(AppStorageEnvironment.shared.applicationSupportRoot.path)) } + + private func temporaryURL() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("audit-\(UUID().uuidString).json") + } + + private func storedChain(_ count: Int, startingAt first: Int = 0, after previous: String? = nil) -> [ExecutionAuditRecord] { + var records: [ExecutionAuditRecord] = [] + for index in first ..< first + count { + records.append(ExecutionAuditRecord( + sequence: index, + recordedAt: Date(timeIntervalSince1970: 1_700_000_000 + Double(index)), + connectionId: UUID(), + kind: "readQuery", + caller: "userInterface", + outcome: .authorized, + effectiveWrite: false, + statementDigest: ExecutionAuditRecord.sha256("select \(index)"), + previousHash: records.last?.hash ?? previous ?? ExecutionAuditRecord.genesisHash + )) + } + return records + } + + private func decodedArray(at url: URL) throws -> [ExecutionAuditRecord] { + try JSONDecoder().decode([ExecutionAuditRecord].self, from: Data(contentsOf: url)) + } + + @Test("a log an earlier build wrote as one array goes on from its last record") + func continuesAnEarlierBuildsArray() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let earlier = storedChain(3) + try JSONEncoder().encode(earlier).write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let entries = await log.entries() + #expect(entries.count == 4) + #expect(entries[3].sequence == 3) + #expect(entries[3].previousHash == earlier[2].hash) + #expect(await log.verify() == .intact(count: 4)) + } + + @Test("after appends the file is still the array an earlier build decodes and rewrites") + func fileStaysAnArray() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + for _ in 0 ..< 3 { + await log.record(request: request(write: false), decision: .denied(reason: "x")) + } + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 3) + #expect(ExecutionAuditRecord.firstBrokenSequence(in: decoded) == nil) + } + + @Test("a log longer than the tail that is read on launch still chains from its last record") + func longLogChainsFromItsTail() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let earlier = storedChain(400) + let data = try JSONEncoder().encode(earlier) + #expect(data.count > 65_536) + try data.write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: true), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 401) + #expect(decoded[400].previousHash == earlier[399].hash) + #expect(await log.verify() == .intact(count: 401)) + } + + @Test("an empty array, padded with whitespace, takes its first record") + func paddedEmptyArrayStartsTheChain() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + try Data("[ ]\n".utf8).write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + #expect(try decodedArray(at: url).count == 2) + #expect(await log.verify() == .intact(count: 2)) + } + + @Test("a file changed by someone else is read again before the next record goes on") + func externalChangeIsReadBeforeAppending() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let replaced = storedChain(5) + try JSONEncoder().encode(replaced).write(to: url, options: .atomic) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let entries = await log.entries() + #expect(entries.count == 6) + #expect(entries[5].sequence == 5) + #expect(entries[5].previousHash == replaced[4].hash) + } + + @Test("an unreadable log is set aside with its bytes intact and a new chain starts") + func unreadableLogIsSetAsideNotOverwritten() async throws { + let url = temporaryURL() + let directory = url.deletingLastPathComponent() + let stem = url.deletingPathExtension().lastPathComponent + defer { + try? FileManager.default.removeItem(at: url) + for name in (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + where name.hasPrefix("\(stem).damaged-") { + try? FileManager.default.removeItem(at: directory.appendingPathComponent(name)) + } + } + let damaged = Data(#"[{"sequence":0,"recordedAt":1,"#.utf8) + try damaged.write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let setAside = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix("\(stem).damaged-") } + #expect(setAside.count == 1) + let kept = try Data(contentsOf: directory.appendingPathComponent(try #require(setAside.first))) + #expect(kept == damaged) + #expect(await log.verify() == .intact(count: 1)) + } + + @Test("a log cut short mid-record keeps its whole records, goes on from them, and keeps the cut file aside") + func tornTailIsRepaired() async throws { + let url = temporaryURL() + let directory = url.deletingLastPathComponent() + let stem = url.deletingPathExtension().lastPathComponent + defer { + try? FileManager.default.removeItem(at: url) + for name in (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + where name.hasPrefix("\(stem).damaged-") { + try? FileManager.default.removeItem(at: directory.appendingPathComponent(name)) + } + } + let earlier = storedChain(3) + var torn = try JSONEncoder().encode(earlier) + torn.removeLast() + torn.append(Data(#",{"sequence":3,"recordedAt":17000"#.utf8)) + try torn.write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 4) + #expect(decoded[3].sequence == 3) + #expect(decoded[3].previousHash == earlier[2].hash) + let kept = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix("\(stem).damaged-") } + #expect(kept.count == 1) + #expect(try Data(contentsOf: directory.appendingPathComponent(try #require(kept.first))) == torn) + } + + @Test("a file replaced by another of the same size is read again rather than written into") + func sameSizeReplacementIsReadAgain() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + let written = try Data(contentsOf: url) + + let first = try decodedArray(at: url)[0] + var replaced = Data("[".utf8) + (try JSONEncoder().encode(first)) + Data("]".utf8) + replaced.append(Data(repeating: UInt8(ascii: " "), count: written.count - replaced.count)) + #expect(replaced.count == written.count) + try replaced.write(to: url) + + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 2) + #expect(decoded[1].previousHash == first.hash) + #expect(await log.verify() == .intact(count: 2)) + } + + @Test("a record that cannot be written is held and written ahead of the next one") + func unwrittenRecordIsHeldNotLost() async throws { + let url = temporaryURL() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let log = ExecutionAuditLog(fileURL: url, retryInterval: .zero) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + await log.record(request: request(write: true), decision: .denied(reason: "x")) + #expect(try decodedArray(at: url).count == 1) + + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.map(\.sequence) == [0, 1, 2]) + #expect(decoded[1].kind == "writeQuery") + #expect(await log.verify() == .intact(count: 3)) + } + + @Test("a record held on its first try after launch goes on from the log's last record once written") + func heldRecordLinksToTheHeadAtWriteTime() async throws { + let url = temporaryURL() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let earlier = storedChain(3) + try JSONEncoder().encode(earlier).write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + + let log = ExecutionAuditLog(fileURL: url, retryInterval: .zero) + await log.record(request: request(write: true), decision: .denied(reason: "x")) + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.map(\.sequence) == [0, 1, 2, 3, 4]) + #expect(decoded[3].previousHash == earlier[2].hash) + #expect(decoded[3].kind == "writeQuery") + #expect(await log.verify() == .intact(count: 5)) + } + + @Test("after a failed write the log waits before trying again, holding what was decided meanwhile") + func failedWriteBacksOff() async throws { + let url = temporaryURL() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let log = ExecutionAuditLog(fileURL: url, retryInterval: .seconds(3_600)) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + #expect(try decodedArray(at: url).count == 1) + } + + @Test("a log replaced by another chain of the same size links to the replacement's last record") + func sameSizeChainReplacementIsReadAgain() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + let written = try decodedArray(at: url) + + var replacement: [ExecutionAuditRecord] = [] + for record in written { + replacement.append(ExecutionAuditRecord( + sequence: record.sequence, + recordedAt: record.recordedAt, + connectionId: UUID(), + kind: record.kind, + caller: record.caller, + outcome: record.outcome, + effectiveWrite: record.effectiveWrite, + statementDigest: record.statementDigest, + previousHash: replacement.last?.hash ?? ExecutionAuditRecord.genesisHash + )) + } + let replaced = try JSONEncoder().encode(replacement) + #expect(replaced.count == (try Data(contentsOf: url)).count) + try replaced.write(to: url, options: .atomic) + + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded[2].previousHash == replacement[1].hash) + #expect(await log.verify() == .intact(count: 3)) + } + + @Test("an unreadable log that cannot be moved aside is left untouched until it can be") + func unreadableLogThatCannotMoveIsKept() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("audit-dir-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) + try? FileManager.default.removeItem(at: directory) + } + let url = directory.appendingPathComponent("ExecutionAudit.json") + let damaged = Data("not an array".utf8) + try damaged.write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: directory.path) + + let log = ExecutionAuditLog(fileURL: url, retryInterval: .zero) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + #expect(try Data(contentsOf: url) == damaged) + + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + #expect(try decodedArray(at: url).map(\.sequence) == [0, 1]) + let setAside = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix("ExecutionAudit.damaged-") } + #expect(setAside.count == 1) + } + + @Test("a damaged log reads as broken, not as an empty intact chain") + func damagedLogVerifiesAsBroken() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + try Data("not json".utf8).write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + + #expect(await log.verify() == .broken(atSequence: 0)) + } + + @Test("the tail reader takes the last record whose opening brace follows a separator") + func tailReaderFindsTheLastRecord() throws { + let records = storedChain(2) + let data = try JSONEncoder().encode(records) + + let head = try #require(ExecutionAuditLog.chainHead(inTail: data, startingAt: 0, fileSize: UInt64(data.count))) + + #expect(head.nextSequence == 2) + #expect(head.lastHash == records[1].hash) + #expect(head.closingBracketOffset == UInt64(data.count - 1)) + #expect(ExecutionAuditLog.chainHead(inTail: Data("[{}".utf8), startingAt: 0, fileSize: 3) == nil) + #expect(ExecutionAuditLog.chainHead(inTail: Data("x[]".utf8), startingAt: 0, fileSize: 3) == nil) + } } diff --git a/apple/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift b/apple/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift index c5ed69ea8e..bf058d176a 100644 --- a/apple/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift +++ b/apple/TableProTests/Core/Services/Highlight/HighlightConditionTests.swift @@ -91,6 +91,20 @@ struct HighlightConditionTests { #expect(matches("1", .equal, "true", type: .integer(rawType: "TINYINT(1)"))) } + @Test("A boolean column holding integers compares them as numbers") + func booleanColumnHoldingIntegers() { + let tinyint = ColumnTypeClassifier(family: .mysql).classify(rawTypeName: "TINYINT(1)") + #expect(tinyint.isBooleanType) + #expect(matches("10", .greaterThan, "9", type: tinyint)) + #expect(!matches("9", .greaterThan, "10", type: tinyint)) + #expect(matches("-2", .lessThan, "-1", type: tinyint)) + #expect(matches("5", .between, "2", second: "10", type: tinyint)) + #expect(matches("1", .equal, "1.0", type: tinyint)) + #expect(matches("1", .equal, "true", type: tinyint)) + #expect(!matches("5", .equal, "true", type: tinyint)) + #expect(matches("t", .greaterThan, "0", type: .boolean(rawType: "BOOLEAN"))) + } + @Test("Pattern operators ignore case by default and honour Match Case") func patterns() { #expect(matches("Hello World", .contains, "world")) diff --git a/apple/TableProTests/Core/Services/ImportRunTests.swift b/apple/TableProTests/Core/Services/ImportRunTests.swift index d43c3a4a25..011073d9ca 100644 --- a/apple/TableProTests/Core/Services/ImportRunTests.swift +++ b/apple/TableProTests/Core/Services/ImportRunTests.swift @@ -208,4 +208,101 @@ struct ImportStopResultTests { #expect(value.hasWrites) #expect(ImportStopResult(ledger: value).fileText(for: 0).contains("emptied")) } + + private final class Interrupts: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + var count: Int { lock.withLock { value } } + + func record() { lock.withLock { value += 1 } } + } + + // Admits one call and holds it inside `interruptible` until `release` opens. + private static func copyInFlight( + on run: ImportRun, + interrupts: Interrupts, + release: AsyncGate + ) async throws -> Task { + #expect(try await run.admit(.data) == .proceed) + let inside = AsyncGate() + let call = Task { + _ = try? await run.interruptible(by: { interrupts.record() }) { + await inside.open() + await release.wait() + } + run.release() + } + await inside.wait() + return call + } + + @Test("Stop during a COPY ends it at once and stops the run with rollback, asking nothing") + func stopInterruptsACopy() async throws { + let run = ImportRun() + let interrupts = Interrupts() + let release = AsyncGate() + let call = try await Self.copyInFlight(on: run, interrupts: interrupts, release: release) + + run.requestPause() + + #expect(interrupts.count == 1) + #expect(run.currentPhase == .stopping(.rollBack)) + #expect(run.questionIfPaused() == nil) + await release.open() + await call.value + await #expect(throws: PluginImportCancellationError.self) { try await run.admit(.data) } + await #expect(throws: PluginImportCancellationError.self) { try await run.admit(.commit) } + #expect(try await run.admit(.rollback) == .proceed) + run.release() + #expect(interrupts.count == 1) + } + + @Test("Closing the import during a COPY ends the COPY too") + func teardownInterruptsACopy() async throws { + let run = ImportRun() + let interrupts = Interrupts() + let release = AsyncGate() + let call = try await Self.copyInFlight(on: run, interrupts: interrupts, release: release) + + run.stop(.rollBack) + + #expect(interrupts.count == 1) + await release.open() + await call.value + } + + @Test("Once the COPY returned, Stop pauses for the question as for any statement") + func stopAfterTheCopyPauses() async throws { + let run = ImportRun() + let interrupts = Interrupts() + #expect(try await run.admit(.data) == .proceed) + try await run.interruptible(by: { interrupts.record() }) {} + + run.requestPause() + + #expect(run.currentPhase == .pausing) + run.release() + #expect(run.questionIfPaused() != nil) + #expect(interrupts.count == 0) + } + + @Test("A Stop between a COPY's admission and its start stops it before a row is sent") + func stopBeforeTheCopyStarts() async throws { + let run = ImportRun() + let interrupts = Interrupts() + #expect(try await run.admit(.data) == .proceed) + run.requestPause() + + let bodyRuns = Interrupts() + await #expect(throws: PluginImportCancellationError.self) { + try await run.interruptible(by: { interrupts.record() }) { bodyRuns.record() } + } + + #expect(bodyRuns.count == 0) + #expect(interrupts.count == 0) + #expect(run.currentPhase == .stopping(.rollBack)) + run.release() + #expect(run.questionIfPaused() == nil) + } } diff --git a/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift b/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift index f009821b40..54e8fc8f20 100644 --- a/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift +++ b/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift @@ -26,9 +26,6 @@ struct AgentSessionRegistryTests { #expect(registry.sessions.isEmpty) } - /// One session, two presentations. The trailing-pane chat and the agent-mode conversation column - /// both resolve through here, so they must land on the same object or the feature is two - /// conversations wearing one name. @Test("Both presentations resolve to the same session") func bothPresentationsShareOneSession() { let registry = AgentSessionRegistry(store: makeStore()) @@ -48,9 +45,7 @@ struct AgentSessionRegistryTests { #expect(session.viewModel.sessionId == session.id) } - /// The conversation flushes a held prompt from a `task`, and a reparent re-runs that task on the - /// same view: a mode toggle and a connection switch are both one. Taking the prompt rather than - /// reading it is what keeps each re-run from sending it again. + // A reparent re-runs the task that flushes the prompt, so taking it is what sends it once. @Test("A held prompt is handed over once, and only once the connection is up") func heldPromptIsTakenOnce() throws { let registry = AgentSessionRegistry(store: makeStore()) @@ -65,10 +60,9 @@ struct AgentSessionRegistryTests { #expect(session.pendingPrompt == nil) } - /// Stopping keeps the transcript. Window close, disconnect and a lost session all reach it, and - /// none of them is the user throwing a conversation away. + // The transcript leaves memory and stays on disk under the id the session still names. @Test("Stopping a session keeps it and its transcript") - func stoppingKeepsTheSession() throws { + func stoppingKeepsTheSession() async throws { let registry = AgentSessionRegistry(store: makeStore()) let connectionId = UUID() let session = try #require(registry.resolveSession(for: connectionId, startingIfNeeded: true)) @@ -77,8 +71,14 @@ struct AgentSessionRegistryTests { registry.stopSessions(for: connectionId) #expect(session.status == .stopped) - #expect(session.viewModel.messages.isEmpty == false) #expect(registry.sessions(for: connectionId).count == 1) + let conversationId = try #require(session.conversationId) + await session.viewModel.lastConversationWrite?.value + let stored = await AIChatStorage.shared.load(id: conversationId) + let storedTexts = stored?.messages.map { $0.plainText } + #expect(storedTexts == ["keep me"]) + #expect(session.viewModel.messages.isEmpty) + await AIChatStorage.shared.delete(conversationId) } @Test("Resolving a stopped session resumes it rather than starting another") @@ -120,9 +120,7 @@ struct AgentSessionRegistryTests { #expect(registry.sessions.count == 2) } - /// Restore has to finish before any window exists. A window that opened while a load was - /// suspended found nothing, minted a session, and was then joined by the stored one: two - /// sessions on one conversation, both persisted, both in the rail. + // A window opened while a load was suspended minted a second session beside the stored one. @Test("Sessions come back from disk in the registry's own initialiser") func restoreIsSynchronous() throws { let store = makeStore() @@ -150,6 +148,32 @@ struct AgentSessionRegistryTests { #expect(restored.viewModel.sessionId == sessionId) } + // Its record used to be written with no conversation, so the next launch reopened it empty. + @Test("A restored session that was never opened keeps its conversation across a save") + func unopenedSessionKeepsItsConversation() throws { + let store = makeStore() + let conversationId = UUID() + store.save([ + AgentSessionRecord( + id: UUID(), + connectionId: UUID(), + conversationId: conversationId, + status: .stopped, + title: "Earlier work", + startedAt: Date(timeIntervalSince1970: 1_000), + lastActiveAt: Date(timeIntervalSince1970: 2_000) + ) + ]) + let registry = AgentSessionRegistry(store: store) + + registry.startSession(for: UUID()) + registry.persistNow() + + let records = store.load() + #expect(records.count == 2) + #expect(records.contains { $0.conversationId == conversationId && $0.title == "Earlier work" }) + } + @Test("A restored session keeps its id rather than being minted again") func restorePreservesIdentity() throws { let store = makeStore() @@ -179,9 +203,7 @@ struct AgentSessionRegistryTests { // MARK: - Order - /// The rail lists the latest first, and going to work is what makes a session the latest. Opening - /// one is not work: ordered by that, the row someone double-clicked in the middle of the list - /// would jump to the top from under the pointer that opened it. + // Opening is not work, or the row double-clicked would jump from under the pointer. @Test("The rail lists the session that last went to work first") func sessionsAreOrderedByActivity() throws { let registry = AgentSessionRegistry(store: makeStore()) @@ -208,7 +230,6 @@ struct AgentSessionRegistryTests { #expect(registry.currentSession(for: connectionId) === first) } - /// With nothing named, the latest live session is the one two panes share, not the oldest. @Test("The shared session is the latest live one") func currentSessionPrefersTheLatestLiveOne() { let registry = AgentSessionRegistry(store: makeStore()) @@ -222,8 +243,6 @@ struct AgentSessionRegistryTests { // MARK: - Closing and deleting the session on screen - /// Close used to stop the session and tell nothing, so the conversation column went on drawing it - /// with a composer that still took messages. @Test("Closing the session on screen hands the window the next live one") func closingHandsOverToTheNextLiveSession() { let registry = AgentSessionRegistry(store: makeStore()) @@ -251,8 +270,6 @@ struct AgentSessionRegistryTests { #expect(registry.sessions(for: connectionId).count == 1) } - /// Nothing on screen is a state the user asked for, and entering the mode again is them asking - /// for a session; resolving is what mints it. @Test("A window with no session open starts one when it is asked to") func resolvingAfterClosingStartsAnother() throws { let registry = AgentSessionRegistry(store: makeStore()) @@ -292,8 +309,6 @@ struct AgentSessionRegistryTests { #expect(registry.currentSession(for: connectionId) === open) } - /// A reply that was still arriving when the app went away did not finish, and saying so is more - /// honest than leaving it reading as busy for ever. @Test("A session still working at terminate is recorded as failed") func terminationMarksWorkingSessionsFailed() throws { let store = makeStore() diff --git a/apple/TableProTests/Core/Services/Infrastructure/ConnectionCloseActionTests.swift b/apple/TableProTests/Core/Services/Infrastructure/ConnectionCloseActionTests.swift index 82b2d215b0..ef18231f34 100644 --- a/apple/TableProTests/Core/Services/Infrastructure/ConnectionCloseActionTests.swift +++ b/apple/TableProTests/Core/Services/Infrastructure/ConnectionCloseActionTests.swift @@ -9,83 +9,52 @@ import Testing @MainActor struct ConnectionCloseActionTests { - /// The case the old command failed on. A connection the window hosts but that has no session - /// yet still has a rail row, and Close on it resolved no coordinator and returned in silence. - /// There is nothing to lose there, so it closes without asking rather than doing nothing. + // A connection the window hosts with no session yet still has a rail row; Close on it once did nothing. @Test("A connection with no session closes without asking") func sessionlessClosesImmediately() { #expect( - ConnectionCloseAction.decision(hasSession: false, holdsTransaction: false, hasUnsavedWork: false) == .closeImmediately + ConnectionCloseAction.decision(hasSession: false, hasRunningWork: false, hasUnsavedWork: false) == .closeImmediately ) } - /// Unsaved work reported for a connection that has no session cannot be acted on, so it must - /// not gate the close behind an alert whose Save button has nothing to call. @Test("Unsaved work without a session still closes without asking") func sessionlessIgnoresUnsavedWork() { #expect( - ConnectionCloseAction.decision(hasSession: false, holdsTransaction: false, hasUnsavedWork: true) == .closeImmediately + ConnectionCloseAction.decision(hasSession: false, hasRunningWork: false, hasUnsavedWork: true) == .closeImmediately ) } - @Test("A clean connection closes without asking") - func cleanSessionClosesImmediately() { + @Test("Running work is asked about even without a session") + func sessionlessStillAsksAboutRunningWork() { #expect( - ConnectionCloseAction.decision(hasSession: true, holdsTransaction: false, hasUnsavedWork: false) == .closeImmediately + ConnectionCloseAction.decision(hasSession: false, hasRunningWork: true, hasUnsavedWork: false) == .confirmRunningWork ) } - @Test("A connection with unsaved work asks first") - func unsavedWorkIsConfirmed() { + // The transaction is read after the answers, so with nothing else to ask it is the next step. + @Test("A clean connection goes straight to the transaction step") + func cleanSessionReadsTheTransaction() { #expect( - ConnectionCloseAction.decision(hasSession: true, holdsTransaction: false, hasUnsavedWork: true) == .confirmUnsavedWork + ConnectionCloseAction.decision(hasSession: true, hasRunningWork: false, hasUnsavedWork: false) + == .confirmEndingTransaction ) } - /// Closing ends the session connection, and the server rolls back what it was holding. - @Test("A connection holding an open transaction asks first") - func openTransactionIsConfirmed() { + @Test("Running work with nothing unsaved gets its own alert") + func runningWorkIsConfirmed() { #expect( - ConnectionCloseAction.decision(hasSession: true, holdsTransaction: true, hasUnsavedWork: false) - == .confirmEndingTransaction + ConnectionCloseAction.decision(hasSession: true, hasRunningWork: true, hasUnsavedWork: false) == .confirmRunningWork ) } - /// Save on the unsaved-work alert writes at once, so the transaction has to be asked about - /// while a Cancel can still leave everything as it was. - @Test("An open transaction is asked about before unsaved work") - func openTransactionComesBeforeUnsavedWork() { + // A Save under Manual Commit writes into a held transaction, which the close must still ask about after it. + @Test("Unsaved work is asked before the transaction, and names the running work on the same alert") + func unsavedWorkComesFirst() { #expect( - ConnectionCloseAction.decision(hasSession: true, holdsTransaction: true, hasUnsavedWork: true) - == .confirmEndingTransaction + ConnectionCloseAction.decision(hasSession: true, hasRunningWork: false, hasUnsavedWork: true) == .confirmUnsavedWork ) - } - - @Test("A transaction reported without a session still closes without asking") - func sessionlessIgnoresTransaction() { #expect( - ConnectionCloseAction.decision(hasSession: false, holdsTransaction: true, hasUnsavedWork: true) - == .closeImmediately + ConnectionCloseAction.decision(hasSession: true, hasRunningWork: true, hasUnsavedWork: true) == .confirmUnsavedWork ) } - - @Test("No open transaction adds nothing to the close prompt") - func noTransactionNoMessage() { - #expect(ConnectionCloseAction.transactionMessage(for: []) == nil) - } - - @Test("One open transaction names its database") - func oneTransactionNamesTheDatabase() throws { - let message = try #require(ConnectionCloseAction.transactionMessage(for: ["app"])) - #expect(message.contains("“app”")) - #expect(message.contains("Closing rolls it back")) - } - - @Test("Several open transactions list every database") - func severalTransactionsListEveryDatabase() throws { - let message = try #require(ConnectionCloseAction.transactionMessage(for: ["app", "logs"])) - #expect(message.contains("“app”")) - #expect(message.contains("“logs”")) - #expect(message.contains("Closing rolls them back")) - } } diff --git a/apple/TableProTests/Core/Services/Infrastructure/ConnectionDisconnectActionTests.swift b/apple/TableProTests/Core/Services/Infrastructure/ConnectionDisconnectActionTests.swift index 9b5ba01fe5..6446cea1ec 100644 --- a/apple/TableProTests/Core/Services/Infrastructure/ConnectionDisconnectActionTests.swift +++ b/apple/TableProTests/Core/Services/Infrastructure/ConnectionDisconnectActionTests.swift @@ -9,44 +9,20 @@ import Testing @MainActor struct ConnectionDisconnectActionTests { - @Test("No open transaction adds nothing to the disconnect prompt") - func noTransactionNoMessage() { - #expect(ConnectionDisconnectAction.transactionMessage(for: []) == nil) - } - - @Test("One open transaction names its database") - func oneTransactionNamesTheDatabase() throws { - let message = try #require(ConnectionDisconnectAction.transactionMessage(for: ["app"])) - #expect(message.contains("“app”")) - #expect(message.contains("Disconnecting rolls it back")) - } - - @Test("Several open transactions list every database") - func severalTransactionsListEveryDatabase() throws { - let message = try #require(ConnectionDisconnectAction.transactionMessage(for: ["app", "logs"])) - #expect(message.contains("“app”")) - #expect(message.contains("“logs”")) - #expect(message.contains("Disconnecting rolls them back")) - } - @Test("A clean connection disconnects without asking") func cleanConnectionAsksNothing() { - #expect(ConnectionDisconnectAction.confirmationMessage(holdingTransaction: [], hasUnsavedWork: false) == nil) - #expect(ConnectionDisconnectAction.otherLosses(holdingTransaction: [], hasUnsavedWork: false).isEmpty) + #expect(ConnectionDisconnectAction.confirmationMessage(hasUnsavedWork: false) == nil) + #expect(ConnectionDisconnectAction.otherLosses(hasUnsavedWork: false).isEmpty) } - @Test("With running work, the transaction and the unsaved changes join the same alert") - func lossesJoinTheRunningWorkAlert() { - let losses = ConnectionDisconnectAction.otherLosses(holdingTransaction: ["app"], hasUnsavedWork: true) - - #expect(losses == ["Open transaction in “app”", "Unsaved changes"]) + // The transaction gets its own alert after this one, with Commit, so it never rides along here. + @Test("With running work, only the unsaved changes join the running-work alert") + func unsavedChangesJoinTheRunningWorkAlert() { + #expect(ConnectionDisconnectAction.otherLosses(hasUnsavedWork: true) == ["Unsaved changes"]) } - @Test("Without running work, an open transaction is asked about before unsaved changes") - func transactionOutranksUnsavedWork() throws { - let message = try #require( - ConnectionDisconnectAction.confirmationMessage(holdingTransaction: ["app"], hasUnsavedWork: true) - ) - #expect(message.contains("open transaction")) + @Test("Unsaved changes alone are asked about") + func unsavedChangesAreConfirmed() { + #expect(ConnectionDisconnectAction.confirmationMessage(hasUnsavedWork: true) == "Unsaved changes will be lost.") } } diff --git a/apple/TableProTests/Core/Services/Infrastructure/ExternalConnectionGateTests.swift b/apple/TableProTests/Core/Services/Infrastructure/ExternalConnectionGateTests.swift index 3dbc86c75b..8d02d9fbad 100644 --- a/apple/TableProTests/Core/Services/Infrastructure/ExternalConnectionGateTests.swift +++ b/apple/TableProTests/Core/Services/Infrastructure/ExternalConnectionGateTests.swift @@ -1,3 +1,4 @@ +import AppKit import Foundation @testable import TablePro import TableProSSHConfig @@ -268,4 +269,71 @@ struct ExternalConnectionGateTests { #expect(prompt.callCount == 1) #expect(prompt.offeredAlwaysAllow == false) } + + // Each connection with the server or method the alert has to name for it. + private func routedConnections() -> [(connection: DatabaseConnection, named: String)] { + var socks = ddevConnection() + socks.socksProxyMode = .inline(SOCKSProxyConfiguration(host: "proxy.example.com")) + var command = ddevConnection() + command.tunnelCommandMode = .inline(TunnelCommandConfiguration(method: .custom, command: "ssh -N -L {port}:db:3306 prod")) + var cloudflare = ddevConnection() + cloudflare.cloudflareTunnelMode = .inline(CloudflareConfiguration(accessHostname: "db.example.com")) + var cloudSQL = ddevConnection() + cloudSQL.cloudSQLProxyMode = .inline(CloudSQLProxyConfiguration(instanceConnectionName: "p:r:i")) + var profile = ddevConnection() + profile.sshTunnelMode = .profile( + id: UUID(), + snapshot: SSHConfiguration(enabled: true, host: "bastion.example.com", username: "ops") + ) + return [ + (socks, "proxy.example.com:1080"), + (command, TunnelCommandMethod.custom.displayName), + (cloudflare, "db.example.com"), + (cloudSQL, "p:r:i"), + (profile, "bastion.example.com"), + ] + } + + @Test("A loopback link reached through a proxy, a tunnel command or an SSH profile asks every time") + func everyTransportIgnoresTrust() async throws { + for (connection, _) in routedConnections() { + let store = try makeStore() + store.trust(ExternalConnectionTrustKey(connection: ddevConnection(), scopeName: "ddev-shop")) + let prompt = SpyPrompt(decision: .alwaysAllow) + let gate = ExternalConnectionGate(trustStore: store, prompt: prompt) + + #expect(await gate.authorize(connection, scopeName: "ddev-shop")) + #expect(await gate.authorize(connection, scopeName: "ddev-shop")) + #expect(prompt.callCount == 2) + #expect(prompt.offeredAlwaysAllow == false) + #expect(store.entries().count == 1) + } + } + + @Test("Trust given to a link alone never covers a saved connection, and one saved connection never covers another") + func savedConnectionsKeepTheirOwnTrust() async throws { + let store = try makeStore() + store.trust(ExternalConnectionTrustKey(connection: ddevConnection(), scopeName: "ddev-shop")) + let prompt = SpyPrompt(decision: .alwaysAllow) + let gate = ExternalConnectionGate(trustStore: store, prompt: prompt) + let saved = ddevConnection() + + #expect(await gate.authorize(saved, isSaved: true, scopeName: "ddev-shop")) + #expect(prompt.callCount == 1) + #expect(prompt.offeredAlwaysAllow == true) + + #expect(await gate.authorize(saved, isSaved: true, scopeName: "ddev-shop")) + #expect(prompt.callCount == 1) + + #expect(await gate.authorize(ddevConnection(), isSaved: true, scopeName: "ddev-shop")) + #expect(prompt.callCount == 2) + } + + @Test("The alert names each transport a loopback link crosses") + func alertNamesTheTransports() { + for (connection, named) in routedConnections() { + let alert = ExternalConnectionAlertPrompt.makeAlert(for: connection, offerAlwaysAllow: false) + #expect(alert.informativeText.contains(named)) + } + } } diff --git a/apple/TableProTests/Core/Services/Infrastructure/LaunchIntentFailureOwnershipTests.swift b/apple/TableProTests/Core/Services/Infrastructure/LaunchIntentFailureOwnershipTests.swift index 240400a114..999c63b1d8 100644 --- a/apple/TableProTests/Core/Services/Infrastructure/LaunchIntentFailureOwnershipTests.swift +++ b/apple/TableProTests/Core/Services/Infrastructure/LaunchIntentFailureOwnershipTests.swift @@ -1,14 +1,8 @@ -// -// LaunchIntentFailureOwnershipTests.swift -// TableProTests -// -// Opening a database file or URL builds its connection inside the router, so the launch intent -// names no connection. A connect that failed there was then alerted over the window that already -// showed the same failure, covering the button that fixed it. -// +// A file or URL open builds its connection in the router, so its failure must not be alerted over the window showing it. import Foundation @testable import TablePro +import TableProMCP import Testing @MainActor @@ -45,4 +39,35 @@ struct LaunchIntentFailureOwnershipTests { #expect(error.localizedDescription == Self.underlying.localizedDescription) } + + @Test("A background launch made only for MCP quits when the server fails to start, as it does when MCP is off") + func failedMCPStartEndsAnIdleLaunch() async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-launch-\(UUID().uuidString)", isDirectory: true) + // Too long for a socket path, so the server falls back to the temporary folder, which is a file here. + let directory = root.appendingPathComponent(String(repeating: "d", count: 110), isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let notAFolder = root.appendingPathComponent("tmp-file", isDirectory: false) + try Data().write(to: notAFolder) + let manager = MCPServerManager( + handshake: MCPHandshakeFile(directory: directory, temporaryDirectory: notAFolder), + isEnabled: { true } + ) + + #expect(await LaunchIntentRouter.startMCPServerLeavesNothingToDo(manager: manager, isIdleMachineSession: { true })) + #expect(manager.state == .failed(.socketFolderUnusable)) + #expect(!(await LaunchIntentRouter.startMCPServerLeavesNothingToDo(manager: manager, isIdleMachineSession: { false }))) + await manager.stop() + } + + @Test("A background launch made only for MCP quits when MCP is off") + func refusedMCPStartEndsAnIdleLaunch() async throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-launch-off-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + + #expect(await LaunchIntentRouter.startMCPServerLeavesNothingToDo(manager: manager, isIdleMachineSession: { true })) + #expect(manager.state == .stopped) + } } diff --git a/apple/TableProTests/Core/Services/Infrastructure/QuitTransactionPromptTests.swift b/apple/TableProTests/Core/Services/Infrastructure/QuitTransactionPromptTests.swift new file mode 100644 index 0000000000..909e0eba18 --- /dev/null +++ b/apple/TableProTests/Core/Services/Infrastructure/QuitTransactionPromptTests.swift @@ -0,0 +1,254 @@ +// +// QuitTransactionPromptTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite(.serialized) +@MainActor +struct QuitTransactionPromptTests { + private static func seed(name: String) -> (connection: DatabaseConnection, driver: QuitStubDriver) { + let connection = TestFixtures.makeConnection(name: name, database: "app", type: .mysql) + let driver = QuitStubDriver() + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.browseDatabase = "app" + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return (connection, driver) + } + + private static func begin(on connectionId: UUID) async throws { + _ = try await DatabaseManager.shared.withScopedDriver( + scope: DatabaseScope(connectionId: connectionId, database: "app", schema: nil), + route: .sessionDriver, + cancellation: .protectedWrite + ) { driver in + try await driver.execute(query: "BEGIN") + } + } + + private static func plan( + registry: RunningWorkRegistry, + hasUnsavedTabs: Bool = false, + answer: AlertHelper.TransactionEndAnswer, + asked: inout [TransactionEndPrompt] + ) -> QuitPlan? { + var seen: [TransactionEndPrompt] = [] + let plan = QuitApproval.approvesStoppingWork( + registry: registry, + action: .quit, + hasUnsavedTabs: { hasUnsavedTabs }, + confirmWork: { _ in true }, + answerTransaction: { prompt in + seen.append(prompt) + return answer + }, + confirmDiscardingTabs: { _ in + Issue.record("The transaction prompt already named the unsaved tabs") + return false + } + ) + asked = seen + return plan + } + + @Test("Each connection holding a transaction is named once, whatever its lanes") + func holdsAreNamedOncePerConnection() { + let prod = UUID() + let staging = UUID() + let driver = PluginDriverAdapter(connection: TestFixtures.makeConnection(), pluginDriver: QuitStubDriver()) + let tracker = SessionTransactionTracker { _, _ in true } + tracker.record(.inTransaction, on: driver, for: .init(connectionId: prod, lane: "sales")) + tracker.record(.abortedTransaction, on: driver, for: .init(connectionId: prod, lane: "hr")) + tracker.record(.inTransaction, on: driver, for: .init(connectionId: staging, lane: nil)) + let names = [prod: "Prod", staging: "Staging"] + + let named = QuitApproval.openTransactionNames( + holds: Array(tracker.holds.values), + connectionName: { names[$0] } + ) + + #expect(named == ["Prod", "Staging"]) + } + + @Test("Commit on the quit prompt plans the commits and stops no work yet") + func commitPlansTheHolds() async throws { + let (connection, _) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + var stopped = 0 + let registry = RunningWorkRegistry(providers: []) + registry.begin(.restore, subject: "sales", connectionName: "Prod", connectionIds: [connection.id]) { stopped += 1 } + var asked: [TransactionEndPrompt] = [] + + let plan = try #require(Self.plan(registry: registry, answer: .commit, asked: &asked)) + + #expect(asked.count == 1) + #expect(asked.first?.offersCommit == true) + #expect(asked.first?.message.contains("“Prod”") == true) + #expect(plan.commits.contains { $0.connectionId == connection.id }) + #expect(plan.work.count == 1) + #expect(stopped == 0) + } + + // Quitting ends the session, and the server rolls back what it held. + @Test("Roll Back on the quit prompt plans no commit") + func rollBackPlansNothing() async throws { + let (connection, _) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + var asked: [TransactionEndPrompt] = [] + + let plan = try #require(Self.plan(registry: RunningWorkRegistry(providers: []), answer: .rollBack, asked: &asked)) + + #expect(plan.commits.isEmpty) + } + + @Test("Cancel on the quit prompt keeps the app open") + func cancelKeepsTheAppOpen() async throws { + let (connection, _) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + var asked: [TransactionEndPrompt] = [] + + #expect(Self.plan(registry: RunningWorkRegistry(providers: []), answer: .cancel, asked: &asked) == nil) + } + + @Test("Unsaved tabs ride along on the transaction question rather than asking again") + func unsavedTabsRideAlong() async throws { + let (connection, _) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + var asked: [TransactionEndPrompt] = [] + + _ = Self.plan(registry: RunningWorkRegistry(providers: []), hasUnsavedTabs: true, answer: .rollBack, asked: &asked) + + #expect(asked.first?.message.hasSuffix("Some tabs have unsaved edits. Quitting will discard these changes.") == true) + } + + @Test("The commit lands before the quit goes ahead") + func commitBeforeQuittingCommits() async throws { + let (connection, driver) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + let plan = QuitPlan(commits: DatabaseManager.shared.sessionTransactions.holds(for: connection.id), work: []) + var failures: [String] = [] + + let quits = await QuitApproval.commitBeforeQuitting( + plan, + commit: { try await ConnectionTransactionActions.commitBeforeEnding($0, gate: AlwaysAllowGate()) }, + reportFailure: { failures.append($0) } + ) + + #expect(quits) + #expect(driver.statements.last == "COMMIT") + #expect(failures.isEmpty) + } + + @Test("Cancelling the Safe Mode question keeps the app open and says nothing") + func gateCancelKeepsTheAppOpen() async throws { + let (connection, driver) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + let plan = QuitPlan(commits: DatabaseManager.shared.sessionTransactions.holds(for: connection.id), work: []) + var failures: [String] = [] + + let quits = await QuitApproval.commitBeforeQuitting( + plan, + commit: { try await ConnectionTransactionActions.commitBeforeEnding($0, gate: AlwaysDenyGate.cancelling) }, + reportFailure: { failures.append($0) } + ) + + #expect(!quits) + #expect(!driver.statements.contains("COMMIT")) + #expect(failures.isEmpty) + } + + @Test("A failed commit keeps the app open and shows why") + func failedCommitKeepsTheAppOpen() async throws { + let (connection, _) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + let plan = QuitPlan(commits: DatabaseManager.shared.sessionTransactions.holds(for: connection.id), work: []) + var failures: [String] = [] + + let quits = await QuitApproval.commitBeforeQuitting( + plan, + commit: { _ in throw SessionTransactionActionError.cannotCommit }, + reportFailure: { failures.append($0) } + ) + + #expect(!quits) + #expect(failures == [SessionTransactionActionError.cannotCommit.localizedDescription]) + } + + // A nested Teradata BT survives one END TRANSACTION; quitting after it would roll back what was committed. + @Test("A transaction still open after the commit keeps the app open and names its connection") + func stillOpenKeepsTheAppOpen() async throws { + let (connection, _) = Self.seed(name: "Prod") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.begin(on: connection.id) + let plan = QuitPlan(commits: DatabaseManager.shared.sessionTransactions.holds(for: connection.id), work: []) + var failures: [String] = [] + + let quits = await QuitApproval.commitBeforeQuitting( + plan, + commit: { _ in true }, + reportFailure: { failures.append($0) } + ) + + #expect(!quits) + #expect(failures.count == 1) + #expect(failures.first?.contains("“Prod”") == true) + } + + @Test("Nothing to commit quits without touching the server") + func nothingToCommitQuits() async { + let quits = await QuitApproval.commitBeforeQuitting( + QuitPlan(commits: [], work: []), + commit: { _ in + Issue.record("Nothing to commit") + return false + }, + reportFailure: { _ in Issue.record("Nothing failed") } + ) + + #expect(quits) + } +} + +private final class QuitStubDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var current: PluginSessionTransactionState = .idle + private var sent: [String] = [] + + var supportsSchemas: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + var statements: [String] { lock.withLock { sent } } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { + sent.append(query) + switch query.uppercased() { + case "BEGIN": current = .inTransaction + case "COMMIT", "ROLLBACK": current = .idle + default: break + } + } + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { + lock.withLock { current } + } + + func switchDatabase(to database: String) async throws {} +} diff --git a/apple/TableProTests/Core/Services/Infrastructure/TransactionEndPromptTests.swift b/apple/TableProTests/Core/Services/Infrastructure/TransactionEndPromptTests.swift new file mode 100644 index 0000000000..cfa4d2ccec --- /dev/null +++ b/apple/TableProTests/Core/Services/Infrastructure/TransactionEndPromptTests.swift @@ -0,0 +1,267 @@ +// +// TransactionEndPromptTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct TransactionEndPromptTests { + private static func hold( + _ connectionId: UUID = UUID(), + lane: String? = nil, + state: SessionTransactionTracker.State = .open + ) -> SessionTransactionTracker.Hold { + SessionTransactionTracker.Hold(key: .init(connectionId: connectionId, lane: lane), state: state, revision: 0) + } + + private static func make( + _ action: TransactionEndPrompt.Action, + names: [String], + holds: [SessionTransactionTracker.Hold], + untracked: Bool = false, + savedUnderManualCommit: Bool = false, + hasUnsavedWork: Bool = false, + readOnly: Set = [], + busy: Set = [] + ) -> TransactionEndPrompt? { + TransactionEndPrompt.make( + action: action, + names: names, + holds: holds, + untracked: untracked, + savedUnderManualCommit: savedUnderManualCommit, + hasUnsavedWork: hasUnsavedWork, + effectiveLevel: { readOnly.contains($0) ? .readOnly : .silent }, + busy: { busy.contains($0.key) } + ) + } + + @Test("Nothing holding a transaction asks nothing") + func noTransactionNoPrompt() { + #expect(Self.make(.disconnect(name: "Prod"), names: [], holds: []) == nil) + #expect(Self.make(.quit, names: [], holds: []) == nil) + } + + @Test("Disconnect asks to commit, names a database with its own session and says what is rolled back") + func disconnectOffersCommit() throws { + let prompt = try #require(Self.make(.disconnect(name: "Prod"), names: ["app"], holds: [Self.hold(lane: "app")])) + + #expect(prompt.offersCommit) + #expect(prompt.title == "Do you want to commit the open transaction before disconnecting from “Prod”?") + #expect(prompt.message == "The database “app” has an open transaction. Changes you don’t commit are rolled back.") + #expect(!prompt.message.contains("\u{2014}")) + } + + // A SQLite connection's database is its file path, which the title's connection name already stands for. + @Test("A connection with one session for every database is not asked about by database name") + func oneSessionConnectionNamesNoDatabase() throws { + let prompt = try #require(Self.make(.disconnect(name: "Fixture"), names: ["/tmp/a.sqlite"], holds: [Self.hold()])) + + #expect(prompt.message == "Changes you don’t commit are rolled back.") + let held = Self.hold() + let busy = try #require(Self.make( + .disconnect(name: "Fixture"), names: ["/tmp/a.sqlite"], holds: [held], busy: [held.key] + )) + #expect(!busy.offersCommit) + #expect(busy.message.hasPrefix("This connection has an open transaction. Disconnecting rolls it back")) + #expect(!busy.message.contains("a.sqlite")) + } + + @Test("Several databases make the question plural and list every one") + func severalDatabasesAreListed() throws { + let id = UUID() + let prompt = try #require(Self.make( + .closeConnection(name: "Prod"), + names: ["app", "logs"], + holds: [Self.hold(id, lane: "app"), Self.hold(id, lane: "logs")] + )) + + #expect(prompt.title == "Do you want to commit the open transactions before closing “Prod”?") + #expect(prompt.message.contains("“app”")) + #expect(prompt.message.contains("“logs”")) + #expect(prompt.message.hasSuffix("Changes you don’t commit are rolled back.")) + } + + @Test("A database entry is named in the question") + func entryCloseNamesTheDatabase() throws { + let prompt = try #require(Self.make(.closeEntry(database: "logs"), names: ["logs"], holds: [Self.hold(lane: "logs")])) + + #expect(prompt.title == "Do you want to commit the open transaction before closing the database “logs”?") + #expect(prompt.message == "Changes you don’t commit are rolled back.") + } + + @Test("Disconnect keeps the unsaved-changes line, since Commit then disconnect still loses them") + func disconnectKeepsUnsavedLine() throws { + let prompt = try #require(Self.make(.disconnect(name: "Prod"), names: ["app"], holds: [Self.hold()], hasUnsavedWork: true)) + + #expect(prompt.message.hasSuffix("Unsaved changes will be lost.")) + } + + @Test("A close asked about unsaved work on its own Save alert, so it carries no unsaved line") + func closeCarriesNoUnsavedLine() throws { + let prompt = try #require(Self.make(.closeConnection(name: "Prod"), names: ["app"], holds: [Self.hold()], hasUnsavedWork: true)) + + #expect(!prompt.message.contains("Unsaved")) + } + + @Test("Under Manual Commit the message says saved edits are not committed yet") + func manualCommitNote() throws { + let prompt = try #require(Self.make( + .closeConnection(name: "Prod"), names: ["app"], holds: [Self.hold()], savedUnderManualCommit: true + )) + + #expect(prompt.message.contains("Edits saved with Manual Commit on are not committed yet.")) + let disconnect = try #require(Self.make( + .disconnect(name: "Prod"), names: ["app"], holds: [Self.hold()], savedUnderManualCommit: true + )) + #expect(!disconnect.message.contains("Manual Commit")) + } + + @Test("Quit names each connection and keeps the unsaved-tabs line") + func quitNamesConnections() throws { + let prompt = try #require(Self.make(.quit, names: ["Prod"], holds: [Self.hold()], hasUnsavedWork: true)) + + #expect(prompt.offersCommit) + #expect(prompt.title == "Do you want to commit the open transaction before quitting?") + #expect(prompt.message.hasPrefix("The connection “Prod” has an open transaction. Changes you don’t commit are rolled back.")) + #expect(prompt.message.hasSuffix("Some tabs have unsaved edits. Quitting will discard these changes.")) + } + + @Test("A connection holding two lanes is one name with plural transactions") + func quitOneConnectionTwoLanes() throws { + let id = UUID() + let prompt = try #require(Self.make(.quit, names: ["Prod"], holds: [Self.hold(id, lane: "app"), Self.hold(id, lane: "logs")])) + + #expect(prompt.title == "Do you want to commit the open transactions before quitting?") + #expect(prompt.message.hasPrefix("The connection “Prod” has open transactions.")) + } + + @Test("Relaunch speaks of relaunching, never of quitting") + func relaunchWording() throws { + let commit = try #require(Self.make(.relaunch, names: ["Prod"], holds: [Self.hold()], hasUnsavedWork: true)) + #expect(commit.title == "Do you want to commit the open transaction before relaunching?") + #expect(commit.message.hasSuffix("Some tabs have unsaved edits. Relaunching will discard these changes.")) + #expect(!commit.message.contains("Quitting")) + + let rollBack = try #require(Self.make( + .relaunch, names: ["Prod"], holds: [Self.hold(state: .aborted)], hasUnsavedWork: true + )) + #expect(rollBack.title == "Relaunch with an Open Transaction?") + #expect(rollBack.confirmButton == "Relaunch Anyway") + #expect(rollBack.message.hasPrefix("Uncommitted changes on “Prod” are rolled back when TablePro relaunches.")) + #expect(!rollBack.message.contains("quits")) + #expect(!rollBack.message.contains("Quitting")) + } + + @Test("Each action keeps its roll-back button when Commit is not offered") + func rollBackButtons() throws { + let aborted = [Self.hold(state: .aborted)] + + #expect(Self.make(.closeEntry(database: "app"), names: ["app"], holds: aborted)?.confirmButton == "Close") + #expect(Self.make(.closeConnection(name: "Prod"), names: ["app"], holds: aborted)?.confirmButton == "Close") + #expect(Self.make(.disconnect(name: "Prod"), names: ["app"], holds: aborted)?.confirmButton == "Disconnect") + #expect(Self.make(.quit, names: ["Prod"], holds: aborted)?.confirmButton == "Quit Anyway") + #expect(Self.make(.relaunch, names: ["Prod"], holds: aborted)?.confirmButton == "Relaunch Anyway") + } + + @Test("A failed statement leaves only Roll Back and says why") + func abortedOffersNoCommit() throws { + let prompt = try #require(Self.make(.disconnect(name: "Prod"), names: ["app"], holds: [Self.hold(lane: "app", state: .aborted)])) + + #expect(prompt.commitBlock == .aborted) + #expect(!prompt.offersCommit) + #expect(prompt.title == "Disconnect from “Prod”?") + #expect(prompt.message.hasPrefix( + "The database “app” has an open transaction. Disconnecting rolls it back and discards its uncommitted changes." + )) + #expect(prompt.message.contains("A statement failed in the transaction, so it can only be rolled back.")) + } + + @Test("Read-Only Safe Mode leaves only Roll Back and says why") + func readOnlyOffersNoCommit() throws { + let id = UUID() + let prompt = try #require(Self.make(.closeConnection(name: "Prod"), names: ["app"], holds: [Self.hold(id)], readOnly: [id])) + + #expect(prompt.commitBlock == .readOnly) + #expect(prompt.title == "Close the connection “Prod”?") + #expect(prompt.message.contains("Read-Only Safe Mode does not allow a commit.")) + } + + // The gate commits at the connection's level raised to the managed floor, so the prompt reads that level. + @Test("A managed Read-Only floor leaves only Roll Back when the connection itself is below it") + func managedFloorOffersNoCommit() throws { + let floor = ManagedFloorPolicy(floor: .readOnly) + let prompt = try #require(TransactionEndPrompt.make( + action: .disconnect(name: "Prod"), + names: ["app"], + holds: [Self.hold()], + effectiveLevel: { _ in ManagedPolicyResolver.effectiveSafeModeLevel(connectionLevel: .silent, policy: floor) }, + busy: { _ in false } + )) + + #expect(prompt.commitBlock == .readOnly) + } + + @Test("A statement still running in the transaction leaves only Roll Back and says why") + func busyOffersNoCommit() throws { + let hold = Self.hold(lane: "app") + let prompt = try #require(Self.make(.closeEntry(database: "app"), names: ["app"], holds: [hold], busy: [hold.key])) + + #expect(prompt.commitBlock == .busy) + #expect(prompt.title == "Close the database “app”?") + #expect(prompt.message.contains("A statement is still running in the transaction, so it cannot be committed now.")) + } + + // The live read can find one the tracker never recorded, and a commit goes through a tracked hold only. + @Test("A database with no tracked hold leaves only Roll Back") + func untrackedOffersNoCommit() throws { + let id = UUID() + let partly = try #require(Self.make( + .disconnect(name: "Prod"), names: ["app", "logs"], holds: [Self.hold(id, lane: "app")], untracked: true + )) + #expect(partly.commitBlock == .untracked) + #expect(partly.message.hasPrefix("These databases have open transactions: ")) + #expect(partly.message.contains("“logs”")) + + let none = try #require(Self.make(.disconnect(name: "Prod"), names: ["app"], holds: [])) + #expect(none.commitBlock == .untracked) + } + + @Test("A failed transaction is named before the other reasons") + func abortedOutranksTheRest() { + let id = UUID() + let aborted = Self.hold(id, lane: "app", state: .aborted) + let block = TransactionEndPrompt.commitBlock( + holds: [aborted], + untracked: true, + effectiveLevel: { _ in .readOnly }, + busy: { _ in true } + ) + + #expect(block == .aborted) + } + + @Test("One hold that cannot commit blocks Commit for all") + func oneBlockedHoldBlocksAll() { + let id = UUID() + let block = TransactionEndPrompt.commitBlock( + holds: [Self.hold(id, lane: "app"), Self.hold(id, lane: "logs", state: .aborted)], + untracked: false, + effectiveLevel: { _ in .silent }, + busy: { _ in false } + ) + + #expect(block == .aborted) + } +} + +private struct ManagedFloorPolicy: ManagedPolicyReading { + let floor: SafeModeLevel + + func string(_ policy: ManagedPolicy) -> String? { + policy == .minimumSafeModeLevel ? floor.rawValue : nil + } +} diff --git a/apple/TableProTests/Core/Services/Infrastructure/TransactionEndingFlowTests.swift b/apple/TableProTests/Core/Services/Infrastructure/TransactionEndingFlowTests.swift new file mode 100644 index 0000000000..837298e97f --- /dev/null +++ b/apple/TableProTests/Core/Services/Infrastructure/TransactionEndingFlowTests.swift @@ -0,0 +1,405 @@ +// +// TransactionEndingFlowTests.swift +// TableProTests +// + +import AppKit +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@Suite(.serialized) +@MainActor +struct TransactionEndingFlowTests { + private static func seed( + name: String = "Prod", + safeModeLevel: SafeModeLevel = .silent + ) -> (connection: DatabaseConnection, driver: EndingScriptedDriver) { + var connection = TestFixtures.makeConnection(name: name, database: "app", type: .mysql) + connection.safeModeLevel = safeModeLevel + let driver = EndingScriptedDriver() + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.browseDatabase = "app" + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return (connection, driver) + } + + private static func run(_ sql: String, on connectionId: UUID) async throws { + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + _ = try await DatabaseManager.shared.withScopedDriver( + scope: scope, + route: .sessionDriver, + cancellation: .protectedWrite + ) { driver in + try await driver.execute(query: sql) + } + } + + private static func holds(_ connectionId: UUID) -> [SessionTransactionTracker.Hold] { + DatabaseManager.shared.sessionTransactions.holds(for: connectionId) + } + + // The flow as a close or disconnect runs it, with the alert answered by the test and the gate injected. + private static func resolve( + _ connectionId: UUID, + answers: [AlertHelper.TransactionEndAnswer], + asked: inout [TransactionEndPrompt], + failures: inout [String], + gate: any ExecutionGate = AlwaysAllowGate(), + commit: (@MainActor ([SessionTransactionTracker.Hold]) async throws -> Bool)? = nil + ) async -> TransactionEndingFlow.Outcome { + var remaining = answers + var seen: [TransactionEndPrompt] = [] + var reported: [String] = [] + let outcome = await TransactionEndingFlow.resolve( + connectionId: connectionId, + action: .disconnect(name: "Prod"), + window: { nil }, + ask: { prompt, _ in + seen.append(prompt) + return remaining.isEmpty ? .cancel : remaining.removeFirst() + }, + commit: { holds in + if let commit { return try await commit(holds) } + return try await ConnectionTransactionActions.commitBeforeEnding(holds, gate: gate) + }, + reportFailure: { message, _ in reported.append(message) } + ) + asked = seen + failures = reported + return outcome + } + + @Test("Nothing open asks nothing and lets the close go ahead") + func nothingOpenAsksNothing() async throws { + let (connection, _) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve(connection.id, answers: [.commit], asked: &asked, failures: &failures) + + #expect(outcome == .nothingOpen) + #expect(asked.isEmpty) + } + + @Test("Commit commits the open transaction, then the close goes ahead") + func commitThenProceed() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve(connection.id, answers: [.commit], asked: &asked, failures: &failures) + + #expect(outcome == .ended) + #expect(asked.count == 1) + #expect(asked.first?.offersCommit == true) + #expect(driver.statements.filter { $0 == "COMMIT" }.count == 1) + #expect(driver.statements.last == "COMMIT") + #expect(Self.holds(connection.id).isEmpty) + #expect(failures.isEmpty) + } + + // The disconnect that follows ends the session, and the server rolls back what it held. + @Test("Roll Back sends nothing and asks no more") + func rollBackProceedsWithoutReasking() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve(connection.id, answers: [.rollBack, .commit], asked: &asked, failures: &failures) + + #expect(outcome == .ended) + #expect(asked.count == 1) + #expect(!driver.statements.contains("COMMIT")) + #expect(!driver.statements.contains("ROLLBACK")) + #expect(Self.holds(connection.id).count == 1) + } + + @Test("Cancel keeps the transaction and the connection") + func cancelKeepsEverything() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve(connection.id, answers: [.cancel], asked: &asked, failures: &failures) + + #expect(outcome == .cancelled) + #expect(!driver.statements.contains("COMMIT")) + #expect(!driver.statements.contains("ROLLBACK")) + #expect(Self.holds(connection.id).count == 1) + } + + @Test("Cancelling the Safe Mode question commits nothing and keeps the connection") + func gateCancelKeepsTheConnection() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve( + connection.id, answers: [.commit], asked: &asked, failures: &failures, gate: AlwaysDenyGate.cancelling + ) + + #expect(outcome == .cancelled) + #expect(!driver.statements.contains("COMMIT")) + #expect(!driver.statements.contains("ROLLBACK")) + #expect(failures.isEmpty) + } + + // A commit that failed and a close that went ahead anyway would be a rollback nobody chose. + @Test("A failed commit shows the error and keeps the connection") + func failedCommitKeepsTheConnection() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve(connection.id, answers: [.commit], asked: &asked, failures: &failures) { _ in + throw SessionTransactionActionError.cannotCommit + } + + #expect(outcome == .cancelled) + #expect(failures == [SessionTransactionActionError.cannotCommit.localizedDescription]) + #expect(!driver.statements.contains("COMMIT")) + #expect(!driver.statements.contains("ROLLBACK")) + #expect(Self.holds(connection.id).count == 1) + } + + // Read-Only is the strictest level, so no managed floor on the test Mac can change this answer. + @Test("The close prompt, the bar and the menu all withhold Commit at the gate's Read-Only level") + func readOnlyWithholdsCommitEverywhere() async throws { + let (connection, _) = Self.seed(safeModeLevel: .readOnly) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let hold = try #require(Self.holds(connection.id).first) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve(connection.id, answers: [.cancel], asked: &asked, failures: &failures) + + #expect(outcome == .cancelled) + #expect(asked.first?.commitBlock == .readOnly) + #expect(!ConnectionTransactionActions.offersCommit(hold)) + let bar = ConnectionTransactionBarContent( + isAborted: false, + connectionName: "Prod", + database: "app", + effectiveSafeModeLevel: ExecutionGateProvider.effectiveSafeModeLevel(for: connection.id) + ) + #expect(!bar.offersCommit) + } + + @Test("A Read-Only refusal of the commit shows the reason and keeps the connection") + func policyDenialKeepsTheConnection() async throws { + let (connection, _) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve( + connection.id, answers: [.commit], asked: &asked, failures: &failures, gate: AlwaysDenyGate() + ) + + #expect(outcome == .cancelled) + #expect(failures == ["Read-Only connection"]) + } + + // A nested Teradata BT survives one END TRANSACTION; closing after it would roll back what the user committed. + @Test("A transaction still open after Commit is asked about again") + func stillOpenAfterCommitAsksAgain() async throws { + let (connection, _) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + + let outcome = await Self.resolve( + connection.id, answers: [.commit, .rollBack], asked: &asked, failures: &failures + ) { _ in true } + + #expect(outcome == .ended) + #expect(asked.count == 2) + } + + @Test("A transaction a second Commit ends lets the close go ahead") + func secondCommitEndsIt() async throws { + let (connection, driver) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + var asked: [TransactionEndPrompt] = [] + var failures: [String] = [] + var commits = 0 + + let outcome = await Self.resolve( + connection.id, answers: [.commit, .commit], asked: &asked, failures: &failures + ) { holds in + commits += 1 + guard commits > 1 else { return true } + return try await ConnectionTransactionActions.commitBeforeEnding(holds, gate: AlwaysAllowGate()) + } + + #expect(outcome == .ended) + #expect(asked.count == 2) + #expect(driver.statements.filter { $0 == "COMMIT" }.count == 1) + #expect(driver.statements.last == "COMMIT") + } + + private static func entry(_ name: String, on connectionId: UUID) -> TransactionEndingFlow.Entry { + TransactionEndingFlow.Entry( + name: name, + hold: SessionTransactionTracker.Hold(key: .init(connectionId: connectionId, lane: name), state: .open, revision: 1) + ) + } + + @Test("A database that starts holding one after Roll Back is asked about, alone") + func newDatabaseAfterRollBackAsksAgain() async { + let id = UUID() + let app = Self.entry("app", on: id) + let logs = Self.entry("logs", on: id) + var reads: [[TransactionEndingFlow.Entry]] = [[app], [app, logs], [app, logs]] + var listed: [[String]] = [] + + let outcome = await TransactionEndingFlow.run( + read: { reads.isEmpty ? [] : reads.removeFirst() }, + prompt: { entries in + if !entries.isEmpty { listed.append(entries.map { $0.name }) } + return TransactionEndPrompt.make( + action: .closeConnection(name: "Prod"), + names: entries.map { $0.name }, + holds: entries.compactMap { $0.hold }, + effectiveLevel: { _ in .silent }, + busy: { _ in false } + ) + }, + ask: { _ in .rollBack }, + commit: { _ in + Issue.record("Roll Back commits nothing") + return true + }, + reportFailure: { _ in } + ) + + #expect(outcome == .ended) + #expect(listed == [["app"], ["logs"]]) + } + + @Test("Commit goes through only the holds the question listed") + func commitTakesTheListedHolds() async { + let app = Self.entry("app", on: UUID()) + var reads: [[TransactionEndingFlow.Entry]] = [[app], []] + var committed: [[SessionTransactionTracker.Hold]] = [] + + let outcome = await TransactionEndingFlow.run( + read: { reads.isEmpty ? [] : reads.removeFirst() }, + prompt: { entries in + TransactionEndPrompt.make( + action: .closeConnection(name: "Prod"), + names: entries.map { $0.name }, + holds: entries.compactMap { $0.hold }, + effectiveLevel: { _ in .silent }, + busy: { _ in false } + ) + }, + ask: { _ in .commit }, + commit: { holds in + committed.append(holds) + return true + }, + reportFailure: { _ in } + ) + + #expect(outcome == .ended) + #expect(committed.count == 1) + #expect(committed.first?.map { $0.key.lane } == ["app"]) + } + + @Test("Commit asks the gate once per connection before anything is sent") + func commitAuthorizesOncePerConnection() async throws { + let (first, _) = Self.seed(name: "First") + let (second, _) = Self.seed(name: "Second") + defer { + DatabaseManager.shared.removeSession(for: first.id) + DatabaseManager.shared.removeSession(for: second.id) + } + let gate = RequestRecordingGate(allows: true) + // Never recorded by the tracker, so the first one refused proves every question came before it. + let holds = [ + SessionTransactionTracker.Hold(key: .init(connectionId: first.id, lane: "app"), state: .open, revision: 9), + SessionTransactionTracker.Hold(key: .init(connectionId: first.id, lane: "logs"), state: .open, revision: 9), + SessionTransactionTracker.Hold(key: .init(connectionId: second.id, lane: nil), state: .open, revision: 9), + ] + + await #expect(throws: SessionTransactionActionError.transactionChanged) { + _ = try await ConnectionTransactionActions.commitBeforeEnding(holds, gate: gate) + } + let requests = await gate.requests + #expect(requests.map { $0.connectionId } == [first.id, second.id]) + #expect(requests.allSatisfy { $0.sql == "COMMIT" }) + } + + // A commit would wait behind the running turn, so the prompt offers Roll Back only while one runs. + @Test("A turn running on the hold's connection makes its lane busy") + func turnMakesTheLaneBusy() async throws { + let (connection, _) = Self.seed() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + try await Self.run("BEGIN", on: connection.id) + let hold = try #require(Self.holds(connection.id).first) + let key = DatabaseManager.shared.browsedGateKey(for: connection.id) + let gate = AsyncGate() + + let turn = Task { + try await DatabaseManager.shared.sessionDriverGate.withExclusiveAccess(key) { + await gate.wait() + } + } + await gate.arrived(1) + #expect(DatabaseManager.shared.isLaneBusy(holding: hold)) + await gate.open() + try await turn.value + + #expect(!DatabaseManager.shared.isLaneBusy(holding: hold)) + } +} + +// Answers its session state from the statements it saw, like a server that reports the flag in every reply. +private final class EndingScriptedDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var current: PluginSessionTransactionState = .idle + private var sent: [String] = [] + + var statements: [String] { lock.withLock { sent } } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { + sent.append(query) + switch query.uppercased() { + case "BEGIN": current = .inTransaction + case "COMMIT", "ROLLBACK": current = .idle + default: break + } + } + return .empty + } + + func sessionTransactionState() async -> PluginSessionTransactionState { + lock.withLock { current } + } + + func switchDatabase(to database: String) async throws {} +} diff --git a/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift b/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift index 3e467e5515..d8c12d30d0 100644 --- a/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift +++ b/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift @@ -153,4 +153,41 @@ struct MaskingExportTests { #expect(rows == [[.text("1"), .text("••••")], [.text("2"), .text("••••")]]) } + + @Test("Export Results loads each chunk of server values during the writer's pull for it") + @MainActor + func exportResultsLoadsValuesAsTheWriterPulls() async throws { + let count = 120 + let tableRows = TableRows.from( + queryRows: (0..(initialState: []) + let source = QueryResultExportDataSource( + tableRows: tableRows, + databaseType: .mysql, + driver: nil, + fillUnloaded: { rows in + let pull = pulls.withLock { $0 } + pullsAtEachFill.withLock { $0.append(pull) } + return Dictionary(uniqueKeysWithValues: rows.map { ($0.id, [1: PluginCellValue.text("x")]) }) + } + ) + let stream = source.streamExportRows(table: "customers", databaseName: "") + + // Off the main actor, so a fill that ran ahead of the writer would not wait behind it. + try await Task.detached { + var iterator = stream.makeAsyncIterator() + while true { + pulls.withLock { $0 += 1 } + guard try await iterator.next() != nil else { break } + } + }.value + + // Pull 1 is the header; the three chunks of 50, 50 and 20 rows are filled during pulls 2, 3 and 4. + #expect(pullsAtEachFill.withLock { $0 } == [2, 3, 4]) + } } diff --git a/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift b/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift index 482b9a163b..830f8ec5be 100644 --- a/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift +++ b/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift @@ -43,6 +43,57 @@ struct MaskingPolicyTests { #expect(policy.style(table: "hosts", column: "address", typeName: nil) == nil) } + // MySQL results name `TINYINT(1)` and `BIT(8)`, and a `tinyint` rule showed those columns in the clear. + @Test("A type rule matches the type a result names with its parameters or a wrapper, ignoring case") + func typeRuleMatchesTheBaseTypeName() { + let policy = MaskingPolicy(rules: [ + MaskingRule(pattern: "tinyint", target: .type), + MaskingRule(pattern: "bit", target: .type, style: .null), + MaskingRule(pattern: "String", target: .type, style: .hash), + MaskingRule(pattern: "var*", target: .type, style: .lastFour), + ]) + #expect(policy.style(table: "users", column: "active", typeName: "TINYINT(1)") == .full) + #expect(policy.style(table: "users", column: "age", typeName: "tinyint(4) unsigned") == .full) + #expect(policy.style(table: "users", column: "flags", typeName: "BIT(8)") == .null) + #expect(policy.style(table: "users", column: "flag", typeName: "BIT") == .null) + #expect(policy.style(table: "events", column: "token", typeName: "LowCardinality(Nullable(String))") == .hash) + #expect(policy.style(table: "users", column: "name", typeName: "VARCHAR(255)") == .lastFour) + #expect(policy.style(table: "users", column: "id", typeName: "INT") == nil) + #expect(policy.style(table: "users", column: "bits", typeName: "BITSTRING(8)") == nil) + } + + @Test("A type rule that names parameters still matches only those parameters") + func parameterizedTypeRuleStaysExact() { + let policy = MaskingPolicy(rules: [MaskingRule(pattern: "tinyint(1)", target: .type)]) + #expect(policy.style(table: "users", column: "active", typeName: "TINYINT(1)") == .full) + #expect(policy.style(table: "users", column: "age", typeName: "TINYINT(4)") == nil) + #expect(policy.style(table: "users", column: "age", typeName: "TINYINT") == nil) + } + + @Test("A column rule after a type rule still wins when the type rule matches only the base name") + func exactMatchesBeatBaseNameMatches() { + let policy = MaskingPolicy(rules: [ + MaskingRule(pattern: "varchar", target: .type, style: .lastFour), + MaskingRule(pattern: "ssn", target: .column), + ]) + #expect(policy.style(table: "people", column: "ssn", typeName: "VARCHAR(11)") == .full) + #expect(policy.style(table: "people", column: "name", typeName: "VARCHAR(11)") == .lastFour) + #expect(policy.style(table: "people", column: "ssn", typeName: "varchar") == .lastFour) + } + + @Test("Column masks match a type rule against the classified type's raw name") + func columnMasksMatchTypeRulesOnParameterizedTypes() { + let policy = MaskingPolicy(rules: [MaskingRule(pattern: "tinyint", target: .type)]) + let classifier = ColumnTypeClassifier(family: .mysql) + let masks = policy.columnMasks( + table: "users", + columns: ["id", "active"], + columnTypes: [classifier.classify(rawTypeName: "INT"), classifier.classify(rawTypeName: "TINYINT(1)")] + ) + #expect(masks.maskedColumns == [1]) + #expect(masks.row([.text("7"), .text("1")]) == [.text("7"), .text("••••")]) + } + @Test("The first matching rule wins, and an empty pattern is ignored") func firstRuleWins() { let policy = MaskingPolicy(rules: [ diff --git a/apple/TableProTests/Core/Services/Query/DatabaseTreeCatalogRefreshPlanTests.swift b/apple/TableProTests/Core/Services/Query/DatabaseTreeCatalogRefreshPlanTests.swift index 4acaa2c342..5c17abcde2 100644 --- a/apple/TableProTests/Core/Services/Query/DatabaseTreeCatalogRefreshPlanTests.swift +++ b/apple/TableProTests/Core/Services/Query/DatabaseTreeCatalogRefreshPlanTests.swift @@ -200,4 +200,74 @@ struct DatabaseTreeCatalogRefreshPlanTests { #expect(result.allSchemaTables.isEmpty) #expect(result.isEmpty) } + + private typealias ServerObjectsKey = DatabaseTreeMetadataService.ServerObjectsKey + + private func serverObjectPlan(_ change: CatalogChange, keys: [ServerObjectsKey]) -> CatalogTreeRefreshPlan { + DatabaseTreeMetadataService.catalogRefreshPlan( + for: change, + hasDatabaseList: false, + schemaListKeys: [], + tableKeys: [], + routineKeys: [], + triggerKeys: [], + typeKeys: [], + serverObjectKeys: keys + ) + } + + @Test("An event change marks the container's events stale and leaves another database current") + func eventChangeReachesItsContainer() { + let shopEvents = ServerObjectsKey(connectionId: connectionId, database: "shop", schema: nil, kindId: "event") + let warehouseEvents = ServerObjectsKey(connectionId: connectionId, database: "warehouse", schema: nil, kindId: "event") + let otherConnection = ServerObjectsKey(connectionId: UUID(), database: "shop", schema: nil, kindId: "event") + + let result = serverObjectPlan( + CatalogChange(connectionId: connectionId, database: "shop", kinds: [.triggers, .serverObjects]), + keys: [shopEvents, warehouseEvents, otherConnection] + ) + + #expect(result.serverObjects == [shopEvents]) + #expect(!result.isEmpty) + } + + @Test("A database-wide list is reached by a change made in any of its schemas") + func databaseScopedListIsReachedFromASchema() { + let extensions = ServerObjectsKey(connectionId: connectionId, database: "app", schema: nil, kindId: "extension") + let auditEvents = ServerObjectsKey(connectionId: connectionId, database: "app", schema: "audit", kindId: "event") + + let result = serverObjectPlan( + CatalogChange(connectionId: connectionId, database: "app", schema: "public", kinds: .objects), + keys: [extensions, auditEvents] + ) + + #expect(result.serverObjects == [extensions]) + } + + @Test("A container lists the declared kinds of its own level") + func serverKindsFollowTheContainerLevel() { + let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + let extensions = SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", iconName: "puzzlepiece.extension", + scope: .database + ) + let declared = [events, extensions] + + #expect(DatabaseTreeMetadataService.serverKinds(declared: declared, schema: "public", listsTablesPerSchema: true) == [events]) + #expect(DatabaseTreeMetadataService.serverKinds(declared: declared, schema: nil, listsTablesPerSchema: true) == [extensions]) + #expect(DatabaseTreeMetadataService.serverKinds(declared: declared, schema: nil, listsTablesPerSchema: false) == declared) + #expect(DatabaseTreeMetadataService.serverKinds(declared: declared, schema: "", listsTablesPerSchema: true) == [extensions]) + } + + @Test("A change that names no server object kind leaves those lists current") + func unrelatedChangeLeavesServerObjects() { + let shopEvents = ServerObjectsKey(connectionId: connectionId, database: "shop", schema: nil, kindId: "event") + + let result = serverObjectPlan(CatalogChange(connectionId: connectionId, database: "shop", kinds: .tables), keys: [shopEvents]) + + #expect(result.serverObjects.isEmpty) + #expect(result.isEmpty) + } } diff --git a/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift b/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift index 9513e89f5e..456bc825e2 100644 --- a/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift +++ b/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift @@ -628,4 +628,66 @@ struct DatabaseTreeMetadataServicePooledScopeTests { #expect(MetadataConnectionPool.shared.pooledDriverCount(for: connection.id) == 1) await tearDown(connection) } + + // Completion reads on the interactive connection, so a Refresh of rows already on screen must not queue there. + private func twoWorkloadSession() -> (DatabaseConnection, interactive: MockDatabaseDriver, bulk: MockDatabaseDriver) { + let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + session.browseDatabase = "shop" + session.browseSchema = "public" + DatabaseManager.shared.injectSession(session, for: connection.id) + let interactive = MockDatabaseDriver(connection: connection) + let bulk = MockDatabaseDriver(connection: connection) + for (driver, workload) in [(interactive, MetadataConnectionPool.Workload.interactive), (bulk, .bulk)] { + MetadataConnectionPool.shared.injectEntry( + driver, scope: DatabaseScope(connectionId: connection.id, database: "shop", schema: "public"), workload: workload + ) + MetadataConnectionPool.shared.injectEntry( + driver, scope: DatabaseScope(connectionId: connection.id, database: "shop", schema: nil), workload: workload + ) + } + return (connection, interactive, bulk) + } + + @Test("A Refresh of tables already on screen reads on the bulk connection") + func refreshOfLoadedTablesReadsOnTheBulkConnection() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.loadTables(connectionId: connection.id, database: "shop", schema: "public") + #expect(interactive.fetchScopedTablesCallCount == 1) + await service.refreshTableObjects(connectionId: connection.id, database: "shop", schema: "public") + + #expect(interactive.fetchScopedTablesCallCount == 1) + #expect(bulk.fetchScopedTablesCallCount == 1) + await tearDown(connection) + } + + @Test("A Refresh of a schema list already on screen reads on the bulk connection") + func refreshOfLoadedSchemaListReadsOnTheBulkConnection() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.loadSchemas(connectionId: connection.id, database: "shop") + let firstReads = interactive.fetchSchemasCallCount + #expect(firstReads == 1) + await service.refreshSchemas(connectionId: connection.id, database: "shop") + + #expect(interactive.fetchSchemasCallCount == firstReads) + #expect(bulk.fetchSchemasCallCount == 1) + await tearDown(connection) + } + + @Test("A Retry with nothing on screen stays on the interactive connection") + func retryWithNothingOnScreenStaysInteractive() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.refreshTableObjects(connectionId: connection.id, database: "shop", schema: "public") + + #expect(interactive.fetchScopedTablesCallCount == 1) + #expect(bulk.fetchScopedTablesCallCount == 0) + await tearDown(connection) + } } diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift index 983f92b642..3e3be93641 100644 --- a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift @@ -5,7 +5,8 @@ // The pool closed at most one idle connection each time a new one was opened, so a burst of opens // on distinct keys, which a tree restore or a Refresh of every expanded database produces, left // every connection it opened standing until the idle sweep: 13 to 16 backends for one saved -// connection on a server 1.5 seconds away, against a limit of 6 (#3103). +// connection on a server 1.5 seconds away, against a limit of 6 (#3103). Trimming at the end of +// the burst still let it dial every key at once, so the limit now holds while the burst runs. // import Foundation @@ -13,34 +14,9 @@ import Foundation import TableProTestSupport import Testing -/// Stands in for opening a pooled connection, remembering each driver by the database it was -/// opened for. @MainActor -private final class CountingOpener { - private let connectDelaySeconds: Double - private(set) var opened: [MockDatabaseDriver] = [] - private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] - var afterOpen: ((DatabaseScope) -> Void)? - var deadOnArrival: Set = [] - - init(connectDelaySeconds: Double = 0) { - self.connectDelaySeconds = connectDelaySeconds - } - - var openCount: Int { - opened.filter { $0.disconnectCallCount == 0 }.count - } - - func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { - let driver = MockDatabaseDriver() - driver.connectDelaySeconds = connectDelaySeconds - driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil - opened.append(driver) - openedByDatabase[scope.database] = driver - try await driver.connect() - afterOpen?(scope) - return driver - } +private final class StartedTaskBox { + var task: Task? } /// Holds a lease's body until the test lets it go, so a test can keep work in flight. @@ -85,12 +61,13 @@ struct MetadataConnectionPoolCapacityTests { private func fill( _ pool: MetadataConnectionPool, _ connection: DatabaseConnection, - databases: [String] + databases: [String], + workload: MetadataConnectionPool.Workload = .interactive ) async throws -> (gates: [LeaseGate], leases: [Task]) { let gates = databases.map { _ in LeaseGate() } let leases = zip(databases, gates).map { database, gate in Task { @MainActor in - try await pool.withDriver(scope: scope(connection, database)) { _ in + try await pool.withDriver(scope: scope(connection, database), workload: workload) { _ in await gate.wait() } } @@ -126,34 +103,248 @@ struct MetadataConnectionPoolCapacityTests { } #expect(opener.opened.count == limit * 2) + #expect(opener.peakOpenCount == limit) #expect(opener.openCount == limit) #expect(pool.heldConnectionCount(for: connection.id) == limit) } - @Test("Each lease that ends past the limit closes the connection unused longest, never one still working") - func trimClosesOnlyIdleConnections() async throws { + @Test("A burst on twice the limit never has more than the limit open while it runs") + func burstNeverRunsPastTheLimit() async throws { let connection = makeSession() defer { DatabaseManager.shared.removeSession(for: connection.id) } let opener = CountingOpener() let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) defer { pool.closeAll(connectionId: connection.id) } - let databases = (0..] = [] + for (database, workload) in arrivals { + let place = scope(connection, database) + waiting.append(Task { @MainActor in + try await pool.withDriver(scope: place, workload: workload) { _ in } + }) + let parked = waiting.count + try await waitUntil { pool.roomWaiterCount(for: connection.id) == parked } + } + + gates[0].open() + _ = try? await leases[0].value + try await waiting[1].value + try await waiting[3].value + #expect(Array(opener.openOrder.dropFirst(limit)) == ["i1", "i2"]) + #expect(pool.roomWaiterCount(for: connection.id) == 2) + + let lateBulk = scope(connection, "b3") + let late = Task { @MainActor in + try await pool.withDriver(scope: lateBulk, workload: .bulk) { _ in } + } + try await waitUntil { pool.roomWaiterCount(for: connection.id) == 3 } + try await pool.withDriver(scope: scope(connection, "i3")) { _ in } + #expect(Array(opener.openOrder.dropFirst(limit)) == ["i1", "i2", "i3"]) + + gates[1].open() + _ = try? await leases[1].value + try await waiting[0].value + try await waiting[2].value + try await late.value + #expect(Array(opener.openOrder.dropFirst(limit)) == ["i1", "i2", "i3", "b1", "b2", "b3"]) + await release(gates, leases) + } + + @Test("Bulk work never takes the last place, so an interactive lease runs while five bulk leases hold the rest") + func bulkLeavesOnePlaceForInteractive() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let (gates, leases) = try await fill(pool, connection, databases: (0.. DatabaseConnection { + var connection = TestFixtures.makeConnection(database: "shop") + connection.startupCommands = startupCommands + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return connection + } + + private func scope(_ connection: DatabaseConnection, _ schema: String?) -> DatabaseScope { + DatabaseScope(connectionId: connection.id, database: "shop", schema: schema) + } + + private func probe(_ pool: MetadataConnectionPool, _ scope: DatabaseScope) async throws -> SchemaSighting { + try await pool.withDriver(scope: scope) { driver in SchemaSighting(driver) } + } + + @Test("Leases on three schemas of one database, one after another, open one connection") + func schemasOfOneDatabaseShareAnIdleConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + for schema in ["a", "b", "c"] { + _ = try await probe(pool, scope(connection, schema)) + } + + #expect(opener.opened.count == 1) + #expect(pool.pooledDriverCount(for: connection.id) == 1) + } + + @Test("Each lease reads on its own schema, so a shared connection is moved before every change of schema") + func leaseRunsOnItsOwnSchema() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + var sightings: [SchemaSighting] = [] + for schema in ["a", "b", "a"] { + sightings.append(try await probe(pool, scope(connection, schema))) + } + let schemas = sightings.map { $0.schema } + let drivers = Set(sightings.map { $0.driver }) + + #expect(schemas == ["a", "b", "a"]) + #expect(drivers.count == 1) + #expect(opener.opened.first?.switchSchemaCallCount == 3) + } + + // One connection per database would make a schema's reads wait behind another schema's long lease. + @Test("A lease on another schema opens its own connection while the first schema's lease still runs", .timeLimit(.minutes(1))) + func busySchemaOpensASiblingForAnotherSchema() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let held = scope(connection, "a") + let holding = Task { @MainActor in + try await pool.withDriver(scope: held) { driver in + await gate.wait() + return ObjectIdentifier(driver) + } + } + await gate.arrived(1) + + let sibling = try await probe(pool, scope(connection, "b")) + #expect(opener.opened.count == 2) + + await gate.open() + let heldOn = try await holding.value + #expect(sibling.driver != heldOn) + #expect(sibling.schema == "b") + } + + @Test("Two leases on one schema at once queue on one connection") + func sameSchemaStillQueuesOnItsConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let log = SchemaLeaseLog() + let place = scope(connection, "a") + let first = Task { @MainActor in + try await pool.withDriver(scope: place) { driver in + await gate.wait() + await log.record("first") + return ObjectIdentifier(driver) + } + } + await gate.arrived(1) + let second = Task { @MainActor in + try await pool.withDriver(scope: place) { driver in + await log.record("second") + return ObjectIdentifier(driver) + } + } + try await waitUntil { pool.leaseCount(for: connection.id) == 2 } + #expect(opener.opened.count == 1) + + await gate.open() + let firstRanOn = try await first.value + let secondRanOn = try await second.value + + #expect(firstRanOn == secondRanOn) + #expect(log.entries == ["first", "second"]) + #expect(opener.opened.count == 1) + } + + // Data Compare and Object Copy lease one side inside the other, and both sides stream at once. + @Test( + "A lease nested in another schema's lease gets its own connection, past the limit when it is full", + .timeLimit(.minutes(1)) + ) + func nestedLeaseOnAnotherSchemaGetsItsOwnConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let fills = (1.. = [], - inserted: Set = [], - modified: [RowID: Set] = [:] + defaulted: [RowID: Set] = [:] ) -> SelectionSummaryInput { SelectionSummaryInput( selection: selection(rects), @@ -39,8 +38,7 @@ struct SelectionSummaryCalculatorTests { columnCount: tableRows.columns.count ), deletedRowIDs: deleted, - insertedRowIDs: inserted, - modifiedCells: modified + defaultedCells: defaulted ) } @@ -113,25 +111,24 @@ struct SelectionSummaryCalculatorTests { #expect(summary.numbers?.sum.decimal == 1) } - @Test("NULL is empty, empty text is a value, and the default marker is empty only where an edit put it") + @Test("NULL is empty, empty text is a value, and DEFAULT is empty only where an edit marked it") func emptyCellRules() async throws { let insertedID = RowID.inserted(UUID()) var table = Self.tableRows( [ [.null, .text("")], - [.text(PluginCellValue.defaultMarkerText), .text(PluginCellValue.defaultMarkerText)], - [.text("7"), .text(PluginCellValue.defaultMarkerText)] + [.text("DEFAULT"), .text("DEFAULT")], + [.text("7"), .columnDefault] ], types: [Self.integer, Self.untyped] ) - table.appendInsertedRow(id: insertedID, values: [.text(PluginCellValue.defaultMarkerText), .text("5")]) + table.appendInsertedRow(id: insertedID, values: [.columnDefault, .text("5")]) let summary = try await Self.summarize( Self.input( table, selecting: [GridRect(rows: 0...3, columns: 0...1)], - inserted: [insertedID], - modified: [.existing(2): [1]] + defaulted: [.existing(2): [1], insertedID: [0]] ) ) diff --git a/apple/TableProTests/Core/Services/Query/SidebarSchemaFilterTests.swift b/apple/TableProTests/Core/Services/Query/SidebarSchemaFilterTests.swift index b0028c93be..d42f233767 100644 --- a/apple/TableProTests/Core/Services/Query/SidebarSchemaFilterTests.swift +++ b/apple/TableProTests/Core/Services/Query/SidebarSchemaFilterTests.swift @@ -88,7 +88,6 @@ struct SidebarSchemaFilterTests { } SharedSidebarState.forConnection(connection.id).setSchemaFilter(["attendance"], for: "shop") let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), @@ -113,7 +112,6 @@ struct SidebarSchemaFilterTests { driver.allSchemaTablesError = DatabaseError.queryFailed("listing timed out") } let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), @@ -153,7 +151,6 @@ struct SidebarSchemaFilterTests { } SharedSidebarState.forConnection(connection.id).setSchemaFilter(["attendance"], for: "shop") let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), @@ -178,7 +175,6 @@ struct SidebarSchemaFilterTests { private func makeViewModel(_ connection: DatabaseConnection) -> SidebarViewModel { SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), diff --git a/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift b/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift index a0e0ecf3b2..7de2740536 100644 --- a/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift @@ -82,7 +82,7 @@ extension GlobalStateSuites { #expect(tableRows.row(withID: nullKept.rowID)?.absentColumns.isEmpty == true) #expect(tableRows.row(withID: missingKept.rowID)?.absentColumns == [2]) #expect(changes.pending.insertedAbsentColumns(forRow: missingKept.rowID) == [2]) - #expect(missingKept.values[0] == .text("__DEFAULT__")) + #expect(missingKept.values[0] == .columnDefault) } @Test("Copying rows records which fields each lacked, in the copied column order") @@ -123,7 +123,7 @@ extension GlobalStateSuites { let (tableRows, result) = paste(clipboard, into: manager) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), "Ada", .null]) + #expect(result.pastedRows.first?.values == ["1", "Ada", .null]) #expect(tableRows.rows[0].absentColumns.isEmpty) } diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift index 80819290f7..e81b6da9bf 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift @@ -20,9 +20,10 @@ struct RowOperationsManagerPasteTests { return RowOperationsManager(changeManager: changeManager) } + // `id` is AUTO_INCREMENT, so the server fills it. private func emptyRows() -> TableRows { let types: [ColumnType] = Array(repeating: .text(rawType: nil), count: Self.columns.count) - return TableRows.from(queryRows: [], columns: Self.columns, columnTypes: types) + return TableRows.from(queryRows: [], columns: Self.columns, columnTypes: types, columnIdentity: ["id": .byDefault]) } private func pasteRows(_ clipboard: RecordingClipboard, manager: RowOperationsManager) -> [[PluginCellValue]] { @@ -51,7 +52,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Smith, John")) #expect(values[2] == .text("a@test.com")) } @@ -85,6 +86,45 @@ struct RowOperationsManagerPasteTests { #expect(rows.first?[2] == .null) } + @Test("A binary column takes back the bytes a copy wrote as hex, from the grid's cells and from text") + func copiedHexPastesAsBytes() { + var types: [ColumnType] = Array(repeating: .text(rawType: nil), count: Self.columns.count) + types[2] = .blob(rawType: "varbinary(16)") + let manager = makeManager() + + let cells = RecordingClipboard() + cells.writeCells(GridCellsClipboardPayload(text: "7\tAnn\t0x4142", fields: [["7", "Ann", "0x4142"]])) + var rows = TableRows.from(queryRows: [], columns: Self.columns, columnTypes: types) + let fromCells = manager.pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: cells + ) + #expect(fromCells.pastedRows.first?.values[2] == .bytes(Data([0x41, 0x42]))) + + let text = RecordingClipboard() + text.text = "7\t0x41\t0x00FF\n8\tBo\t0x" + let fromText = manager.pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: text + ) + #expect(fromText.pastedRows.map { $0.values[1] } == [.text("0x41"), .text("Bo")]) + #expect(fromText.pastedRows.map { $0.values[2] } == [.bytes(Data([0x00, 0xFF])), .bytes(Data())]) + } + + @Test("A binary field that starts with 0x but is not hex, such as a copy cut short, pastes nothing") + func brokenHexRefusesThePaste() { + var types: [ColumnType] = Array(repeating: .text(rawType: nil), count: Self.columns.count) + types[2] = .blob(rawType: "varbinary(16)") + var rows = TableRows.from(queryRows: [], columns: Self.columns, columnTypes: types) + let clipboard = RecordingClipboard() + clipboard.text = "7\tAnn\t0x4142\n8\tBo\t0x41…" + + let result = makeManager().pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard + ) + + #expect(result.pastedRows.isEmpty) + #expect(rows.count == 0) + } + @Test("Copied cells stale against the clipboard's text are not used") func staleCopiedCellsAreIgnored() { let clipboard = RecordingClipboard() @@ -120,7 +160,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Jo")) #expect(values[2] == .text("e@x.com")) } @@ -136,7 +176,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Jo")) #expect(values[2] == .text("e@x.com")) } @@ -151,7 +191,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("2")) #expect(values[2] == .text("3")) } @@ -170,8 +210,8 @@ struct RowOperationsManagerPasteTests { let rows = pasteRows(clipboard, manager: makeManager()) #expect(rows.count == 2) - #expect(rows[0] == [.text("__DEFAULT__"), .text("Alice"), .text("a@test.com")]) - #expect(rows[1] == [.text("__DEFAULT__"), .text("Bob"), .null]) + #expect(rows[0] == [.columnDefault, .text("Alice"), .text("a@test.com")]) + #expect(rows[1] == [.columnDefault, .text("Bob"), .null]) } @Test("Plain-text TSV with commas inside a value parses as TSV, not CSV") @@ -182,7 +222,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Mozilla/5.0 (KHTML, like Gecko) Safari")) #expect(values[2] == .text("a@test.com")) } @@ -204,6 +244,180 @@ struct RowOperationsManagerPasteTests { } } +// A key the server cannot fill keeps the pasted value: reset to DEFAULT it went out as NULL. +@MainActor +struct RowOperationsManagerPasteKeyTests { + private static let columns = ["sku", "name"] + + private func paste( + _ clipboard: RecordingClipboard, + databaseType: DatabaseType = .postgresql, + declaredTypes: [String: String] = [:], + defaults: [String: String?] = [:], + uuidKeyForms: [Int: UUIDCellForm] = [:] + ) -> [[PluginCellValue]] { + let changeManager = DataChangeManager() + changeManager.configureForTable( + tableName: "products", + columns: Self.columns, + primaryKeyColumns: ["sku"], + databaseType: databaseType, + generatedColumns: [] + ) + var rows = TableRows.from( + queryRows: [], + columns: Self.columns, + columnTypes: Array(repeating: .text(rawType: "VARCHAR"), count: Self.columns.count), + columnDefaults: defaults, + columnNullable: ["sku": false], + columnDeclaredTypes: declaredTypes + ) + let result = RowOperationsManager(changeManager: changeManager).pasteRowsFromClipboard( + columns: Self.columns, + primaryKeyColumns: ["sku"], + tableRows: &rows, + uuidKeyForms: uuidKeyForms, + clipboard: clipboard + ) + return result.pastedRows.map(\.values) + } + + @Test("Pasted text keeps a natural key") + func textKeepsNaturalKey() { + let clipboard = RecordingClipboard() + clipboard.text = "SKU-1\tLamp\nSKU-2\tDesk" + + let keys = paste(clipboard).map { $0.first } + + #expect(keys == [PluginCellValue.text("SKU-1"), PluginCellValue.text("SKU-2")]) + } + + @Test("Pasted CSV keeps a natural key") + func csvKeepsNaturalKey() { + let clipboard = RecordingClipboard() + clipboard.text = "SKU-1,Lamp" + + #expect(paste(clipboard).first?.first == .text("SKU-1")) + } + + @Test("Structured rows keep a natural key") + func structuredKeepsNaturalKey() { + let clipboard = RecordingClipboard() + clipboard.gridRows = GridRowsClipboardPayload(columns: Self.columns, rows: [[.text("SKU-1"), .text("Lamp")]]) + + #expect(paste(clipboard).first == [.text("SKU-1"), .text("Lamp")]) + } + + @Test("Copied cells keep a natural key") + func copiedCellsKeepNaturalKey() { + let clipboard = RecordingClipboard() + clipboard.writeCells(GridCellsClipboardPayload(text: "SKU-1\tLamp", fields: [["SKU-1", "Lamp"]])) + + #expect(paste(clipboard).first?.first == .text("SKU-1")) + } + + @Test("A SQLite INTEGER PRIMARY KEY is the rowid, so the server fills it") + func sqliteRowidKeyGoesToTheServer() { + let clipboard = RecordingClipboard() + clipboard.text = "7\tLamp" + + let rows = paste(clipboard, databaseType: .sqlite, declaredTypes: ["sku": "INTEGER"]) + + #expect(rows.first?.first == .columnDefault) + } + + @Test("A SQLite TEXT key is not the rowid and keeps the pasted value") + func sqliteTextKeyIsKept() { + let clipboard = RecordingClipboard() + clipboard.text = "SKU-1\tLamp" + + let rows = paste(clipboard, databaseType: .sqlite, declaredTypes: ["sku": "TEXT"]) + + #expect(rows.first?.first == .text("SKU-1")) + } + + @Test("A key whose default calls a function goes to the server; a constant default does not replace the pasted key") + func onlyAFunctionDefaultFillsTheKey() { + let clipboard = RecordingClipboard() + clipboard.text = "SKU-1\tLamp" + + #expect(paste(clipboard, defaults: ["sku": "gen_random_uuid()"]).first?.first == .columnDefault) + #expect(paste(clipboard, defaults: ["sku": "(newid())"]).first?.first == .columnDefault) + #expect(paste(clipboard, defaults: ["sku": "'X'::text"]).first?.first == .text("SKU-1")) + #expect(paste(clipboard, defaults: ["sku": "((1))"]).first?.first == .text("SKU-1")) + #expect(paste(clipboard, defaults: ["sku": "(NEXT VALUE FOR [sku_seq])"]).first?.first == .columnDefault) + #expect(paste(clipboard, defaults: ["sku": "\"SKU_SEQ\".\"NEXTVAL\""]).first?.first == .columnDefault) + #expect(paste(clipboard, defaults: ["sku": "now()"]).first?.first == .text("SKU-1")) + #expect(paste(clipboard, defaults: ["sku": "(getdate())"]).first?.first == .text("SKU-1")) + #expect(paste(clipboard, defaults: ["sku": "timezone('utc'::text, now())"]).first?.first == .text("SKU-1")) + } + + @Test("A blank key that refuses NULL stays NULL rather than an empty string") + func blankKeyIsNotEmptyText() { + let clipboard = RecordingClipboard() + clipboard.text = "\tLamp" + + #expect(paste(clipboard).first?.first == .null) + } + + @Test("A UUID key the server does not fill takes a fresh UUID on every pasted row") + func uuidKeyTakesAFreshUUID() { + let clipboard = RecordingClipboard() + clipboard.text = "\tLamp\n6f1c2b0e-3c1a-4d0e-9a7b-1f2e3d4c5b6a\tDesk" + + let rows = paste(clipboard, uuidKeyForms: [0: .hyphenated]) + + let keys = rows.compactMap { $0.first?.asText } + #expect(keys.count == 2) + #expect(keys.allSatisfy { UUID(uuidString: $0) != nil }) + #expect(!keys.contains("6f1c2b0e-3c1a-4d0e-9a7b-1f2e3d4c5b6a")) + } + + @Test("CSV keeps the spaces inside a quoted field") + func csvKeepsQuotedSpaces() { + let clipboard = RecordingClipboard() + clipboard.text = "SKU-1,\" indented note \"" + + #expect(paste(clipboard).first?.last == .text(" indented note ")) + } +} + +@MainActor +struct RowOperationsManagerDuplicateRowsTests { + @Test("Duplicating several rows copies each of them in one undo step") + func duplicatesEverySelectedRow() { + let changeManager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + changeManager.undoManagerProvider = { undoManager } + changeManager.configureForTable( + tableName: "t", + columns: ["id", "name"], + primaryKeyColumns: ["id"], + databaseType: .postgresql, + generatedColumns: [] + ) + var rows = TableRows.from( + queryRows: [[.text("1"), .text("a")], [.text("2"), .text("b")], [.text("3"), .text("c")]], + columns: ["id", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] + ) + + let result = RowOperationsManager(changeManager: changeManager) + .duplicateRows(sourceRowIndices: [0, 1, 2], tableRows: &rows) + + #expect(result.rows.count == 3) + #expect(result.rows.map { $0.values[1] } == [.text("a"), .text("b"), .text("c")]) + #expect(result.rows.allSatisfy { $0.values[0] == .columnDefault }) + #expect(rows.count == 6) + #expect(changeManager.insertedRowIDs.count == 3) + + undoManager.undo() + #expect(changeManager.insertedRowIDs.isEmpty) + #expect(!undoManager.canUndo) + } +} + /// A pasted row arrives whole, so it carries values for columns the server owns. Those never reach /// the cell-edit boundary that refuses them, and `SQLStatementGenerator` drops them without a word, /// so the grid showed a pasted identity value the row was never saved with. @@ -257,7 +471,7 @@ struct RowOperationsManagerPasteServerOwnedTests { func resetsNonKeyIdentity() { let values = paste(payload, columnIdentity: ["code": .always]) - #expect(values[1] == .text("__DEFAULT__")) + #expect(values[1] == .columnDefault) #expect(values[2] == .text("Ada")) } @@ -265,7 +479,7 @@ struct RowOperationsManagerPasteServerOwnedTests { func resetsGeneratedColumn() { let values = paste(payload, generatedColumns: ["code"]) - #expect(values[1] == .text("__DEFAULT__")) + #expect(values[1] == .columnDefault) #expect(values[2] == .text("Ada")) } diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift index 7358b7b600..c56df2acad 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift @@ -118,8 +118,8 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) #expect(result != nil) - #expect(result?.values[0] == "__DEFAULT__") - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) + #expect(result?.values[2] == .columnDefault) } // PostgreSQL leaves `column_default` null for an identity column, so a new row sent an explicit NULL it refuses. @@ -133,7 +133,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) - #expect(result?.values[0] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) #expect(result?.values[1] == .null) } @@ -146,7 +146,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(result?.values[1] == .null) } @@ -159,7 +159,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) #expect(result?.values[1] == .null) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(tableRows.serverAssignsValue(forColumn: "name") == false) } @@ -246,7 +246,7 @@ struct RowOperationsManagerTests { ) #expect(result != nil) - #expect(result?.values[0] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) if case .rowsInserted(let indices) = result?.delta { #expect(indices == IndexSet(integer: 3)) } else { @@ -263,10 +263,40 @@ struct RowOperationsManagerTests { let result = manager.duplicateRow(sourceRowIndex: 1, tableRows: &tableRows) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(result?.values[1] == sourceValues[1]) } + @Test("The change manager names the cells a new row leaves to their default, for the JSON view and Copy as JSON") + func defaultedCellsOfANewRow() throws { + let (manager, changeManager) = makeManager() + var tableRows = emptyTableRows(columnDefaults: ["id": "auto_increment", "email": "'x'"]) + + let added = manager.addNewRow(tableRows: &tableRows) + let result = try #require(added) + + #expect(changeManager.defaultedCells(in: tableRows) == [result.rowID: [0, 2]]) + } + + @Test("A duplicate writes stored text that reads like the default as text, and leaves only its key to the server") + func duplicateRowKeepsDefaultLookingText() throws { + let (manager, changeManager) = makeManager() + var tableRows = TableRows.from( + queryRows: [["7", "__DEFAULT__", "DEFAULT"]], + columns: Self.testColumns, + columnTypes: Self.testColumnTypes + ) + + let duplicated = manager.duplicateRow(sourceRowIndex: 0, tableRows: &tableRows) + let result = try #require(duplicated) + + #expect(changeManager.expressionColumns(forRow: result.rowID) == [0]) + let statements = try changeManager.generateSQL() + #expect(statements.count == 1) + #expect(statements.first?.sql.contains("`id`") == false) + #expect(statements.first?.parameters.compactMap { $0 as? String } == ["__DEFAULT__", "DEFAULT"]) + } + @Test("duplicateRow returns nil for invalid source index") func duplicateRowReturnsNilForInvalidIndex() { let (manager, _) = makeManager() diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift index 5601c64f35..bfcfd74caf 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift @@ -80,7 +80,7 @@ struct RowOperationsManagerUUIDKeyTests { let result = try #require(manager.addNewRow(tableRows: &rows, uuidKeyForms: Self.uuidKey)) - #expect(result.values[0] == .text("__DEFAULT__")) + #expect(result.values[0] == .columnDefault) #expect(result.prefilledColumns.isEmpty) } @@ -99,7 +99,7 @@ struct RowOperationsManagerUUIDKeyTests { #expect(copyKey != sourceKey) #expect(copy.values[1] == .text("a")) #expect(copy.prefilledColumns == [0]) - #expect(plain.values[0] == .text("__DEFAULT__")) + #expect(plain.values[0] == .columnDefault) } @Test("Duplicating a pending row keeps the SQL staged in it, except in the columns the copy resets") @@ -118,14 +118,16 @@ struct RowOperationsManagerUUIDKeyTests { #expect(uuidText(copy.values[0]) != nil) } - @Test("Duplicating a saved row stages no SQL") + @Test("Duplicating a saved row stages no SQL but the default its reset key takes") func duplicateOfASavedRowHasNoMarks() throws { let (manager, changeManager) = makeManager() var rows = makeRows([[.text("0b1e7c9a-58f4-4b8e-9c55-1f1c86f0a001"), .text("now()"), .null]]) let copy = try #require(manager.duplicateRow(sourceRowIndex: 0, tableRows: &rows)) - #expect(changeManager.expressionColumns(forRow: copy.rowID).isEmpty) + #expect(changeManager.expressionColumns(forRow: copy.rowID) == [0]) + #expect(copy.values[0] == .columnDefault) + #expect(copy.values[1] == .text("now()")) } // MARK: - Paste rows @@ -174,8 +176,8 @@ struct RowOperationsManagerUUIDKeyTests { #expect(key != "0b1e7c9a-58f4-4b8e-9c55-1f1c86f0a001") } - @Test("Pasted rows on a key that is not a UUID key still hand it to the server") - func pasteWithoutUUIDKeysKeepsDefault() { + @Test("A pasted key that is neither a UUID key nor filled by the server keeps its value") + func pasteWithoutUUIDKeysKeepsTheKey() { let (manager, _) = makeManager() var rows = makeRows() let clipboard = RecordingClipboard() @@ -188,7 +190,7 @@ struct RowOperationsManagerUUIDKeyTests { clipboard: clipboard ) - #expect(result.pastedRows.first?.values[0] == .text("__DEFAULT__")) + #expect(result.pastedRows.first?.values[0] == .text("k1")) } } @@ -210,11 +212,13 @@ extension GlobalStateSuites { return (RowOperationsManager(changeManager: changeManager), changeManager) } + // `id` is an identity, so a pasted row hands it back to the server. private func makeRows(_ queryRows: [[PluginCellValue]] = []) -> TableRows { TableRows.from( queryRows: queryRows, columns: Self.columns, columnTypes: [.text(rawType: "text"), .text(rawType: "text"), .timestamp(rawType: "timestamptz")], + columnIdentity: ["id": .byDefault], hasAuthoritativeSchema: true ) } @@ -249,8 +253,8 @@ extension GlobalStateSuites { let pasted = result.pastedRows.map(\.rowID) #expect(pasted.count == 2) - #expect(changeManager.expressionColumns(forRow: pasted[0]).isEmpty) - #expect(changeManager.expressionColumns(forRow: pasted[1]) == [2]) + #expect(changeManager.expressionColumns(forRow: pasted[0]) == [0]) + #expect(changeManager.expressionColumns(forRow: pasted[1]) == [0, 2]) #expect(result.pastedRows[0].values[2] == .text("CURRENT_TIMESTAMP")) #expect(result.pastedRows[1].values[2] == .text("now()")) } @@ -268,8 +272,9 @@ extension GlobalStateSuites { let result = try paste(manager, into: &rows) - let rowID = try #require(result.pastedRows.first?.rowID) - #expect(changeManager.expressionColumns(forRow: rowID) == [2]) + let pastedRow = try #require(result.pastedRows.first) + #expect(pastedRow.values[0] == .columnDefault) + #expect(changeManager.expressionColumns(forRow: pastedRow.rowID) == [0, 2]) } @Test("Cells the grid copied as wide as the table paste as rows with their marks") @@ -287,8 +292,8 @@ extension GlobalStateSuites { let pasted = result.pastedRows.map(\.rowID) #expect(pasted.count == 2) - #expect(changeManager.expressionColumns(forRow: pasted[0]) == [2]) - #expect(changeManager.expressionColumns(forRow: pasted[1]).isEmpty) + #expect(changeManager.expressionColumns(forRow: pasted[0]) == [0, 2]) + #expect(changeManager.expressionColumns(forRow: pasted[1]) == [0]) } } } diff --git a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift index 9c1a9b1dd4..d74fe0ddb7 100644 --- a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift +++ b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift @@ -22,16 +22,16 @@ struct RowPasteForeignTextTests { let rows = try parse("\tAnn\tann@x.com\n\tBob\tbob@x.com") #expect(rows == [ - [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")], - [.text("__DEFAULT__"), .text("Bob"), .text("bob@x.com")] + [.null, .text("Ann"), .text("ann@x.com")], + [.null, .text("Bob"), .text("bob@x.com")] ]) } - @Test("An empty first cell reads as NULL when the table has no key to reset") - func leadingEmptyFieldIsNullWithoutAKey() throws { - let rows = try parse("\tAnn\tann@x.com", primaryKey: []) + @Test("The parser keeps a pasted key; only the paste knows whether the server fills it") + func parserKeepsTheKey() throws { + let rows = try parse("7\tAnn\tann@x.com") - #expect(rows == [[.null, .text("Ann"), .text("ann@x.com")]]) + #expect(rows == [[.text("7"), .text("Ann"), .text("ann@x.com")]]) } @Test("Trailing empty cells stay NULL and blank lines are not rows") @@ -67,14 +67,16 @@ struct RowPasteForeignTextTests { var rows = TableRows.from( queryRows: [], columns: Self.columns, - columnTypes: Array(repeating: .text(rawType: nil), count: Self.columns.count) + columnTypes: Array(repeating: .text(rawType: nil), count: Self.columns.count), + columnIdentity: ["id": .byDefault] ) let result = manager.pasteRowsFromClipboard( columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")]) + #expect(result.pastedRows.first?.values == [.columnDefault, .text("Ann"), .text("ann@x.com")]) + #expect(result.pastedRows.first.map { changeManager.expressionColumns(forRow: $0.rowID) } == [0]) } @Test("A blank pasted cell takes the column's default, an empty string where NULL is refused, else NULL") @@ -100,6 +102,7 @@ struct RowPasteForeignTextTests { ], columnDefaults: ["status": "'active'"], columnNullable: ["name": false, "status": false, "note": false, "born": false, "nickname": true], + columnIdentity: ["id": .byDefault], hasAuthoritativeSchema: true ) @@ -107,11 +110,12 @@ struct RowPasteForeignTextTests { columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - let unfilled: [PluginCellValue] = [.text("__DEFAULT__"), .text(""), .null, .null] + let unfilled: [PluginCellValue] = [.columnDefault, .text(""), .null, .null] #expect(result.pastedRows.map { $0.values } == [ - [.text("__DEFAULT__"), .text("Ann")] + unfilled, - [.text("__DEFAULT__"), .text("Bob")] + unfilled, + [.columnDefault, .text("Ann")] + unfilled, + [.columnDefault, .text("Bob")] + unfilled, ]) + #expect(result.pastedRows.map { changeManager.expressionColumns(forRow: $0.rowID) } == [[0, 2], [0, 2]]) } @Test("A pasted NULL stays NULL in a column with a default") @@ -129,6 +133,7 @@ struct RowPasteForeignTextTests { columns: columns, columnTypes: [.integer(rawType: "INT"), .text(rawType: "VARCHAR")], columnDefaults: ["status": "'active'"], + columnIdentity: ["id": .byDefault], hasAuthoritativeSchema: true ) @@ -136,7 +141,7 @@ struct RowPasteForeignTextTests { columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .null]) + #expect(result.pastedRows.first?.values == [.columnDefault, .null]) } } diff --git a/apple/TableProTests/Core/Services/SQL/FileSystemEventWatcherTests.swift b/apple/TableProTests/Core/Services/SQL/FileSystemEventWatcherTests.swift new file mode 100644 index 0000000000..1f5de5306f --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/FileSystemEventWatcherTests.swift @@ -0,0 +1,45 @@ +// +// FileSystemEventWatcherTests.swift +// TableProTests +// +// The stream used to hold its watcher unretained, so an event that arrived after the watcher +// was released read freed memory. +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +@MainActor +struct FileSystemEventWatcherTests { + @Test("A change in the folder reaches a live watcher, and none reaches one cancelled or released before delivery") + func onlyALiveWatcherIsCalled() async throws { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("FileSystemEventWatcherTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: folder) } + + var silenced = 0 + let cancelled = FileSystemEventWatcher(latency: 0.05) + cancelled.onChange = { silenced += 1 } + cancelled.watch([folder.path]) + var released: FileSystemEventWatcher? = FileSystemEventWatcher(latency: 0.05) + released?.onChange = { silenced += 1 } + released?.watch([folder.path]) + + var reported = 0 + let live = FileSystemEventWatcher(latency: 0.05) + live.onChange = { reported += 1 } + live.watch([folder.path]) + defer { live.cancel() } + + try Data("{}".utf8).write(to: folder.appendingPathComponent("theme.json")) + // Stopped before FSEvents delivers, so this pins that a stopped or released watcher is never called. + cancelled.cancel() + released = nil + + try await waitUntil { reported > 0 } + #expect(silenced == 0) + } +} diff --git a/apple/TableProTests/Core/Services/SQL/LinkedFolderGitStatusStoreTests.swift b/apple/TableProTests/Core/Services/SQL/LinkedFolderGitStatusStoreTests.swift new file mode 100644 index 0000000000..fd9544165c --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/LinkedFolderGitStatusStoreTests.swift @@ -0,0 +1,44 @@ +import Foundation +@testable import TablePro +import Testing + +@MainActor +@Suite(.enabled(if: GitClient.make() != nil)) +struct LinkedFolderGitStatusStoreTests { + @Test("Switching back asks Git only about folders outside a repository, once each") + func activationAsksOnlyFoldersOutsideARepository() async throws { + let real = try #require(GitClient.make()) + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("LinkedFolderGitStatusStoreTests-\(UUID().uuidString)", isDirectory: true) + .resolvingSymlinksInPath() + let repository = root.appendingPathComponent("repository", isDirectory: true) + let plain = root.appendingPathComponent("plain", isDirectory: true) + try FileManager.default.createDirectory(at: repository, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: plain, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let initialized = try await real.runner.run( + GitCommand(arguments: ["init", "-q"], workingDirectory: repository, access: .write) + ) + #expect(initialized.succeeded) + + let log = root.appendingPathComponent("invocations.log") + let wrapper = root.appendingPathComponent("git") + let script = "#!/bin/sh\npwd -P >> '\(log.path)'\nexec '\(real.runner.executableURL.path)' \"$@\"\n" + try Data(script.utf8).write(to: wrapper) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: wrapper.path) + let client = GitClient(runner: GitProcessRunner(executableURL: wrapper)) + let folders = [LinkedSQLFolder(path: repository.path), LinkedSQLFolder(path: plain.path)] + let store = LinkedFolderGitStatusStore(clientFactory: { client }, foldersProvider: { folders }) + + await store.refresh() + #expect(store.snapshots.count == 1) + try Data().write(to: log) + + await store.lookForNewRepositories()?.value + + // `pwd -P` prints the /private/var path; resolving strips that prefix again. + let asked = try String(contentsOf: log, encoding: .utf8).split(separator: "\n") + .map { URL(fileURLWithPath: String($0)).resolvingSymlinksInPath().path } + #expect(asked == [plain.path]) + } +} diff --git a/apple/TableProTests/Core/Services/SQL/LinkedSQLFolderScannerTests.swift b/apple/TableProTests/Core/Services/SQL/LinkedSQLFolderScannerTests.swift new file mode 100644 index 0000000000..c9e87738d3 --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/LinkedSQLFolderScannerTests.swift @@ -0,0 +1,172 @@ +// +// LinkedSQLFolderScannerTests.swift +// TableProTests +// +// Any file event under a linked folder used to read every SQL file's header again and rewrite +// the folder's whole index. A scan now reads only files whose size or modification time moved, +// and writes only rows that differ. +// + +import Foundation +import os +@testable import TablePro +import TableProUtilities +import Testing + +struct LinkedSQLFolderScannerTests { + private final class Fixture { + let root: URL + let folder: URL + let folderId = UUID() + let index: LinkedSQLIndex + private let reads = OSAllocatedUnfairLock<[String]>(initialState: []) + + init() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("LinkedSQLFolderScannerTests-\(UUID().uuidString)", isDirectory: true) + folder = root.appendingPathComponent("Queries", isDirectory: true) + try FileManager.default.createDirectory( + at: folder.appendingPathComponent("reports", isDirectory: true), + withIntermediateDirectories: true + ) + index = LinkedSQLIndex(databaseURL: root.appendingPathComponent("index.db"), removeDatabaseOnDeinit: true) + } + + var scanner: LinkedSQLFolderScanner { + let reads = reads + return LinkedSQLFolderScanner(index: index) { url in + reads.withLock { $0.append(url.lastPathComponent) } + return FileTextLoader.loadHeader(url) + } + } + + func scan() async -> Bool { + await scanner.scan(folderId: folderId, at: folder) + } + + // The header reads since the last call. + func takeReads() -> [String] { + reads.withLock { recorded in + defer { recorded = [] } + return recorded.sorted() + } + } + + func write(_ relativePath: String, _ text: String, modified: Date? = nil) throws { + let url = folder.appendingPathComponent(relativePath) + try Data(text.utf8).write(to: url) + if let modified { + try FileManager.default.setAttributes([.modificationDate: modified], ofItemAtPath: url.path) + } + } + + func stored() async -> [String: LinkedSQLIndex.IndexedFile] { + await index.storedFiles(folderId: folderId) + } + + func remove() { + try? FileManager.default.removeItem(at: root) + } + } + + private static func seed(_ fixture: Fixture) throws { + try fixture.write("revenue.sql", "-- @name: Revenue\n-- @keyword: rev\nSELECT 1;") + try fixture.write("users.sql", "-- @description: Active users\nSELECT 2;") + try fixture.write("reports/daily.psql", "SELECT 3;") + try fixture.write("notes.txt", "not a query") + } + + @Test("The first scan reads every SQL file and indexes its header") + func firstScanReadsEverySQLFile() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + + #expect(await fixture.scan()) + + #expect(fixture.takeReads() == ["daily.psql", "revenue.sql", "users.sql"]) + let stored = await fixture.stored() + #expect(Set(stored.keys) == ["revenue.sql", "users.sql", "reports/daily.psql"]) + #expect(stored["revenue.sql"]?.name == "Revenue") + #expect(stored["revenue.sql"]?.keyword == "rev") + #expect(stored["users.sql"]?.name == "users") + #expect(stored["users.sql"]?.description == "Active users") + } + + @Test("A second scan of an unchanged folder reads no file and changes no row") + func unchangedFolderReadsNothing() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + _ = fixture.takeReads() + let before = await fixture.stored() + + #expect(await fixture.scan() == false) + + #expect(fixture.takeReads().isEmpty) + #expect(await fixture.stored() == before) + } + + @Test("A file whose modification time moved is the only one read again") + func touchedFileIsReadAgain() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + _ = fixture.takeReads() + let before = await fixture.stored() + + let modified = Date(timeIntervalSince1970: 1_800_000_000) + try fixture.write("users.sql", "-- @keyword: usr\nSELECT 2;", modified: modified) + + #expect(await fixture.scan()) + + #expect(fixture.takeReads() == ["users.sql"]) + let after = await fixture.stored() + #expect(after["users.sql"]?.keyword == "usr") + #expect(after["users.sql"]?.mtime == modified.timeIntervalSince1970) + #expect(after["revenue.sql"] == before["revenue.sql"]) + #expect(after["reports/daily.psql"] == before["reports/daily.psql"]) + } + + @Test("A deleted file's row is removed and a new file's row is added") + func deletedAndAddedFiles() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + _ = fixture.takeReads() + let before = await fixture.stored() + + try FileManager.default.removeItem(at: fixture.folder.appendingPathComponent("revenue.sql")) + try fixture.write("reports/weekly.sql", "-- @name: Weekly\nSELECT 4;") + + #expect(await fixture.scan()) + + #expect(fixture.takeReads() == ["weekly.sql"]) + let after = await fixture.stored() + #expect(Set(after.keys) == ["users.sql", "reports/daily.psql", "reports/weekly.sql"]) + #expect(after["reports/weekly.sql"]?.name == "Weekly") + #expect(after["users.sql"] == before["users.sql"]) + } + + @Test("A folder no longer linked, or gone from disk, is dropped and reported once") + func removedFoldersAreDropped() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + + #expect(await fixture.scanner.pruneFolders(keeping: [fixture.folderId]) == false) + #expect(await fixture.scanner.pruneFolders(keeping: [])) + #expect(await fixture.stored().isEmpty) + #expect(await fixture.scanner.pruneFolders(keeping: []) == false) + + #expect(await fixture.scan()) + try FileManager.default.removeItem(at: fixture.folder) + #expect(await fixture.scan()) + #expect(await fixture.stored().isEmpty) + #expect(await fixture.scan() == false) + } +} diff --git a/apple/TableProTests/Core/Services/SQL/SQLFolderWatcherTests.swift b/apple/TableProTests/Core/Services/SQL/SQLFolderWatcherTests.swift new file mode 100644 index 0000000000..b3877acf9e --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/SQLFolderWatcherTests.swift @@ -0,0 +1,73 @@ +// +// SQLFolderWatcherTests.swift +// TableProTests +// +// Every event under a linked folder used to rescan it, so a `.DS_Store` write or a build tool's +// output in a linked repository read every SQL file again. +// + +import CoreServices +import Foundation +@testable import TablePro +import Testing + +struct SQLFolderWatcherTests { + private static func event(_ path: String, _ flags: Int) -> FileSystemEvent { + FileSystemEvent(path: path, flags: FSEventStreamEventFlags(flags)) + } + + private static func file(_ path: String) -> FileSystemEvent { + event(path, kFSEventStreamEventFlagItemIsFile | kFSEventStreamEventFlagItemModified) + } + + @Test("A batch of only non-SQL files and repository metadata is skipped") + func nonSQLBatchIsSkipped() { + let batch = [ + Self.file("/repo/.DS_Store"), + Self.file("/repo/build/output.log"), + Self.file("/repo/queries/.revenue.sql.swp"), + Self.file("/repo/.git/index"), + Self.event("/repo/.git/refs/heads", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemCreated), + ] + + #expect(SQLFolderWatcher.shouldRescan(after: batch) == false) + } + + @Test("A repository made or removed in the folder, or an ignore file edited, counts as a Git change") + func gitStateChanges() { + let created = Self.event("/repo/.git", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemCreated) + #expect(SQLFolderWatcher.touchesGitState([created])) + #expect(SQLFolderWatcher.touchesGitState([Self.file("/repo/.DS_Store"), Self.file("/repo/queries/.gitignore")])) + #expect(!SQLFolderWatcher.touchesGitState([Self.file("/repo/.git/index"), Self.file("/repo/.DS_Store")])) + } + + @Test("A batch holding a SQL file is kept, whatever the case of its extension") + func sqlFileIsKept() { + #expect(SQLFolderWatcher.shouldRescan(after: [Self.file("/repo/notes.txt"), Self.file("/repo/q/Revenue.SQL")])) + #expect(SQLFolderWatcher.shouldRescan(after: [Self.file("/repo/q/daily.pgsql")])) + } + + @Test("A directory event is kept, a renamed directory included") + func directoryEventIsKept() { + let created = Self.event("/repo/queries", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemCreated) + let renamed = Self.event("/repo/reports", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemRenamed) + + #expect(SQLFolderWatcher.shouldRescan(after: [Self.file("/repo/.DS_Store"), created])) + #expect(SQLFolderWatcher.shouldRescan(after: [renamed])) + } + + @Test("A batch whose paths no longer say what changed is kept") + func coalescedBatchIsKept() { + let flags = [ + kFSEventStreamEventFlagMustScanSubDirs, + kFSEventStreamEventFlagRootChanged, + kFSEventStreamEventFlagUserDropped, + kFSEventStreamEventFlagKernelDropped, + kFSEventStreamEventFlagEventIdsWrapped, + ] + for flag in flags { + let batch = [Self.event("/repo/build.log", kFSEventStreamEventFlagItemIsFile | flag)] + #expect(SQLFolderWatcher.shouldRescan(after: batch), "flag \(flag)") + } + } +} diff --git a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift index 7de7e4234b..305a631b6c 100644 --- a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift +++ b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift @@ -1235,6 +1235,25 @@ struct SQLFormatterServiceTests { ) } + @Test("A pg_dump COPY block keeps its rows, its \\. line and the statements after it") + func copyBlockStaysAsWritten() throws { + let copy = "COPY public.t (a, b) FROM stdin;\n\tit's\n2\tselect -- x\n\\." + let sql = "\\restrict k\n\(copy)\nselect 1;" + let formatted = try formatter.format(sql, dialect: .postgresql, options: .default).formattedSQL + #expect(formatted.contains(copy + "\n")) + #expect(formatted.hasPrefix("\\restrict k\n")) + let grammar = DatabaseType.postgresql.lexicalGrammar + #expect(SQLStatementScanner.executableStatements(in: formatted, grammar: grammar).map { $0.sql.uppercased() } == [ + "COPY PUBLIC.T (A, B) FROM STDIN", "SELECT 1", + ]) + } + + @Test("COPY rows that end the text keep a trailing tab") + func copyRowsAtTheEndKeepTheirTab() throws { + let sql = "COPY t (a, b) FROM stdin;\n1\t" + #expect(try formatter.format(sql, dialect: .postgresql, options: .default).formattedSQL == sql) + } + @Test("PostgreSQL operators stay whole") func postgresOperators() throws { try expectFormat( diff --git a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift index 7cb207c89e..13e35076d3 100644 --- a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift +++ b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift @@ -305,6 +305,22 @@ struct SQLTokenizerTests { #expect(code("select 1;\ndelimiter $$", .mysql).last?.type == .directive) } + @Test("A DELIMITER line sets the first word after it, as the mysql client reads it") + func delimiterArgumentIsTheFirstWord() { + let tokens = code("DELIMITER // -- set\nselect 1//\nDELIMITER\nselect 2;", .mysql) + #expect(tokens.map(\.value) == ["DELIMITER // -- set", "select", "1", "//", "DELIMITER", "select", "2", ";"]) + #expect(tokens.first?.type == .directive) + #expect(tokens[4].type == .directive) + } + + @Test("A COPY with its rows and a psql \\restrict line keep their text") + func psqlScriptText() { + let tokens = code("\\restrict k\nCOPY t FROM stdin;\n\tselect 'x\n\\.\nselect 2;", .postgresql) + #expect(tokens.map(\.value) == ["\\restrict k", "COPY t FROM stdin;\n\tselect 'x\n\\.", "select", "2", ";"]) + #expect(tokens[0].type == .directive) + #expect(tokens[1].type == .directive) + } + @Test("A MySQL # comment runs to the end of its line") func hashComment() { #expect(code("a # b and\nc", .mysql).map(\.value) == ["a", "# b and", "c"]) diff --git a/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift b/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift new file mode 100644 index 0000000000..a34eb3e553 --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift @@ -0,0 +1,59 @@ +// +// PayloadRowPlannerHeaderMatchTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct PayloadRowPlannerHeaderMatchTests { + private func columns(_ names: [String]) -> [PayloadColumn] { + names.map { PayloadColumn(name: $0, isPrimaryKey: false, isServerOwned: false) } + } + + private func planned(_ rows: [PayloadRow], into names: [String]) throws -> [[String]] { + try PayloadRowPlanner.values(table: "people", columns: columns(names), rows: rows) + .map { row in row.map { "\($0.column)=\($0.value ?? "NULL")" } } + } + + // SQL Server exports its headers bracketed, and Shortcuts has no sheet to map them by hand. + @Test("A CSV with bracketed headers inserts into the columns they name") + func bracketedHeadersFindTheirColumns() async throws { + let rows = try await RowPayload.parse(data: "[id],[name]\n1,Bob\n", file: nil) + #expect(try planned(rows, into: ["id", "name"]) == [["id=1", "name=Bob"]]) + } + + @Test("A doubled closing bracket inside the brackets is one bracket of the name") + func doubledBracketIsUnescaped() throws { + let row = PayloadRow(values: ["[a]]b]": .text("x")]) + #expect(try planned([row], into: ["a]b"]) == [["a]b=x"]]) + } + + @Test("A lone closing bracket inside means the key is not a bracketed name") + func loneBracketIsNotUnwrapped() { + #expect(throws: IntentDataError.unknownColumns(["[a]b]"], "people")) { + _ = try planned([PayloadRow(values: ["[a]b]": .text("x")])], into: ["a]b"]) + } + } + + @Test("A key differing only in case finds its column, as the import sheet matches headers") + func caseInsensitiveMatch() throws { + #expect(try planned([PayloadRow(values: ["Name": .text("Ada")])], into: ["name"]) == [["name=Ada"]]) + } + + @Test("A case match two columns share is refused rather than guessed") + func ambiguousCaseMatchIsUnknown() { + #expect(throws: IntentDataError.unknownColumns(["NAME"], "people")) { + _ = try planned([PayloadRow(values: ["NAME": .text("Ada")])], into: ["Name", "name"]) + } + } + + @Test("A key spelled exactly claims its column before a bracketed one can") + func exactSpellingClaimsFirst() { + #expect(throws: IntentDataError.unknownColumns(["[id]"], "people")) { + _ = try planned([PayloadRow(values: ["id": .text("1"), "[id]": .text("2")])], into: ["id"]) + } + } +} diff --git a/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift b/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift new file mode 100644 index 0000000000..f5f9743a48 --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift @@ -0,0 +1,60 @@ +// +// RowPayloadFormatTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct RowPayloadFormatTests { + // SQL Server exports its headers bracketed, so a valid CSV opens with `[`. + @Test("CSV with bracketed headers is read as CSV") + func bracketedHeaderIsCSV() async throws { + let rows = try await RowPayload.parse(data: "[id],[name]\n1,Bob\n", file: nil) + #expect(rows == [PayloadRow(values: ["[id]": .text("1"), "[name]": .text("Bob")])]) + } + + @Test("CSV whose first header opens with a brace is read as CSV") + func braceHeaderIsCSV() async throws { + let rows = try await RowPayload.parse(data: "{key},value\na,1\n", file: nil) + #expect(rows == [PayloadRow(values: ["{key}": .text("a"), "value": .text("1")])]) + } + + @Test("A JSON object and a JSON array of objects are still read as JSON") + func jsonStaysJSON() async throws { + let object = try await RowPayload.parse(data: #"{"name":"Ada"}"#, file: nil) + #expect(object == [PayloadRow(values: ["name": .text("Ada")])]) + let array = try await RowPayload.parse(data: #"[{"name":"Ada"},{"name":"Grace"}]"#, file: nil) + #expect(array.count == 2) + } + + @Test("Malformed JSON that is no CSV either reports the JSON error") + func malformedJSONKeepsItsError() async { + await #expect { + _ = try await RowPayload.parse(data: #"{"name": tru}"#, file: nil) + } throws: { error in + guard case .malformedPayload = error as? IntentDataError else { return false } + return true + } + } + + // Each line splits into the same number of CSV fields, so the text read as CSV with `{"id": 1` for a header. + @Test("JSON with a trailing comma after each line's object keeps the JSON error") + func jsonThatSplitsEvenlyKeepsItsError() async { + await #expect { + _ = try await RowPayload.parse(data: "{\"id\": 1, \"name\": \"a\"},\n{\"id\": 2, \"name\": \"b\"},\n", file: nil) + } throws: { error in + guard case .malformedPayload = error as? IntentDataError else { return false } + return true + } + } + + @Test("A JSON array of something other than objects keeps its own error") + func jsonArrayOfValuesIsNotRetriedAsCSV() async { + await #expect(throws: IntentDataError.jsonArrayContainsNonObject) { + _ = try await RowPayload.parse(data: "[1,2]", file: nil) + } + } +} diff --git a/apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift b/apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift new file mode 100644 index 0000000000..cb077ddcd6 --- /dev/null +++ b/apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift @@ -0,0 +1,64 @@ +// +// AIChatStorageSummaryTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct AIChatStorageSummaryTests { + private let storage = AIChatStorage.shared + + private static func conversation(_ title: String, connectionId: UUID?, updatedAt seconds: TimeInterval) -> AIConversation { + AIConversation( + title: title, + messages: [ + ChatTurnWire(role: .user, blocks: [.text(title)]), + ChatTurnWire(role: .assistant, blocks: [.text("An answer.")]) + ], + createdAt: Date(timeIntervalSince1970: seconds - 500), + updatedAt: Date(timeIntervalSince1970: seconds), + connectionId: connectionId, + connectionName: "Shop" + ) + } + + // A conversation stored before conversations carried a connection id is listed everywhere. + @Test("Summaries carry the stored titles newest first, without messages, for one connection") + func summariesAreFilteredAndOrdered() async throws { + let connectionId = UUID() + let older = Self.conversation("Older question", connectionId: connectionId, updatedAt: 1_000) + let newer = Self.conversation("Newer question", connectionId: connectionId, updatedAt: 3_000) + let legacy = Self.conversation("Legacy question", connectionId: nil, updatedAt: 2_000) + let elsewhere = Self.conversation("Elsewhere", connectionId: UUID(), updatedAt: 4_000) + let written = [older, newer, legacy, elsewhere] + for conversation in written { + await storage.save(conversation) + } + let ids = Set(written.map { $0.id }) + + let summaries = await storage.loadSummaries(connectionId: connectionId) + let listed = summaries.filter { ids.contains($0.id) } + + #expect(listed.map { $0.title } == ["Newer question", "Legacy question", "Older question"]) + #expect(summaries.allSatisfy { $0.messages.isEmpty }) + #expect(summaries.allSatisfy { $0.connectionId == nil || $0.connectionId == connectionId }) + let first = try #require(listed.first) + #expect(first.id == newer.id) + #expect(first.createdAt == newer.createdAt) + #expect(first.updatedAt == newer.updatedAt) + #expect(first.connectionName == "Shop") + #expect(first.schemaVersion == AIConversation.currentSchemaVersion) + + let everywhere = await storage.loadSummaries(connectionId: nil) + #expect(ids.isSubset(of: Set(everywhere.map { $0.id }))) + + let full = await storage.load(id: newer.id) + #expect(full?.messages.count == 2) + + for conversation in written { + await storage.delete(conversation.id) + } + } +} diff --git a/apple/TableProTests/Core/Storage/AppSettingsStorageResetTests.swift b/apple/TableProTests/Core/Storage/AppSettingsStorageResetTests.swift index 3a986cf84a..b7b4918911 100644 --- a/apple/TableProTests/Core/Storage/AppSettingsStorageResetTests.swift +++ b/apple/TableProTests/Core/Storage/AppSettingsStorageResetTests.swift @@ -23,6 +23,29 @@ struct AppSettingsStorageResetTests { #expect(defaults.string(forKey: SidebarPersistenceKey.defaultLayout) == nil) } + @Test("Reset keeps the AI providers, the active one and the AI switch, and resets every other AI setting") + func resetKeepsAIProviders() throws { + let defaults = try pool.make().defaults + let storage = AppSettingsStorage(userDefaults: defaults) + let provider = AIProviderConfig(name: "Work", type: .claude) + var ai = AISettings.default + ai.providers = [provider] + ai.activeProviderID = provider.id + ai.enabled = false + ai.chatMode = .agent + ai.inlineSuggestionsEnabled = true + storage.saveAI(ai) + + storage.resetToDefaults() + + let reset = storage.loadAI() + #expect(reset.providers.map(\.id) == [provider.id]) + #expect(reset.activeProviderID == provider.id) + #expect(!reset.enabled) + #expect(reset.chatMode == AISettings.default.chatMode) + #expect(reset.inlineSuggestionsEnabled == AISettings.default.inlineSuggestionsEnabled) + } + @Test("Reset clears every remembered inspector field height and the geometry field's mode") func resetClearsInspectorFieldPreferences() throws { let defaults = try pool.make().defaults diff --git a/apple/TableProTests/Core/Storage/AppSettingsSyncSideEffectsTests.swift b/apple/TableProTests/Core/Storage/AppSettingsSyncSideEffectsTests.swift new file mode 100644 index 0000000000..0dc7026d58 --- /dev/null +++ b/apple/TableProTests/Core/Storage/AppSettingsSyncSideEffectsTests.swift @@ -0,0 +1,78 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import TableProTheme +import Testing + +@MainActor +private final class SyncSwitchRecorder { + var calls: [Bool] = [] +} + +@MainActor +struct AppSettingsSyncSideEffectsTests { + private let pool = EphemeralDefaultsPool() + + private func makeManager( + flagDefaults: UserDefaults, + recorder: SyncSwitchRecorder, + root: URL + ) throws -> AppSettingsManager { + let tracker = SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: try pool.make().defaults)) + let catalog = ThemeCatalog( + userDirectory: root.appendingPathComponent("Themes", isDirectory: true), + bundleDirectory: nil, + fileTrash: .folder(root.appendingPathComponent("Trash", isDirectory: true)), + syncTracker: tracker + ) + return AppSettingsManager( + storage: AppSettingsStorage(userDefaults: try pool.make().defaults), + themeEngine: ThemeEngine(catalog: catalog), + syncTracker: tracker, + appEvents: AppEvents(), + passwordSyncDefaults: flagDefaults, + switchSync: { recorder.calls.append($0) } + ) + } + + private static func makeRoot() -> URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("AppSettingsSyncSideEffectsTests-\(UUID().uuidString)", isDirectory: true) + } + + @Test("Turning sync off from anywhere, a reset included, stops password sync and the coordinator") + func turningSyncOffReachesTheKeychainFlag() throws { + let flagDefaults = try pool.make().defaults + let recorder = SyncSwitchRecorder() + let root = Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let manager = try makeManager(flagDefaults: flagDefaults, recorder: recorder, root: root) + + manager.sync.enabled = true + manager.sync.syncConnections = true + manager.sync.syncPasswords = true + #expect(flagDefaults.bool(forKey: KeychainHelper.passwordSyncEnabledKey)) + #expect(recorder.calls == [true]) + + manager.sync = .default + + #expect(!flagDefaults.bool(forKey: KeychainHelper.passwordSyncEnabledKey)) + #expect(recorder.calls == [true, false]) + } + + @Test("A change that leaves sync on or off as it was does not touch the coordinator") + func categoryChangeLeavesTheCoordinatorAlone() throws { + let flagDefaults = try pool.make().defaults + let recorder = SyncSwitchRecorder() + let root = Self.makeRoot() + defer { try? FileManager.default.removeItem(at: root) } + let manager = try makeManager(flagDefaults: flagDefaults, recorder: recorder, root: root) + + manager.sync.syncPasswords = true + manager.sync.syncTableFavorites = false + + #expect(recorder.calls.isEmpty) + #expect(!flagDefaults.bool(forKey: KeychainHelper.passwordSyncEnabledKey)) + } +} diff --git a/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift b/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift new file mode 100644 index 0000000000..b485eb8cd0 --- /dev/null +++ b/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift @@ -0,0 +1,170 @@ +// +// AppSettingsValueColorsTests.swift +// TableProTests +// + +import CloudKit +import Combine +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct AppSettingsValueColorsTests { + @MainActor + private final class Harness { + let environment: SyncTestEnvironment + let storage: AppSettingsStorage + let settings: AppSettingsManager + let coordinator: SyncCoordinator + private(set) var sentEvents = 0 + private var subscription: AnyCancellable? + + var tracker: SyncChangeTracker { environment.tracker } + + init(label: String, saved: ValueColorSettings? = nil) throws { + let environment = try SyncTestEnvironment(label: label) + let storage = AppSettingsStorage(userDefaults: environment.defaults) + if let saved { storage.saveValueColors(saved) } + let events = AppEvents() + let catalog = ThemeCatalog( + userDirectory: environment.directory.appendingPathComponent("Themes", isDirectory: true), + bundleDirectory: nil, + fileTrash: .folder(environment.directory.appendingPathComponent("Trash", isDirectory: true)), + syncTracker: environment.tracker + ) + let settings = AppSettingsManager( + storage: storage, + themeEngine: ThemeEngine(catalog: catalog), + syncTracker: environment.tracker, + appEvents: events + ) + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let columnLayouts = environment.columnLayouts + let base = environment.makeCoordinator(transport: transport) + self.environment = environment + self.storage = storage + self.settings = settings + coordinator = SyncCoordinator( + services: base.services.withAppSettings(settings, storage: storage), + recordCache: environment.recordCache, + transport: transport, + makeNetworkMonitor: { nil }, + columnLayouts: columnLayouts, + themeCatalog: catalog, + licenseGate: { nil }, + pushRegistrar: .inert + ) + subscription = events.valueColorSettingsChanged.sink { [weak self] in self?.sentEvents += 1 } + } + + func pull(_ category: String, _ data: Data) async -> Bool { + let record = SyncRecordMapper.toCKRecord(category: category, settingsData: data, in: SyncTestEnvironment.zoneID) + return await coordinator.applyPullResult(PullResult(changedRecords: [record], deletedRecordIDs: [], newToken: nil)) + } + } + + // MARK: - Manager + + @Test("Turning Color values by type on saves it, tells the open grids and marks its record for sync") + func changeSavesSendsAndMarks() throws { + let harness = try Harness(label: "value-colors-change") + + harness.settings.valueColors = ValueColorSettings(colorsValuesByType: true) + + #expect(harness.storage.loadValueColors().colorsValuesByType) + #expect(harness.sentEvents == 1) + #expect(harness.tracker.dirtyRecords(for: .settings) == [AppSettingsCategory.valueColors]) + } + + @Test("Writing the value colors it already holds tells no grid and marks nothing") + func unchangedWriteDoesNothing() throws { + let harness = try Harness(label: "value-colors-same", saved: ValueColorSettings(colorsValuesByType: true)) + + harness.settings.valueColors = ValueColorSettings(colorsValuesByType: true) + + #expect(harness.sentEvents == 0) + #expect(harness.tracker.dirtyRecords(for: .settings).isEmpty) + } + + // MARK: - Sync + + @Test("The outgoing value colors record carries what this Mac saved") + func outgoingRecordCarriesTheSavedSetting() throws { + let harness = try Harness(label: "value-colors-out", saved: ValueColorSettings(colorsValuesByType: true)) + + let data = try #require(harness.coordinator.settingsData(for: AppSettingsCategory.valueColors)) + + #expect(try JSONDecoder().decode(ValueColorSettings.self, from: data) == ValueColorSettings(colorsValuesByType: true)) + } + + @Test("A pulled value colors record turns the setting on and recolors the grids, without marking it for push") + func pulledRecordAppliesWithoutMarking() async throws { + let harness = try Harness(label: "value-colors-in") + let data = try JSONEncoder().encode(ValueColorSettings(colorsValuesByType: true)) + + let acknowledged = await harness.pull(AppSettingsCategory.valueColors, data) + + #expect(acknowledged) + #expect(harness.settings.valueColors.colorsValuesByType) + #expect(harness.storage.loadValueColors().colorsValuesByType) + #expect(harness.sentEvents == 1) + #expect(harness.tracker.dirtyRecords(for: .settings).isEmpty) + } + + @Test("A Data Grid record from a Mac that predates value colors leaves them as they are") + func olderDataGridRecordKeepsValueColors() async throws { + let harness = try Harness(label: "value-colors-older-grid", saved: ValueColorSettings(colorsValuesByType: true)) + var dataGrid = DataGridSettings.default + dataGrid.showRowNumbers.toggle() + + let acknowledged = await harness.pull(AppSettingsCategory.dataGrid, try JSONEncoder().encode(dataGrid)) + + #expect(acknowledged) + #expect(harness.settings.dataGrid.showRowNumbers == dataGrid.showRowNumbers) + #expect(harness.settings.valueColors.colorsValuesByType) + #expect(harness.storage.loadValueColors().colorsValuesByType) + #expect(harness.sentEvents == 0) + } +} + +@MainActor +private extension AppServices { + func withAppSettings(_ appSettings: AppSettingsManager, storage: AppSettingsStorage) -> AppServices { + AppServices( + appEvents: appEvents, + appSettings: appSettings, + appSettingsStorage: storage, + connectionStorage: connectionStorage, + databaseManager: databaseManager, + pluginManager: pluginManager, + schemaService: schemaService, + schemaRefreshService: schemaRefreshService, + schemaProviderRegistry: schemaProviderRegistry, + catalogChangeService: catalogChangeService, + sqlFavoriteManager: sqlFavoriteManager, + favoriteTablesStorage: favoriteTablesStorage, + favoriteDatabasesStorage: favoriteDatabasesStorage, + tableFolderStorage: tableFolderStorage, + aiChatStorage: aiChatStorage, + aiKeyStorage: aiKeyStorage, + aiAccessApprovals: aiAccessApprovals, + groupStorage: groupStorage, + tagStorage: tagStorage, + sshProfileStorage: sshProfileStorage, + credentialProfileStorage: credentialProfileStorage, + featureAccess: featureAccess, + syncMetadataStorage: syncMetadataStorage, + favoritesExpansionState: favoritesExpansionState, + linkedFolderWatcher: linkedFolderWatcher, + queryHistoryManager: queryHistoryManager, + dateFormattingService: dateFormattingService, + copilotService: copilotService, + mcpServerManager: mcpServerManager, + syncTracker: syncTracker, + welcomeRouter: welcomeRouter, + telemetry: telemetry + ) + } +} diff --git a/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift b/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift index 88944ae2a0..3a2d06c3ed 100644 --- a/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift +++ b/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift @@ -100,7 +100,6 @@ struct AppearanceModeStorageTests { let catalog = ThemeCatalog( userDirectory: root.appendingPathComponent("Themes", isDirectory: true), bundleDirectory: nil, - notificationCenter: NotificationCenter(), fileTrash: .folder(root.appendingPathComponent("Trash", isDirectory: true)), syncTracker: tracker ) diff --git a/apple/TableProTests/Core/Storage/ConnectionApprovalStoreTests.swift b/apple/TableProTests/Core/Storage/ConnectionApprovalStoreTests.swift index 445e2a85d3..7de11bae94 100644 --- a/apple/TableProTests/Core/Storage/ConnectionApprovalStoreTests.swift +++ b/apple/TableProTests/Core/Storage/ConnectionApprovalStoreTests.swift @@ -6,6 +6,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSSHConfig import Testing @MainActor @@ -143,6 +144,25 @@ struct ConnectionApprovalStoreTests { #expect(!store.isApproved(connection)) } + @Test("An approval through an SSH profile asks again once the profile moves to another server") + func movedProfileAsksAgain() { + let profiles = SSHProfileStorage.shared + let saved = profiles.loadProfiles() + var profile = SSHProfile(name: "Jump", host: "bastion-a.example", port: 22, username: "ops", authMethod: .password) + #expect(profiles.saveProfilesWithoutSync(saved + [profile])) + defer { profiles.saveProfilesWithoutSync(saved) } + var connection = referencing() + connection.sshTunnelMode = .profile(id: profile.id, snapshot: profile.toSSHConfiguration()) + let store = makeStore() + store.approve(connection) + #expect(store.isApproved(connection)) + + profile.host = "bastion-b.example" + #expect(profiles.saveProfilesWithoutSync(saved + [profile])) + + #expect(!store.isApproved(connection)) + } + @Test("Approving another destination keeps the earlier one") func approvalsAccumulatePerDestination() { let store = makeStore() diff --git a/apple/TableProTests/Core/Storage/ConnectionLocalStatePurgeTests.swift b/apple/TableProTests/Core/Storage/ConnectionLocalStatePurgeTests.swift index ee2543d04e..206db0c446 100644 --- a/apple/TableProTests/Core/Storage/ConnectionLocalStatePurgeTests.swift +++ b/apple/TableProTests/Core/Storage/ConnectionLocalStatePurgeTests.swift @@ -256,4 +256,29 @@ struct ConnectionLocalStatePurgeTests { defaults.string(forKey: RowInspectorState.viewModeKey(kept)) == InspectorViewMode.json.rawValue ) } + + @Test("Purging a connection removes its query tabs' versions and leaves another's") + func purgeClearsQueryTabVersions() async { + let (history, _) = makeHistory() + let versions = QueryTabVersionStorage( + databaseURL: FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("purge_tab_versions_\(UUID().uuidString).db"), + announce: { _ in }, + removeDatabaseOnDeinit: true + ) + let deleted = UUID() + let kept = UUID() + let deletedTab = UUID() + let keptTab = UUID() + await versions.record(tabId: deletedTab, connectionId: deleted, text: "SELECT secret", reason: .run) + await versions.record(tabId: keptTab, connectionId: kept, text: "SELECT 1", reason: .run) + + await ConnectionLocalState.purgeAsyncStores( + [deleted], origin: .remote, queryHistory: history, queryTabVersions: versions + ) + + #expect(versions.summaries(tabId: deletedTab).isEmpty) + #expect(versions.summaries(tabId: keptTab).count == 1) + } } diff --git a/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift b/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift index 9c7ebc52b3..d4b968352a 100644 --- a/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift +++ b/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift @@ -1,102 +1,14 @@ import Foundation @testable import TablePro import TableProSyncTransport +import TableProTestSupport import Testing -// One synced and one local value per key, as the keychain holds them. Nothing here removes a synced value except delete. -private final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { - private let lock = NSLock() - private var synced: [String: String] = [:] - private var local: [String: String] = [:] - private var unreadableKeys: Set = [] - private var failingKeys: Set = [] - - func seedSynced(_ value: String, forKey key: String) { - lock.lock() - defer { lock.unlock() } - synced[key] = value - } - - func seedLocal(_ value: String, forKey key: String) { - lock.lock() - defer { lock.unlock() } - local[key] = value - } - - func makeUnreadable(_ key: String) { - lock.lock() - defer { lock.unlock() } - unreadableKeys.insert(key) - } - - func makeReadable(_ key: String) { - lock.lock() - defer { lock.unlock() } - unreadableKeys.remove(key) - } - - func failWrites(to key: String) { - lock.lock() - defer { lock.unlock() } - failingKeys.insert(key) - } - - func syncedValue(forKey key: String) -> String? { - lock.lock() - defer { lock.unlock() } - return synced[key] - } - - func localValue(forKey key: String) -> String? { - lock.lock() - defer { lock.unlock() } - return local[key] - } - - @discardableResult - func writeString(_ value: String, forKey key: String) -> Bool { - writeString(value, forKey: key, deviceOnly: false) - } - - // Password sync is on in this model, so a write that is not device-only is a synced one. - @discardableResult - func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { - lock.lock() - defer { lock.unlock() } - guard !failingKeys.contains(key) else { return false } - if deviceOnly { - local[key] = value - } else { - synced[key] = value - local[key] = nil - } - return true - } - - func readStringResult(forKey key: String) -> KeychainStringResult { - readStringResult(forKey: key, deviceOnly: false) - } - - func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult { - lock.lock() - defer { lock.unlock() } - guard !unreadableKeys.contains(key) else { return .locked } - let value = deviceOnly ? local[key] ?? synced[key] : synced[key] ?? local[key] - return value.map { .found($0) } ?? .notFound - } - - func delete(forKey key: String) { - lock.lock() - defer { lock.unlock() } - synced[key] = nil - local[key] = nil - } -} - @MainActor struct ConnectionSecretScopeTests { private let keychain = ScopeRecordingKeychain() private let suite: EphemeralDefaults + private let tracker: SyncChangeTracker private let storage: ConnectionStorage init() throws { @@ -105,10 +17,11 @@ struct ConnectionSecretScopeTests { .appendingPathComponent("tablepro-tests") .appendingPathComponent("secret-scope-\(UUID().uuidString)", isDirectory: true) try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + tracker = SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)) storage = ConnectionStorage( fileURL: directory.appendingPathComponent("connections.json"), userDefaults: suite.defaults, - syncTracker: SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)), + syncTracker: tracker, keychain: keychain, appEvents: AppEvents(), integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: InMemoryKeychain())) @@ -150,8 +63,8 @@ struct ConnectionSecretScopeTests { #expect(keychain.localValue(forKey: passwordKey(connection.id)) == nil) } - @Test("Marking a connection Local only copies its secrets to this Mac and leaves the synced ones for other devices") - func markingLocalOnlyCopiesAndKeepsTheSyncedSecrets() { + @Test("Marking a connection Local only moves its secrets to this Mac and off iCloud Keychain") + func markingLocalOnlyMovesTheSecretsOffICloud() { let connection = makeConnection(localOnly: false) #expect(storage.addConnection(connection, password: "hunter2")) storage.saveSSLClientKeyPassphrase("passphrase", for: connection.id) @@ -161,8 +74,78 @@ struct ConnectionSecretScopeTests { #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") #expect(keychain.localValue(forKey: sslKey(connection.id)) == "passphrase") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == nil) + #expect(keychain.syncedValue(forKey: sslKey(connection.id)) == nil) + } + + @Test("A local write that fails keeps the synced copy") + func aLocalWriteThatFailsKeepsTheSyncedCopy() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + keychain.failWrites(to: passwordKey(connection.id)) + + _ = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == nil) + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + } + + @Test("A secret that cannot be read keeps its synced copy") + func anUnreadableSecretKeepsItsSyncedCopy() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + keychain.makeUnreadable(passwordKey(connection.id)) + + _ = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + + keychain.makeReadable(passwordKey(connection.id)) #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") - #expect(keychain.syncedValue(forKey: sslKey(connection.id)) == "passphrase") + } + + @Test("A synced copy the keychain refused to remove is removed at the next launch") + func aRefusedSyncedRemovalIsRetriedAtLaunch() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + keychain.failSyncedRemoval(of: passwordKey(connection.id)) + _ = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + + keychain.allowSyncedRemoval(of: passwordKey(connection.id)) + storage.keepLocalOnlySecretsOnDeviceIfNeeded() + + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == nil) + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") + } + + @Test("A refused removal is dropped once the connection is back in sync") + func aRefusedRemovalIsDroppedWhenBackInSync() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + keychain.failSyncedRemoval(of: passwordKey(connection.id)) + _ = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + _ = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = false } + // A local copy would let a stale retry go through, so its absence is not what keeps the synced one. + keychain.seedLocal("hunter2", forKey: passwordKey(connection.id)) + + keychain.allowSyncedRemoval(of: passwordKey(connection.id)) + storage.keepLocalOnlySecretsOnDeviceIfNeeded() + + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + } + + @Test("A synced copy that cannot be removed leaves the other secrets moved") + func aFailedSyncedRemovalStillMovesTheOtherSecrets() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + storage.saveTOTPSecret("JBSWY3DP", for: connection.id) + keychain.failSyncedRemoval(of: passwordKey(connection.id)) + + _ = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.localValue(forKey: totpKey(connection.id)) == "JBSWY3DP") + #expect(keychain.syncedValue(forKey: totpKey(connection.id)) == nil) } @Test("A Local only connection reads and changes its own copy, never the one another device synced") @@ -185,7 +168,7 @@ struct ConnectionSecretScopeTests { let marked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } #expect(marked) storage.savePassword("v2", for: connection.id) - #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "v1") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == nil) let unmarked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = false } #expect(unmarked) @@ -221,7 +204,8 @@ struct ConnectionSecretScopeTests { #expect(marked) #expect(storage.loadPassword(for: connection.id) == "in-use") - #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "in-use") + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "in-use") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == nil) } @Test("The launch move copies an older Local only connection's secrets and leaves the synced ones in place") @@ -373,4 +357,82 @@ struct ConnectionSecretScopeTests { #expect(keychain.syncedValue(forKey: passwordKey(id)) == nil) #expect(keychain.syncedValue(forKey: tokenSecretKey) == nil) } + + @Test("A delete from another device removes this Mac's own secrets and leaves the ones iCloud Keychain holds") + func remoteDeleteKeepsTheSyncedSecrets() { + let connection = TestFixtures.makeConnection(type: .mongodb) + let tokenKey = "com.TablePro.plugin.mongoAWSSessionToken.\(connection.id.uuidString)" + for key in [passwordKey(connection.id), totpKey(connection.id), tokenKey] { + keychain.seedLocal("local", forKey: key) + keychain.seedSynced("synced", forKey: key) + } + + storage.deleteSecrets(of: [connection], origin: .remote) + + for key in [passwordKey(connection.id), totpKey(connection.id), tokenKey] { + #expect(keychain.localValue(forKey: key) == nil) + #expect(keychain.syncedValue(forKey: key) == "synced") + } + } + + @Test("Deleting a connection here removes every secret it has, the synced ones too") + func localDeleteRemovesEverySecret() { + let connection = TestFixtures.makeConnection(type: .mongodb) + #expect(storage.addConnection(connection, password: "hunter2")) + let tokenKey = "com.TablePro.plugin.mongoAWSSessionToken.\(connection.id.uuidString)" + for key in [totpKey(connection.id), sslKey(connection.id), tokenKey] { + keychain.seedLocal("local", forKey: key) + keychain.seedSynced("synced", forKey: key) + } + + #expect(storage.deleteConnection(connection)) + + for key in [passwordKey(connection.id), totpKey(connection.id), sslKey(connection.id), tokenKey] { + #expect(keychain.localValue(forKey: key) == nil) + #expect(keychain.syncedValue(forKey: key) == nil) + } + } + + enum LocalOnlyHistory: CaseIterable { + case pushedBeforeMarkedLocalOnly + case pulledFromAnotherDevice + case localOnlyFromTheStart + } + + // Local only from the start too: nothing on this Mac can prove no other device holds the row. + @Test("A Local only delete never removes the synced secrets, whatever the row's history", + arguments: LocalOnlyHistory.allCases, [false, true]) + func localOnlyDeleteKeepsTheSyncedSecrets(_ history: LocalOnlyHistory, inBatch: Bool) async throws { + var localOnly = makeConnection(localOnly: history == .localOnlyFromTheStart) + if history == .pulledFromAnotherDevice { + localOnly.origin = .synced + localOnly.localOnly = true + } + let synced = makeConnection(localOnly: false) + #expect(storage.saveConnections([localOnly, synced])) + if history == .pushedBeforeMarkedLocalOnly { + let marked = storage.mutateConnections(ids: [localOnly.id]) { $0.localOnly = true } + #expect(marked) + localOnly.localOnly = true + } + for connection in [localOnly, synced] { + keychain.seedLocal("local", forKey: totpKey(connection.id)) + keychain.seedSynced("synced", forKey: totpKey(connection.id)) + } + + if inBatch { + #expect(storage.deleteConnections([localOnly, synced])) + } else { + #expect(storage.deleteConnection(localOnly)) + } + + #expect(keychain.localValue(forKey: totpKey(localOnly.id)) == nil) + #expect(keychain.syncedValue(forKey: totpKey(localOnly.id)) == "synced") + if inBatch { + #expect(keychain.localValue(forKey: totpKey(synced.id)) == nil) + #expect(keychain.syncedValue(forKey: totpKey(synced.id)) == nil) + } + // The purge releases a Local only owner from a Task; waiting keeps that defaults write inside the case. + try await waitUntil { tracker.ownersKeptOffSync.isEmpty } + } } diff --git a/apple/TableProTests/Core/Storage/ConnectionStoragePersistenceTests.swift b/apple/TableProTests/Core/Storage/ConnectionStoragePersistenceTests.swift index 3dab06d1c9..791d5a49b0 100644 --- a/apple/TableProTests/Core/Storage/ConnectionStoragePersistenceTests.swift +++ b/apple/TableProTests/Core/Storage/ConnectionStoragePersistenceTests.swift @@ -51,6 +51,28 @@ struct ConnectionStoragePersistenceTests { #expect(reloaded.contains { $0.id == connection.id }) } + @Test("A missing connections.json is a fresh, trusted library") + func missingFileIsAFreshLibrary() { + #expect(storage.loadConnections().isEmpty) + #expect(!storage.lastLoadFailed) + #expect(storage.storeIsTrusted) + } + + @Test("A connections.json that cannot be read is a failed load, never saved over") + func unreadableFileIsNeverOverwritten() throws { + try FileManager.default.createDirectory(at: fileURL, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: fileURL) } + + #expect(storage.loadConnections().isEmpty) + #expect(storage.lastLoadFailed) + #expect(!storage.storeIsTrusted) + #expect(!storage.saveConnections([DatabaseConnection(name: "Fresh")])) + + var isDirectory: ObjCBool = false + #expect(FileManager.default.fileExists(atPath: fileURL.path, isDirectory: &isDirectory)) + #expect(isDirectory.boolValue) + } + @Test("updateSafeModeLevel writes the new level through to disk") func updateSafeModeLevelWritesThrough() { let connection = DatabaseConnection( diff --git a/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift b/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift index e364b79c35..df046e9ad8 100644 --- a/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift +++ b/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift @@ -246,4 +246,35 @@ struct KeychainItemScopeTests { #expect(keychain.readStringResult(forKey: key) == .notFound) } + + @Test("A device-only delete removes this Mac's item and leaves the one iCloud Keychain holds", arguments: [true, false]) + func deviceOnlyDeleteKeepsTheSyncedItem(passwordSync: Bool) { + store.seed("local", account: key, synchronizable: false) + store.seed("synced", account: key, synchronizable: true) + + helper(passwordSync: passwordSync).delete(forKey: key, deviceOnly: true) + + #expect(store.value(account: key, synchronizable: false) == nil) + #expect(store.value(account: key, synchronizable: true) == "synced") + } + + @Test("Removing the synced copy keeps this Mac's own item, whatever the password sync setting", arguments: [true, false]) + func removingTheSyncedCopyKeepsTheLocalItem(passwordSync: Bool) { + store.seed("local", account: key, synchronizable: false) + store.seed("synced", account: key, synchronizable: true) + + #expect(helper(passwordSync: passwordSync).removeSyncedCopy(forKey: key)) + + #expect(store.value(account: key, synchronizable: false) == "local") + #expect(store.value(account: key, synchronizable: true) == nil) + } + + @Test("Removing the synced copy is refused while this Mac has no item of its own") + func removingTheSyncedCopyIsRefusedWithoutALocalItem() { + store.seed("synced", account: key, synchronizable: true) + + #expect(!helper(passwordSync: true).removeSyncedCopy(forKey: key)) + + #expect(store.value(account: key, synchronizable: true) == "synced") + } } diff --git a/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift b/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift new file mode 100644 index 0000000000..ca4bd4c8e1 --- /dev/null +++ b/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift @@ -0,0 +1,33 @@ +import Foundation +@testable import TablePro +import Testing + +struct LinkedSQLIndexDamageTests { + @Test("A damaged linked SQL index is discarded and indexing works again") + func damagedIndexIsRebuilt() async throws { + let scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("linked-index-damage-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + let url = scratch.appendingPathComponent("linked_sql_index.db") + try Data(repeating: 0x2A, count: 4_096).write(to: url) + let index = LinkedSQLIndex(databaseURL: url, removeDatabaseOnDeinit: true) + let folderId = UUID() + let file = LinkedSQLIndex.IndexedFile( + relativePath: "revenue.sql", + name: "revenue", + keyword: "rev", + description: nil, + mtime: 1_800_000_000, + fileSize: 8, + encodingName: "utf-8" + ) + + await index.synchronize(folderId: folderId, files: [file]) + + #expect(await index.allFolderIds() == [folderId]) + let leftovers = try FileManager.default.contentsOfDirectory(atPath: scratch.path(percentEncoded: false)) + .filter { $0.contains(".damaged-") } + #expect(leftovers.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Storage/LocalSQLiteStoreOpenerTests.swift b/apple/TableProTests/Core/Storage/LocalSQLiteStoreOpenerTests.swift new file mode 100644 index 0000000000..2c496a36d8 --- /dev/null +++ b/apple/TableProTests/Core/Storage/LocalSQLiteStoreOpenerTests.swift @@ -0,0 +1,147 @@ +import Foundation +import SQLite3 +@testable import TablePro +import Testing + +struct LocalSQLiteStoreOpenerTests { + private let scratch: URL + private let fixedDate = Date(timeIntervalSince1970: 1_800_000_000) + + init() throws { + scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("local-store-opener-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + } + + private func makeDatabase(at url: URL, rows: Int) { + var db: OpaquePointer? + guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return } + defer { sqlite3_close_v2(db) } + sqlite3_exec(db, "CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT);", nil, nil, nil) + sqlite3_exec(db, """ + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < \(rows)) + INSERT INTO t (v) SELECT hex(randomblob(100)) FROM n; + """, nil, nil, nil) + } + + private func scalarInt(_ db: OpaquePointer, _ sql: String) -> Int? { + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK, + sqlite3_step(statement) == SQLITE_ROW + else { return nil } + return Int(sqlite3_column_int64(statement, 0)) + } + + private func directoryEntries() throws -> [String] { + try FileManager.default.contentsOfDirectory(atPath: scratch.path(percentEncoded: false)).sorted() + } + + @Test("A healthy store opens in place and nothing is set aside") + func healthyStoreOpensInPlace() throws { + let url = scratch.appendingPathComponent("store.db") + makeDatabase(at: url, rows: 10) + + let opening = LocalSQLiteStoreOpener.open(at: url, now: fixedDate) + guard case .opened(let handle, let setAside) = opening else { + Issue.record("A healthy store did not open") + return + } + defer { sqlite3_close_v2(handle) } + + #expect(setAside == nil) + #expect(scalarInt(handle, "SELECT COUNT(*) FROM t;") == 10) + } + + @Test("A file that is not a database is set aside unchanged and a fresh store opens in its place") + func notADatabaseIsSetAside() throws { + let url = scratch.appendingPathComponent("store.db") + let damaged = Data(repeating: 0x2A, count: 4_096) + try damaged.write(to: url) + let expectedSetAside = LocalSQLiteStoreOpener.setAsideURL(for: url, at: fixedDate) + + let opening = LocalSQLiteStoreOpener.open(at: url, now: fixedDate) + guard case .opened(let handle, let setAside) = opening else { + Issue.record("No fresh store opened after setting the damaged one aside") + return + } + defer { sqlite3_close_v2(handle) } + + let moved = try #require(setAside) + #expect(moved.storeURL == url) + #expect(moved.setAsideURL == expectedSetAside) + #expect(try Data(contentsOf: moved.setAsideURL) == damaged) + #expect(sqlite3_exec(handle, "CREATE TABLE fresh (id INTEGER);", nil, nil, nil) == SQLITE_OK) + #expect(scalarInt(handle, "SELECT COUNT(*) FROM fresh;") == 0) + } + + @Test("A database with a damaged page is set aside byte for byte") + func damagedPageIsSetAside() throws { + let url = scratch.appendingPathComponent("store.db") + makeDatabase(at: url, rows: 2_000) + let handle = try FileHandle(forWritingTo: url) + try handle.seek(toOffset: 4_096 * 2) + try handle.write(contentsOf: Data(repeating: 0xFF, count: 4_096)) + try handle.close() + let damaged = try Data(contentsOf: url) + + let opening = LocalSQLiteStoreOpener.open(at: url, now: fixedDate) + guard case .opened(let fresh, let setAside) = opening else { + Issue.record("No fresh store opened after setting the damaged one aside") + return + } + defer { sqlite3_close_v2(fresh) } + + let moved = try #require(setAside) + #expect(try Data(contentsOf: moved.setAsideURL) == damaged) + #expect(scalarInt(fresh, "SELECT COUNT(*) FROM sqlite_master;") == 0) + } + + @Test("A store that cannot be opened is reported unavailable and nothing is created") + func unopenableStoreIsUnavailable() throws { + let blocker = scratch.appendingPathComponent("not-a-directory") + #expect(FileManager.default.createFile(atPath: blocker.path(percentEncoded: false), contents: Data())) + + let opening = LocalSQLiteStoreOpener.open(at: blocker.appendingPathComponent("store.db"), now: fixedDate) + + guard case .unavailable = opening else { + Issue.record("A store under a file opened") + return + } + #expect(try directoryEntries() == ["not-a-directory"]) + } + + @Test("A damaged store that cannot be moved stays where it is and nothing replaces it") + func damagedStoreThatCannotMoveStaysInPlace() throws { + let url = scratch.appendingPathComponent("store.db") + let damaged = Data(repeating: 0x2A, count: 4_096) + try damaged.write(to: url) + let scratchPath = scratch.path(percentEncoded: false) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: scratchPath) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: scratchPath) } + + let opening = LocalSQLiteStoreOpener.open(at: url, now: fixedDate) + + guard case .unavailable = opening else { + Issue.record("A damaged store was replaced although it could not be set aside") + return + } + #expect(try Data(contentsOf: url) == damaged) + #expect(try directoryEntries() == ["store.db"]) + } + + @Test("A set-aside name already taken gets the next free one") + func setAsideNameSkipsATakenOne() throws { + let url = scratch.appendingPathComponent("store.db") + let first = LocalSQLiteStoreOpener.setAsideURL(for: url, at: fixedDate) + #expect(FileManager.default.createFile(atPath: first.path(percentEncoded: false) + "-wal", contents: Data())) + + let second = LocalSQLiteStoreOpener.setAsideURL(for: url, at: fixedDate) + + #expect(second != first) + #expect(second.deletingLastPathComponent() == first.deletingLastPathComponent()) + #expect(second.lastPathComponent.hasPrefix("store.damaged-")) + #expect(second.lastPathComponent.hasSuffix("-1.db")) + } +} diff --git a/apple/TableProTests/Core/Storage/PluginSecureFieldMigrationTests.swift b/apple/TableProTests/Core/Storage/PluginSecureFieldMigrationTests.swift new file mode 100644 index 0000000000..8bf5826f86 --- /dev/null +++ b/apple/TableProTests/Core/Storage/PluginSecureFieldMigrationTests.swift @@ -0,0 +1,156 @@ +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct PluginSecureFieldMigrationTests { + private static let fieldId = "mongoAWSSessionToken" + private static let doneKey = "com.TablePro.pluginSecureFieldsMovedToKeychain" + + private let keychain = ScopeRecordingKeychain() + private let suite: EphemeralDefaults + private let fileURL: URL + private let storage: ConnectionStorage + + init() throws { + suite = try EphemeralDefaults() + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("plugin-secure-fields-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + fileURL = directory.appendingPathComponent("connections.json") + storage = ConnectionStorage( + fileURL: fileURL, + userDefaults: suite.defaults, + syncTracker: SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)), + keychain: keychain, + appEvents: AppEvents(), + integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: InMemoryKeychain())) + ) + } + + private func tokenKey(_ id: UUID) -> String { + "com.TablePro.plugin.\(Self.fieldId).\(id.uuidString)" + } + + private func legacyConnection() -> DatabaseConnection { + var connection = TestFixtures.makeConnection(type: .mongodb) + connection.additionalFields = [Self.fieldId: "FQoG"] + return connection + } + + private func fieldOnDisk(_ id: UUID) -> String? { + storage.invalidateCache() + return storage.loadConnection(id: id)?.additionalFields[Self.fieldId] + } + + private var isDone: Bool { + suite.defaults.bool(forKey: Self.doneKey) + } + + @Test("A plain-text secure field moves to the Keychain and leaves the record") + func fieldMovesToTheKeychain() { + let connection = legacyConnection() + #expect(storage.saveConnections([connection])) + + storage.migratePluginSecureFieldsIfNeeded() + + #expect(keychain.syncedValue(forKey: tokenKey(connection.id)) == "FQoG") + #expect(fieldOnDisk(connection.id) == nil) + #expect(isDone) + } + + @Test("A field whose Keychain write fails stays in the record, and the next launch moves it") + func failedWriteKeepsTheField() { + let connection = legacyConnection() + #expect(storage.saveConnections([connection])) + keychain.failWrites(to: tokenKey(connection.id)) + + storage.migratePluginSecureFieldsIfNeeded() + + #expect(fieldOnDisk(connection.id) == "FQoG") + #expect(!isDone) + + keychain.allowWrites(to: tokenKey(connection.id)) + storage.migratePluginSecureFieldsIfNeeded() + + #expect(keychain.syncedValue(forKey: tokenKey(connection.id)) == "FQoG") + #expect(fieldOnDisk(connection.id) == nil) + #expect(isDone) + } + + @Test("A value already in the Keychain wins over the plain-text copy, which only leaves the record") + func storedValueWins() { + let connection = legacyConnection() + #expect(storage.saveConnections([connection])) + keychain.seedSynced("saved-on-another-device", forKey: tokenKey(connection.id)) + + storage.migratePluginSecureFieldsIfNeeded() + + #expect(keychain.syncedValue(forKey: tokenKey(connection.id)) == "saved-on-another-device") + #expect(fieldOnDisk(connection.id) == nil) + #expect(isDone) + } + + @Test("A Keychain item that cannot be read keeps the plain-text field until it can") + func unreadableItemKeepsTheField() { + let connection = legacyConnection() + #expect(storage.saveConnections([connection])) + keychain.seedSynced("saved-on-another-device", forKey: tokenKey(connection.id)) + keychain.makeUnreadable(tokenKey(connection.id)) + + storage.migratePluginSecureFieldsIfNeeded() + + #expect(keychain.syncedValue(forKey: tokenKey(connection.id)) == "saved-on-another-device") + #expect(fieldOnDisk(connection.id) == "FQoG") + #expect(!isDone) + } + + @Test("An unreadable connections.json is not marked done, so the move runs once it reads again") + func unreadableLibraryRetries() throws { + try Data("not json".utf8).write(to: fileURL) + + storage.migratePluginSecureFieldsIfNeeded() + #expect(!isDone) + + try FileManager.default.removeItem(at: fileURL) + storage.invalidateCache() + let connection = legacyConnection() + #expect(storage.saveConnections([connection])) + storage.migratePluginSecureFieldsIfNeeded() + + #expect(keychain.syncedValue(forKey: tokenKey(connection.id)) == "FQoG") + #expect(fieldOnDisk(connection.id) == nil) + } + + @Test("A connection whose plugin is not registered keeps the move pending") + func unregisteredPluginKeepsItPending() { + let known = legacyConnection() + let unknown = TestFixtures.makeConnection(type: DatabaseType(rawValue: "UnregisteredEngine-\(UUID().uuidString)")) + #expect(storage.saveConnections([known, unknown])) + + storage.migratePluginSecureFieldsIfNeeded() + + #expect(keychain.syncedValue(forKey: tokenKey(known.id)) == "FQoG") + #expect(fieldOnDisk(known.id) == nil) + #expect(!isDone) + } + + @Test("A connections.json changed outside TablePro moves nothing into the Keychain") + func untrustedLibraryMovesNothing() throws { + let connection = legacyConnection() + #expect(storage.saveConnections([connection])) + var edited = try Data(contentsOf: fileURL) + edited.append(contentsOf: Array(" ".utf8)) + try edited.write(to: fileURL) + storage.invalidateCache() + + storage.migratePluginSecureFieldsIfNeeded() + + #expect(!storage.storeIsTrusted) + #expect(keychain.syncedValue(forKey: tokenKey(connection.id)) == nil) + #expect(fieldOnDisk(connection.id) == "FQoG") + #expect(!isDone) + } +} diff --git a/apple/TableProTests/Core/Storage/QueryHistoryStoreRecoveryTests.swift b/apple/TableProTests/Core/Storage/QueryHistoryStoreRecoveryTests.swift new file mode 100644 index 0000000000..6890b01285 --- /dev/null +++ b/apple/TableProTests/Core/Storage/QueryHistoryStoreRecoveryTests.swift @@ -0,0 +1,140 @@ +import Foundation +import os +import SQLite3 +@testable import TablePro +import Testing + +struct QueryHistoryStoreRecoveryTests { + private static let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + private let scratch: URL + + init() throws { + scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("history-recovery-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + } + + // The pre-v3 shape, except that database_name allows NULL, which the v3 table refuses. + private func makeLegacyDatabase(rows: [(query: String, databaseName: String?)], connectionId: UUID) -> URL { + let url = scratch.appendingPathComponent("query_history.db") + var db: OpaquePointer? + guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return url } + defer { sqlite3_close_v2(db) } + + sqlite3_exec(db, """ + CREATE TABLE history ( + id TEXT PRIMARY KEY, + query TEXT NOT NULL, + connection_id TEXT NOT NULL, + database_name TEXT, + executed_at REAL NOT NULL, + execution_time REAL NOT NULL, + row_count INTEGER NOT NULL, + was_successful INTEGER NOT NULL, + error_message TEXT + ); + """, nil, nil, nil) + sqlite3_exec(db, "PRAGMA user_version = 2;", nil, nil, nil) + + for row in rows { + var statement: OpaquePointer? + let insert = """ + INSERT INTO history (id, query, connection_id, database_name, executed_at, + execution_time, row_count, was_successful, error_message) + VALUES (?, ?, ?, ?, 1700000000, 0.5, 1, 1, NULL); + """ + guard sqlite3_prepare_v2(db, insert, -1, &statement, nil) == SQLITE_OK else { continue } + sqlite3_bind_text(statement, 1, UUID().uuidString, -1, Self.transient) + sqlite3_bind_text(statement, 2, row.query, -1, Self.transient) + sqlite3_bind_text(statement, 3, connectionId.uuidString, -1, Self.transient) + if let databaseName = row.databaseName { + sqlite3_bind_text(statement, 4, databaseName, -1, Self.transient) + } else { + sqlite3_bind_null(statement, 4) + } + sqlite3_step(statement) + sqlite3_finalize(statement) + } + return url + } + + private func scalarInt(in url: URL, _ sql: String) -> Int? { + var db: OpaquePointer? + guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return nil } + defer { sqlite3_close_v2(db) } + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK, + sqlite3_step(statement) == SQLITE_ROW + else { return nil } + return Int(sqlite3_column_int64(statement, 0)) + } + + @Test("A version 3 migration whose copy fails keeps every legacy row and leaves the store closed") + func failedMigrationKeepsLegacyRows() async { + let url = makeLegacyDatabase( + rows: [(query: "SELECT 1", databaseName: "legacydb"), (query: "SELECT 2", databaseName: nil)], + connectionId: UUID() + ) + let storage = QueryHistoryStorage(databaseURL: url) + + #expect(await storage.isStoreAvailable() == false) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM history;") == 2) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM sqlite_master WHERE name = 'history_legacy';") == 0) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM pragma_table_info('history') WHERE name = 'source';") == 0) + #expect(scalarInt(in: url, "PRAGMA user_version;") == 2) + } + + @Test("A damaged history database is set aside once and recording works again") + func damagedStoreIsSetAside() async throws { + let url = scratch.appendingPathComponent("query_history.db") + let damaged = Data(repeating: 0x2A, count: 4_096) + try damaged.write(to: url) + let reports = OSAllocatedUnfairLock(initialState: [SetAsideDamagedStore]()) + let storage = QueryHistoryStorage( + databaseURL: url, + removeDatabaseOnDeinit: true, + onSetAside: { report in reports.withLock { $0.append(report) } } + ) + let entry = QueryHistoryEntry( + query: "SELECT 1", + connectionId: UUID(), + databaseName: "app", + databaseType: .postgresql, + source: .editor, + executionTime: 0.1, + rowCount: 1, + wasSuccessful: true + ) + + #expect(await storage.record(entry)) + #expect(await storage.isStoreAvailable()) + #expect(await storage.count(scope: .all) == 1) + + let setAside = try #require(reports.withLock { $0 }.first) + #expect(reports.withLock { $0.count } == 1) + #expect(setAside.storeURL == url) + #expect(try Data(contentsOf: setAside.setAsideURL) == damaged) + } + + @Test("Clearing every row also deletes the damaged copy set aside earlier, and a narrower clear keeps it") + func fullClearDeletesTheSetAsideCopy() async throws { + let url = scratch.appendingPathComponent("query_history.db") + try Data(repeating: 0x2A, count: 4_096).write(to: url) + let reports = OSAllocatedUnfairLock(initialState: [SetAsideDamagedStore]()) + let storage = QueryHistoryStorage( + databaseURL: url, + removeDatabaseOnDeinit: true, + onSetAside: { report in reports.withLock { $0.append(report) } } + ) + #expect(await storage.isStoreAvailable()) + let setAsidePath = try #require(reports.withLock { $0 }.first).setAsideURL.path(percentEncoded: false) + + #expect(await storage.clear(matching: QueryHistoryFilter(scope: .all, sources: QueryHistorySource.userAuthored))) + #expect(FileManager.default.fileExists(atPath: setAsidePath)) + + #expect(await storage.clear(matching: QueryHistoryFilter(scope: .all))) + #expect(!FileManager.default.fileExists(atPath: setAsidePath)) + } +} diff --git a/apple/TableProTests/Core/Storage/QueryTabVersionStorageTests.swift b/apple/TableProTests/Core/Storage/QueryTabVersionStorageTests.swift new file mode 100644 index 0000000000..b6d8f3c451 --- /dev/null +++ b/apple/TableProTests/Core/Storage/QueryTabVersionStorageTests.swift @@ -0,0 +1,255 @@ +// +// QueryTabVersionStorageTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import Testing + +struct QueryTabVersionStorageTests { + private static let start = Date(timeIntervalSince1970: 1_800_000_000) + + private let url: URL + private let now = OSAllocatedUnfairLock(initialState: QueryTabVersionStorageTests.start) + private let historyMaxDays = OSAllocatedUnfairLock(initialState: 90) + private let announced = OSAllocatedUnfairLock(initialState: [UUID?]()) + private let storage: QueryTabVersionStorage + private let tabId = UUID() + private let connectionId = UUID() + + init() { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("query_tab_versions_\(UUID().uuidString).db") + self.url = url + storage = Self.makeStorage( + url: url, + now: now, + historyMaxDays: historyMaxDays, + announced: announced, + removeDatabaseOnDeinit: true + ) + } + + private static func makeStorage( + url: URL, + now: OSAllocatedUnfairLock, + historyMaxDays: OSAllocatedUnfairLock, + announced: OSAllocatedUnfairLock<[UUID?]>, + removeDatabaseOnDeinit: Bool + ) -> QueryTabVersionStorage { + QueryTabVersionStorage( + databaseURL: url, + now: { now.withLock { $0 } }, + historyMaxDays: { historyMaxDays.withLock { $0 } }, + announce: { tabId in announced.withLock { $0.append(tabId) } }, + removeDatabaseOnDeinit: removeDatabaseOnDeinit + ) + } + + private func reopenedStorage() -> QueryTabVersionStorage { + Self.makeStorage( + url: url, + now: now, + historyMaxDays: historyMaxDays, + announced: announced, + removeDatabaseOnDeinit: false + ) + } + + private func advance(by seconds: TimeInterval) { + now.withLock { $0 = $0.addingTimeInterval(seconds) } + } + + @discardableResult + private func record(_ text: String, _ reason: QueryTabVersionReason = .run, tab: UUID? = nil) async -> Bool { + await storage.record(tabId: tab ?? tabId, connectionId: connectionId, text: text, reason: reason) + } + + private func texts(tab: UUID? = nil) -> [String] { + let id = tab ?? tabId + return storage.summaries(tabId: id, limit: 200).compactMap { storage.version(id: $0.id, tabId: id)?.query } + } + + @Test("Blank text is not recorded") + func blankTextIsSkipped() async { + #expect(await record(" \n\t ") == false) + #expect(storage.summaries(tabId: tabId).isEmpty) + } + + @Test("Text larger than a tab can persist is not recorded") + func oversizeTextIsSkipped() async { + let oversize = String(repeating: "x", count: TabQueryContent.maxPersistableQuerySize + 1) + #expect(await record(oversize) == false) + #expect(storage.summaries(tabId: tabId).isEmpty) + } + + @Test("Text identical to the tab's newest version is not recorded again") + func duplicateOfNewestIsSkipped() async { + #expect(await record("SELECT 1")) + #expect(await record("SELECT 1", .beforeReplace) == false) + #expect(await record("SELECT 2")) + #expect(await record("SELECT 1")) + #expect(texts() == ["SELECT 1", "SELECT 2", "SELECT 1"]) + } + + @Test("A tab keeps its 50 newest versions") + func capKeepsTheNewest() async { + for index in 1...55 { + await record("SELECT \(index)") + } + let kept = texts() + #expect(kept.count == QueryTabVersionStorage.retainedVersionCount) + #expect(kept.first == "SELECT 55") + #expect(kept.last == "SELECT 6") + } + + @Test("The cap counts each tab on its own") + func capIsPerTab() async { + let other = UUID() + for index in 1...55 { + await record("SELECT \(index)") + } + await record("SELECT other", tab: other) + #expect(texts(tab: other) == ["SELECT other"]) + } + + @Test("Versions older than 30 days are gone, read or written") + func versionsExpireAfterThirtyDays() async { + await record("SELECT 1") + advance(by: 29 * 86_400) + await record("SELECT 2") + #expect(texts() == ["SELECT 2", "SELECT 1"]) + + advance(by: 2 * 86_400) + #expect(texts() == ["SELECT 2"]) + + await record("SELECT 3") + let reopened = reopenedStorage() + #expect(reopened.summaries(tabId: tabId).count == 2) + } + + @Test("A shorter Query History window shortens the retention", arguments: [(7, 7), (90, 30), (0, 30)]) + func historyWindowCapsRetention(historyDays: Int, expected: Int) async { + historyMaxDays.withLock { $0 = historyDays } + #expect(storage.retentionDays == expected) + + await record("SELECT 1") + advance(by: TimeInterval(expected) * 86_400 + 1) + #expect(storage.summaries(tabId: tabId).isEmpty) + } + + @Test("Idle snapshots within five minutes of the first become one version holding the latest text") + func idleSnapshotsCoalesce() async throws { + await record("SELECT 1", .idle) + advance(by: 60) + await record("SELECT 12", .idle) + advance(by: 200) + await record("SELECT 123", .idle) + + let summaries = storage.summaries(tabId: tabId) + let only = try #require(summaries.first) + #expect(summaries.count == 1) + #expect(only.reason == .idle) + #expect(only.savedAt == Self.start.addingTimeInterval(260)) + #expect(texts() == ["SELECT 123"]) + } + + @Test("Steady typing still leaves a version every five minutes") + func idleWindowCountsFromTheFirstSnapshot() async { + await record("SELECT 1", .idle) + for step in 2...4 { + advance(by: 120) + await record("SELECT \(step)", .idle) + } + #expect(texts() == ["SELECT 4", "SELECT 3"]) + } + + @Test("A run between idle snapshots keeps both sides of it") + func runBreaksTheIdleWindow() async { + await record("SELECT 1", .idle) + advance(by: 10) + await record("SELECT 2", .run) + advance(by: 10) + await record("SELECT 3", .idle) + #expect(texts() == ["SELECT 3", "SELECT 2", "SELECT 1"]) + #expect(storage.summaries(tabId: tabId).map { $0.reason } == [.idle, .run, .idle]) + } + + @Test("A restore of the idle text seals it, so the next pause cannot overwrite it") + func restoreSealsTheIdleVersion() async { + await record("SELECT before", .idle) + advance(by: 10) + await record("SELECT before", .beforeRestore) + advance(by: 10) + await record("SELECT restored", .idle) + #expect(texts() == ["SELECT restored", "SELECT before"]) + #expect(storage.summaries(tabId: tabId).map { $0.reason } == [.idle, .beforeRestore]) + } + + @Test("A coalesced pause takes the connection the tab moved to") + func coalescedPauseFollowsTheConnection() async { + let moved = UUID() + await record("SELECT 1", .idle) + advance(by: 10) + _ = await storage.record(tabId: tabId, connectionId: moved, text: "SELECT 2", reason: .idle) + #expect(await storage.removeVersions(forConnection: moved)) + #expect(storage.summaries(tabId: tabId).isEmpty) + } + + @Test("A run, a replace and a restore always add a version") + func otherReasonsAlwaysInsert() async { + await record("SELECT 1", .run) + await record("SELECT 2", .beforeReplace) + await record("SELECT 3", .beforeRestore) + #expect(storage.summaries(tabId: tabId).map { $0.reason } == [.beforeRestore, .beforeReplace, .run]) + } + + @Test("Deleting a connection removes its tabs' versions and leaves another's") + func purgeByConnection() async { + let otherTab = UUID() + await record("SELECT 1") + _ = await storage.record(tabId: otherTab, connectionId: UUID(), text: "SELECT kept", reason: .run) + + #expect(await storage.removeVersions(forConnection: connectionId)) + #expect(storage.summaries(tabId: tabId).isEmpty) + #expect(texts(tab: otherTab) == ["SELECT kept"]) + } + + @Test("Clear History removes every version") + func removeAllClearsEveryTab() async { + let otherTab = UUID() + await record("SELECT 1") + await record("SELECT 2", tab: otherTab) + + #expect(await storage.removeAll()) + #expect(storage.summaries(tabId: tabId).isEmpty) + #expect(storage.summaries(tabId: otherTab).isEmpty) + } + + @Test("Each write is announced with its tab, and a purge with none") + func writesAreAnnounced() async { + await record("SELECT 1") + await record("SELECT 1") + await storage.removeAll() + #expect(announced.withLock { $0 } == [tabId, nil]) + } + + @Test("A second store on the same file reads what the first wrote") + func secondInstanceReadsTheRows() async throws { + await record("SELECT 1") + let second = reopenedStorage() + + let summary = try #require(second.summaries(tabId: tabId).first) + #expect(second.version(id: summary.id, tabId: tabId)?.query == "SELECT 1") + #expect(second.version(id: summary.id, tabId: UUID()) == nil) + } + + @Test("Before the first write there is nothing to read and no file is made") + func readBeforeAnyWriteIsEmpty() { + #expect(storage.summaries(tabId: tabId).isEmpty) + #expect(!FileManager.default.fileExists(atPath: url.path(percentEncoded: false))) + } +} diff --git a/apple/TableProTests/Core/Storage/SQLFavoriteMigrationTests.swift b/apple/TableProTests/Core/Storage/SQLFavoriteMigrationTests.swift new file mode 100644 index 0000000000..16f6403c00 --- /dev/null +++ b/apple/TableProTests/Core/Storage/SQLFavoriteMigrationTests.swift @@ -0,0 +1,114 @@ +import Foundation +import SQLite3 +@testable import TablePro +import Testing + +struct SQLFavoriteMigrationTests { + private static let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + private let scratch: URL + + init() throws { + scratch = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("sql-favorite-migration-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: scratch, withIntermediateDirectories: true) + } + + // A version 1 store whose favorites.name allows NULL, which the version 2 table refuses. + private func makeVersion1Database(favorites: [(id: UUID, name: String?, query: String)], folderId: UUID) -> URL { + let url = scratch.appendingPathComponent("sql_favorites.db") + var db: OpaquePointer? + guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return url } + defer { sqlite3_close_v2(db) } + + let schema = [ + """ + CREATE TABLE favorites ( + id TEXT PRIMARY KEY, name TEXT, query TEXT NOT NULL, + keyword TEXT, folder_id TEXT, connection_id TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL + ); + """, + """ + CREATE TABLE folders ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, parent_id TEXT, connection_id TEXT, + sort_order INTEGER NOT NULL DEFAULT 0, created_at REAL NOT NULL, updated_at REAL NOT NULL + ); + """, + "INSERT INTO folders (id, name, created_at, updated_at) VALUES ('\(folderId.uuidString)', 'Reports', 100, 100);", + "PRAGMA user_version = 1;", + ] + for sql in schema { + sqlite3_exec(db, sql, nil, nil, nil) + } + + for favorite in favorites { + var statement: OpaquePointer? + let insert = "INSERT INTO favorites (id, name, query, created_at, updated_at) VALUES (?, ?, ?, 100, 100);" + guard sqlite3_prepare_v2(db, insert, -1, &statement, nil) == SQLITE_OK else { continue } + sqlite3_bind_text(statement, 1, favorite.id.uuidString, -1, Self.transient) + if let name = favorite.name { + sqlite3_bind_text(statement, 2, name, -1, Self.transient) + } else { + sqlite3_bind_null(statement, 2) + } + sqlite3_bind_text(statement, 3, favorite.query, -1, Self.transient) + sqlite3_step(statement) + sqlite3_finalize(statement) + } + return url + } + + private func scalarInt(in url: URL, _ sql: String) -> Int? { + var db: OpaquePointer? + guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return nil } + defer { sqlite3_close_v2(db) } + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK, + sqlite3_step(statement) == SQLITE_ROW + else { return nil } + return Int(sqlite3_column_int64(statement, 0)) + } + + @Test("A version 1 store migrates to version 2 with every saved query, folder and search entry") + func version1StoreMigrates() async { + let revenue = UUID() + let churn = UUID() + let folderId = UUID() + let url = makeVersion1Database( + favorites: [(id: revenue, name: "Revenue", query: "SELECT 1"), (id: churn, name: "Churn", query: "SELECT 2")], + folderId: folderId + ) + let storage = SQLFavoriteStorage(databaseURL: url) + + let favoriteIds = await storage.readAllFavorites()?.map(\.id) + let folderIds = await storage.readAllFolders()?.map(\.id) + let searched = await storage.fetchFavorites(searchText: "Churn").map(\.id) + + #expect(Set(favoriteIds ?? []) == [revenue, churn]) + #expect(folderIds == [folderId]) + #expect(searched == [churn]) + #expect(scalarInt(in: url, "PRAGMA user_version;") == 2) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM sqlite_master WHERE name IN ('favorites_old', 'folders_old');") == 0) + } + + @Test("A version 2 migration whose copy fails rolls back and keeps every saved query") + func failedMigrationKeepsEverySavedQuery() async { + let named = UUID() + let unnamed = UUID() + let url = makeVersion1Database( + favorites: [(id: named, name: "Revenue", query: "SELECT 1"), (id: unnamed, name: nil, query: "SELECT 2")], + folderId: UUID() + ) + let storage = SQLFavoriteStorage(databaseURL: url) + + let readable = await storage.readAllFavorites()?.map(\.id) + + #expect(readable == [named]) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM favorites;") == 2) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM folders;") == 1) + #expect(scalarInt(in: url, "SELECT COUNT(*) FROM sqlite_master WHERE name IN ('favorites_old', 'folders_old');") == 0) + #expect(scalarInt(in: url, "PRAGMA user_version;") == 1) + } +} diff --git a/apple/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift b/apple/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift index beadb3cc17..01d54bf85f 100644 --- a/apple/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift +++ b/apple/TableProTests/Core/Storage/SQLFavoriteStorageOpenTests.swift @@ -1,4 +1,5 @@ import Foundation +import os @testable import TablePro import Testing @@ -57,13 +58,72 @@ struct SQLFavoriteStorageOpenTests { #expect(await storage.readAllFolders()?.map(\.id) == [folder.id]) } - @Test("A file that is not a database reads as unreadable, not as an empty store") - func notADatabaseReadsAsUnreadable() async throws { - let url = scratch.appendingPathComponent("sql_favorites.db") - try Data(repeating: 0x2A, count: 4_096).write(to: url) - let storage = SQLFavoriteStorage(databaseURL: url, removeDatabaseOnDeinit: true) + @Test("A store that cannot be opened reads as unreadable, not as an empty store") + func unopenableStoreReadsAsUnreadable() async throws { + let blocker = scratch.appendingPathComponent("favorites-directory") + #expect(FileManager.default.createFile(atPath: blocker.path(percentEncoded: false), contents: Data())) + let storage = SQLFavoriteStorage( + databaseURL: blocker.appendingPathComponent("sql_favorites.db"), + removeDatabaseOnDeinit: true + ) #expect(await storage.readAllFavorites() == nil) #expect(await storage.readAllFolders() == nil) } + + @Test("A file that is not a database is set aside once, and saving queries works again") + func notADatabaseIsSetAside() async throws { + let url = scratch.appendingPathComponent("sql_favorites.db") + let damaged = Data(repeating: 0x2A, count: 4_096) + try damaged.write(to: url) + let reports = OSAllocatedUnfairLock(initialState: [SetAsideDamagedStore]()) + let storage = SQLFavoriteStorage( + databaseURL: url, + removeDatabaseOnDeinit: true, + onSetAside: { report in reports.withLock { $0.append(report) } } + ) + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") + + #expect(await storage.addFavorite(favorite)) + #expect(await storage.readAllFavorites()?.map(\.id) == [favorite.id]) + + let setAside = try #require(reports.withLock { $0 }.first) + #expect(reports.withLock { $0.count } == 1) + #expect(setAside.storeURL == url) + #expect(try Data(contentsOf: setAside.setAsideURL) == damaged) + } + + @Test("A store that replaced a damaged one reads as unreadable for sync, across a relaunch, until refilled") + func replacedStoreWaitsForRefill() async throws { + let url = scratch.appendingPathComponent("sql_favorites.db") + try Data(repeating: 0x2A, count: 4_096).write(to: url) + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") + let first = SQLFavoriteStorage(databaseURL: url) + #expect(await first.addFavorite(favorite)) + + #expect(await first.isAwaitingSyncRefill()) + #expect(await first.readAllFavoritesForSync() == nil) + #expect(await first.readAllFoldersForSync() == nil) + #expect(await first.readAllFavorites()?.map(\.id) == [favorite.id]) + + let relaunched = SQLFavoriteStorage(databaseURL: url) + #expect(await relaunched.isAwaitingSyncRefill()) + await relaunched.finishSyncRefill() + #expect(await relaunched.isAwaitingSyncRefill() == false) + #expect(await relaunched.readAllFavoritesForSync()?.map(\.id) == [favorite.id]) + + let afterRefill = SQLFavoriteStorage(databaseURL: url) + #expect(await afterRefill.isAwaitingSyncRefill() == false) + } + + @Test("A healthy store is never waiting for a refill") + func healthyStoreIsNotWaiting() async { + let storage = SQLFavoriteStorage( + databaseURL: scratch.appendingPathComponent("sql_favorites.db"), + removeDatabaseOnDeinit: true + ) + + #expect(await storage.isAwaitingSyncRefill() == false) + #expect(await storage.readAllFavoritesForSync()?.isEmpty == true) + } } diff --git a/apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift b/apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift new file mode 100644 index 0000000000..d028251bd7 --- /dev/null +++ b/apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift @@ -0,0 +1,173 @@ +import Foundation +@testable import TablePro +import Testing + +struct TabDiskUnchangedSaveTests { + private let tabStateDirectory = AppStorageEnvironment.shared.applicationSupportRoot + .appendingPathComponent("TablePro", isDirectory: true) + .appendingPathComponent("TabState", isDirectory: true) + + private func stateURL(_ connectionId: UUID) -> URL { + tabStateDirectory.appendingPathComponent("\(connectionId.uuidString).json") + } + + private func sidecarURL(_ connectionId: UUID, tabId: UUID) -> URL { + TabQueryOverflowStore.directory(inside: tabStateDirectory) + .appendingPathComponent("\(connectionId.uuidString)_\(tabId.uuidString).sql") + } + + private func fileNumber(_ url: URL) throws -> Int { + try #require(FileManager.default.attributesOfItem(atPath: url.path)[.systemFileNumber] as? Int) + } + + private func tab(id: UUID = UUID(), title: String, query: String) -> PersistedTab { + PersistedTab( + id: id, + title: title, + query: query, + tabType: .query, + tableName: nil, + columnWidths: ["b": 120, "a": 80, "c": 64] + ) + } + + @discardableResult + private func save(_ tabs: [PersistedTab], connectionId: UUID, selected: UUID? = nil) async throws -> Bool { + try await TabDiskActor.shared.save( + connectionId: connectionId, + tabs: tabs, + selectedTabId: selected ?? tabs.first?.id, + writeToken: TabDiskActor.issueWriteToken(for: connectionId) + ) + } + + @Test("A save that changes nothing leaves the tab file in place, and a change replaces it") + func unchangedSaveKeepsTheFile() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1"), tab(title: "Two", query: "SELECT 2")] + + try await save(tabs, connectionId: connectionId) + let written = try fileNumber(stateURL(connectionId)) + try await save(tabs, connectionId: connectionId) + #expect(try fileNumber(stateURL(connectionId)) == written) + + try await save(tabs, connectionId: connectionId, selected: tabs[1].id) + #expect(try fileNumber(stateURL(connectionId)) != written) + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.selectedTabId == tabs[1].id) + } + + @Test("A tab file removed behind the store is written again by an unchanged save") + func removedFileIsWrittenAgain() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + try FileManager.default.removeItem(at: stateURL(connectionId)) + try await save(tabs, connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["One"]) + } + + @Test("A tab file a load found unreadable is written again by the next unchanged save") + func unreadableFileIsRepairedBySave() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + try Data("{".utf8).write(to: stateURL(connectionId)) + #expect(await TabDiskActor.shared.load(connectionId: connectionId) == nil) + try await save(tabs, connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["One"]) + } + + @Test("An unchanged oversized query keeps its sidecar, and an edited one is rewritten") + func sidecarIsRewrittenOnlyWhenTheQueryChanges() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabId = UUID() + let large = String(repeating: "x", count: TabQueryContent.maxPersistableQuerySize + 1) + + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + let sidecar = sidecarURL(connectionId, tabId: tabId) + let written = try fileNumber(sidecar) + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + #expect(try fileNumber(sidecar) == written) + + let edited = large + "y" + try await save([tab(id: tabId, title: "Dump", query: edited)], connectionId: connectionId) + #expect(try fileNumber(sidecar) != written) + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.first?.query == edited) + } + + @Test("A sidecar removed behind the store is written again by an unchanged save") + func removedSidecarIsWrittenAgain() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabId = UUID() + let large = String(repeating: "z", count: TabQueryContent.maxPersistableQuerySize + 1) + + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + try FileManager.default.removeItem(at: sidecarURL(connectionId, tabId: tabId)) + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.first?.query == large) + } + + @Test("A closed tab's sidecar is removed even though the other sidecars did not change") + func closedTabSidecarIsPruned() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let kept = UUID() + let closed = UUID() + let large = String(repeating: "q", count: TabQueryContent.maxPersistableQuerySize + 1) + + try await save( + [tab(id: kept, title: "Kept", query: large), tab(id: closed, title: "Closed", query: large)], + connectionId: connectionId + ) + try await save([tab(id: kept, title: "Kept", query: large)], connectionId: connectionId) + + #expect(FileManager.default.fileExists(atPath: sidecarURL(connectionId, tabId: kept).path)) + #expect(!FileManager.default.fileExists(atPath: sidecarURL(connectionId, tabId: closed).path)) + } + + @Test("After a clear, saving the same tabs writes them again") + func saveAfterClearWrites() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + TabDiskActor.clearSync(connectionId: connectionId) + try await save(tabs, connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["One"]) + } + + @Test("The quit-time save skips an unchanged state too, and still writes a change") + func syncSaveSkipsUnchangedState() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + let written = try fileNumber(stateURL(connectionId)) + TabDiskActor.saveSync(connectionId: connectionId, tabs: tabs, selectedTabId: tabs.first?.id) + #expect(try fileNumber(stateURL(connectionId)) == written) + + let renamed = [tab(id: tabs[0].id, title: "Renamed", query: "SELECT 1")] + TabDiskActor.saveSync(connectionId: connectionId, tabs: renamed, selectedTabId: renamed.first?.id) + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["Renamed"]) + } +} diff --git a/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift b/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift index f3d157d2eb..40dc170262 100644 --- a/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift +++ b/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift @@ -27,9 +27,23 @@ struct SyncCategoryRefetchTests { #expect(!content.contains(SyncRecordType.settings.rawValue)) #expect(!content.contains(SyncCoordinator.themeContentKind)) #expect(!content.contains(SyncCoordinator.formatterContentKind)) + #expect(!content.contains(SyncCoordinator.valueColorsContentKind)) #expect(content.contains(SyncRecordType.connection.rawValue)) } + @Test("A Mac upgraded from a build that skipped value colors records fetches the zone once") + func upgradeRefetchesSkippedValueColorsRecords() throws { + let pool = EphemeralDefaultsPool() + let metadata = SyncMetadataStorage(userDefaults: try pool.make().defaults) + var older = SyncCoordinator.readableContent + older.remove(SyncCoordinator.valueColorsContentKind) + _ = metadata.adoptReadableRecordTypes(older) + + #expect(SyncCoordinator.readableContent(for: .default).contains(SyncCoordinator.valueColorsContentKind)) + #expect(metadata.adoptReadableRecordTypes(SyncCoordinator.readableContent)) + #expect(!metadata.adoptReadableRecordTypes(SyncCoordinator.readableContent)) + } + @Test("Turning Saved Queries on fetches the zone once and brings back a query saved elsewhere while it was off") func turningACategoryOnRefetches() async throws { let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") @@ -214,7 +228,8 @@ struct SyncCategoryRefetchTests { transport: server, makeNetworkMonitor: { nil }, columnLayouts: columnLayouts, - licenseGate: { nil } + licenseGate: { nil }, + pushRegistrar: .inert ) } @@ -272,6 +287,8 @@ private actor ZoneServer: SyncTransport { func ensureZoneExists() async throws {} + func saveSubscription() async throws {} + func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome { var outcome = PushOutcome() for record in records { diff --git a/apple/TableProTests/Core/Sync/SyncConnectionPullBatchTests.swift b/apple/TableProTests/Core/Sync/SyncConnectionPullBatchTests.swift new file mode 100644 index 0000000000..e113a8e61d --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncConnectionPullBatchTests.swift @@ -0,0 +1,106 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct SyncConnectionPullBatchTests { + private static let zoneID = SyncTestEnvironment.zoneID + + private struct LibraryWork: Equatable { + let reads: Int + let writes: Int + } + + private static func pull(_ records: [CKRecord]) -> PullResult { + PullResult(changedRecords: records, deletedRecordIDs: [], newToken: nil) + } + + private static func remoteConnections(_ count: Int) -> [DatabaseConnection] { + (0.. (LibraryWork, [DatabaseConnection]) { + let environment = try SyncTestEnvironment(label: "sync-connection-pull-batch") + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let readsBefore = environment.connections.fileReadCount + let writesBefore = environment.connections.fileWriteCount + + let applied = await coordinator.applyPullResult( + Self.pull(connections.map { SyncRecordMapper.toCKRecord($0, in: Self.zoneID) }), + settings: .default + ) + + #expect(applied) + let work = LibraryWork( + reads: environment.connections.fileReadCount - readsBefore, + writes: environment.connections.fileWriteCount - writesBefore + ) + return (work, environment.connections.loadConnections()) + } + + // Each record used to read, verify, decode, encode and write the whole library: 300 of each at 300 connections. + @Test("A pull reads and writes the library the same number of times for 10 connections as for 300") + func pullWorkDoesNotGrowWithTheBatch() async throws { + let few = Self.remoteConnections(10) + let many = Self.remoteConnections(300) + + let (fewWork, fewStored) = try await pullIntoEmptyLibrary(few) + let (manyWork, manyStored) = try await pullIntoEmptyLibrary(many) + + #expect(fewWork.writes == 1) + #expect(manyWork == fewWork) + let fewIds = fewStored.map(\.id) + let manyIds = manyStored.map(\.id) + #expect(fewIds == few.map(\.id)) + #expect(manyIds == many.map(\.id)) + let allSynced = manyStored.allSatisfy { $0.origin == .synced } + #expect(allSynced) + } + + @Test("A credential profile that arrives after a connection still writes its username through to it") + func profileAfterAConnectionSeesTheConnection() async throws { + let environment = try SyncTestEnvironment(label: "sync-connection-pull-order") + let profile = CredentialProfile(name: "Prod", username: "profile-user") + var local = TestFixtures.makeConnection(name: "Linked") + local.username = "local-user" + local.credentialMode = .profile(id: profile.id) + #expect(environment.connections.saveConnections([local])) + var remote = local + remote.username = "remote-user" + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + + let applied = await coordinator.applyPullResult( + Self.pull([ + SyncRecordMapper.toCKRecord(remote, in: Self.zoneID), + SyncRecordMapper.toCKRecord(profile, in: Self.zoneID), + ]), + settings: .default + ) + + #expect(applied) + #expect(environment.connections.loadConnection(id: local.id)?.username == "profile-user") + } + + @Test("A pull whose library cannot be read writes none of its connections and is not acknowledged") + func unreadableLibraryRefusesTheWholeBatch() async throws { + let environment = try SyncTestEnvironment(label: "sync-connection-pull-unreadable") + let fileURL = environment.directory.appendingPathComponent("connections.json") + let garbage = Data("not a connection list".utf8) + try garbage.write(to: fileURL) + environment.connections.invalidateCache() + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let writesBefore = environment.connections.fileWriteCount + + let applied = await coordinator.applyPullResult( + Self.pull(Self.remoteConnections(3).map { SyncRecordMapper.toCKRecord($0, in: Self.zoneID) }), + settings: .default + ) + + #expect(!applied) + #expect(environment.connections.fileWriteCount == writesBefore) + #expect(try Data(contentsOf: fileURL) == garbage) + } +} diff --git a/apple/TableProTests/Core/Sync/SyncCoordinatorActivationTests.swift b/apple/TableProTests/Core/Sync/SyncCoordinatorActivationTests.swift new file mode 100644 index 0000000000..9eaf3ecb93 --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncCoordinatorActivationTests.swift @@ -0,0 +1,143 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import TableProTestSupport +import Testing + +@MainActor +struct SyncCoordinatorActivationTests { + private let environment: SyncTestEnvironment + private let clock = ManualClock() + + init() throws { + environment = try SyncTestEnvironment(label: "sync-activation") + var settings = SyncSettings.default + settings.enabled = true + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + } + + private func makeCoordinator(_ transport: ScriptedSyncTransport) -> SyncCoordinator { + environment.makeCoordinator(transport: transport, activationClock: clock) + } + + @Test("An activation while the launch run has yet to download adds no second run") + func activationDuringLaunchRunIsDropped() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + let activated = ActivationLatch() + await transport.whileZoneIsRead { + guard activated.claim() else { return } + await coordinator.sync(.activation) + } + + await coordinator.sync(.launch) + #expect(activated.isClaimed) + + // Held instead of dropped, it would run once the interval ends. + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { !coordinator.isHoldingActivations } + try await waitUntil { coordinator.syncStatus != .syncing } + #expect(await transport.pullCount == 1) + } + + @Test("Activations inside the interval wait for its end and then download once") + func activationsInsideTheIntervalRunOnceAtItsEnd() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + #expect(await transport.pullCount == 1) + + await coordinator.sync(.activation) + await coordinator.sync(.activation) + #expect(await transport.pullCount == 1) + + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { await transport.pullCount == 2 } + try await waitUntil { coordinator.syncStatus != .syncing } + #expect(await transport.pullCount == 2) + } + + @Test("An activation after the interval downloads at once") + func activationAfterTheIntervalRuns() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { !coordinator.isHoldingActivations } + await coordinator.sync(.activation) + #expect(await transport.pullCount == 2) + } + + @Test("An edit queued behind an activation during a download is not held with it") + func queuedEditIsNotHeldBehindAnActivation() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { !coordinator.isHoldingActivations } + + // The second zone read of a run comes after its download, where an activation is queued, not dropped. + let reads = ZoneReadCounter() + await transport.whileZoneIsRead { + guard reads.next() == 2 else { return } + await coordinator.sync(.activation) + await coordinator.sync(.localChange) + } + await coordinator.sync(.userRequest) + try await waitUntil { await transport.pullCount == 3 } + } + + @Test("Turning sync off ends the activation hold") + func syncOffEndsTheHold() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + #expect(coordinator.isHoldingActivations) + + var settings = SyncSettings.default + settings.enabled = false + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + coordinator.start() + #expect(!coordinator.isHoldingActivations) + } + + @Test("A failed download leaves the next activation free to run") + func failedDownloadDoesNotHoldActivation() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + await transport.failPulls(with: CKError(.networkFailure)) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + #expect(await transport.pullCount == 1) + + await transport.failPulls(with: nil) + await coordinator.sync(.activation) + #expect(await transport.pullCount == 2) + } +} + +@MainActor +private final class ActivationLatch { + private(set) var isClaimed = false + + func claim() -> Bool { + guard !isClaimed else { return false } + isClaimed = true + return true + } +} + +@MainActor +private final class ZoneReadCounter { + private var count = 0 + + func next() -> Int { + count += 1 + return count + } +} diff --git a/apple/TableProTests/Core/Sync/SyncCoordinatorPushTests.swift b/apple/TableProTests/Core/Sync/SyncCoordinatorPushTests.swift new file mode 100644 index 0000000000..45872c6346 --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncCoordinatorPushTests.swift @@ -0,0 +1,353 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import TableProTestSupport +import Testing + +@MainActor +struct SyncCoordinatorPushTests { + private let environment: SyncTestEnvironment + private let clock = ManualClock() + private let registrar = RecordingPushRegistrar() + + init() throws { + environment = try SyncTestEnvironment(label: "sync-push") + setSyncEnabled(true) + } + + private func setSyncEnabled(_ enabled: Bool) { + var settings = SyncSettings.default + settings.enabled = enabled + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + } + + private func makeCoordinator(_ transport: any SyncTransport) -> SyncCoordinator { + let base = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID)) + return SyncCoordinator( + services: base.services, + recordCache: environment.recordCache, + transport: transport, + makeNetworkMonitor: { nil }, + columnLayouts: environment.columnLayouts, + licenseGate: { nil }, + activationClock: clock, + pushRegistrar: registrar.registrar + ) + } + + // What CloudKit sends for the database subscription, in the form its own parser reads. + private static func notice(subscriptionID: String = SyncPushSubscription.id) -> [AnyHashable: Any] { + [ + "aps": ["content-available": 1], + "ck": [ + "ce": 2, + "cid": CloudKitSyncEngine.defaultContainerID, + "nid": UUID().uuidString, + "met": ["dbs": 1, "sid": subscriptionID] + ] as [String: Any] + ] + } + + // The launch run of an app whose registration CloudKit answered. + private func launch(_ coordinator: SyncCoordinator, registered: Bool, pulls: @escaping () async -> Int) async throws { + coordinator.start() + coordinator.pushRegistrationDidFinish(registered: registered) + try await waitUntil { await pulls() == 1 } + try await settle(coordinator) + } + + // A run ends in a new hold, or in none when nothing was held. + private func settle(_ coordinator: SyncCoordinator) async throws { + try await waitUntil { + coordinator.syncStatus != .syncing && (clock.pendingSleeperCount > 0 || !coordinator.isHoldingActivations) + } + } + + @Test("A change notice downloads at once, inside the hold that keeps activations back") + func noticeDownloadsInsideTheHold() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: true) { await transport.pullCount } + await coordinator.sync(.activation) + #expect(coordinator.isHoldingActivations) + #expect(await transport.pullCount == 1) + + coordinator.handleRemoteNotification(Self.notice()) + + try await waitUntil { await transport.pullCount == 2 } + #expect(clock.now.offset == .zero) + } + + @Test("Notices before a run's download are answered by it") + func noticesBeforeTheDownloadAddNoRun() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + let reads = ZoneReadCount() + // The first zone read of a run comes before its download. + await transport.whileZoneIsRead { + guard reads.next() == 1 else { return } + for _ in 0..<5 { + await coordinator.sync(.remoteChange) + } + } + + await coordinator.sync(.userRequest) + try await settle(coordinator) + + #expect(await transport.pullCount == 1) + } + + @Test("Notices after a run's download began add one more run, never one each") + func noticesDuringTheDownloadAddOneRun() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + let reads = ZoneReadCount() + // The second zone read of a run comes after its download. + await transport.whileZoneIsRead { + guard reads.next() == 2 else { return } + for _ in 0..<5 { + await coordinator.sync(.remoteChange) + } + } + + await coordinator.sync(.userRequest) + try await waitUntil { await transport.pullCount == 2 } + try await settle(coordinator) + + #expect(await transport.pullCount == 2) + } + + @Test("With sync off nothing registers, no subscription is saved, and notices are ignored") + func syncOffDoesNothing() async throws { + setSyncEnabled(false) + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + coordinator.start() + coordinator.pushRegistrationDidFinish(registered: true) + coordinator.handleRemoteNotification(Self.notice()) + await coordinator.sync(.remoteChange) + + #expect(registrar.registrations == 0) + #expect(registrar.unregistrations == 0) + #expect(await transport.subscriptionSaveCount == 0) + #expect(await transport.pullCount == 0) + #expect(!coordinator.isPushLive) + } + + @Test("Turning sync on registers, and turning it off unregisters and ends push") + func registrationFollowsTheSwitch() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: true) { await transport.pullCount } + #expect(registrar.registrations == 1) + #expect(coordinator.isPushLive) + + setSyncEnabled(false) + coordinator.disableSync() + #expect(registrar.unregistrations == 1) + #expect(!coordinator.isPushLive) + + setSyncEnabled(true) + coordinator.enableSync() + #expect(registrar.registrations == 2) + #expect(!coordinator.isPushLive) + } + + @Test("The subscription is saved once per account, and again for a new account") + func subscriptionIsSavedOncePerAccount() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + await coordinator.sync(.userRequest) + #expect(await transport.subscriptionSaveCount == 1) + #expect(environment.metadata.subscribedAccountId == ScriptedSyncTransport.accountId) + + await transport.setAccountId("another") + await coordinator.sync(.accountChange) + #expect(await transport.subscriptionSaveCount == 2) + #expect(environment.metadata.subscribedAccountId == "another") + } + + @Test("A save that fails leaves the run alone, and activations keep the one minute hold") + func failedSaveKeepsTheMinute() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + await transport.failSubscriptionSaves(with: CKError(.networkFailure)) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: true) { await transport.pullCount } + + #expect(coordinator.syncStatus == .idle) + #expect(!coordinator.isPushLive) + #expect(coordinator.activationHoldInterval == SyncCoordinator.activationInterval) + + await transport.failSubscriptionSaves(with: nil) + await coordinator.sync(.userRequest) + #expect(await transport.subscriptionSaveCount == 2) + #expect(coordinator.isPushLive) + } + + @Test("An activation every 30 s for an hour downloads 60 times without push and 4 times with it", + arguments: [(false, 60), (true, 4)]) + func activationsForAnHour(registered: Bool, downloads: Int) async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: registered) { await transport.pullCount } + #expect(coordinator.isPushLive == registered) + + for _ in 1..<120 { + clock.advance(by: .seconds(30)) + try await settle(coordinator) + await coordinator.sync(.activation) + try await settle(coordinator) + } + + #expect(await transport.pullCount == downloads) + } + + // A transport whose deltas carry what `remote` holds, the way another Mac's saves arrive. + private func makeRemoteServer(_ remote: RemoteChanges) throws -> ScriptedSyncTransport { + environment.metadata.saveToken(try Self.makeChangeToken()) + let token = try Self.makeChangeToken() + return ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID, echoing: { _, _ in + PullResult(changedRecords: remote.take(), deletedRecordIDs: [], newToken: token) + }) + } + + private func endHold(_ coordinator: SyncCoordinator) async throws { + await clock.waitUntilSleeping() + clock.advance(by: coordinator.activationHoldInterval) + try await waitUntil { !coordinator.isHoldingActivations } + } + + private static var remoteTag: CKRecord { + SyncRecordMapper.toCKRecord(ConnectionTag(name: "from another Mac"), in: SyncTestEnvironment.zoneID) + } + + @Test("An activation that brings a change no notice announced goes back to the minute, and the next notice ends that") + func missedNoticeFallsBackToTheMinute() async throws { + let remote = RemoteChanges() + let transport = try makeRemoteServer(remote) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: true) { await transport.pullCount } + #expect(coordinator.activationHoldInterval == SyncCoordinator.pushFallbackInterval) + + remote.put(Self.remoteTag) + try await endHold(coordinator) + await coordinator.sync(.activation) + try await settle(coordinator) + + #expect(await transport.pullCount == 2) + #expect(!coordinator.isPushLive) + #expect(coordinator.activationHoldInterval == SyncCoordinator.activationInterval) + + // Counted synchronously, so a payload the filter let through shows before any run starts. + let noticesBefore = coordinator.pushChannel.noticesReceived + coordinator.handleRemoteNotification(Self.notice(subscriptionID: "other")) + coordinator.handleRemoteNotification(["aps": ["alert": "Hello"]]) + #expect(coordinator.pushChannel.noticesReceived == noticesBefore) + + coordinator.handleRemoteNotification(Self.notice()) + try await waitUntil { await transport.pullCount == 3 } + try await settle(coordinator) + #expect(coordinator.isPushLive) + #expect(await transport.subscriptionSaveCount == 2) + } + + @Test("A notice that lands during an activation's run keeps the changes it brings announced") + func noticeDuringTheActivationKeepsPushLive() async throws { + let remote = RemoteChanges() + let transport = try makeRemoteServer(remote) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: true) { await transport.pullCount } + + remote.put(Self.remoteTag) + try await endHold(coordinator) + let reads = ZoneReadCount() + await transport.whileZoneIsRead { + guard reads.next() == 1 else { return } + coordinator.handleRemoteNotification(Self.notice()) + } + await coordinator.sync(.activation) + try await settle(coordinator) + + #expect(coordinator.isPushLive) + #expect(coordinator.activationHoldInterval == SyncCoordinator.pushFallbackInterval) + } + + @Test("A notice whose download failed still announces the change the next activation brings") + func noticeBeforeAFailedDownloadKeepsPushLive() async throws { + let remote = RemoteChanges() + let transport = try makeRemoteServer(remote) + let coordinator = makeCoordinator(transport) + try await launch(coordinator, registered: true) { await transport.pullCount } + + remote.put(Self.remoteTag) + try await endHold(coordinator) + await transport.failPulls(with: CKError(.networkFailure)) + let reads = ZoneReadCount() + await transport.whileZoneIsRead { + guard reads.next() == 1 else { return } + coordinator.handleRemoteNotification(Self.notice()) + } + await coordinator.sync(.userRequest) + #expect(coordinator.syncStatus == .error(.offline)) + + await transport.failPulls(with: nil) + await coordinator.sync(.activation) + try await settle(coordinator) + + #expect(coordinator.isPushLive) + #expect(coordinator.activationHoldInterval == SyncCoordinator.pushFallbackInterval) + } + + private static func makeChangeToken() throws -> CKServerChangeToken { + let archiver = NSKeyedArchiver(requiringSecureCoding: true) + archiver.setClassName("CKServerChangeToken", for: PushTestsChangeTokenArchive.self) + archiver.encode(PushTestsChangeTokenArchive(), forKey: NSKeyedArchiveRootObjectKey) + archiver.finishEncoding() + let token = try NSKeyedUnarchiver.unarchivedObject(ofClass: CKServerChangeToken.self, from: archiver.encodedData) + return try #require(token) + } +} + +@MainActor +private final class ZoneReadCount { + private var count = 0 + + func next() -> Int { + count += 1 + return count + } +} + +// What another device saved since the last fetch, handed out once. +private final class RemoteChanges: @unchecked Sendable { + private let lock = NSLock() + private var records: [CKRecord] = [] + + func put(_ record: CKRecord) { + lock.withLock { records.append(record) } + } + + func take() -> [CKRecord] { + lock.withLock { + defer { records = [] } + return records + } + } +} + +@objc(SyncCoordinatorPushTestsChangeTokenArchive) +private final class PushTestsChangeTokenArchive: NSObject, NSSecureCoding { + static var supportsSecureCoding: Bool { true } + + override init() { + super.init() + } + + required init?(coder: NSCoder) { + super.init() + } + + func encode(with coder: NSCoder) {} +} diff --git a/apple/TableProTests/Core/Sync/SyncCoordinatorSQLFavoritePullTests.swift b/apple/TableProTests/Core/Sync/SyncCoordinatorSQLFavoritePullTests.swift index b4f2821c73..9c6a005d17 100644 --- a/apple/TableProTests/Core/Sync/SyncCoordinatorSQLFavoritePullTests.swift +++ b/apple/TableProTests/Core/Sync/SyncCoordinatorSQLFavoritePullTests.swift @@ -182,7 +182,7 @@ struct SyncCoordinatorSQLFavoritePullTests { welcomeRouter: live.welcomeRouter, telemetry: live.telemetry ) - return SyncCoordinator(services: services, recordCache: recordCache) + return SyncCoordinator(services: services, recordCache: recordCache, pushRegistrar: .inert) } private static func makeChangeToken() throws -> CKServerChangeToken { diff --git a/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift b/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift index 06087c619b..182a671afa 100644 --- a/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift +++ b/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift @@ -47,7 +47,6 @@ struct SyncCoordinatorThemeTests { catalog = ThemeCatalog( userDirectory: themes, bundleDirectory: nil, - notificationCenter: NotificationCenter(), fileTrash: .folder(trash), syncTracker: environment.tracker ) @@ -68,7 +67,8 @@ struct SyncCoordinatorThemeTests { makeNetworkMonitor: { nil }, columnLayouts: columnLayouts, themeCatalog: catalog, - licenseGate: { nil } + licenseGate: { nil }, + pushRegistrar: .inert ) } @@ -99,6 +99,17 @@ struct SyncCoordinatorThemeTests { ) } + nonisolated private static func valueTheme(_ id: String) throws -> ThemeDefinition { + var values = try theme(id) + values.overrides[.gridString] = try ThemeColor(validating: "#F1FA8C") + values.overrides[.gridNumber] = try ThemeColor(validating: "#BD93F9") + return values + } + + nonisolated private static func schema(of payload: Data) throws -> Int? { + try (JSONSerialization.jsonObject(with: payload) as? [String: Any])?["schema"] as? Int + } + private static func themeRecord(_ theme: ThemeDefinition) throws -> CKRecord { SyncRecordMapper.toCKRecord( category: ThemeCatalog.syncCategory(for: theme.id), @@ -143,6 +154,20 @@ struct SyncCoordinatorThemeTests { #expect(harness.tracker.dirtyRecords(for: .settings).isEmpty) } + @Test("A pulled format 4 theme record lands with its value colors") + func pulledFormatFourThemeKeepsItsValueColors() async throws { + let harness = try Harness(label: "theme-values-arrival") + let synced = try Self.valueTheme("user.values") + + let acknowledged = await harness.coordinator.applyPullResult(Self.pull([try Self.themeRecord(synced)])) + + #expect(acknowledged) + #expect(harness.catalog.storedTheme(id: synced.id) == synced) + let onDisk = try Data(contentsOf: harness.themes.appendingPathComponent("\(synced.id).json")) + #expect(try Self.schema(of: onDisk) == 4) + #expect(harness.tracker.dirtyRecords(for: .settings).isEmpty) + } + @Test("A theme record that fails validation is skipped, and the pull is still acknowledged") func invalidThemeIsSkipped() async throws { let harness = try Harness(label: "theme-invalid") @@ -234,6 +259,32 @@ struct SyncCoordinatorThemeTests { #expect(harness.tracker.tombstonedIds(for: .settings).isEmpty) } + @Test("Each theme pushes one record, theme.: format 4 holding its value colors, or format 3 without them") + func pushWritesTheOldestFormatHoldingTheTheme() async throws { + let harness = try Harness(label: "theme-push-format") + let night = try Self.theme("user.night") + let values = try Self.valueTheme("user.values") + try harness.catalog.save(night) + try harness.catalog.save(values) + + #expect(await harness.coordinator.runSyncCycle() == nil) + + let pushed = await harness.transport.pushedRecords + let themes = Dictionary(pushed.compactMap { record -> (String, Data)? in + guard let category = SyncRecordMapper.settingsCategory(from: record), + ThemeCatalog.themeId(forSyncCategory: category) != nil, + let data = SyncRecordMapper.settingsData(from: record) + else { return nil } + return (category, data) + }, uniquingKeysWith: { _, last in last }) + #expect(Set(themes.keys) == [ThemeCatalog.syncCategory(for: night.id), ThemeCatalog.syncCategory(for: values.id)]) + let plain = try #require(themes[ThemeCatalog.syncCategory(for: night.id)]) + #expect(try Self.schema(of: plain) == 3) + let colored = try #require(themes[ThemeCatalog.syncCategory(for: values.id)]) + #expect(try Self.schema(of: colored) == 4) + #expect(try ThemeFileFormat.decode(colored) == values) + } + @Test("Turning sync on marks every theme on this Mac") func enablingSyncMarksThemes() throws { let harness = try Harness(label: "theme-enable") @@ -285,7 +336,8 @@ struct SyncCoordinatorThemeTests { makeNetworkMonitor: { nil }, columnLayouts: columnLayouts, themeCatalog: harness.catalog, - licenseGate: { nil } + licenseGate: { nil }, + pushRegistrar: .inert ) relaunched.start() #expect(harness.metadata.loadToken() != nil) diff --git a/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift b/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift index c7a6b21ec5..11d6d22cd3 100644 --- a/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift +++ b/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift @@ -95,11 +95,12 @@ struct SyncFullFetchDeletionTests { #expect(await environment.favorites.fetchFavorite(id: edited.id) != nil) } - @Test("A full fetch that could not read a record deletes nothing") - func unreadRecordStopsTheDeletions() async throws { + @Test("A record a full fetch listed but could not read is kept, and the deletions the fetch proves still apply") + func unreadRecordIsKeptAndDeletionsStillApply() async throws { let kept = SQLFavorite(name: "Churn", query: "SELECT 2") let unread = SQLFavorite(name: "Revenue", query: "SELECT 1") - let coordinator = try await seededCoordinator(with: [kept, unread]) + let deleted = SQLFavorite(name: "Retention", query: "SELECT 3") + let coordinator = try await seededCoordinator(with: [kept, unread, deleted]) let result = PullResult( changedRecords: [Self.record(kept)], deletedRecordIDs: [], @@ -109,7 +110,9 @@ struct SyncFullFetchDeletionTests { #expect(await coordinator.applyPullResult(result, settings: Self.syncSettings(), isFullFetch: true)) + #expect(await environment.favorites.fetchFavorite(id: kept.id) != nil) #expect(await environment.favorites.fetchFavorite(id: unread.id) != nil) + #expect(await environment.favorites.fetchFavorite(id: deleted.id) == nil) } @Test("A delta is never read for what it does not list") @@ -170,7 +173,8 @@ struct SyncFullFetchDeletionTests { makeNetworkMonitor: { nil }, columnLayouts: environment.columnLayouts, licenseGate: { nil }, - recordCreationDate: { dates.date(for: $0) } + recordCreationDate: { dates.date(for: $0) }, + pushRegistrar: .inert ) } @@ -252,6 +256,8 @@ private actor ExpiringZoneServer: SyncTransport { func ensureZoneExists() async throws {} + func saveSubscription() async throws {} + // Saves without listing the record in the next fetch, as a fetch that lags the save would. func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome { var outcome = PushOutcome() diff --git a/apple/TableProTests/Core/Sync/SyncRemoteConnectionSecretsTests.swift b/apple/TableProTests/Core/Sync/SyncRemoteConnectionSecretsTests.swift new file mode 100644 index 0000000000..d3c157479d --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncRemoteConnectionSecretsTests.swift @@ -0,0 +1,75 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct SyncRemoteConnectionSecretsTests { + private static let zoneID = SyncTestEnvironment.zoneID + + private let environment: SyncTestEnvironment + + init() throws { + environment = try SyncTestEnvironment(label: "sync-remote-connection-secrets") + } + + private var storage: ConnectionStorage { environment.connections } + + private func secretKeys(of id: UUID) -> [String] { + [ + "password", "sshpassword", "keypassphrase", "totpsecret", "cloudflaretokensecret", + "cloudsqlproxyserviceaccountkey", "socksproxypassword", "plugin.mongoAWSSessionToken" + ].map { "com.TablePro.\($0).\(id.uuidString)" } + } + + private func addConnectionWithSecrets(localOnly: Bool = false) -> DatabaseConnection { + var connection = TestFixtures.makeConnection(type: .mongodb) + connection.localOnly = localOnly + #expect(storage.addConnection(connection, password: "hunter2")) + storage.saveSSHPassword("ssh", for: connection.id) + storage.saveKeyPassphrase("passphrase", for: connection.id) + storage.saveTOTPSecret("JBSWY3DP", for: connection.id) + storage.saveCloudflareTokenSecret("cf", for: connection.id) + storage.saveCloudSQLProxyServiceAccountKey("{}", for: connection.id) + storage.saveSOCKSProxyPassword("proxy", for: connection.id) + storage.savePluginSecureField("FQoG", fieldId: "mongoAWSSessionToken", for: connection.id) + return connection + } + + private func deletion(of connection: DatabaseConnection) -> PullResult { + PullResult( + changedRecords: [], + deletedRecordIDs: [SyncRecordMapper.recordID(type: .connection, id: connection.id.uuidString, in: Self.zoneID)], + newToken: nil + ) + } + + @Test("A connection deleted on another device takes its secrets out of this Mac's Keychain") + func remoteDeletionRemovesTheSecrets() async { + let connection = addConnectionWithSecrets() + + let acknowledged = await environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + .applyPullResult(deletion(of: connection)) + + #expect(acknowledged) + #expect(storage.loadConnection(id: connection.id) == nil) + for key in secretKeys(of: connection.id) { + #expect(environment.keychain.readStringResult(forKey: key) == .notFound, "\(key)") + } + } + + @Test("A Local only connection named by a deletion from another device keeps its row and its secrets") + func localOnlyConnectionKeepsItsSecrets() async { + let connection = addConnectionWithSecrets(localOnly: true) + + let acknowledged = await environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + .applyPullResult(deletion(of: connection)) + + #expect(acknowledged) + #expect(storage.loadConnection(id: connection.id) != nil) + for key in secretKeys(of: connection.id) { + #expect(environment.keychain.readStringResult(forKey: key) != .notFound, "\(key)") + } + } +} diff --git a/apple/TableProTests/Core/Sync/SyncSavedQueriesRefillTests.swift b/apple/TableProTests/Core/Sync/SyncSavedQueriesRefillTests.swift new file mode 100644 index 0000000000..091b9fa11e --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncSavedQueriesRefillTests.swift @@ -0,0 +1,198 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct SyncSavedQueriesRefillTests { + private static let zoneID = SyncTestEnvironment.zoneID + + private let environment: SyncTestEnvironment + + init() throws { + environment = try SyncTestEnvironment(label: "sync-saved-queries-refill") + var settings = SyncSettings.default + settings.enabled = true + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + _ = environment.metadata.adoptReadableRecordTypes(SyncCoordinator.readableContent(for: settings)) + } + + @Test("A damaged saved queries store is refilled by a full fetch although a sync token is stored") + func damagedStoreIsRefilledByAFullFetch() async throws { + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") + let token = try Self.makeChangeToken() + environment.metadata.saveToken(token) + let server = RefillZoneServer(zoneID: Self.zoneID, records: [Self.record(favorite)], token: token) + let damaged = try damagedFavorites() + let coordinator = makeCoordinator(transport: server, favorites: damaged) + + #expect(await coordinator.runSyncCycle() == nil) + + #expect(await server.pulls == [.full]) + #expect(await damaged.fetchFavorite(id: favorite.id)?.name == "Revenue") + #expect(await damaged.isAwaitingSyncRefill() == false) + + #expect(await coordinator.runSyncCycle() == nil) + #expect(await server.pulls == [.full, .delta]) + } + + @Test("A saved query deleted and then edited again before the damage comes back instead of being deleted everywhere") + func restoredSavedQuerySurvivesTheRefill() async throws { + let restored = SQLFavorite(name: "Revenue", query: "SELECT 1") + let id = restored.id.uuidString + environment.tracker.markDeleted(.favorite, id: id) + environment.tracker.markDirty(.favorite, id: id) + let token = try Self.makeChangeToken() + environment.metadata.saveToken(token) + let server = RefillZoneServer(zoneID: Self.zoneID, records: [Self.record(restored)], token: token) + let damaged = try damagedFavorites() + let coordinator = makeCoordinator(transport: server, favorites: damaged) + + #expect(await coordinator.runSyncCycle() == nil) + #expect(await coordinator.runSyncCycle() == nil) + + #expect(await server.pushedDeletionNames.isEmpty) + #expect(await damaged.fetchFavorite(id: restored.id)?.name == "Revenue") + } + + @Test("A full fetch that left a saved query unread keeps the refill going until a fetch delivers it") + func unreadSavedQueryKeepsTheRefill() async throws { + let delivered = SQLFavorite(name: "Revenue", query: "SELECT 1") + let late = SQLFavorite(name: "Churn", query: "SELECT 2") + let token = try Self.makeChangeToken() + environment.metadata.saveToken(token) + let server = RefillZoneServer( + zoneID: Self.zoneID, + records: [Self.record(delivered)], + token: token, + unread: [Self.record(late).recordID] + ) + let damaged = try damagedFavorites() + let coordinator = makeCoordinator(transport: server, favorites: damaged) + + #expect(await coordinator.runSyncCycle() == nil) + #expect(await damaged.isAwaitingSyncRefill()) + + await server.serve([Self.record(delivered), Self.record(late)], unread: []) + #expect(await coordinator.runSyncCycle() == nil) + + #expect(await server.pulls == [.full, .full]) + #expect(await damaged.isAwaitingSyncRefill() == false) + #expect(await damaged.fetchFavorite(id: late.id)?.name == "Churn") + } + + private func damagedFavorites() throws -> SQLFavoriteManager { + let url = environment.directory.appendingPathComponent("damaged-sql_favorites.db") + try Data(repeating: 0x2A, count: 4_096).write(to: url) + return SQLFavoriteManager( + storage: SQLFavoriteStorage(databaseURL: url, removeDatabaseOnDeinit: true), + syncTracker: environment.tracker + ) + } + + private func makeCoordinator(transport: any SyncTransport, favorites: SQLFavoriteManager) -> SyncCoordinator { + let base = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID), favorites: favorites) + return SyncCoordinator( + services: base.services, + recordCache: environment.recordCache, + transport: transport, + makeNetworkMonitor: { nil }, + columnLayouts: environment.columnLayouts, + licenseGate: { nil }, + pushRegistrar: .inert + ) + } + + private static func record(_ favorite: SQLFavorite) -> CKRecord { + SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: zoneID) + } + + private static func makeChangeToken() throws -> CKServerChangeToken { + let archiver = NSKeyedArchiver(requiringSecureCoding: true) + archiver.setClassName("CKServerChangeToken", for: RefillChangeTokenArchive.self) + archiver.encode(RefillChangeTokenArchive(), forKey: NSKeyedArchiveRootObjectKey) + archiver.finishEncoding() + let token = try NSKeyedUnarchiver.unarchivedObject(ofClass: CKServerChangeToken.self, from: archiver.encodedData) + return try #require(token) + } +} + +// Serves every record on a full fetch and nothing on a delta, and records what was asked and pushed. +private actor RefillZoneServer: SyncTransport { + enum Pull: Equatable { + case full + case delta + } + + let currentZoneID: CKRecordZone.ID + private var records: [CKRecord] + private var unread: [CKRecord.ID] + private let token: CKServerChangeToken + private(set) var pulls: [Pull] = [] + private(set) var pushedDeletionNames: [String] = [] + + init(zoneID: CKRecordZone.ID, records: [CKRecord], token: CKServerChangeToken, unread: [CKRecord.ID] = []) { + currentZoneID = zoneID + self.records = records + self.token = token + self.unread = unread + } + + func serve(_ records: [CKRecord], unread: [CKRecord.ID]) { + self.records = records + self.unread = unread + } + + func accountStatus() async throws -> CKAccountStatus { + .available + } + + func currentAccountId() async throws -> String { + ScriptedSyncTransport.accountId + } + + func ensureZoneExists() async throws {} + + func saveSubscription() async throws {} + + func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome { + var outcome = PushOutcome() + for record in records { + outcome.recordSave(record) + } + for recordID in deletions { + pushedDeletionNames.append(recordID.recordName) + outcome.recordDeletion(recordID) + } + return outcome + } + + func fetch(recordIDs: [CKRecord.ID]) async throws -> [CKRecord.ID: CKRecord] { + [:] + } + + func pull(since token: CKServerChangeToken?) async throws -> PullResult { + guard token != nil else { + pulls.append(.full) + return PullResult(changedRecords: records, deletedRecordIDs: [], newToken: self.token, unreadRecordIDs: unread) + } + pulls.append(.delta) + return PullResult(changedRecords: [], deletedRecordIDs: [], newToken: self.token) + } +} + +@objc(SyncSavedQueriesRefillTestsChangeTokenArchive) +private final class RefillChangeTokenArchive: NSObject, NSSecureCoding { + static var supportsSecureCoding: Bool { true } + + override init() { + super.init() + } + + required init?(coder: NSCoder) { + super.init() + } + + func encode(with coder: NSCoder) {} +} diff --git a/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift b/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift index 4cb996f0e4..0ee582a754 100644 --- a/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift +++ b/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift @@ -62,3 +62,47 @@ struct PrincipalApplyOpenTransactionTests { #expect(driver.executedQueries.isEmpty) } } + +// The review sheet showed a Cancel at the gate as a failed apply. +@MainActor +struct PrincipalApplyGateDenialTests { + private func apply(gate: any ExecutionGate) async -> (failure: GatedOperationFailure?, executed: [String]) { + let driver = TransactionHoldingPrincipalDriver(rollsBack: true) + let connection = TestFixtures.makeConnection(type: .postgresql) + DatabaseManager.shared.injectSession( + ConnectionSession(connection: connection, driver: PluginDriverAdapter(connection: connection, pluginDriver: driver)), + for: connection.id + ) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let change = PrincipalChange.create(PluginPrincipalDefinition(ref: PluginPrincipalRef(name: "X"), password: "p")) + do { + try await DatabaseManager.shared.executePrincipalChanges( + changes: [change], + databaseType: .postgresql, + connectionId: connection.id, + gate: gate + ) + return (nil, driver.executedQueries) + } catch { + return (GatedOperationFailure(error), driver.executedQueries) + } + } + + @Test("A Cancel at the gate ends the apply quietly, with nothing run") + func cancelIsQuiet() async { + let result = await apply(gate: AlwaysDenyGate.cancelling) + + #expect(result.failure == .cancelledByUser) + #expect(result.failure?.message == nil) + #expect(result.executed.isEmpty) + } + + @Test("A refused apply says why and is not a failed statement") + func refusalExplainsItself() async { + let result = await apply(gate: AlwaysDenyGate(reason: "Safe Mode is read-only")) + + #expect(result.failure == .refused("Safe Mode is read-only")) + #expect(result.executed.isEmpty) + } +} diff --git a/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift b/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift index fcdd2c5053..d8c628c9eb 100644 --- a/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift +++ b/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift @@ -329,6 +329,46 @@ struct PrincipalChangeManagerTests { #expect(manager.changeCount == 1) } + @Test("A staged drop runs only the DROP and counts as one change") + func dropSupersedesOtherEdits() { + let manager = makeManager() + manager.stageAlter(PluginPrincipalDefinition(ref: alice, canLogin: false), for: alice) + manager.stageSetPassword("s3cret", for: alice) + manager.setGranted(true, privilege: "CREATE", scope: app, for: alice) + #expect(manager.changeCount == 3) + + manager.stageDrop(alice, options: PluginPrincipalDropOptions()) + + let changes = manager.pendingChanges() + #expect(changes.count == 1) + guard case .drop(let ref, _) = changes.first else { + Issue.record("Expected only the DROP, got \(changes)") + return + } + #expect(ref == alice) + #expect(manager.changeCount == 1) + } + + @Test("Undoing a staged drop brings the principal's other edits back") + func unstagingDropRestoresEdits() { + let manager = makeManager() + manager.stageAlter(PluginPrincipalDefinition(ref: alice, canLogin: false), for: alice) + manager.stageSetPassword("s3cret", for: alice) + manager.setGranted(true, privilege: "CREATE", scope: app, for: alice) + manager.stageDrop(alice, options: PluginPrincipalDropOptions()) + + manager.unstageDrop(alice) + + let changes = manager.pendingChanges() + let dropsAnything = changes.contains { change in + if case .drop = change { return true } + return false + } + #expect(changes.count == 3) + #expect(!dropsAnything) + #expect(manager.changeCount == 3) + } + @Test("Dropping the connected account is reported as self-impact") func reportsSelfImpact() { let manager = makeManager() diff --git a/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift b/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift index 5d3279d1d6..ca411b21b4 100644 --- a/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift @@ -135,4 +135,32 @@ struct InClauseConverterTests { let result = converter.generateInClause(rows: [["alice", "1"], ["bob", "2"]]) #expect(result == "(1, 2)") } + + @Test("An engine without TRUE and FALSE gets 1 and 0") + func numericBooleans() { + var converter = makeConverter(columnIndex: 0, columnTypes: [.boolean(rawType: "bit")]) + converter.booleanLiteralStyle = .numeric + #expect(converter.generateInClause(rows: [["1"], ["false"], ["yes"]]) == "(1, 0, 1)") + } + + @Test("Binary values are written as the engine's binary literal, not dropped") + func binaryValues() { + var converter = makeConverter(columnIndex: 0, columnTypes: [.blob(rawType: "BINARY(16)")]) + converter.databaseType = .mysql + let rows: [[PluginCellValue]] = [[.bytes(Data([0x41, 0x42]))], [.bytes(Data([0x00]))], [.null]] + #expect(converter.generateInClause(rows: rows) == "(X'4142', X'00')") + + converter.databaseType = .mssql + #expect(converter.generateInClause(rows: rows) == "(0x4142, 0x00)") + + converter.databaseType = .postgresql + #expect(converter.generateInClause(rows: [[.bytes(Data([0x41]))]]) == "(decode('41', 'hex'))") + } + + // SQLite keeps text in a BLOB column, and text never equals a blob there. + @Test("Text in a binary column stays text, and with no engine bytes take the standard literal") + func binaryColumnTextWithoutEngine() { + let converter = makeConverter(columnIndex: 0, columnTypes: [.blob(rawType: "blob")]) + #expect(converter.generateInClause(rows: [["AB"], [.bytes(Data([0xFF]))]]) == "('AB', X'FF')") + } } diff --git a/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift b/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift index 759614a413..975bf5bd2a 100644 --- a/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift @@ -250,6 +250,14 @@ struct JsonRowConverterTests { #expect(result.contains("\"maybe\"")) } + @Test("A boolean column holding another integer writes that number") + func booleanColumnOtherInteger() { + let converter = makeConverter(columns: ["flag"], columnTypes: [.boolean(rawType: "TINYINT(1)")]) + let result = converter.generateJson(rows: [["2"]]) + #expect(result.contains(": 2")) + #expect(!result.contains("\"2\"")) + } + // MARK: - JSON @Test("Valid JSON column is embedded verbatim") @@ -384,20 +392,16 @@ struct JsonRowConverterTests { // MARK: - Default-value marker - /// A pending inserted row carries the default marker for columns the server fills in. The grid - /// draws it as a placeholder, so JSON has to agree instead of printing the internal token. - @Test("The default-value marker renders as null, not as literal text") - func defaultMarkerRendersAsNull() { + // The default is marked beside the value, so text a table holds that spells the old marker is a value. + @Test("Stored text that spells the old default marker renders as that text") + func oldMarkerTextIsAValue() { let converter = makeConverter( - columns: ["id", "name"], - columnTypes: [.integer(rawType: nil), .text(rawType: nil)] - ) - let result = converter.generateJson( - rows: [[.text(PluginCellValue.defaultMarkerText), .text("Ada")]] + columns: ["status", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] ) + let result = converter.generateJson(rows: [[.text("__DEFAULT__"), .text("Ada")]]) - #expect(!result.contains(PluginCellValue.defaultMarkerText)) - #expect(result.contains("\"id\": null")) + #expect(result.contains("\"status\": \"__DEFAULT__\"")) #expect(result.contains("\"Ada\"")) } diff --git a/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift b/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift index 0cb1ce7b76..78b83b244f 100644 --- a/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift +++ b/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift @@ -259,8 +259,8 @@ struct PasswordSourceResolverTests { ) #expect(try await PasswordSourceResolver.resolve(source, for: context, cache: cache) == "1") - #expect(try await PasswordSourceResolver.test(source, for: context) == "2") - #expect(try await PasswordSourceResolver.test(source, for: context) == "3") + #expect(try await PasswordSourceResolver.test(source, for: context, leaseOwner: context.connectionId) == "2") + #expect(try await PasswordSourceResolver.test(source, for: context, leaseOwner: context.connectionId) == "3") #expect(try await PasswordSourceResolver.resolve(source, for: context, cache: cache) == "1") } @@ -270,7 +270,11 @@ struct PasswordSourceResolverTests { connectionId: UUID(), connectionName: "c", host: "db.example.com", port: 5432, user: "u", database: "d" ) - let output = try await PasswordSourceResolver.test(.command(shell: "echo {host}:$TABLEPRO_PORT"), for: context) + let output = try await PasswordSourceResolver.test( + .command(shell: "echo {host}:$TABLEPRO_PORT"), + for: context, + leaseOwner: context.connectionId + ) #expect(output == "db.example.com:5432") } diff --git a/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift index 4b4a7a6637..129021733c 100644 --- a/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift @@ -392,6 +392,23 @@ struct EtcdCommandParserDelTests { try EtcdCommandParser.parse("del") } } + + @Test("A del of one key is a write, and a del of a prefix is destructive") + func delTier() throws { + #expect(try EtcdCommandParser.parse("del mykey").tier == .write) + #expect(try EtcdCommandParser.parse("delete mykey").tier == .write) + #expect(try EtcdCommandParser.parse("del --prefix -- --cache/").tier == .destructive) + #expect(try EtcdCommandParser.parse("del /app/ --prefix").tier == .destructive) + #expect(try EtcdCommandParser.parse(#"del "" --prefix"#).tier == .destructive) + } + + @Test("A del etcdctl reads as a range is refused, and graded destructive", arguments: [ + "del /a /z", "del --from-key /a", "del /a --from-key=true", "del /a --prefix=true", "del /a --prefix=false", + ]) + func delOfARangeIsRefused(line: String) { + #expect(throws: EtcdParseError.self) { try EtcdCommandParser.parse(line) } + #expect(QueryClassifier.classifyTier(line, databaseType: .etcd) == .destructive) + } } struct EtcdCommandParserWatchTests { @@ -1162,10 +1179,10 @@ private let etcdReadLines = [ ] private let etcdWriteLines = [ "put /a v", "lease grant 60", "lease keep-alive 1", "auth enable", "user add bob pw", "role add readers", - "user grant-role bob readers", "user revoke-role bob readers", "create /a v", + "user grant-role bob readers", "user revoke-role bob readers", "create /a v", "del /a", "delete /a", ] private let etcdDeleteLines = [ - "del /a", #"del "" --prefix"#, "del /app/ --prefix", "compaction 5", "lease revoke 1", "auth disable", + #"del "" --prefix"#, "del /app/ --prefix", "compaction 5", "lease revoke 1", "auth disable", "user delete bob", "role delete readers", "mv /a /b v --mod-rev=3", "mv /a /b v", ] diff --git a/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift b/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift index a85513aba3..8c1667f800 100644 --- a/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift @@ -71,11 +71,22 @@ struct InvisibleCharacterRemoverTests { #expect(clean(text, grammar: TestGrammar.postgres) == text) } - /// Measured on PostgreSQL 17: `SELECT$$a b$$` is a syntax error, because every byte over 0x7F continues an - /// identifier there and a `$` glued to one opens no literal. So the text is code, and its blanks are cleaned. - @Test("A dollar sign glued to a non-ASCII blank opens no literal") - func dollarQuoteBehindABlankIsCode() { - #expect(clean("SELECT\u{A0}$$a\u{A0}b$$", grammar: TestGrammar.postgres) == "SELECT $$a b$$") + // PostgreSQL reads the raw NBSP into the name, but the cleaned text that runs opens a dollar quote there. + @Test("A dollar-quoted body after a non-ASCII blank keeps its characters") + func dollarQuoteBehindABlankKeepsItsBody() { + #expect(clean("SELECT\u{A0}$$a\u{A0}b$$", grammar: TestGrammar.postgres) == "SELECT $$a\u{A0}b$$") + #expect(clean("SELECT\u{1680}$$a\u{A0}b$$", grammar: TestGrammar.duckdb) == "SELECT $$a\u{A0}b$$") + } + + @Test("A line separator before an escape string ends the name, as the line break it becomes does") + func lineSeparatorBeforeEscapeString() { + #expect(clean("SELECT\u{2028}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "SELECT\nE'a\\'\u{A0}b'") + } + + @Test("A removed character joins its neighbours when deciding where an escape string starts") + func removedCharacterBeforeEscapeString() { + #expect(clean("SELECT \u{200B}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "SELECT E'a\\'\u{A0}b'") + #expect(clean("x\u{8}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "xE'a\\' b'") } @Test("A MySQL conditional comment runs, so it is cleaned like code") @@ -97,6 +108,13 @@ struct InvisibleCharacterRemoverTests { #expect(clean("/* a /* b */\u{A0} */ SELECT\u{A0}1", grammar: grammar) == "/* a /* b */\u{A0} */ SELECT 1") } + // Measured on PostgreSQL 17.11: the NBSP left a syntax error, and the cleaned text returns `ab`. + @Test("A non-ASCII space between the parts of a PostgreSQL string is cleaned, so the parts join as they run") + func nonASCIISpaceBeforeAContinuedString() { + #expect(clean("SELECT E'a'\u{A0}\n'b'", grammar: TestGrammar.postgres) == "SELECT E'a' \n'b'") + #expect(clean("SELECT E'a'\n\u{3000}'b\u{A0}c'", grammar: TestGrammar.postgres) == "SELECT E'a'\n 'b\u{A0}c'") + } + @Test("A bracketed identifier keeps its characters where brackets quote names") func bracketedIdentifier() { let grammar = TestGrammar.sqlite diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierClientScriptTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierClientScriptTests.swift new file mode 100644 index 0000000000..8b95fdee77 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierClientScriptTests.swift @@ -0,0 +1,35 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +// A script line the client reads, a DELIMITER or a COPY's rows, never lets a second statement past the gate. +struct QueryClassifierClientScriptTests { + @Test("A routine between DELIMITER lines is one statement for the gate") + func delimitedRoutineIsOneStatement() { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END//\nDELIMITER ;" + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mysql)) + } + + @Test("A ; the server splits at is still a second statement under a custom delimiter") + func customDelimiterHidesNoStatement() { + let sql = "DELIMITER //\nSELECT 1; DROP TABLE users//" + #expect(QueryClassifier.isMultiStatement(sql, databaseType: .mysql)) + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: .mysql)) + } + + @Test("A COPY's rows are never statements, and the COPY is never safe") + func copyRowsAreNotStatements() { + let sql = "COPY t FROM stdin;\nx\tDROP TABLE users;\n\\.\n" + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .postgresql)) + #expect(QueryClassifier.classifyTier(sql, databaseType: .postgresql) != .safe) + #expect(QueryClassifier.isMultiStatement(sql + "SELECT 1;", databaseType: .postgresql)) + } + + @Test("An IF ... ELSE with a ; before the ELSE is one statement, and both branches are tiered") + func ifElseIsOneStatement() { + let sql = "IF @env = 'dev' SELECT 1; ELSE DROP TABLE dbo.t;" + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + #expect(QueryClassifier.classifyTier(sql, databaseType: .mssql) == .destructive) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierEmbeddedStatementTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierEmbeddedStatementTests.swift index 4233b88457..00c8c9d25d 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierEmbeddedStatementTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierEmbeddedStatementTests.swift @@ -67,6 +67,17 @@ struct QueryClassifierEmbeddedStatementTests { } } + @Test("A SurrealQL DELETE of one record by id is a row change, on its own or inside another statement") + func surrealQLRecordDeleteIsAWrite() { + for sql in ["DELETE person:tobie", "DELETE ONLY person:⟨tobie⟩ RETURN BEFORE", "SELECT * FROM (DELETE person:tobie)"] { + #expect(QueryClassifier.classifyTier(sql, databaseType: .surrealdb) == .write, "\(sql)") + #expect(!QueryClassifier.isDangerousQuery(sql, databaseType: .surrealdb), "\(sql)") + } + for sql in ["DELETE person", "DELETE person:1..10", "DELETE person:a, person:b", "SELECT * FROM (DELETE person:..)"] { + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: .surrealdb), "\(sql)") + } + } + @Test("A CockroachDB SELECT from a bracketed statement that writes is a write", arguments: [ "SELECT * FROM [DELETE FROM orders RETURNING id]", "SELECT * FROM [ UPDATE orders SET total = 0 RETURNING id ]", diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index 01da6e95ba..cab145eb8f 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -685,7 +685,8 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery("get /keys", databaseType: .etcd)) #expect(!QueryClassifier.isWriteQuery("watch /keys", databaseType: .etcd)) #expect(QueryClassifier.isWriteQuery("put /keys value", databaseType: .etcd)) - #expect(QueryClassifier.classifyTier("del /keys", databaseType: .etcd) == .destructive) + #expect(QueryClassifier.classifyTier("del /keys", databaseType: .etcd) == .write) + #expect(QueryClassifier.classifyTier("del /keys --prefix", databaseType: .etcd) == .destructive) #expect(QueryClassifier.classifyTier("compact 5", databaseType: .etcd) == .destructive) #expect(QueryClassifier.reachesFilesystemOrExecutesCode("snapshot save backup.db", databaseType: .etcd)) } @@ -698,7 +699,7 @@ struct QueryClassifierNonSqlTests { "user delete alice", "role delete readers", "auth disable", - "delete /keys", + "delete /keys --prefix", "compaction 5 --physical" ] for statement in destructive { @@ -827,8 +828,9 @@ struct QueryClassifierNonSqlTests { QueryClassifier.classifyTier( "DELETE /collections/books/documents/1", databaseType: .typesense - ) == .destructive + ) == .write ) + #expect(QueryClassifier.classifyTier("DELETE /collections/books", databaseType: .typesense) == .destructive) } /// The body of a Typesense console request is the caller's own JSON. Scanning the whole @@ -1037,7 +1039,7 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.reachesFilesystemOrExecutesCode("GET /collections", databaseType: .qdrant)) } - @Test("Memcached reads are safe, stores write, and delete and flush_all are destructive") + @Test("Memcached reads are safe, stores and delete write, and flush_all is destructive") func memcachedTiers() { #expect(!QueryClassifier.isWriteQuery("MEMCACHED_BROWSE:e30=", databaseType: .memcached)) #expect(!QueryClassifier.isWriteQuery("get user:1 user:2", databaseType: .memcached)) @@ -1054,12 +1056,16 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.classifyTier("set k 0 0 \"v\"", databaseType: .memcached) == .write) #expect(QueryClassifier.classifyTier("gat 60 k", databaseType: .memcached) == .write) #expect(QueryClassifier.classifyTier("touch k 0", databaseType: .memcached) == .write) - #expect(QueryClassifier.classifyTier("delete k", databaseType: .memcached) == .destructive) + #expect(QueryClassifier.classifyTier("delete k", databaseType: .memcached) == .write) #expect(QueryClassifier.classifyTier("FLUSH_ALL", databaseType: .memcached) == .destructive) #expect(QueryClassifier.classifyTier("get k; flush_all", databaseType: .memcached) == .destructive) + // The driver decodes an escape inside a quoted word before it names the command. + #expect(QueryClassifier.classifyTier(#""flush\x5fall""#, databaseType: .memcached) == .destructive) + #expect(QueryClassifier.classifyTier(#""flush\_all""#, databaseType: .memcached) == .destructive) + #expect(QueryClassifier.classifyTier(#""\x67et" k"#, databaseType: .memcached) == .safe) } - @Test("Bigtable request lines are tiered by their verb, and a PUT that nulls a cell is destructive") + @Test("Bigtable request lines are tiered by their verb, and a change to one row is a write, deletes included") func bigtableTiers() { #expect(!QueryClassifier.isWriteQuery("BIGTABLE_BROWSE:e30=", databaseType: .bigtable)) #expect(!QueryClassifier.isWriteQuery("GET events/user#42", databaseType: .bigtable)) @@ -1067,9 +1073,9 @@ struct QueryClassifierNonSqlTests { #expect(!QueryClassifier.isWriteQuery("READ events {\"rowsLimit\": \"5\"}", databaseType: .bigtable)) #expect(QueryClassifier.classifyTier("INSERT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": \"1\"}", databaseType: .bigtable) == .write) - #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": null}", databaseType: .bigtable) == .destructive) - #expect(QueryClassifier.classifyTier("PUT events/\"a{b\" {\"cf:a\": null}", databaseType: .bigtable) == .destructive) - #expect(QueryClassifier.classifyTier("DELETE events/k", databaseType: .bigtable) == .destructive) + #expect(QueryClassifier.classifyTier("PUT events/k {\"cf:a\": null}", databaseType: .bigtable) == .write) + #expect(QueryClassifier.classifyTier("PUT events/\"a{b\" {\"cf:a\": null}", databaseType: .bigtable) == .write) + #expect(QueryClassifier.classifyTier("DELETE events/k", databaseType: .bigtable) == .write) #expect(QueryClassifier.isWriteQuery("SELECT * FROM events", databaseType: .bigtable)) } @@ -1166,19 +1172,27 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.isWriteQuery("POST users {}", databaseType: type)) #expect(QueryClassifier.isWriteQuery("PUT /config/maintenance false", databaseType: type)) #expect(QueryClassifier.isWriteQuery("SELECT * FROM users", databaseType: type)) - #expect(QueryClassifier.classifyTier("DELETE users/alice", databaseType: type) == .destructive) + #expect(QueryClassifier.classifyTier("DELETE users/alice", databaseType: type) == .write) + #expect(QueryClassifier.classifyTier("DELETE users", databaseType: type) == .destructive) } } - @Test("A Firebase write that replaces the root or leaves a node empty ranks with DELETE") + @Test("A Firebase write that replaces the root or a Realtime Database table, or leaves a node empty, ranks with DELETE") func firebaseWritesThatDelete() { for type in [DatabaseType.firestore, .firebaseRealtimeDatabase] { #expect(QueryClassifier.classifyTier("PUT / {\"a\": 1}", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier("PUT %2F 1", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier("PUT users null", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier("PUT users\n{\"a\": {}, \"b\": [null]}", databaseType: type) == .destructive) - #expect(QueryClassifier.classifyTier("PUT users {\"a\": 1}", databaseType: type) == .write) + #expect(QueryClassifier.classifyTier("PUT users/alice {\"a\": 1}", databaseType: type) == .write) } + #expect(QueryClassifier.classifyTier("PUT users {\"a\": 1}", databaseType: .firestore) == .write) + // A top-level node is a table: replacing it drops every record under it. + #expect(QueryClassifier.classifyTier("PUT users {\"a\": 1}", databaseType: .firebaseRealtimeDatabase) == .destructive) + #expect(QueryClassifier.classifyTier("PATCH / {\"users\": {\"x\": 1}}", databaseType: .firebaseRealtimeDatabase) == .destructive) + #expect(QueryClassifier.classifyTier("PATCH / {\"users/x\": 1}", databaseType: .firebaseRealtimeDatabase) == .write) + #expect(QueryClassifier.classifyTier("PATCH users {\"x\": {\"n\": 1}}", databaseType: .firebaseRealtimeDatabase) == .write) + #expect(QueryClassifier.classifyTier("POST users {\"n\": 1}", databaseType: .firebaseRealtimeDatabase) == .write) #expect(QueryClassifier.classifyTier("PATCH / {\"users\": null}", databaseType: .firebaseRealtimeDatabase) == .destructive) #expect(QueryClassifier.classifyTier("PATCH users/a {\"n\": 1}", databaseType: .firebaseRealtimeDatabase) == .write) #expect(QueryClassifier.classifyTier("PATCH users/a {\"n\": null}", databaseType: .firestore) == .write) @@ -1191,12 +1205,12 @@ struct QueryClassifierNonSqlTests { #expect(QueryClassifier.classifyTier("PUT users {\".value\": null, \".priority\": 1}", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier("PUT users {\"a\": {\".priority\": 1}}", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier("PUT users {\".value\": {\".priority\": 1}}", databaseType: type) == .destructive) - #expect(QueryClassifier.classifyTier("PUT users {\".value\": 5, \".priority\": 1}", databaseType: type) == .write) - #expect(QueryClassifier.classifyTier("PUT users {\".sv\": \"timestamp\"}", databaseType: type) == .write) - #expect(QueryClassifier.classifyTier("PUT users {\"a\": 1, \".priority\": 1}", databaseType: type) == .write) + #expect(QueryClassifier.classifyTier("PUT users/a {\".value\": 5, \".priority\": 1}", databaseType: type) == .write) + #expect(QueryClassifier.classifyTier("PUT users/a {\".sv\": \"timestamp\"}", databaseType: type) == .write) + #expect(QueryClassifier.classifyTier("PUT users/a {\"a\": 1, \".priority\": 1}", databaseType: type) == .write) } #expect(QueryClassifier.classifyTier("PATCH / {\"users\": {\".priority\": 1}}", databaseType: .firebaseRealtimeDatabase) == .destructive) - #expect(QueryClassifier.classifyTier("PATCH users {\".priority\": 1}", databaseType: .firebaseRealtimeDatabase) == .write) + #expect(QueryClassifier.classifyTier("PATCH users/a {\".priority\": 1}", databaseType: .firebaseRealtimeDatabase) == .write) } @Test("A Firebase path that starts with a mark is tiered as the driver reads it", arguments: [ @@ -1206,11 +1220,13 @@ struct QueryClassifierNonSqlTests { for type in [DatabaseType.firestore, .firebaseRealtimeDatabase] { #expect(QueryClassifier.classifyTier("PUT \(name) null", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier(FirebaseRequest(method: .put, path: name, body: "null").text, databaseType: type) == .destructive) - #expect(QueryClassifier.classifyTier("PUT \(name) {\"a\": 1}", databaseType: type) == .write) + #expect(QueryClassifier.classifyTier("PUT \(name)/a {\"a\": 1}", databaseType: type) == .write) #expect(QueryClassifier.classifyTier("DELETE \(name)", databaseType: type) == .destructive) #expect(QueryClassifier.classifyTier("GET \(name)", databaseType: type) == .safe) } - #expect(QueryClassifier.classifyTier("PATCH \(name) {\"x\": null}", databaseType: .firebaseRealtimeDatabase) == .destructive) + #expect(QueryClassifier.classifyTier("PUT \(name) {\"a\": 1}", databaseType: .firestore) == .write) + // Read as the root, this PATCH would remove a top-level node; read as the driver does, it removes one record. + #expect(QueryClassifier.classifyTier("PATCH \(name) {\"x\": null}", databaseType: .firebaseRealtimeDatabase) == .write) } @Test("A console body declaring a mutation is classified the same as a bare one") @@ -1247,13 +1263,12 @@ struct QueryClassifierNonSqlTests { } @Test("A request text is tiered by the requests the scanner hands the driver", arguments: [ - (DatabaseType.firestore, "GET users", "DELETE users/alice"), - (.firebaseRealtimeDatabase, "GET users", "DELETE users/alice"), - (.bigtable, "GET events/k", "DELETE events/k"), + (DatabaseType.firestore, "GET users", "DELETE users"), + (.firebaseRealtimeDatabase, "GET users", "DELETE users"), (.typesense, "GET /collections", "DELETE /collections/books"), (.weaviate, "GET /v1/schema", "DELETE /v1/schema/Article"), (.elasticsearch, "GET /_cat/indices", "DELETE /orders"), - (.etcd, "get /keys", "del /keys") + (.etcd, "get /keys", "del /keys --prefix") ]) func requestTextTiersWhatTheDriverReceives(type: DatabaseType, read: String, delete: String) { for text in [";\(delete)", "\(read);\(delete)", "--\n;\(delete)", "--\r\(read)\n;\(delete)", "/**/;\(delete)"] { @@ -1264,6 +1279,19 @@ struct QueryClassifierNonSqlTests { } } + // Every Bigtable change touches one row, so the hidden one is a write. + @Test("A Bigtable request text is tiered by the requests the scanner hands the driver") + func bigtableRequestTextTiersWhatTheDriverReceives() { + let read = "GET events/k" + let delete = "DELETE events/k" + for text in [";\(delete)", "\(read);\(delete)", "--\n;\(delete)", "--\r\(read)\n;\(delete)", "/**/;\(delete)"] { + #expect(QueryClassifier.classifyTier(text, databaseType: .bigtable) == .write, "\(text) must be a write") + } + for text in ["\(read);", ";\(read)", "-- note\n;\(read)", "-- \(delete)", ";"] { + #expect(QueryClassifier.classifyTier(text, databaseType: .bigtable) == .safe, "\(text) must be safe") + } + } + @Test("Elasticsearch stored scripts are flagged as code execution on both verbs") func elasticsearchScriptsAreFlagged() { #expect( diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift index 6fc2683ea8..e65f1ad426 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift @@ -146,6 +146,33 @@ struct QueryClassifierLexicalTests { #expect(QueryClassifier.isDangerousQuery(bypass.sql, databaseType: bypass.engine)) } + // Measured on 2026-11-13: MariaDB 13.0.2 stores each routine body whole and alone, psql 17.11 ran the statement + // after `'\'`, the NBSP being part of a name, and PostgreSQL 17.11 ran the DROP after the function in one query. + @Test("The statement after a MySQL SET or IF body, or after a string a non-ASCII space ends, is its own", arguments: [ + (DatabaseType.mysql, "CREATE PROCEDURE p() SET @x = begin IS NULL; DROP TABLE users"), + (DatabaseType.mariadb, "CREATE PROCEDURE p() IF 1 THEN SELECT begin FROM t; END IF; DROP TABLE users"), + (DatabaseType.postgresql, "SELECT E'a'\u{A0}\n'\\'; DROP TABLE users; --'"), + ( + DatabaseType.postgresql, + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; END; " + + "DROP TABLE users; END" + ), + (DatabaseType(rawValue: "Nonesuch"), "IF @x = 1 SELECT 1; DROP TABLE users"), + ]) + func statementAfterABodyIsItsOwn(engine: DatabaseType, sql: String) { + #expect(QueryClassifier.classifyTier(sql, databaseType: engine) == .destructive) + #expect(QueryClassifier.isMultiStatement(sql, databaseType: engine)) + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: engine)) + } + + @Test("A MySQL IF body is one statement for an external client, and one left open is several") + func mySQLIfBodyForExternalClients() { + let closed = "CREATE PROCEDURE p() IF 1 THEN SELECT begin FROM t; SELECT 2; END IF" + #expect(!QueryClassifier.isMultiStatement(closed, databaseType: .mysql)) + #expect(QueryClassifier.classifyTier(closed, databaseType: .mysql) == .write) + #expect(QueryClassifier.isMultiStatement("CREATE PROCEDURE p() IF 1 THEN SELECT 1; DROP TABLE users", databaseType: .mysql)) + } + @Test("A COPY TO PROGRAM hidden after a literal backslash reaches the filesystem, measured on PostgreSQL 17") func hiddenCopyToProgramRunsCode() { let sql = "SELECT 'a\\'; COPY users TO PROGRAM 'touch /tmp/pwned'; --'" diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierOneItemDeleteTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierOneItemDeleteTests.swift new file mode 100644 index 0000000000..b33a43fd38 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierOneItemDeleteTests.swift @@ -0,0 +1,112 @@ +// +// QueryClassifierOneItemDeleteTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProSQLGrammar +import TableProWeaviateCore +import Testing + +struct QueryClassifierOneItemDeleteTests { + @Test("A delete of one item by its id or key is a write", arguments: [ + (DatabaseType.elasticsearch, "DELETE /orders/_doc/1"), + (.elasticsearch, "DELETE orders/_doc/1?refresh=true&routing=a"), + (.elasticsearch, "DELETE /orders/_doc/1\n{}"), + (.typesense, "DELETE /collections/books/documents/1"), + (.typesense, "DELETE /collections/books/documents/1?ignore_not_found=true"), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [1]}"#), + (.qdrant, #"POST /collections/docs/points/delete?wait=true {"points": ["5c56c793-69f3-4fbf-87e6-c4bf54c28c26"]}"#), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [7], "shard_key": "eu"}"#), + (.weaviate, "DELETE /v1/objects/Article/36ddd591-2dee-4e7e-a3cc-eb86d30a4303"), + (.weaviate, "DELETE /objects/Article/36ddd591-2dee-4e7e-a3cc-eb86d30a4303?tenant=t1"), + (.weaviate, #"DELETE /v1/objects/Article/36ddd591-2dee-4e7e-a3cc-eb86d30a4303/references/author\#n{"beacon": "weaviate://localhost/Author/1"}"#), + (.weaviate, "DELETE /v1/objects/36ddd591-2dee-4e7e-a3cc-eb86d30a4303/references/author"), + (.firestore, "DELETE users/alice"), + (.firestore, "DELETE users/alice/orders/1"), + (.firebaseRealtimeDatabase, "DELETE users/alice"), + (.firebaseRealtimeDatabase, "DELETE /users/alice/age"), + (.firebaseRealtimeDatabase, "PUT users/alice null"), + (.firebaseRealtimeDatabase, #"PUT users/alice {".priority": 1}"#), + (.firebaseRealtimeDatabase, #"PATCH users {"alice": null}"#), + (.firebaseRealtimeDatabase, #"PATCH / {"users/alice": null, "users/bob/age": 3}"#), + (.firebaseRealtimeDatabase, #"PATCH users/alice {"age": null}"#), + (.bigtable, "DELETE events/user#42"), + (.bigtable, "DELETE events/user#42 profile:name"), + (.bigtable, #"PUT events/user#42 {"profile:name": null}"#), + (.bigtable, #"PUT events/"a{b" {"cf:a": null}"#), + ]) + func oneItemDeleteIsAWrite(type: DatabaseType, request: String) { + #expect(QueryClassifier.classifyTier(request, databaseType: type) == .write) + #expect(!QueryClassifier.isDangerousQuery(request, databaseType: type)) + } + + @Test("A delete of a container, every item, a batch, a filter or a range stays destructive", arguments: [ + (DatabaseType.elasticsearch, "DELETE /orders"), + (.elasticsearch, "DELETE /orders/_doc"), + (.elasticsearch, "DELETE /orders/_doc/"), + (.elasticsearch, "DELETE /orders/_doc/1/extra"), + (.elasticsearch, "DELETE /orders/_doc/_query"), + (.elasticsearch, "DELETE /_all/_doc/1"), + (.elasticsearch, "DELETE /a,b/_doc/1"), + (.elasticsearch, "DELETE /orders*/_doc/1"), + (.elasticsearch, "DELETE /remote:orders/_doc/1"), + (.elasticsearch, "DELETE /-orders/_doc/1"), + (.elasticsearch, "DELETE /.security-7/_doc/1"), + (.elasticsearch, "DELETE /%3Clogs-%7Bnow%7D%3E/_doc/1"), + (.elasticsearch, "DELETE /orders/_doc/../_doc/1"), + (.elasticsearch, "DELETE //orders/_doc/1"), + (.typesense, "DELETE /collections/books"), + (.typesense, "DELETE /collections/books/documents?filter_by=x:1"), + (.typesense, "DELETE /collections/books/documents/1/../"), + (.typesense, "DELETE /aliases/a"), + (.typesense, "DELETE /keys/1"), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [1, 2]}"#), + (.qdrant, #"POST /collections/docs/points/delete\#n{"filter": {"must": [{"key": "city", "match": {"value": "x"}}]}}"#), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [1], "filter": {}}"#), + (.qdrant, #"POST /collections/docs/points/delete\#n{"points": [[1]]}"#), + (.qdrant, "POST /collections/docs/points/delete"), + (.qdrant, #"POST /collections/docs/points/payload/delete\#n{"keys": ["a"], "points": [1]}"#), + (.qdrant, #"POST /collections/docs/points/vectors/delete\#n{"vector": ["v"], "points": [1]}"#), + (.qdrant, #"POST /collections/docs/points/batch\#n{"operations": [{"delete": {"points": [1]}}]}"#), + (.qdrant, "DELETE /collections/docs"), + (.weaviate, "DELETE /v1/batch/objects"), + (.weaviate, "DELETE /v1/schema/Article"), + (.weaviate, "DELETE /v1/schema/Article/tenants"), + (.weaviate, "DELETE /v1/objects/36ddd591-2dee-4e7e-a3cc-eb86d30a4303"), + (.weaviate, "DELETE /v1/objects/Article/../36ddd591-2dee-4e7e-a3cc-eb86d30a4303"), + (.firestore, "DELETE users"), + (.firestore, "DELETE /"), + (.firestore, "DELETE ./alice"), + (.firebaseRealtimeDatabase, "DELETE users"), + (.firebaseRealtimeDatabase, "DELETE /"), + (.firebaseRealtimeDatabase, "PUT users null"), + (.firebaseRealtimeDatabase, #"PATCH / {"users": null}"#), + (.firebaseRealtimeDatabase, #"PATCH / {"users": {"alice": null}}"#), + (.firebaseRealtimeDatabase, #"PATCH / {"users/alice": null, "orders": null}"#), + (.firebaseRealtimeDatabase, #"PATCH users {"": null}"#), + (.firebaseRealtimeDatabase, "DELETE .settings/rules"), + ]) + func manyItemDeleteStaysDestructive(type: DatabaseType, request: String) { + #expect(QueryClassifier.classifyTier(request, databaseType: type) == .destructive) + #expect(QueryClassifier.isDangerousQuery(request, databaseType: type)) + } + + // The server deletes a class-less `/v1/objects/` in every class, so only the class route is one object. + @Test("A grid delete of a Weaviate object is tiered by the path it sends", arguments: [ + ("/v1/objects/Article/36ddd591-2dee-4e7e-a3cc-eb86d30a4303", QueryTier.write), + ("/v1/objects/36ddd591-2dee-4e7e-a3cc-eb86d30a4303", .destructive), + ("/v1/batch/objects", .destructive), + ]) + func weaviateTaggedDeleteFollowsItsPath(path: String, tier: QueryTier) { + let write = WeaviateWriteRequest(method: "DELETE", path: path, query: ["class": "Article"], body: nil) + #expect(QueryClassifier.weaviateClassification(WeaviateWriteCodec.encode(write)).tier == tier) + } + + @Test("Firestore PUT and PATCH keep their tiers") + func firestorePutAndPatchUnchanged() { + #expect(QueryClassifier.classifyTier("PUT users/alice null", databaseType: .firestore) == .destructive) + #expect(QueryClassifier.classifyTier(#"PATCH users/alice {"age": null}"#, databaseType: .firestore) == .write) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift new file mode 100644 index 0000000000..86fc85825a --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift @@ -0,0 +1,39 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +// A T-SQL block runs as one statement, and a caller allowed one statement at a time is still refused one holding more. +struct QueryClassifierTSQLBlockTests { + @Test("A block holding more than one statement is multi-statement", arguments: [ + "IF @x = 1 BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", + "BEGIN TRY DELETE FROM a WHERE id = 1; DELETE FROM b WHERE id = 1; END TRY BEGIN CATCH SELECT 1; END CATCH", + "WHILE @i < 3 BEGIN SET @i += 1; PRINT @i; END", + "IF @x = 1 BEGIN SELECT 1; WITH c AS (SELECT id FROM t) DELETE FROM c; END", + "IF @x = 1 BEGIN SELECT 1; DISABLE TRIGGER trg ON t; END", + "BEGIN TRY SELECT 1; THROW 50000, 'x', 1; END TRY BEGIN CATCH SELECT 2; END CATCH", + "IF @x = 1 BEGIN SELECT 1; done: SELECT 2; END", + "IF @x = 1 BEGIN SELECT 1; (SELECT 2); END", + ]) + func blockOfSeveralIsMulti(sql: String) { + #expect(QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + } + + @Test("A conditional statement, or a block holding one, is a single statement", arguments: [ + "IF @x = 1 SELECT 1", + "IF @x = 1 BEGIN SELECT 1; END", + "BEGIN SELECT 1 END", + "BEGIN IF @x = 1 SELECT 1; ELSE SELECT 2; END", + "BEGIN TRY SELECT 1; END TRY BEGIN CATCH THROW; END CATCH", + ]) + func singleStatementIsNotMulti(sql: String) { + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + } + + @Test("A stored routine's body runs nothing, so it is one statement however many it holds") + func routineBodyIsNotMulti() { + #expect(!QueryClassifier.isMultiStatement( + "CREATE PROCEDURE dbo.p AS BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", databaseType: .mssql + )) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBytesReadTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBytesReadTests.swift new file mode 100644 index 0000000000..a517d3b548 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBytesReadTests.swift @@ -0,0 +1,151 @@ +// +// SQLFileParserBytesReadTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +private final class ReadReports: Sendable { + struct Report: Equatable, Sendable { + let read: Int64 + let size: Int64 + } + + private let reports = OSAllocatedUnfairLock<[Report]>(initialState: []) + + func record(read: Int64, size: Int64) { + reports.withLock { $0.append(Report(read: read, size: size)) } + } + + var all: [Report] { + reports.withLock { $0 } + } +} + +struct SQLFileParserBytesReadTests { + private static func withFile(_ sql: String, _ body: (URL) async throws -> T) async throws -> T { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("SQLFileParserBytesReadTests-\(UUID().uuidString).sql") + try Data(sql.utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + return try await body(url) + } + + private static func expectGrowsToTheEnd( + _ reports: [ReadReports.Report], + of sql: String, + sourceLocation: SourceLocation = #_sourceLocation + ) { + let fileSize = Int64(Data(sql.utf8).count) + let grows = zip(reports, reports.dropFirst()).allSatisfy { pair in pair.0.read < pair.1.read } + let sizes = Set(reports.map { report in report.size }) + + #expect(reports.count > 1, sourceLocation: sourceLocation) + #expect(grows, "\(reports.map { report in report.read })", sourceLocation: sourceLocation) + #expect(reports.last?.read == fileSize, sourceLocation: sourceLocation) + #expect(sizes == [fileSize], sourceLocation: sourceLocation) + } + + @Test("Each chunk read reports the bytes read so far and the file's size, up to the whole file") + func reportsEachChunk() async throws { + let sql = (1...200).map { "SELECT \($0);\n" }.joined() + let reports = ReadReports() + let parser = SQLFileParser(chunkSize: 64) + + let (elements, count): (Int, Int) = try await Self.withFile(sql) { url in + var yielded = 0 + let stream = parser.parseElements( + url: url, + encoding: .utf8, + grammar: { TestGrammar.postgres }, + onRead: { read, size in reports.record(read: read, size: size) } + ) + for try await _ in stream { + yielded += 1 + } + let counted = try await parser.countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + return (yielded, counted) + } + + #expect(elements == 200) + #expect(count == 200) + Self.expectGrowsToTheEnd(reports.all, of: sql) + } + + @Test("The statement stream reports reading the same way") + func statementStreamReports() async throws { + let sql = (1...100).map { "SELECT \($0);\n" }.joined() + let reports = ReadReports() + + try await Self.withFile(sql) { url in + let stream = SQLFileParser(chunkSize: 50).parseFile( + url: url, + encoding: .utf8, + grammar: { TestGrammar.postgres }, + onRead: { read, size in reports.record(read: read, size: size) } + ) + for try await _ in stream {} + } + + Self.expectGrowsToTheEnd(reports.all, of: sql) + } + + @Test("Bytes read grow while a COPY's rows are read, before the next element is asked for") + func bytesGrowWhileCopyRowsAreRead() async throws { + let rows = (1...2_000).map { "\($0)\trow \($0)\n" }.joined() + let sql = "CREATE TABLE t (a int, b text);\nCOPY t (a, b) FROM stdin;\n\(rows)\\.\nSELECT 1;\n" + let reports = ReadReports() + let parser = SQLFileParser(chunkSize: 256, copyPieceLength: 128) + + let copies: Int = try await Self.withFile(sql) { url in + var copiesRead = 0 + let stream = parser.parseElements( + url: url, + encoding: .utf8, + grammar: { TestGrammar.postgres }, + onRead: { read, size in reports.record(read: read, size: size) } + ) + for try await element in stream { + guard case let .copyFromStandardInput(_, _, copyRows) = element else { continue } + copiesRead += 1 + let before = reports.all + var data = Data() + while let piece = try copyRows.nextPiece() { + data.append(piece) + } + let after = reports.all + + #expect(data == Data(rows.utf8)) + #expect(after.count > before.count + 1) + #expect((after.last?.read ?? 0) > (before.last?.read ?? 0)) + } + return copiesRead + } + + #expect(copies == 1) + Self.expectGrowsToTheEnd(reports.all, of: sql) + } + + @Test("The scan for GO lines before a SQL Server script is cut reports nothing") + func batchSeparatorScanReportsNothing() async throws { + let sql = (1...100).map { "SELECT \($0);\n" }.joined() + "GO\n" + let reports = ReadReports() + + try await Self.withFile(sql) { url in + let stream = SQLFileParser(chunkSize: 64).parseElements( + url: url, + encoding: .utf8, + grammar: { TestGrammar.sqlServer }, + onRead: { read, size in reports.record(read: read, size: size) } + ) + for try await _ in stream {} + } + + Self.expectGrowsToTheEnd(reports.all, of: sql) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift new file mode 100644 index 0000000000..e227718272 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift @@ -0,0 +1,256 @@ +// +// SQLFileParserCopyRowsTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@testable import TablePro + +struct SQLFileParserCopyRowsTests { + private enum Element: Equatable { + case statement(String, line: Int) + case copy(String, line: Int, rows: Data) + + static func copying(_ statement: String, line: Int, rows: String) -> Element { + .copy(statement, line: line, rows: Data(rows.utf8)) + } + } + + private static func withFile(_ sql: String, _ body: (URL) async throws -> T) async throws -> T { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("SQLFileParserCopyRowsTests-\(UUID().uuidString).sql") + try Data(sql.utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + return try await body(url) + } + + private static func pieces(of rows: PluginCopyData) throws -> [Data] { + var pieces: [Data] = [] + while let piece = try rows.nextPiece() { + pieces.append(piece) + } + return pieces + } + + private static func read(_ element: PluginImportElement) throws -> Element { + switch element { + case let .statement(statement, lineNumber): + return .statement(statement, line: lineNumber) + case let .copyFromStandardInput(statement, lineNumber, rows): + return .copy(statement, line: lineNumber, rows: Data(try pieces(of: rows).joined())) + @unknown default: + Issue.record("An element this test does not read: \(element)") + return .statement("", line: 0) + } + } + + private static func elements( + _ sql: String, + parser: SQLFileParser = SQLFileParser(), + grammar: SQLLexicalGrammar = TestGrammar.postgres + ) async throws -> [Element] { + try await withFile(sql) { url in + var result: [Element] = [] + for try await element in parser.parseElements(url: url, encoding: .utf8, grammar: grammar) { + result.append(try read(element)) + } + return result + } + } + + // The pieces of every COPY the file holds, in order. + private static func copyPieces(_ sql: String, parser: SQLFileParser = SQLFileParser()) async throws -> [Data] { + try await withFile(sql) { url in + var result: [Data] = [] + for try await element in parser.parseElements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + if case let .copyFromStandardInput(_, _, rows) = element { + result += try pieces(of: rows) + } + } + return result + } + } + + @Test("COPY FROM STDIN yields its rows up to the \\. line, and the statements after it keep their file lines") + func linefeedRows() async throws { + let sql = "CREATE TABLE t (a int, b text);\nCOPY public.t (a, b) FROM stdin;\n1\tone\n2\tDROP TABLE users;\n\\.\n" + + "SELECT count(*) FROM t;\nSELECT 2;\n" + #expect(try await Self.elements(sql) == [ + .statement("CREATE TABLE t (a int, b text)", line: 1), + .copying("COPY public.t (a, b) FROM stdin", line: 2, rows: "1\tone\n2\tDROP TABLE users;\n"), + .statement("SELECT count(*) FROM t", line: 6), + .statement("SELECT 2", line: 7), + ]) + } + + @Test("CRLF rows keep their CRLF, and a \\. line ending in CRLF ends them") + func carriageReturnRows() async throws { + let sql = "COPY t (a) FROM stdin;\r\n1\r\n2\r\n\\.\r\nSELECT 1;\r\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY t (a) FROM stdin", line: 1, rows: "1\r\n2\r\n"), + .statement("SELECT 1", line: 5), + ]) + } + + @Test("Rows end at the end of the file when no \\. line ends them first", arguments: [ + ("COPY t (a) FROM stdin;\n1\n2\n", "1\n2\n"), + ("COPY t (a) FROM stdin;\n1\n2", "1\n2"), + ("COPY t (a) FROM stdin;\n1\n\\.", "1\n"), + ("COPY t (a) FROM stdin;\n1\n\\.\r", "1\n"), + ("COPY t (a) FROM stdin;\n\\.\n", ""), + ("COPY t (a) FROM stdin;\n", ""), + ("COPY t (a) FROM stdin;", ""), + ]) + func rowsToTheEndOfTheFile(sql: String, rows: String) async throws { + #expect(try await Self.elements(sql) == [.copying("COPY t (a) FROM stdin", line: 1, rows: rows)]) + } + + @Test("Text after the ; on the COPY line is neither data nor run") + func restOfTheCopyLineRunsNothing() async throws { + let sql = "COPY t (a) FROM stdin; SELECT 'not run';\n1\n\\.\nSELECT 2;\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY t (a) FROM stdin", line: 1, rows: "1\n"), + .statement("SELECT 2", line: 4), + ]) + } + + @Test("Two COPY blocks each get their own rows") + func twoCopyBlocks() async throws { + let sql = "COPY a (x) FROM stdin;\n1\n\\.\nCOPY b (y) FROM stdin;\n2\n3\n\\.\nSELECT 3;\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY a (x) FROM stdin", line: 1, rows: "1\n"), + .copying("COPY b (y) FROM stdin", line: 4, rows: "2\n3\n"), + .statement("SELECT 3", line: 8), + ]) + } + + @Test("Rows left unread are passed over, and their reader answers nil once the next element is asked for") + func unreadRowsArePassedOver() async throws { + let sql = "COPY t (a) FROM stdin;\n1\n2\n\\.\nSELECT 5;\n" + try await Self.withFile(sql) { url in + var iterator = SQLFileParser().parseElements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + .makeAsyncIterator() + let copy = try #require(try await iterator.next()) + guard case let .copyFromStandardInput(_, _, rows) = copy else { + Issue.record("Expected the COPY, got \(copy)") + return + } + let next = try #require(try await iterator.next()) + #expect(try Self.read(next) == .statement("SELECT 5", line: 5)) + #expect(try rows.nextPiece() == nil) + let end = try await iterator.next().map { try Self.read($0) } + #expect(end == nil) + } + } + + @Test("Rows read half way are passed over from where the reading stopped") + func halfReadRowsArePassedOver() async throws { + let sql = "COPY t (a) FROM stdin;\n1\n2\n3\n4\n5\n\\.\nSELECT 5;\n" + try await Self.withFile(sql) { url in + let parser = SQLFileParser(copyPieceLength: 4) + var iterator = parser.parseElements(url: url, encoding: .utf8, grammar: TestGrammar.postgres).makeAsyncIterator() + let copy = try #require(try await iterator.next()) + guard case let .copyFromStandardInput(_, _, rows) = copy else { + Issue.record("Expected the COPY, got \(copy)") + return + } + #expect(try rows.nextPiece() == Data("1\n2\n".utf8)) + let next = try #require(try await iterator.next()) + #expect(try Self.read(next) == .statement("SELECT 5", line: 8)) + #expect(try rows.nextPiece() == nil) + } + } + + @Test("A CSV header line is data, sent as the server reads it") + func csvHeaderIsData() async throws { + let sql = "COPY t (a, b) FROM stdin WITH (FORMAT csv, HEADER true);\na,b\n1,\"x\"\n\\.\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY t (a, b) FROM stdin WITH (FORMAT csv, HEADER true)", line: 1, rows: "a,b\n1,\"x\"\n"), + ]) + } + + @Test("Rows keep non-ASCII text and emoji byte for byte, whatever the chunk and piece size", + arguments: [2, 3, 5, SQLFileParser.defaultCopyPieceLength]) + func nonASCIIRowsAreByteExact(pieceLength: Int) async throws { + let rows = "1\tcafé\n2\t日本語\n3\t😀😀😀\n" + let sql = "COPY t (a, b) FROM stdin;\n\(rows)\\.\nSELECT 1;\n" + for chunkSize in [1, 2, 3, 7, SQLFileParser.defaultChunkSize] { + let parser = SQLFileParser(chunkSize: chunkSize, copyPieceLength: pieceLength) + let pieces = try await Self.copyPieces(sql, parser: parser) + #expect(Data(pieces.joined()) == Data(rows.utf8), "chunk \(chunkSize)") + } + } + + @Test("A chunk end anywhere, inside a \\. line too, changes nothing") + func chunkEndAnywhere() async throws { + let sql = "SET a = 1;\nCOPY t (a) FROM stdin;\r\n1\r\n\\.x\n\\.\r2\n\\\n22\n\\.\r\nSELECT 2;\nCOPY u FROM stdin;\n3\n\\." + let expected: [Element] = [ + .statement("SET a = 1", line: 1), + .copying("COPY t (a) FROM stdin", line: 2, rows: "1\r\n\\.x\n\\.\r2\n\\\n22\n"), + .statement("SELECT 2", line: 9), + .copying("COPY u FROM stdin", line: 10, rows: "3\n"), + ] + for chunkSize in 1...12 { + for pieceLength in [2, 3, 5, 64] { + let parser = SQLFileParser(chunkSize: chunkSize, copyPieceLength: pieceLength) + #expect(try await Self.elements(sql, parser: parser) == expected, "chunk \(chunkSize), piece \(pieceLength)") + } + } + } + + @Test("A row longer than a piece arrives in pieces no longer than the limit") + func rowLongerThanAPiece() async throws { + let row = String(repeating: "x", count: 50) + "\n" + let pieces = try await Self.copyPieces("COPY t (a) FROM stdin;\n\(row)\\.\n", parser: SQLFileParser(copyPieceLength: 8)) + #expect(pieces.count == 7) + #expect(pieces.allSatisfy { piece in piece.count <= 8 }) + #expect(Data(pieces.joined()) == Data(row.utf8)) + } + + @Test("Pieces end at a line end when the line fits") + func piecesEndAtLineEnds() async throws { + let rows = (1...500).map { "\($0)\tvalue \($0)\n" }.joined() + let pieces = try await Self.copyPieces("COPY t (a, b) FROM stdin;\n\(rows)\\.\n", parser: SQLFileParser(copyPieceLength: 100)) + #expect(pieces.count > 1) + #expect(pieces.allSatisfy { piece in piece.count <= 100 && piece.last == 0x0A }) + #expect(Data(pieces.joined()) == Data(rows.utf8)) + } + + @Test("A piece holds at most 1 MiB of UTF-8, even of three-byte characters") + func pieceStaysWithinOneMebibyte() async throws { + let row = String(repeating: "日", count: 400_000) + "\n" + let pieces = try await Self.copyPieces("COPY t (a) FROM stdin;\n\(row)\\.\n") + #expect(pieces.count > 1) + #expect(pieces.allSatisfy { piece in piece.count <= 1_048_576 }) + #expect(Data(pieces.joined()) == Data(row.utf8)) + } + + @Test("The count reads each COPY as one statement and none of its rows") + func countReadsCopyAsOne() async throws { + let sql = "CREATE TABLE t (a text);\nCOPY t (a) FROM stdin;\nSELECT 1;\nDROP TABLE t;\n\\.\n" + + "COPY t (a) FROM stdin;\nx\n\\.\nSELECT 2;\n" + let count = try await Self.withFile(sql) { url in + try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + } + #expect(count == 4) + #expect(try await Self.elements(sql).count == count) + } + + @Test("The element stream yields each run the statement stream yields, GO n repeats included") + func elementsRepeatLikeStatements() async throws { + let sql = "SELECT 1\nGO 3\nSELECT 2\nGO\n" + let statements = try await Self.withFile(sql) { url in + var result: [Element] = [] + for try await (statement, line) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: TestGrammar.sqlServer) { + result.append(.statement(statement, line: line)) + } + return result + } + #expect(statements.count == 4) + #expect(try await Self.elements(sql, grammar: TestGrammar.sqlServer) == statements) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 70208995de..f2be88f9fb 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -74,6 +74,39 @@ struct SQLFileParserTests { #expect(stmts[1] == "SELECT 3") } + @Test("Postgres: an e glued to a name by $ or a non-ASCII unit starts a plain string, as the scanner reads it") + func postgresPrefixGluedToAName() async throws { + for glue in ["$", "\u{00A0}", "\u{2019}"] { + let sql = "SELECT x\(glue)e'\\'; DROP TABLE t; SELECT 'x';" + let stmts = try await Self.parse(sql, grammar: TestGrammar.postgres) + #expect(stmts == ["SELECT x\(glue)e'\\'", "DROP TABLE t", "SELECT 'x'"], "\(glue)") + } + } + + @Test("DuckDB: an E after a non-ASCII space DuckDB blanks starts an escape string") + func duckDBPrefixAfterABlank() async throws { + let sql = "SELECT varchar\u{200B}E'\\''; DROP TABLE t; SELECT 'x';" + let stmts = try await Self.parse(sql, grammar: TestGrammar.duckdb) + #expect(stmts == ["SELECT varchar\u{200B}E'\\''", "DROP TABLE t", "SELECT 'x'"]) + let dollar = try await Self.parse("SELECT 1 AS a,\u{200B}$$'$$; DROP TABLE t; SELECT 'x';", grammar: TestGrammar.duckdb) + #expect(dollar == ["SELECT 1 AS a,\u{200B}$$'$$", "DROP TABLE t", "SELECT 'x'"]) + } + + @Test("MySQL: a begin column in a one-statement procedure leaves the next statements their own") + func mysqlBeginColumnInAOneStatementBody() async throws { + let sql = """ + CREATE PROCEDURE latest_shift() SELECT id FROM shifts ORDER BY begin DESC LIMIT 1; + CALL latest_shift(); + UPDATE shifts SET closed = 1 WHERE id = 7; + """ + let stmts = try await Self.parse(sql, grammar: TestGrammar.mysql) + #expect(stmts == [ + "CREATE PROCEDURE latest_shift() SELECT id FROM shifts ORDER BY begin DESC LIMIT 1", + "CALL latest_shift()", + "UPDATE shifts SET closed = 1 WHERE id = 7", + ]) + } + @Test("CockroachDB byte literal: backslash escape active") func cockroach_byte_literal_backslash_escape() async throws { let sql = """ @@ -382,6 +415,46 @@ struct SQLFileParserTests { #expect(count == statementCount) } + @Test("An empty file holds no statement, parsed or counted") + func emptyFile() async throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try Data().write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let parser = SQLFileParser() + var statements: [String] = [] + for try await (statement, _) in parser.parseFile(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + statements.append(statement) + } + #expect(statements.isEmpty) + #expect(try await parser.countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) == 0) + } + + @Test("A last statement with no ; arrives when the file ends on a full read chunk", arguments: [1, 6, 9, 18]) + func lastStatementEndsOnChunkBoundary(chunkSize: Int) async throws { + let sql = "SELECT 1;\nSELECT 2" + try #require(sql.utf8.count % chunkSize == 0) + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try Data(sql.utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let parser = SQLFileParser(chunkSize: chunkSize) + var statements: [String] = [] + for try await (statement, _) in parser.parseFile(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + statements.append(statement) + } + #expect(statements == ["SELECT 1", "SELECT 2"]) + #expect(try await parser.countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) == 2) + } + + @Test("A UTF-8 file cut inside its last character fails rather than dropping the bytes") + func fileCutInsideItsLastCharacter() async throws { + let cut = Data(Data("SELECT 'é".utf8).dropLast()) + await #expect(throws: DecompressionError.self) { + _ = try await Self.parse(cut, encoding: .utf8) + } + } + @Test("Dialect.from maps known database type ids") func dialect_from_database_type_id() { #expect(SqlDialect.from(databaseTypeId: "PostgreSQL") == .postgres) @@ -519,4 +592,138 @@ struct SQLFileParserTests { #expect(statements == Self.cqlStatements, "boundary at \(boundary)") } } + + private static func parseWithLines(_ sql: String, grammar: SQLLexicalGrammar) async throws -> [ParsedLine] { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + var statements: [ParsedLine] = [] + for try await (statement, line) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: grammar) { + statements.append(ParsedLine(statement: statement, line: line)) + } + return statements + } + + private struct ParsedLine: Equatable { + let statement: String + let line: Int + } + + @Test("MySQL: a DELIMITER line is read as the mysql client reads it", arguments: [ + ("DELIMITER\t//\nSELECT 1//\nDELIMITER ;\nSELECT 2;\n", ["SELECT 1", "SELECT 2"]), + ("DELIMITER // -- set\nSELECT 1//\nDELIMITER ;\nSELECT 2;\n", ["SELECT 1", "SELECT 2"]), + ("DELIMITER // SELECT 99\nSELECT 1//\n", ["SELECT 1"]), + ("DELIMITER\nSELECT 1;\n", ["SELECT 1"]), + (" delimiter $$\nCREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END$$\ndelimiter ;\nSELECT f();\n", [ + "CREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END", "SELECT f()", + ]), + ("SELECT 1; DELIMITER //\nSELECT 2;\n", ["SELECT 1", "DELIMITER //\nSELECT 2"]), + ]) + func mysqlDelimiterLines(sql: String, expected: [String]) async throws { + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql) == expected) + } + + @Test("Import reads a line ending in a comment, a CockroachDB byte literal and a \\restrict line as the editor does") + func linesTheEditorReadsTheSameWay() async throws { + #expect(try await Self.parse("DELIMITER // -- set\nSELECT 1//\nSELECT 2//\n", grammar: TestGrammar.mysql) == [ + "SELECT 1", "SELECT 2", + ]) + let cockroach = DatabaseType.cockroachdb.lexicalGrammar + #expect(try await Self.parse("INSERT INTO t VALUES (b'\\''); DELETE FROM x; SELECT 'a';", grammar: cockroach) == [ + "INSERT INTO t VALUES (b'\\'')", "DELETE FROM x", "SELECT 'a'", + ]) + #expect(try await Self.parse("\\restrict k\rDROP TABLE t;\nSELECT 1;\n", grammar: TestGrammar.postgres) == ["SELECT 1"]) + } + + @Test("MySQL: the statement count reads DELIMITER lines as the import does") + func mysqlCountReadsDelimiterLines() async throws { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nSELECT 3;\n" + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + let count = try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.mysql) + #expect(count == 2) + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql).count == count) + } + + @Test("Postgres: a plain pg_dump opens with \\restrict, which runs nothing") + func postgresRestrictLinesRunNothing() async throws { + let sql = "\\restrict abc123\n\nSET statement_timeout = 0;\nSELECT 1;\n\n\\unrestrict abc123\n" + #expect(try await Self.parseWithLines(sql, grammar: TestGrammar.postgres) == [ + ParsedLine(statement: "SET statement_timeout = 0", line: 3), + ParsedLine(statement: "SELECT 1", line: 4), + ]) + } + + @Test("Postgres: the statement stream stops at COPY FROM STDIN before the COPY or its rows are sent, and the count reads the COPY as one statement") + func postgresStatementStreamRefusesCopyFromStandardInputAndCountReadsIt() async throws { + let sql = """ + CREATE TABLE t (a int, b text); + COPY public.t (a, b) FROM stdin; + 1\tit's + 2\tDROP TABLE users; + \\. + SELECT 1; + """ + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + var statements: [String] = [] + await #expect(throws: SQLFileParseError.copyFromStandardInput(line: 2)) { + for try await (statement, _) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + statements.append(statement) + } + } + #expect(statements == ["CREATE TABLE t (a int, b text)"]) + #expect(try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) == 3) + } + + @Test("Postgres: a COPY that reads a server file or writes rows out still runs", arguments: [ + "COPY t FROM '/tmp/stdin';\nSELECT 1;", + "COPY (SELECT 1) TO STDOUT;\nSELECT 1;", + ]) + func postgresCopyWithoutScriptRowsRuns(sql: String) async throws { + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres).count == 2) + } + + @Test("Postgres: a BEGIN ATOMIC body and a rule's action list arrive whole, as the editor reads them") + func postgresBodiesArriveWhole() async throws { + let function = "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT 1; SELECT 2; END" + let rule = "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2))" + let sql = "\(function);\n\(rule);\nSELECT 3;\n" + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres) == [function, rule, "SELECT 3"]) + } + + @Test("SQL Server: a ; before an IF's ELSE stays inside the IF, and the next statement keeps its line") + func sqlServerSemicolonBeforeElse() async throws { + let sql = """ + IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday') + SELECT 'Weekend'; + -- weekday + ELSE + SELECT 'Weekday'; + IF @a = 1 SELECT 1; + SELECT 2; + """ + #expect(try await Self.parseWithLines(sql, grammar: TestGrammar.sqlServer) == [ + ParsedLine( + statement: "IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday')\n SELECT 'Weekend';\n-- weekday\nELSE\n SELECT 'Weekday'", + line: 1 + ), + ParsedLine(statement: "IF @a = 1 SELECT 1", line: 6), + ParsedLine(statement: "SELECT 2", line: 7), + ]) + } + + @Test("SQL Server: a chunk boundary anywhere around a held ; changes nothing") + func sqlServerHeldSemicolonAcrossChunks() async throws { + let script = "IF @a = 1 SELECT 1;; ELSE SELECT 2;\nIF @b = 1 SELECT 3; SELECT 4;" + let expected = ["IF @a = 1 SELECT 1;; ELSE SELECT 2", "IF @b = 1 SELECT 3", "SELECT 4"] + let chunkSize = 65_536 + for boundary in 0..<(script as NSString).length { + let padding = "--" + String(repeating: "x", count: chunkSize - boundary - 3) + "\n" + #expect(try await Self.parse(padding + script, grammar: TestGrammar.sqlServer) == expected, "boundary at \(boundary)") + } + } } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift index 3e44d78a03..677d512ffb 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift @@ -335,6 +335,55 @@ struct SQLFoldScannerTests { #expect(blocks.map { $0.startLine } == [0]) #expect(blocks.map { $0.endLine } == [3]) } + + // MARK: - Client lines + + private func statementLines(_ sql: String, grammar: SQLLexicalGrammar) -> [ClosedRange] { + regions(sql, grammar: grammar).filter { $0.kind == .statement }.map { $0.startLine...$0.endLine } + } + + @Test("A DELIMITER line folds nothing, and the delimiter ends the routine's fold") + func delimiterLines() { + let sql = """ + DELIMITER // + CREATE PROCEDURE p() + BEGIN + SELECT 1; + END// + DELIMITER ; + SELECT 2, + 3; + """ + #expect(statementLines(sql, grammar: TestGrammar.mysql) == [1...4, 6...7]) + } + + @Test("A COPY's fold runs through its rows, which open no fold of their own") + func copyRows() { + let sql = """ + \\restrict k + COPY t (a, b) FROM stdin; + 1\t(it's + 2\tBEGIN /* + \\. + SELECT 2, + 3; + """ + #expect(statementLines(sql, grammar: TestGrammar.postgres) == [1...4, 5...6]) + #expect(regions(sql, grammar: TestGrammar.postgres).allSatisfy { $0.kind == .statement }) + } + + @Test("A ; before an IF's ELSE does not end the IF's fold") + func semicolonBeforeElse() { + let sql = """ + IF @a = 1 + SELECT 1; + ELSE + SELECT 2; + SELECT 3, + 4; + """ + #expect(statementLines(sql, grammar: TestGrammar.sqlServer) == [0...3, 4...5]) + } } struct SQLFoldEventOrderingTests { diff --git a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift index 17f6c3c005..12629b8c26 100644 --- a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift @@ -132,8 +132,16 @@ struct SQLParameterExtractorTests { ) #expect(result.sql == "WHERE id = ? AND name = ?") #expect(result.values.count == 2) - #expect((result.values[0] as? String) == "42") - #expect((result.values[1] as? String) == "Alice") + #expect((result.values[0] as? TypedParameterValue) == TypedParameterValue(text: "42", kind: .integer)) + #expect((result.values[1] as? TypedParameterValue) == TypedParameterValue(text: "Alice", kind: .text)) + } + + @Test("A String parameter holding a plain number keeps the reading by shape; any other text is text", arguments: [ + ("42", PluginColumnKind?.none), ("-3.5", nil), ("00123", nil), ("1E5", .text), ("12e3", .text), ("1.", .text), + (".5", nil), ("abc", .text), ("", .text), ("-", .text), + ]) + func stringKindFollowsTheValue(value: String, kind: PluginColumnKind?) { + #expect(QueryParameterType.string.pluginKind(for: value) == kind) } @Test @@ -161,8 +169,8 @@ struct SQLParameterExtractorTests { ) #expect(result.sql == "WHERE ? = ?") #expect(result.values.count == 2) - #expect((result.values[0] as? String) == "42") - #expect((result.values[1] as? String) == "42") + #expect((result.values[0] as? TypedParameterValue)?.text == "42") + #expect((result.values[1] as? TypedParameterValue)?.text == "42") } @Test diff --git a/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift b/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift index b5df30bfd0..8fc75fa0d1 100644 --- a/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift @@ -384,8 +384,9 @@ struct SQLRowToStatementConverterTests { #expect(lines.count == 50_000) } - @Test("PostgreSQL: binary cell renders as bytea hex literal in INSERT") - func postgresBinaryInsertEmitsByteaLiteral() throws { + // `'\x…'::bytea` reads its backslash by `standard_conforming_strings`; `decode` means the same bytes either way. + @Test("PostgreSQL: binary cell decodes hex in INSERT") + func postgresBinaryInsertDecodesHex() throws { let converter = try SQLRowToStatementConverter( tableName: "documents", columns: ["id", "payload"], @@ -396,10 +397,28 @@ struct SQLRowToStatementConverterTests { ) let bytes = Data([0xD3, 0x8C, 0xE5, 0x66]) let result = converter.generateInserts(rows: plain([[.text("1"), .bytes(bytes)]])) - #expect(result.contains("'\\xD38CE566'::bytea")) + #expect(result.contains("decode('D38CE566', 'hex')")) #expect(!result.contains("NULL")) } + // DuckDB reads `X'…'` as the text `x…` and Oracle has no such literal. + @Test("DuckDB and Oracle binary cells use their own hex functions in INSERT", arguments: [ + (DatabaseType.duckdb, "unhex('CAFE')"), (DatabaseType.oracle, "HEXTORAW('CAFE')"), + ]) + func binaryInsertUsesTheEnginesHexFunction(databaseType: DatabaseType, literal: String) throws { + let converter = try SQLRowToStatementConverter( + tableName: "documents", + columns: ["id", "payload"], + primaryKeyColumns: ["id"], + databaseType: databaseType, + quoteIdentifier: { "\"\($0)\"" }, + escapeStringLiteral: { $0.replacingOccurrences(of: "'", with: "''") } + ) + let result = converter.generateInserts(rows: plain([[.text("1"), .bytes(Data([0xCA, 0xFE]))]])) + #expect(result.contains(literal)) + #expect(!result.contains("X'CAFE'")) + } + @Test("MySQL: binary cell renders as X'...' literal in INSERT") func mysqlBinaryInsertEmitsXLiteral() throws { let converter = try makeConverter( @@ -441,7 +460,7 @@ struct SQLRowToStatementConverterTests { ) let bytes = Data([0xAB, 0xCD]) let result = converter.generateUpdates(rows: plain([[.text("42"), .bytes(bytes)]])) - #expect(result.contains("\"payload\" = '\\xABCD'::bytea")) + #expect(result.contains("\"payload\" = decode('ABCD', 'hex')")) #expect(result.contains("WHERE \"id\" = '42'")) } @@ -656,8 +675,8 @@ struct SQLRowToStatementConverterTests { databaseType: .pglite, dialect: Self.postgresDialect ) let rows = plain([[.text("1"), .bytes(Data([0xDE, 0xAD]))]]) - #expect(converter.generateInserts(rows: rows).contains("'\\xDEAD'::bytea")) - #expect(converter.generateUpdates(rows: rows).contains("\"payload\" = '\\xDEAD'::bytea")) + #expect(converter.generateInserts(rows: rows).contains("decode('DEAD', 'hex')")) + #expect(converter.generateUpdates(rows: rows).contains("\"payload\" = decode('DEAD', 'hex')")) #expect(!converter.generateInserts(rows: rows).contains("X'DEAD'")) } } diff --git a/apple/TableProTests/Core/Utilities/TransactionEndAlertTests.swift b/apple/TableProTests/Core/Utilities/TransactionEndAlertTests.swift new file mode 100644 index 0000000000..923b8a9006 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/TransactionEndAlertTests.swift @@ -0,0 +1,76 @@ +// +// TransactionEndAlertTests.swift +// TableProTests +// + +import AppKit +@testable import TablePro +import Testing + +@MainActor +struct TransactionEndAlertTests { + private static let escape = "\u{1B}" + private static let returnKey = "\r" + + private static func prompt(state: SessionTransactionTracker.State) throws -> TransactionEndPrompt { + let hold = SessionTransactionTracker.Hold(key: .init(connectionId: UUID(), lane: nil), state: state, revision: 0) + return try #require(TransactionEndPrompt.make( + action: .disconnect(name: "Prod"), + names: ["app"], + holds: [hold], + effectiveLevel: { _ in .silent }, + busy: { _ in false } + )) + } + + // The Save alert's order: the default first, Cancel second, the destructive choice third. + @Test("Commit, Cancel, then Roll Back, in the order NSDocument uses for Save") + func buttonOrder() throws { + let alert = AlertHelper.makeTransactionEndAlert(try Self.prompt(state: .open)) + + #expect(alert.buttons.map { $0.title } == ["Commit", "Cancel", "Roll Back"]) + #expect(alert.alertStyle == .warning) + } + + @Test("Return commits, Escape cancels, and Roll Back takes a deliberate click") + func keyBindings() throws { + let alert = AlertHelper.makeTransactionEndAlert(try Self.prompt(state: .open)) + + #expect(alert.buttons[0].keyEquivalent == Self.returnKey) + #expect(alert.buttons[1].keyEquivalent == Self.escape) + #expect(alert.buttons[2].keyEquivalent == "") + #expect(alert.buttons.filter { $0.keyEquivalent == Self.returnKey }.count == 1) + #expect(alert.buttons.filter { $0.keyEquivalent == Self.escape }.count == 1) + } + + @Test("Roll Back is the destructive button and Commit is not") + func rollBackIsDestructive() throws { + let alert = AlertHelper.makeTransactionEndAlert(try Self.prompt(state: .open)) + + #expect(alert.buttons[2].hasDestructiveAction) + #expect(!alert.buttons[0].hasDestructiveAction) + #expect(!alert.buttons[1].hasDestructiveAction) + } + + @Test("Each button maps to its answer") + func answers() { + #expect(AlertHelper.transactionEndAnswer(to: .alertFirstButtonReturn, offersCommit: true) == .commit) + #expect(AlertHelper.transactionEndAnswer(to: .alertSecondButtonReturn, offersCommit: true) == .cancel) + #expect(AlertHelper.transactionEndAnswer(to: .alertThirdButtonReturn, offersCommit: true) == .rollBack) + #expect(AlertHelper.transactionEndAnswer(to: .abort, offersCommit: true) == .cancel) + } + + @Test("Without Commit, the roll-back alert keeps Escape on Cancel and takes Return off its button") + func rollBackOnlyAlert() throws { + let prompt = try Self.prompt(state: .aborted) + let alert = AlertHelper.makeTransactionEndAlert(prompt) + + #expect(alert.buttons.map { $0.title } == ["Disconnect", "Cancel"]) + #expect(alert.buttons[0].hasDestructiveAction) + #expect(alert.buttons[0].keyEquivalent == "") + #expect(alert.buttons[1].keyEquivalent == Self.escape) + #expect(AlertHelper.transactionEndAnswer(to: .alertFirstButtonReturn, offersCommit: false) == .rollBack) + #expect(AlertHelper.transactionEndAnswer(to: .alertSecondButtonReturn, offersCommit: false) == .cancel) + #expect(AlertHelper.transactionEndAnswer(to: .alertThirdButtonReturn, offersCommit: false) == .cancel) + } +} diff --git a/apple/TableProTests/Core/VersionHistory/QueryTabVersionHistoryProviderTests.swift b/apple/TableProTests/Core/VersionHistory/QueryTabVersionHistoryProviderTests.swift new file mode 100644 index 0000000000..cc68ed7f09 --- /dev/null +++ b/apple/TableProTests/Core/VersionHistory/QueryTabVersionHistoryProviderTests.swift @@ -0,0 +1,158 @@ +// +// QueryTabVersionHistoryProviderTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +private final class FakeQueryTab { + let connectionId = UUID() + var text: String? + var isCapturePaused = false + private(set) var writes: [String] = [] + private(set) var reasonsAtWrite: [[QueryTabVersionReason]] = [] + + init(text: String?) { + self.text = text + } + + func write(_ newText: String, storage: QueryTabVersionStorage, tabId: UUID) -> Bool { + guard text != nil else { return false } + reasonsAtWrite.append(storage.summaries(tabId: tabId).map { $0.reason }) + writes.append(newText) + text = newText + return true + } +} + +@MainActor +struct QueryTabVersionHistoryProviderTests { + private let storage = QueryTabVersionStorage( + databaseURL: FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("query_tab_provider_\(UUID().uuidString).db"), + announce: { _ in }, + removeDatabaseOnDeinit: true + ) + private let tabId = UUID() + + private func provider(for tab: FakeQueryTab) -> QueryTabVersionHistoryProvider { + QueryTabVersionHistoryProvider( + tabId: tabId, + storage: storage, + liveTab: { tab.text.map { LiveQueryTab(connectionId: tab.connectionId, text: $0) } }, + write: { tab.write($0, storage: storage, tabId: tabId) }, + isCapturePaused: { tab.isCapturePaused } + ) + } + + private func recordPast(_ texts: [String], connectionId: UUID) async { + for text in texts { + await storage.record(tabId: tabId, connectionId: connectionId, text: text, reason: .run) + } + } + + @Test("The page is the current text, then each version newest first, with the retention notice") + func pageListsCurrentThenNewestFirst() async throws { + let tab = FakeQueryTab(text: "SELECT 3") + await recordPast(["SELECT 1", "SELECT 2"], connectionId: tab.connectionId) + + let page = try await provider(for: tab).loadHistory() + + #expect(page.entries.count == 3) + #expect(page.entries.first?.reference == .current) + #expect(page.entries.dropFirst().allSatisfy { $0.summary == String(localized: "Run") }) + #expect(page.notice == .keepsRecentVersions(count: 50, days: storage.retentionDays)) + let newest = try #require(page.entries.dropFirst().first) + #expect(try await provider(for: tab).content(of: newest.reference) == "SELECT 2") + } + + @Test("The current entry reads the tab's live text") + func currentContentIsTheLiveText() async throws { + let tab = FakeQueryTab(text: "SELECT live") + #expect(try await provider(for: tab).content(of: .current) == "SELECT live") + } + + @Test("A version of another tab, or one that is gone, is not found") + func foreignVersionIsNotFound() async throws { + let tab = FakeQueryTab(text: "SELECT 1") + let otherTab = UUID() + await storage.record(tabId: otherTab, connectionId: tab.connectionId, text: "SELECT other", reason: .run) + let otherId = try #require(storage.summaries(tabId: otherTab).first?.id) + + await #expect(throws: VersionHistoryError.versionNotFound) { + try await provider(for: tab).content(of: .tabVersion(id: otherId)) + } + await #expect(throws: VersionHistoryError.versionNotFound) { + try await provider(for: tab).prepareRestore(.tabVersion(id: 999)) + } + } + + @Test("Restore records the text it replaces first, then writes the version into the tab") + func restoreRecordsThenWrites() async throws { + let tab = FakeQueryTab(text: "SELECT 1") + await recordPast(["SELECT 1"], connectionId: tab.connectionId) + let version = try #require(storage.summaries(tabId: tabId).first) + tab.text = "SELECT typed since" + + let plan = try await provider(for: tab).prepareRestore(.tabVersion(id: version.id)) + #expect(plan.replacesUncommittedChanges == false) + try await plan.apply() + + #expect(tab.writes == ["SELECT 1"]) + #expect(tab.reasonsAtWrite == [[.beforeRestore, .run]]) + let replaced = try #require(storage.summaries(tabId: tabId).first) + #expect(storage.version(id: replaced.id, tabId: tabId)?.query == "SELECT typed since") + } + + @Test("With capture paused, restore writes the version and records nothing") + func pausedRestoreRecordsNothing() async throws { + let tab = FakeQueryTab(text: "SELECT 1") + await recordPast(["SELECT 1"], connectionId: tab.connectionId) + let version = try #require(storage.summaries(tabId: tabId).first) + tab.text = "SELECT typed since" + tab.isCapturePaused = true + + try await provider(for: tab).prepareRestore(.tabVersion(id: version.id)).apply() + + #expect(tab.writes == ["SELECT 1"]) + #expect(storage.summaries(tabId: tabId).map { $0.id } == [version.id]) + } + + @Test("A closed tab says so for its list, its text and a restore") + func closedTabThrowsTabClosed() async throws { + let tab = FakeQueryTab(text: nil) + await recordPast(["SELECT 1"], connectionId: tab.connectionId) + let version = try #require(storage.summaries(tabId: tabId).first) + + await #expect(throws: VersionHistoryError.tabClosed) { try await provider(for: tab).loadHistory() } + await #expect(throws: VersionHistoryError.tabClosed) { try await provider(for: tab).content(of: .current) } + await #expect(throws: VersionHistoryError.tabClosed) { + try await provider(for: tab).prepareRestore(.tabVersion(id: version.id)) + } + #expect(try await provider(for: tab).content(of: .tabVersion(id: version.id)) == "SELECT 1") + } + + @Test("A tab that closes between prepare and apply is not written") + func tabClosedBeforeApplyIsNotWritten() async throws { + let tab = FakeQueryTab(text: "SELECT 2") + await recordPast(["SELECT 1"], connectionId: tab.connectionId) + let version = try #require(storage.summaries(tabId: tabId).first) + let plan = try await provider(for: tab).prepareRestore(.tabVersion(id: version.id)) + tab.text = nil + + await #expect(throws: VersionHistoryError.tabClosed) { try await plan.apply() } + #expect(tab.writes.isEmpty) + } + + @Test("Each reason names its version, and an idle snapshot is titled by its date") + func reasonSummaries() { + #expect(QueryTabVersionReason.run.summary == String(localized: "Run")) + #expect(QueryTabVersionReason.beforeReplace.summary == String(localized: "Before Replace")) + #expect(QueryTabVersionReason.beforeRestore.summary == String(localized: "Before Restore")) + #expect(QueryTabVersionReason.idle.summary == nil) + } +} diff --git a/apple/TableProTests/Core/VersionHistory/QueryTabVersionSubjectTests.swift b/apple/TableProTests/Core/VersionHistory/QueryTabVersionSubjectTests.swift new file mode 100644 index 0000000000..b1ad41bca3 --- /dev/null +++ b/apple/TableProTests/Core/VersionHistory/QueryTabVersionSubjectTests.swift @@ -0,0 +1,66 @@ +// +// QueryTabVersionSubjectTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct QueryTabVersionSubjectTests { + @Test("A history tab for a query tab round-trips the tab it shows") + func queryTabSubjectRoundTrips() throws { + let subject = VersionHistorySubject.queryTab(id: UUID()) + let tab = PersistedTab( + id: UUID(), + title: "History: Query 1", + query: "", + tabType: .versionHistory, + tableName: nil, + versionHistorySubject: subject + ) + + let decoded = try JSONDecoder().decode(PersistedTab.self, from: try JSONEncoder().encode(tab)) + + #expect(decoded.tabType == .versionHistory) + #expect(decoded.versionHistorySubject == subject) + } + + @Test("The saved query and file subjects keep the encoding older builds wrote") + func existingSubjectsDecodeUnchanged() throws { + let favoriteId = UUID() + let json = Data(#"{"savedQuery":{"id":"\#(favoriteId.uuidString)"}}"#.utf8) + #expect(try JSONDecoder().decode(VersionHistorySubject.self, from: json) == .savedQuery(id: favoriteId)) + } + + @Test("The notice names both the count and the days") + func noticeNamesCountAndDays() { + let text = VersionHistoryFormatting.noticeText(.keepsRecentVersions(count: 50, days: 7)) + #expect(text == String( + format: String(localized: "Keeps up to %1$d versions from the last %2$d days on this Mac."), + 50, + 7 + )) + } + + @Test("A Revert To row leads with the date, then names a run") + func menuTitleLeadsWithTheDate() { + let date = Date(timeIntervalSince1970: 1_800_000_000) + let dateText = date.formatted(date: .abbreviated, time: .shortened) + let run = VersionHistoryEntry(reference: .tabVersion(id: 1), summary: String(localized: "Run"), date: date) + let idle = VersionHistoryEntry(reference: .tabVersion(id: 2), summary: nil, date: date) + + #expect(VersionHistoryFormatting.menuTitle(for: run) == String( + format: String(localized: "%1$@ (%2$@)"), + dateText, + String(localized: "Run") + )) + #expect(VersionHistoryFormatting.menuTitle(for: idle) == dateText) + } + + @Test("A closed tab's message tells the reader how to get the history back") + func closedTabMessage() { + #expect(VersionHistoryError.tabClosed.errorDescription + == String(localized: "This tab is closed. Reopen it to see its history.")) + } +} diff --git a/apple/TableProTests/Core/Vim/VimEngineComposedCharacterTests.swift b/apple/TableProTests/Core/Vim/VimEngineComposedCharacterTests.swift new file mode 100644 index 0000000000..c86aa02130 --- /dev/null +++ b/apple/TableProTests/Core/Vim/VimEngineComposedCharacterTests.swift @@ -0,0 +1,579 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProTextEngine +import Testing + +private let thumbsUp = "\u{1F44D}" +private let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}" +private let accented = "e\u{301}" +private let composedCharacters = [thumbsUp, family, accented] + +private func units(_ text: String) -> Int { + (text as NSString).length +} + +extension GlobalStateSuites { + @Suite(.recordingClipboard) + @MainActor + struct VimEngineComposedCharacterTests { + @MainActor + private final class Session { + let buffer: VimTextBufferMock + let engine: VimEngine + + init(_ text: String, caret: Int) { + buffer = VimTextBufferMock(text: text, selectedRange: NSRange(location: caret, length: 0)) + engine = VimEngine(buffer: buffer) + } + + func press(_ keys: String) { + for key in keys { _ = engine.process(key, shift: false) } + } + + func escape() { _ = engine.process("\u{1B}", shift: false) } + + func backspace() { _ = engine.process("\u{7F}", shift: false) } + + // Insert mode leaves typing to the text view, which puts the text at the caret. + func typeInTextView(_ text: String) { + buffer.replaceCharacters(in: buffer.selectedRange(), with: text) + } + + var text: String { buffer.text } + var caret: Int { buffer.selectedRange().location } + var selection: NSRange { buffer.selectedRange() } + } + + // MARK: - x, X, s + + @Test(arguments: composedCharacters) + func xDeletesTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("x") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.register.text == character) + } + + @Test + func xOnAnEmojiInAStatement() { + let session = Session("INSERT INTO notes VALUES ('ok \(thumbsUp)')\n", caret: 30) + session.press("x") + #expect(session.text == "INSERT INTO notes VALUES ('ok ')\n") + #expect(session.caret == 30) + } + + @Test + func countedXDeletesWholeCharactersOfEachKind() { + let session = Session("a\(thumbsUp)\(family)\(accented)z\n", caret: 1) + session.press("3x") + #expect(session.text == "az\n") + #expect(session.caret == 1) + #expect(session.engine.register.text == thumbsUp + family + accented) + } + + @Test + func xOnTheLastCharacterLandsOnTheStartOfTheNewLastOne() { + let session = Session("a\(thumbsUp)\(thumbsUp)\n", caret: 3) + session.press("x") + #expect(session.text == "a\(thumbsUp)\n") + #expect(session.caret == 1) + } + + @Test(arguments: composedCharacters) + func capitalXDeletesTheWholeCharacterBeforeTheCaret(character: String) { + let session = Session("a\(character)b\n", caret: 1 + units(character)) + session.press("X") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test + func countedCapitalXDeletesWholeCharacters() { + let session = Session("a\(family)\(accented)b\n", caret: 1 + units(family) + units(accented)) + session.press("2X") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test(arguments: composedCharacters) + func sSubstitutesTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("s") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.mode == .insert) + } + + // MARK: - r + + @Test(arguments: composedCharacters) + func rReplacesTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("r-") + #expect(session.text == "a-b\n") + #expect(session.caret == 1) + } + + @Test + func countedRReplacesEmojiByCharacter() { + let session = Session("\(thumbsUp)\(thumbsUp)z\n", caret: 0) + session.press("2r*") + #expect(session.text == "**z\n") + #expect(session.caret == 1) + } + + @Test + func rWithAnEmojiLeavesTheCaretOnTheLastOneWritten() { + let session = Session("abcd\n", caret: 0) + session.press("3r\(thumbsUp)") + #expect(session.text == "\(thumbsUp)\(thumbsUp)\(thumbsUp)d\n") + #expect(session.caret == 4) + } + + @Test + func rCountsCharactersLeftOnTheLine() { + let refused = Session("a\(thumbsUp)\n", caret: 0) + refused.press("3rx") + #expect(refused.text == "a\(thumbsUp)\n", "Two characters remain, so 3r is refused") + #expect(refused.caret == 0) + + let replaced = Session("a\(thumbsUp)\n", caret: 0) + replaced.press("2rx") + #expect(replaced.text == "xx\n") + #expect(replaced.caret == 1) + } + + // MARK: - ~ + + @Test + func tildeTogglesAcrossAnEmoji() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("3~") + #expect(session.text == "A\(thumbsUp)B\n") + #expect(session.caret == 3) + } + + @Test + func tildeKeepsTheMarkOnItsLetter() { + let session = Session("\(accented)x\n", caret: 0) + session.press("~") + #expect(session.text == "E\u{301}x\n") + #expect(session.caret == 2) + } + + @Test + func tildeLandsAfterACaseChangeThatGrowsTheText() { + let session = Session("\u{DF}a\n", caret: 0) + session.press("~") + #expect(session.text == "SSa\n") + #expect(session.caret == 2) + } + + // MARK: - h, l, $, j, e + + @Test + func hAndLStepOneCharacterAtATime() { + // a 0, thumbs up 1, b 3, family 4, c 12, space 13, accented e 14, x 16 + let session = Session("a\(thumbsUp)b\(family)c \(accented)x\n", caret: 0) + var forward: [Int] = [] + for _ in 0..<8 { + session.press("l") + forward.append(session.caret) + } + #expect(forward == [1, 3, 4, 12, 13, 14, 16, 16]) + var backward: [Int] = [] + for _ in 0..<8 { + session.press("h") + backward.append(session.caret) + } + #expect(backward == [14, 13, 12, 4, 3, 1, 0, 0]) + } + + @Test + func countedLAndHCrossSeveralCharacters() { + let session = Session("a\(thumbsUp)b\(family)c \(accented)x\n", caret: 0) + session.press("3l") + #expect(session.caret == 4) + session.press("$") + session.press("3h") + #expect(session.caret == 12) + } + + @Test(arguments: composedCharacters) + func dollarLandsOnTheStartOfAFinalCharacter(character: String) { + let session = Session("ok \(character)\nnext\n", caret: 0) + session.press("$") + #expect(session.caret == 3) + } + + @Test + func jLandsOnACharacterStart() { + // Column 2 on the second line is the second half of the emoji. + let session = Session("abcdef\nx\(thumbsUp)yz\n", caret: 2) + session.press("j") + #expect(session.caret == 8) + session.press("k") + #expect(session.caret == 2) + } + + @Test + func eLandsOnTheStartOfAnEmojiWord() { + let session = Session("ab \(thumbsUp) cd\n", caret: 0) + session.press("ee") + #expect(session.caret == 3) + } + + // MARK: - Insert and append + + @Test(arguments: composedCharacters) + func aAppendsAfterTheWholeCharacter(character: String) { + let session = Session("x\(character)\n", caret: 1) + session.press("a") + #expect(session.engine.mode == .insert) + #expect(session.caret == 1 + units(character)) + session.typeInTextView("!") + #expect(session.text == "x\(character)!\n") + session.escape() + #expect(session.caret == 1 + units(character)) + } + + @Test(arguments: composedCharacters) + func escapeFromInsertStepsBackOverTheWholeCharacter(character: String) { + let session = Session("ab\(character)\n", caret: 2 + units(character)) + session.press("i") + session.escape() + #expect(session.engine.mode == .normal) + #expect(session.caret == 2) + } + + // MARK: - p + + @Test + func pPastesAfterTheEmojiUnderTheCaret() { + let session = Session("a\(thumbsUp)b\n", caret: 3) + session.press("x") + #expect(session.text == "a\(thumbsUp)\n") + #expect(session.caret == 1) + session.press("p") + #expect(session.text == "a\(thumbsUp)b\n") + #expect(session.caret == 3) + } + + @Test + func pLeavesTheCaretOnTheStartOfAPastedEmoji() { + let session = Session("\(thumbsUp)ab\n", caret: 0) + session.press("x") + session.press("p") + #expect(session.text == "a\(thumbsUp)b\n") + #expect(session.caret == 1) + } + + // MARK: - Operators with inclusive motions + + @Test + func deDeletesThroughAnEmojiWord() { + let session = Session("say \(thumbsUp) now\n", caret: 3) + session.press("de") + #expect(session.text == "say now\n") + #expect(session.caret == 3) + } + + @Test + func deDeletesThroughALetterAndItsMark() { + let session = Session("caf\(accented) ok\n", caret: 0) + session.press("de") + #expect(session.text == " ok\n") + #expect(session.caret == 0) + } + + @Test(arguments: composedCharacters) + func dfDeletesThroughTheTargetCharacter(character: String) { + let session = Session("go \(character) now\n", caret: 0) + session.press("df\(character)") + #expect(session.text == " now\n") + #expect(session.caret == 0) + } + + // MARK: - Operators with h and l + + @Test(arguments: composedCharacters + ["q"]) + func dlDeletesTheCharacterUnderTheCaret(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("dl") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.register.text == character) + } + + @Test(arguments: composedCharacters + ["q"]) + func dlOnTheLastCharacterDeletesItAndStaysOnTheLine(character: String) { + let session = Session("a\(character)\nnext\n", caret: 1) + session.press("dl") + #expect(session.text == "a\nnext\n") + #expect(session.caret == 0) + } + + @Test(arguments: composedCharacters + ["q"]) + func dhDeletesTheCharacterBeforeTheCaret(character: String) { + let session = Session("a\(character)b\n", caret: 1 + units(character)) + session.press("dh") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test + func countedDlStopsAtTheLineEnd() { + let session = Session("a\(thumbsUp)\(family)\nnext\n", caret: 1) + session.press("5dl") + #expect(session.text == "a\nnext\n") + #expect(session.engine.mode == .normal) + } + + @Test + func clChangesTheWholeCharacter() { + let session = Session("a\(family)b\n", caret: 1) + session.press("cl") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.mode == .insert) + } + + @Test + func dDollarLeavesTheCaretOnTheNewLastCharacter() { + let session = Session("ab\(thumbsUp)cd\nnext\n", caret: 4) + session.press("d$") + #expect(session.text == "ab\(thumbsUp)\nnext\n") + #expect(session.caret == 2) + } + + // MARK: - f, F, t, T + + @Test + func fFindsAnEmojiAndRepeats() { + // thumbs up at 3 and 10 + let session = Session("go \(thumbsUp) now \(thumbsUp)!\n", caret: 0) + session.press("f\(thumbsUp)") + #expect(session.caret == 3) + session.press(";") + #expect(session.caret == 10) + session.press(",") + #expect(session.caret == 3) + } + + @Test + func countedFFindsTheSecondEmoji() { + let session = Session("go \(thumbsUp) now \(thumbsUp)!\n", caret: 0) + session.press("2f\(thumbsUp)") + #expect(session.caret == 10) + } + + @Test + func fFindsAFamilyAsOneCharacter() { + let session = Session("a\(thumbsUp) \(family)b\n", caret: 0) + session.press("f\(family)") + #expect(session.caret == 4) + } + + @Test + func fWithAPrecomposedLetterFindsTheDecomposedOne() { + let session = Session("caf\(accented)!\n", caret: 0) + session.press("f\u{E9}") + #expect(session.caret == 3) + } + + @Test + func tStopsOnTheCharacterBeforeAnEmoji() { + let session = Session("go \(thumbsUp) now\n", caret: 0) + session.press("t\(thumbsUp)") + #expect(session.caret == 2) + } + + @Test + func tLandsOnAnEmojiBeforeTheTarget() { + let session = Session("a\(thumbsUp)x\n", caret: 0) + session.press("tx") + #expect(session.caret == 1) + } + + @Test + func capitalFAndTFindAnEmojiBackward() { + // thumbs up 3, space 5, w 8 + let find = Session("go \(thumbsUp) now\n", caret: 8) + find.press("F\(thumbsUp)") + #expect(find.caret == 3) + + let till = Session("go \(thumbsUp) now\n", caret: 8) + till.press("T\(thumbsUp)") + #expect(till.caret == 5) + } + + // MARK: - R + + @Test(arguments: composedCharacters) + func replaceModeOverwritesTheWholeCharacterAndBackspaceRestoresIt(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("R") + session.press("x") + #expect(session.text == "axb\n") + #expect(session.caret == 2) + session.backspace() + #expect(session.text == "a\(character)b\n") + #expect(session.caret == 1) + } + + @Test(arguments: composedCharacters) + func replaceModeTypesACharacterAndBackspaceRestoresTheLetter(character: String) { + let session = Session("abc\n", caret: 1) + session.press("R") + session.press(character) + #expect(session.text == "a\(character)c\n") + #expect(session.caret == 1 + units(character)) + session.backspace() + #expect(session.text == "abc\n") + #expect(session.caret == 1) + } + + @Test + func replaceModeBackspaceRemovesAnAppendedEmoji() { + let session = Session("ab\n", caret: 2) + session.press("R") + session.press(thumbsUp) + #expect(session.text == "ab\(thumbsUp)\n") + #expect(session.caret == 4) + session.backspace() + #expect(session.text == "ab\n") + #expect(session.caret == 2) + } + + @Test + func escapeFromReplaceModeStepsBackOverAnEmoji() { + let session = Session("abc\n", caret: 1) + session.press("R") + session.press(thumbsUp) + session.escape() + #expect(session.text == "a\(thumbsUp)c\n") + #expect(session.caret == 1) + } + + // MARK: - Visual + + @Test(arguments: composedCharacters) + func vSelectsTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("v") + #expect(session.selection == NSRange(location: 1, length: units(character))) + } + + @Test + func visualLExtendsOverAnEmojiAndDeletes() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("vl") + #expect(session.selection == NSRange(location: 0, length: 3)) + session.press("l") + #expect(session.selection == NSRange(location: 0, length: 4)) + session.press("d") + #expect(session.text == "\n") + #expect(session.engine.mode == .normal) + } + + @Test + func visualHExtendsBackOverAnEmoji() { + let session = Session("a\(thumbsUp)b\n", caret: 3) + session.press("vh") + #expect(session.selection == NSRange(location: 1, length: 3)) + session.press("h") + #expect(session.selection == NSRange(location: 0, length: 4)) + } + + @Test + func visualDeleteOfAnEmoji() { + let session = Session("a\(family)b\n", caret: 1) + session.press("vd") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test + func visualReplaceWritesOneCharacterPerCharacter() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("vllr-") + #expect(session.text == "---\n") + } + + @Test + func visualSwapKeepsTheEmojiWhole() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("vlo") + #expect(session.selection == NSRange(location: 0, length: 3)) + session.press("l") + #expect(session.selection == NSRange(location: 1, length: 2)) + } + + // MARK: - The text view adapter + + private final class StubDelegate: TextViewDelegate {} + + private func makeTextView(_ string: String) -> TextView { + let textView = TextView( + string: string, + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + textColor: .labelColor, + lineHeightMultiplier: 1.0, + wrapLines: false, + isEditable: true, + isSelectable: true, + letterSpacing: 1.0, + delegate: StubDelegate() + ) + textView.frame = NSRect(x: 0, y: 0, width: 500, height: 500) + textView.layout() + return textView + } + + @Test + func adapterCharacterRangeIsTheComposedSequence() { + let textView = makeTextView("a\(thumbsUp)b") + let adapter = VimTextBufferAdapter(textView: textView) + withExtendedLifetime(textView) { + #expect(adapter.characterRange(at: 2) == NSRange(location: 1, length: 2)) + #expect(adapter.characterRange(at: 3) == NSRange(location: 3, length: 1)) + #expect(adapter.characterRange(at: 4) == NSRange(location: 4, length: 0)) + #expect(adapter.characterRange(at: -1) == NSRange(location: 0, length: 0)) + } + } + + @Test + func adapterKeepsTheSelectionOnWholeCharacters() { + let textView = makeTextView("a\(thumbsUp)b") + let adapter = VimTextBufferAdapter(textView: textView) + withExtendedLifetime(textView) { + adapter.setSelectedRange(NSRange(location: 2, length: 0)) + #expect(textView.selectedRange() == NSRange(location: 1, length: 0)) + adapter.setSelectedRange(NSRange(location: 2, length: 1)) + #expect(textView.selectedRange() == NSRange(location: 1, length: 2)) + adapter.setSelectedRange(NSRange(location: 0, length: 2)) + #expect(textView.selectedRange() == NSRange(location: 0, length: 3)) + } + } + + @Test + func engineOverTheTextViewStepsAndDeletesWholeEmoji() { + let textView = makeTextView("INSERT INTO notes VALUES ('ok \(thumbsUp)')") + let adapter = VimTextBufferAdapter(textView: textView) + let engine = VimEngine(buffer: adapter) + withExtendedLifetime(textView) { + adapter.setSelectedRange(NSRange(location: 29, length: 0)) + _ = engine.process("l", shift: false) + #expect(adapter.selectedRange().location == 30) + _ = engine.process("l", shift: false) + #expect(adapter.selectedRange().location == 32) + _ = engine.process("h", shift: false) + _ = engine.process("x", shift: false) + #expect(textView.string == "INSERT INTO notes VALUES ('ok ')") + #expect(adapter.selectedRange().location == 30) + } + } + } +} diff --git a/apple/TableProTests/Core/Vim/VimTextBufferAdapterLineIndexTests.swift b/apple/TableProTests/Core/Vim/VimTextBufferAdapterLineIndexTests.swift new file mode 100644 index 0000000000..407c69ed29 --- /dev/null +++ b/apple/TableProTests/Core/Vim/VimTextBufferAdapterLineIndexTests.swift @@ -0,0 +1,322 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTextEngine +import Testing + +private let lineIndexDocuments = [ + "", + "SELECT 1", + "a\nb\n", + "\n", + "\n\n", + "a\n\n\nb\n\n", + "line one\r\nline two\r\n\r\nline four", + "a\r\n", + "\r\n\r\n", + "a\r\nb\nc\r\n\nd", + "x\u{1F600}y\n\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467} z\ne\u{301}\n\u{1F600}", + "\tindent\n\t\tmore\n spaces\t\n\t", +] + +@MainActor +struct VimTextBufferAdapterLineIndexTests { + private final class StubDelegate: TextViewDelegate {} + + private func makeTextView(string: String, layingOut: Bool = true) -> TextView { + let textView = TextView( + string: string, + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + textColor: .labelColor, + lineHeightMultiplier: 1.0, + wrapLines: false, + isEditable: true, + isSelectable: true, + letterSpacing: 1.0, + delegate: StubDelegate() + ) + textView.frame = NSRect(x: 0, y: 0, width: 500, height: 500) + if layingOut { + textView.layout() + } + return textView + } + + // MARK: - Same answers as the line walk + + @Test("Line queries answer as the line walk does", arguments: lineIndexDocuments) + func freshDocumentMatchesTheLineWalk(text: String) { + let textView = makeTextView(string: text) + let adapter = VimTextBufferAdapter(textView: textView) + + let mismatches = withExtendedLifetime(textView) { Self.mismatches(adapter, against: textView.string as NSString) } + + #expect(mismatches.isEmpty, "\(mismatches.prefix(8))") + } + + @Test("Line queries answer as the line walk does after each of a run of edits") + func editsKeepLineQueriesInStep() { + let textView = makeTextView(string: "SELECT 1\nFROM t\r\n\nWHERE a = 1\n") + let adapter = VimTextBufferAdapter(textView: textView) + let replacements = ["", "", "a", "xy", "\n", "\r", "\r\n", "\n\n", "q\nr\r\ns", "\u{1F600}", "\t", "--\n"] + var generator = SeededGenerator(seed: 0x51DE) + var mismatches: [String] = [] + + withExtendedLifetime(textView) { + for step in 0..<60 { + let text = textView.string as NSString + // Edits land on character boundaries, as the editor's own do, so none leaves half a surrogate pair. + let start = Self.characterBoundary(Int.random(in: 0...text.length, using: &generator), in: text) + let end = max( + start, + Self.characterBoundary(min(text.length, start + Int.random(in: 0...6, using: &generator)), in: text) + ) + let replacement = replacements.randomElement(using: &generator) ?? "" + textView.replaceCharacters(in: NSRange(location: start, length: end - start), with: replacement) + // The editor drops the adapter's cache on every edit. + adapter.invalidateLineCache() + + let edited = textView.string as NSString + mismatches += Self.mismatches(adapter, against: edited).map { + "step \(step) \(String(reflecting: edited as String)): \($0)" + } + } + } + + #expect(mismatches.isEmpty, "\(mismatches.prefix(8))") + } + + @Test("Line queries walk the text while the line storage still describes the text before an edit", arguments: [ + ("a\nb\r\nc\n", NSRange(location: 2, length: 0), "x\n\ny\r\n"), + ("one\ntwo\nthree", NSRange(location: 0, length: 4), ""), + ("SELECT 1\r\n\r\nSELECT 2", NSRange(location: 10, length: 2), "--\n"), + ]) + func staleLineStorageFallsBackToTheText(original: String, range: NSRange, replacement: String) { + let textView = makeTextView(string: original) + let adapter = VimTextBufferAdapter(textView: textView) + + withExtendedLifetime(textView) { + textView.layoutManager.processesEdits = false + textView.textStorage.replaceCharacters(in: range, with: replacement) + #expect(textView.layoutManager.lineStorage.length != textView.textStorage.length) + + let stale = Self.mismatches(adapter, against: textView.string as NSString) + #expect(stale.isEmpty, "\(stale.prefix(8))") + + textView.layoutManager.processesEdits = true + textView.layoutManager.reset() + let rebuilt = Self.mismatches(adapter, against: textView.string as NSString) + #expect(rebuilt.isEmpty, "\(rebuilt.prefix(8))") + } + } + + // The line walk this replaced counted only line feeds for `lineAndColumn`, while `lineRange` and + // `offset(forLine:column:)` also break at a lone carriage return. Every query now numbers lines the same way. + @Test("A lone carriage return ends a line for every line query", arguments: [ + "a\rb\rc", + "a\rb\r", + "a\r\rb\nc\r\nd", + ]) + func carriageReturnEndsALine(text: String) { + let textView = makeTextView(string: text) + let adapter = VimTextBufferAdapter(textView: textView) + let nsText = text as NSString + var lineRanges: [NSRange] = [] + var location = 0 + while location < nsText.length { + let range = nsText.lineRangeBreakingAtLineEndings(for: NSRange(location: location, length: 0)) + lineRanges.append(range) + location = NSMaxRange(range) + } + + var mismatches: [String] = [] + withExtendedLifetime(textView) { + if adapter.lineCount != lineRanges.count { + mismatches.append("lineCount \(adapter.lineCount), expected \(lineRanges.count)") + } + for (line, range) in lineRanges.enumerated() { + if adapter.offset(forLine: line, column: 0) != range.location { + mismatches.append("offset(forLine: \(line)) \(adapter.offset(forLine: line, column: 0))") + } + for offset in range.location.. SectionScript { + let script = String(repeating: "SELECT id, name FROM users WHERE id = 1;\n", count: lines) + let textView = makeTextView(string: script, layingOut: false) + let adapter = VimTextBufferAdapter(textView: textView) + return SectionScript( + textView: textView, + adapter: adapter, + engine: VimEngine(buffer: adapter), + lastOffset: (script as NSString).length - 1 + ) + } + + private static func pressSectionForward(_ engine: VimEngine, _ adapter: VimTextBufferAdapter) -> Int { + adapter.setSelectedRange(NSRange(location: 0, length: 0)) + _ = engine.process("]", shift: false) + _ = engine.process("]", shift: false) + return adapter.selectedRange().location + } + + private static func secondsSpent(_ work: () -> Void) -> TimeInterval { + let start = ProcessInfo.processInfo.systemUptime + work() + return ProcessInfo.processInfo.systemUptime - start + } + + // MARK: - Comparison + + private static func characterBoundary(_ offset: Int, in text: NSString) -> Int { + guard offset > 0, offset < text.length else { return offset } + return text.rangeOfComposedCharacterSequence(at: offset).location + } + + private static func mismatches(_ adapter: VimTextBufferAdapter, against text: NSString) -> [String] { + var found: [String] = [] + if adapter.length != text.length { + found.append("length \(adapter.length), expected \(text.length)") + } + let lineCount = LineWalk.lineCount(of: text) + if adapter.lineCount != lineCount { + found.append("lineCount \(adapter.lineCount), expected \(lineCount)") + } + for offset in -1...(text.length + 1) { + let position = adapter.lineAndColumn(forOffset: offset) + let expectedPosition = LineWalk.lineAndColumn(of: text, forOffset: offset) + if position != expectedPosition { + found.append("lineAndColumn(\(offset)) \(position), expected \(expectedPosition)") + } + let range = adapter.lineRange(forOffset: offset) + let expectedRange = LineWalk.lineRange(of: text, forOffset: offset) + if range != expectedRange { + found.append("lineRange(\(offset)) \(range), expected \(expectedRange)") + } + if offset >= 0 && offset < text.length && adapter.character(at: offset) != text.character(at: offset) { + found.append("character(at: \(offset)) \(adapter.character(at: offset))") + } + } + for line in -1...(lineCount + 1) { + for column in [-1, 0, 1, 2, 5, 1_000] { + let offset = adapter.offset(forLine: line, column: column) + let expectedOffset = LineWalk.offset(of: text, forLine: line, column: column) + if offset != expectedOffset { + found.append("offset(forLine: \(line), column: \(column)) \(offset), expected \(expectedOffset)") + } + } + } + return found + } + +} + +// The line walk `VimTextBufferAdapter` used before it read the layout manager's line storage, kept as the reference +// its answers are compared with. `lineAndColumn` counts every line break here, as the other queries always did; the +// old one counted only line feeds, which `carriageReturnEndsALine` covers. +private enum LineWalk { + static func lineCount(of text: NSString) -> Int { + if text.length == 0 { return 1 } + var count = 0 + var index = 0 + while index < text.length { + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) + count += 1 + index = lineRange.location + lineRange.length + } + return max(1, count) + } + + static func lineRange(of text: NSString, forOffset offset: Int) -> NSRange { + let clampedOffset = min(max(0, offset), text.length) + return text.lineRangeBreakingAtLineEndings(for: NSRange(location: clampedOffset, length: 0)) + } + + static func lineAndColumn(of text: NSString, forOffset offset: Int) -> (line: Int, column: Int) { + let clampedOffset = min(max(0, offset), text.length) + if text.length == 0 { return (0, 0) } + let safeOffset = min(clampedOffset, max(0, text.length - 1)) + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: safeOffset, length: 0)) + var line = 0 + var index = 0 + while index < lineRange.location { + index = NSMaxRange(text.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0))) + line += 1 + } + return (line, clampedOffset - lineRange.location) + } + + static func offset(of text: NSString, forLine line: Int, column: Int) -> Int { + var currentLine = 0 + var index = 0 + while index < text.length && currentLine < line { + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: index, length: 0)) + currentLine += 1 + index = lineRange.location + lineRange.length + } + let lineRange = text.lineRangeBreakingAtLineEndings(for: NSRange(location: min(index, text.length), length: 0)) + let contentLength: Int + let lineEnd = lineRange.location + lineRange.length + if lineEnd > lineRange.location && lineEnd <= text.length && text.character(at: lineEnd - 1) == 0x0A { + contentLength = lineRange.length - 1 + } else { + contentLength = lineRange.length + } + let clampedCol = min(column, max(0, contentLength - 1)) + return lineRange.location + max(0, clampedCol) + } +} diff --git a/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift b/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift index 6fadfaf773..48c97aac20 100644 --- a/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift +++ b/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift @@ -2,8 +2,7 @@ // VimTextBufferAdapterPerfTests.swift // TableProTests // -// Regression tests for VimTextBufferAdapter incremental lineCount -// and setSelectedRange guard +// Regression tests for VimTextBufferAdapter lineCount and setSelectedRange guard // import AppKit @@ -65,83 +64,29 @@ struct VimTextBufferAdapterPerfTests { #expect(adapter.lineCount == 2) } - // MARK: - textDidChange incremental (pure insertion) - - @Test("textDidChange with pure insertion updates line count incrementally") - func insertionUpdatesLineCount() { - let (adapter, textView) = makeAdapter(string: "hello") - - // Prime the cache - let initial = adapter.lineCount - #expect(initial == 1) - - // Simulate inserting "\nworld" at offset 5 - textView.replaceCharacters(in: NSRange(location: 5, length: 0), with: "\nworld") - adapter.textDidChange(in: NSRange(location: 5, length: 0), replacementLength: 6) - - #expect(adapter.lineCount == 2) - } - - @Test("textDidChange with multi-newline insertion") - func multiNewlineInsertion() { - let (adapter, textView) = makeAdapter(string: "hello") - - _ = adapter.lineCount // prime cache - - textView.replaceCharacters(in: NSRange(location: 5, length: 0), with: "\n\n\n") - adapter.textDidChange(in: NSRange(location: 5, length: 0), replacementLength: 3) - - #expect(adapter.lineCount == 4) - } - - // MARK: - textDidChange fallback on deletion - - @Test("textDidChange with deletion invalidates cache") - func deletionInvalidatesCache() { - let (adapter, textView) = makeAdapter(string: "a\nb\nc") - - let initial = adapter.lineCount - #expect(initial == 3) - - // Simulate deleting "\nb" (range.length > 0 means it's not pure insertion) - textView.replaceCharacters(in: NSRange(location: 1, length: 2), with: "") - adapter.textDidChange(in: NSRange(location: 1, length: 2), replacementLength: 0) - - // Cache should be invalidated; next access does a full recount - #expect(adapter.lineCount == 2) - } - - // MARK: - textDidChange(oldText:...) incremental replacement - - @Test("textDidChange with oldText correctly computes delta for replacement") - func oldTextReplacementDelta() { - let originalText = "a\nb\nc" - let (adapter, textView) = makeAdapter(string: originalText) - - let initial = adapter.lineCount - #expect(initial == 3) - - // Replace "b\nc" (range 2..4, contains 1 newline) with "x" (0 newlines) - textView.replaceCharacters(in: NSRange(location: 2, length: 3), with: "x") - adapter.textDidChange(oldText: originalText, in: NSRange(location: 2, length: 3), replacementLength: 1) - - // 3 original - 1 removed newline + 0 added = 2 - #expect(adapter.lineCount == 2) - } - - @Test("textDidChange with oldText adding newlines") - func oldTextAddingNewlines() { - let originalText = "abc" - let (adapter, textView) = makeAdapter(string: originalText) - - _ = adapter.lineCount // prime: 1 - - // Replace "b" with "x\ny\nz" (adding 2 newlines) - textView.replaceCharacters(in: NSRange(location: 1, length: 1), with: "x\ny\nz") - adapter.textDidChange(oldText: originalText, in: NSRange(location: 1, length: 1), replacementLength: 5) - - // 1 original - 0 removed + 2 added = 3 - #expect(adapter.lineCount == 3) + // MARK: - Line count after an edit + + // The editor drops the cache on every edit, so the next count reads the text as it is now. + @Test("After an edit the line count matches a fresh count of the new text", arguments: [ + (NSRange(location: 5, length: 0), "\nworld", "hello", 2), + (NSRange(location: 5, length: 0), "\n\n\n", "hello", 3), + (NSRange(location: 1, length: 2), "", "a\nb\nc", 2), + (NSRange(location: 2, length: 3), "x", "a\nb\nc", 2), + (NSRange(location: 1, length: 1), "x\ny\nz", "abc", 3), + (NSRange(location: 1, length: 0), "\r", "ab", 2), + ]) + func lineCountAfterAnEdit(range: NSRange, replacement: String, original: String, expected: Int) { + let (adapter, textView) = makeAdapter(string: original) + _ = adapter.lineCount + + textView.replaceCharacters(in: range, with: replacement) + adapter.invalidateLineCache() + + let counted = withExtendedLifetime(textView) { adapter.lineCount } + let (fresh, freshTextView) = makeAdapter(string: textView.string) + let freshCount = withExtendedLifetime(freshTextView) { fresh.lineCount } + #expect(counted == expected) + #expect(counted == freshCount) } // MARK: - setSelectedRange guard diff --git a/apple/TableProTests/Core/Vim/VimTextBufferMock.swift b/apple/TableProTests/Core/Vim/VimTextBufferMock.swift index 441955690b..7beb32f778 100644 --- a/apple/TableProTests/Core/Vim/VimTextBufferMock.swift +++ b/apple/TableProTests/Core/Vim/VimTextBufferMock.swift @@ -36,9 +36,7 @@ final class VimTextBufferMock: VimTextBuffer { return max(1, count) } - func invalidateLineCache() { - // No-op in mock — lineCount is always computed from current text - } + func invalidateLineCache() {} func lineRange(forOffset offset: Int) -> NSRange { let nsString = text as NSString @@ -97,6 +95,14 @@ final class VimTextBufferMock: VimTextBuffer { return nsString.character(at: offset) } + func characterRange(at offset: Int) -> NSRange { + let nsString = text as NSString + guard offset >= 0 && offset < nsString.length else { + return NSRange(location: min(max(0, offset), nsString.length), length: 0) + } + return nsString.rangeOfComposedCharacterSequence(at: offset) + } + func wordBoundary(forward: Bool, from offset: Int) -> Int { let nsString = text as NSString guard nsString.length > 0 else { return 0 } @@ -285,14 +291,16 @@ final class VimTextBufferMock: VimTextBuffer { let clampedLocation = max(0, min(range.location, nsString.length)) let maxLength = nsString.length - clampedLocation let clampedLength = max(0, min(range.length, maxLength)) - _selectedRange = NSRange(location: clampedLocation, length: clampedLength) + let clamped = NSRange(location: clampedLocation, length: clampedLength) + _selectedRange = clamped.length > 0 + ? nsString.rangeOfComposedCharacterSequences(for: clamped) + : NSRange(location: characterRange(at: clampedLocation).location, length: 0) } func replaceCharacters(in range: NSRange, with string: String) { let mutable = NSMutableString(string: text) mutable.replaceCharacters(in: range, with: string) text = mutable as String - // Update selection to end of inserted text _selectedRange = NSRange(location: range.location + (string as NSString).length, length: 0) } diff --git a/apple/TableProTests/Core/WritePolicy/AgentWritePolicyMatrixTests.swift b/apple/TableProTests/Core/WritePolicy/AgentWritePolicyMatrixTests.swift index 4b441b8b19..b9b3679e59 100644 --- a/apple/TableProTests/Core/WritePolicy/AgentWritePolicyMatrixTests.swift +++ b/apple/TableProTests/Core/WritePolicy/AgentWritePolicyMatrixTests.swift @@ -189,6 +189,7 @@ extension GlobalStateSuites { connectionId: rig.connection.id, bridge: MCPConnectionBridge(), authPolicy: rig.authPolicy, + chatMode: .agent, sessionId: UUID() ), explicitlyApproved: clicked ? [block.id] : [], diff --git a/apple/TableProTests/Core/WritePolicy/DestructiveStatementRoutingTests.swift b/apple/TableProTests/Core/WritePolicy/DestructiveStatementRoutingTests.swift index 95755d9282..82c1c64a6a 100644 --- a/apple/TableProTests/Core/WritePolicy/DestructiveStatementRoutingTests.swift +++ b/apple/TableProTests/Core/WritePolicy/DestructiveStatementRoutingTests.swift @@ -59,13 +59,14 @@ extension GlobalStateSuites { private static let unfilteredDelete = DestructiveRoute(sql: "DELETE FROM matrix_rows", destructive: true) - private func makeRig(_ prompts: WritePolicyPrompts, _ databaseType: DatabaseType) -> Rig { - let connection = WritePolicyRig.connection( + private func makeRig(_ prompts: WritePolicyPrompts, _ databaseType: DatabaseType, database: String? = nil) -> Rig { + var connection = WritePolicyRig.connection( type: databaseType, externalAccess: .readWrite, aiPolicy: .alwaysAllow, safeMode: .silent ) + if let database { connection.database = database } guard databaseType == .mssql else { let rig = WritePolicyRig(connection: connection, prompts: prompts, live: true) return Rig(rig: rig, received: { rig.driver.received }, rowCaps: { [] }) @@ -121,16 +122,19 @@ extension GlobalStateSuites { private func assistant( _ tool: any ChatTool, _ route: DestructiveRoute, - answer: WritePolicyPrompts.Answer = .approveAll + answer: WritePolicyPrompts.Answer = .approveAll, + mode: AIChatMode = .agent, + database: String? = nil ) async -> Outcome { let prompts = WritePolicyPrompts(answer) - let setup = makeRig(prompts, route.databaseType) + let setup = makeRig(prompts, route.databaseType, database: database) let rig = setup.rig defer { rig.tearDown() } let context = ChatToolContext( connectionId: rig.connection.id, bridge: MCPConnectionBridge(), authPolicy: rig.authPolicy, + chatMode: mode, sessionId: UUID(), policyLookup: { _ in .alwaysAllow } ) @@ -199,6 +203,52 @@ extension GlobalStateSuites { #expect(outcome.received.isEmpty) } + @Test("Edit runs a row change and refuses a schema, privilege or session statement that Agent runs", arguments: [ + ("UPDATE matrix_rows SET note = 'x' WHERE id = 1", true), + ("INSERT INTO matrix_rows (id) VALUES (2)", true), + ("ALTER ROLE app_user WITH SUPERUSER", false), + ("GRANT ALL ON matrix_rows TO app_user", false), + ("SET session_replication_role = replica", false), + ("CREATE TABLE notes (id int)", false), + ]) + func editRunsOnlyDataStatements(sql: String, isData: Bool) async { + let route = DestructiveRoute(sql: sql, destructive: false) + let edit = await assistant(ExecuteQueryChatTool(), route, mode: .edit) + let agent = await assistant(ExecuteQueryChatTool(), route, mode: .agent) + + #expect(edit.ran == isData, "\(edit.text)") + #expect(edit.received.contains { $0.contains(sql) } == isData) + if !isData { + #expect(edit.text == ExternalStatementGate.dataStatementRefusal) + #expect(edit.prompts.isEmpty) + } + #expect(agent.ran, "\(agent.text)") + #expect(agent.received.contains { $0.contains(sql) }) + } + + @Test("Edit refuses a row change that lands in the engine's own schema, which Agent runs") + func editRefusesARowChangeInASystemSchema() async { + let route = DestructiveRoute( + sql: "UPDATE user SET Super_priv = 'Y' WHERE User = 'app'", destructive: false, databaseType: .mysql + ) + let edit = await assistant(ExecuteQueryChatTool(), route, mode: .edit, database: "mysql") + let agent = await assistant(ExecuteQueryChatTool(), route, mode: .agent, database: "mysql") + + #expect(!edit.ran) + #expect(edit.text == ExternalStatementGate.dataStatementRefusal) + #expect(edit.received.isEmpty) + #expect(agent.ran, "\(agent.text)") + } + + @Test("Edit refuses a transaction statement before anything reaches the database", arguments: ["BEGIN", "COMMIT"]) + func editRefusesTransactionStatements(sql: String) async { + let edit = await assistant(ExecuteQueryChatTool(), DestructiveRoute(sql: sql, destructive: false), mode: .edit) + + #expect(!edit.ran) + #expect(edit.text == ExternalStatementGate.dataStatementRefusal) + #expect(edit.received.isEmpty) + } + @Test("A DELETE with no WHERE the user declines never reaches the database") func declinedUnfilteredDeleteDoesNotRun() async { let overMCP = await mcp(ConfirmDestructiveOperationTool.name, Self.unfilteredDelete, answer: .refuseAll) diff --git a/apple/TableProTests/Core/WritePolicy/WritePolicyMatrix.swift b/apple/TableProTests/Core/WritePolicy/WritePolicyMatrix.swift index c455669a61..1b366f3a6d 100644 --- a/apple/TableProTests/Core/WritePolicy/WritePolicyMatrix.swift +++ b/apple/TableProTests/Core/WritePolicy/WritePolicyMatrix.swift @@ -102,7 +102,8 @@ enum MCPWritePath: String, CaseIterable, Sendable, CustomTestStringConvertible { case session // The connection stays unsaved, so a call the gates pass stops at the tool's lookup before a window opens. case window - // Names no connection: the tab list hides the ones the client may not reach. A hosted test has no window to raise. + // Names a tab, and asks about the tab's connection; the tab list hides the ones the client may not reach. + // A hosted test has no window to raise. case tab } @@ -209,7 +210,6 @@ enum MCPWritePath: String, CaseIterable, Sendable, CustomTestStringConvertible { static func expected(_ path: MCPWritePath, _ cell: MCPWriteCell) -> WriteOutcome { guard path.scopes.isSubset(of: cell.token.scopes) else { return .refused } guard cell.aiPolicy != .never, cell.externalAccess != .blocked else { return .refused } - guard path.reach != .tab else { return .runs(after: []) } var prompts: [WritePrompt] = cell.aiPolicy == .askEachTime ? [.connectionAccess] : [] guard path.reach == .statement else { return .runs(after: prompts) } guard cell.externalAccess == .readWrite, cell.safeMode != .readOnly else { return .refused } diff --git a/apple/TableProTests/Database/NativeDumpRegistryTests.swift b/apple/TableProTests/Database/NativeDumpRegistryTests.swift index 67cb591f32..b86d33d134 100644 --- a/apple/TableProTests/Database/NativeDumpRegistryTests.swift +++ b/apple/TableProTests/Database/NativeDumpRegistryTests.swift @@ -47,10 +47,21 @@ struct NativeDumpRegistryTests { localFilePath: String? = nil, flavor: NativeDumpToolFlavor = .mysql, toolVersionText: String? = nil, - serverVersion: String? = nil + serverVersion: String? = nil, + systemTrustAnchorsFile: URL? = nil ) throws -> NativeDumpCommand { let tool = try #require(NativeDumpRegistry.descriptor(for: type)?.commandLineTool) let effective = overrideConnection ?? connection(type: type) + var request = NativeDumpDescriptor.Request( + connection: effective, + database: database, + fileURL: fileURL, + password: password, + scope: scope, + localFilePath: localFilePath ?? effective.database, + serverVersion: serverVersion + ) + request.systemTrustAnchorsFile = systemTrustAnchorsFile let built = try NativeDumpService.buildCommand( kind: kind, tool: tool, @@ -60,15 +71,7 @@ struct NativeDumpRegistryTests { flavor: flavor, versionText: toolVersionText ), - request: NativeDumpDescriptor.Request( - connection: effective, - database: database, - fileURL: fileURL, - password: password, - scope: scope, - localFilePath: localFilePath ?? effective.database, - serverVersion: serverVersion - ) + request: request ) if let folder = built.clientPluginFolder { try? FileManager.default.removeItem(at: folder.url) @@ -125,8 +128,8 @@ struct NativeDumpRegistryTests { #expect(try #require(NativeDumpRegistry.descriptor(for: .sqlite)).archiveFormat.fileExtension == "sql") } - /// Anything in `argv` is readable by every process on the machine through `ps`, so no - /// descriptor may put a password there. + // Anything in `argv` is readable by every process on the machine through `ps`, so no + // descriptor may put a password there. @Test("No engine puts the password in the argument list") func passwordNeverReachesArgv() throws { for type in [DatabaseType.postgresql, .mysql, .mongodb, .sqlite] { @@ -149,7 +152,7 @@ struct NativeDumpRegistryTests { #expect(built.arguments.last == "sales") } - /// `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. + // `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. @Test("MySQL is redirected through standard output in both directions") func mysqlRedirects() throws { let backup = try command(.mysql, kind: .backup) @@ -163,7 +166,7 @@ struct NativeDumpRegistryTests { #expect(!restore.arguments.contains("--single-transaction")) } - /// `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. + // `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. @Test("PostgreSQL writes its own file") func postgresWritesItsOwnFile() throws { let built = try command(.postgresql) @@ -171,8 +174,8 @@ struct NativeDumpRegistryTests { #expect(built.redirectedFileURL == nil) } - /// `mongodump` reads a password from neither the environment nor standard input, so the only - /// channel left is a config file, which must be owner-only and must not survive the process. + // `mongodump` reads a password from neither the environment nor standard input, so the only + // channel left is a config file, which must be owner-only and must not survive the process. @Test("MongoDB writes an owner-only credentials file and points at it") func mongoUsesACredentialsFile() throws { let built = try command(.mongodb) @@ -352,7 +355,7 @@ struct NativeDumpRegistryTests { ) } - /// The database is a file the tool opens, so there is nothing to authenticate to. + // The database is a file the tool opens, so there is nothing to authenticate to. @Test("SQLite passes the file path and no network arguments") func sqliteUsesTheFilePath() throws { let file = connection(type: .sqlite, host: "", port: 0, database: "/tmp/app.sqlite", username: "") @@ -390,9 +393,9 @@ struct NativeDumpRegistryTests { #expect(!FileManager.default.fileExists(atPath: marker.path)) } - /// libSQL claims the SQLite descriptor and keeps its path in a plugin-declared additional - /// field, leaving `database` empty. Reading `database` handed `sqlite3` an empty path, which - /// exits 0 after writing a 52-byte file the result sheet then reported as a backup. + // libSQL claims the SQLite descriptor and keeps its path in a plugin-declared additional + // field, leaving `database` empty. Reading `database` handed `sqlite3` an empty path, which + // exits 0 after writing a 52-byte file the result sheet then reported as a backup. @Test("libSQL dumps the file its driver opens, not the empty database field") func libsqlUsesItsResolvedPath() throws { var remote = connection(type: .libsql, host: "", port: 0, database: "", username: "") @@ -437,8 +440,8 @@ struct NativeDumpRegistryTests { } } - /// A libSQL connection to a Turso URL has no local file, so `sqlite3` cannot reach it at all. - /// Answering by type alone is what offered Backup Dump on one and wrote nothing. + // A libSQL connection to a Turso URL has no local file, so `sqlite3` cannot reach it at all. + // Answering by type alone is what offered Backup Dump on one and wrote nothing. @Test("A connection with no local file is not offered a dump") func remoteFileBackedConnectionIsUnsupported() { var remote = connection(type: .libsql, host: "", port: 0, database: "", username: "") @@ -595,9 +598,9 @@ struct NativeDumpRegistryTests { #expect(fromDirectory.arguments.starts(with: ["--no-password", "-h", "/var/run/postgresql", "-p", "5434"])) } - /// Measured, MariaDB 12.3.3 answers any `--ssl-mode` with `unknown variable` and exit 7, and - /// MySQL 8.4.11 answers `--ssl` with `unknown option` and exit 2, so neither spelling may reach - /// the other family's tool in either direction (#3046). + // Measured, MariaDB 12.3.3 answers any `--ssl-mode` with `unknown variable` and exit 7, and + // MySQL 8.4.11 answers `--ssl` with `unknown option` and exit 2, so neither spelling may reach + // the other family's tool in either direction (#3046). @Test("Neither client family is ever handed the other one's SSL flags", arguments: [ NativeDumpKind.backup, .restore ]) @@ -614,8 +617,8 @@ struct NativeDumpRegistryTests { #expect(!mysql.arguments.contains("--skip-ssl")) } - /// SSL off is the other half of the same defect: the old code sent `--ssl-mode=DISABLED`, which - /// MariaDB rejects exactly as it rejects the rest. + // SSL off is the other half of the same defect: the old code sent `--ssl-mode=DISABLED`, which + // MariaDB rejects exactly as it rejects the rest. @Test("An SSL-off connection is disabled in the tool's own spelling") func sslDisabledFollowsTheResolvedTool() throws { let maria = try command(.mysql, flavor: .mariadb) @@ -626,8 +629,8 @@ struct NativeDumpRegistryTests { #expect(mysql.arguments.contains("--ssl-mode=DISABLED")) } - /// `--ssl-cert` implies `--ssl` on MariaDB, so a connection whose SSL is off must not carry the - /// certificate the form still holds. + // `--ssl-cert` implies `--ssl` on MariaDB, so a connection whose SSL is off must not carry the + // certificate the form still holds. @Test("Certificate paths reach the tool, and only while SSL is on") func certificatePathsFollowTheMode() throws { var secured = connection(type: .mysql, sslMode: .verifyCa, sslEnabled: true) @@ -647,8 +650,8 @@ struct NativeDumpRegistryTests { #expect(!disabled.arguments.contains { $0.hasPrefix("--ssl-key") }) } - /// A tool that answered nothing is not guessed at once the connection asks for encryption: - /// MariaDB accepts `--loose-ssl-mode=REQUIRED` and ignores it, which is a cleartext dump. + // A tool that answered nothing is not guessed at once the connection asks for encryption: + // MariaDB accepts `--loose-ssl-mode=REQUIRED` and ignores it, which is a cleartext dump. @Test("An unidentified client refuses an encrypted connection rather than guessing") func unidentifiedToolRefusesEncryptedModes() throws { for mode in [SSLMode.required, .verifyCa, .verifyIdentity] { @@ -662,8 +665,8 @@ struct NativeDumpRegistryTests { #expect(!built.arguments.contains { $0.hasPrefix("--ssl") }) } - /// `mysqldump` 8.0 reads a table no MariaDB server and no MySQL before 8.0 has, and exits 2 - /// after writing part of the file. The flag that skips it is MySQL's own and backup only. + // `mysqldump` 8.0 reads a table no MariaDB server and no MySQL before 8.0 has, and exits 2 + // after writing part of the file. The flag that skips it is MySQL's own and backup only. @Test("A MySQL 8 dump tool skips column statistics on a server that has none") func columnStatisticsFlagFollowsTheServer() throws { let mysql8 = "mysqldump Ver 8.4.11 for macos26.6 on arm64 (Homebrew)" @@ -691,9 +694,9 @@ struct NativeDumpRegistryTests { #expect(!mariaTool.arguments.contains("--skip-column-statistics")) } - /// `my_getopt` parses past a positional argument, so a database named `--no-data` was read as - /// the option: measured on MySQL 8.4.11 and MariaDB 12.3.3, that dumped a different database - /// with no rows and exited 0, which the result sheet reports as a successful backup. + // `my_getopt` parses past a positional argument, so a database named `--no-data` was read as + // the option: measured on MySQL 8.4.11 and MariaDB 12.3.3, that dumped a different database + // with no rows and exited 0, which the result sheet reports as a successful backup. @Test("The option terminator comes before the database, in both directions", arguments: [ NativeDumpKind.backup, .restore ]) @@ -715,9 +718,9 @@ struct NativeDumpRegistryTests { #expect(Array(built.arguments[terminator...]) == ["--", "sales", "orders", "customers"]) } - /// libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, so a - /// Verify CA connection that opens in the app could never be dumped: measured with pg_dump - /// 17.11, it fails with `root certificate file "..." does not exist`. + // libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, so a + // Verify CA connection that opens in the app could never be dumped: measured with pg_dump + // 17.11, it fails with `root certificate file "..." does not exist`. @Test("PostgreSQL sends the whole SSL configuration, not just the mode") func postgresSendsCertificatePaths() throws { var secured = connection(type: .postgresql, sslMode: .verifyCa, sslEnabled: true) @@ -739,8 +742,8 @@ struct NativeDumpRegistryTests { #expect(disabled.environment["PGSSLKEY"] == nil) } - /// The CA belongs to the modes that verify one, which is the rule the live connection already - /// follows in `LibPQConnectionString`. + // The CA belongs to the modes that verify one, which is the rule the live connection already + // follows in `LibPQConnectionString`. @Test("A required connection sends its client certificate but not a CA it does not check") func postgresRequiredKeepsTheClientCertificate() throws { var secured = connection(type: .postgresql, sslMode: .required, sslEnabled: true) @@ -752,8 +755,96 @@ struct NativeDumpRegistryTests { #expect(built.environment["PGSSLCERT"] == "/certs/client.pem") } - /// MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both - /// spellings have to be tried before reporting the tool missing. + private func dumpRequest(_ connection: DatabaseConnection) -> NativeDumpDescriptor.Request { + NativeDumpDescriptor.Request( + connection: connection, + database: "sales", + fileURL: URL(fileURLWithPath: "/tmp/out.dump"), + password: nil + ) + } + + @Test("Verify Identity with no CA file hands pg_dump this Mac's trusted roots") + func verifyIdentityWithoutACAGetsTheSystemAnchors() throws { + let verifying = connection(type: .postgresql, sslMode: .verifyIdentity, sslEnabled: true) + let tool = try #require(NativeDumpRegistry.descriptor(for: .postgresql)?.commandLineTool) + let roots = URL(fileURLWithPath: "/tmp/tablepro-trust-test/roots.pem") + + #expect(tool.needsSystemTrustAnchors(dumpRequest(verifying)) == NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(verifying) + )) + let built = try command(.postgresql, connection: verifying, systemTrustAnchorsFile: roots) + #expect(built.environment["PGSSLMODE"] == "verify-full") + #expect(built.environment["PGSSLROOTCERT"] == roots.path) + + var withCA = verifying + withCA.sslConfig.caCertificatePath = "/certs/ca.pem" + let chosen = try command(.postgresql, connection: withCA, systemTrustAnchorsFile: roots) + #expect(chosen.environment["PGSSLROOTCERT"] == "/certs/ca.pem") + } + + @Test("Verify CA with no CA file still names no root file, as libpq refuses this Mac's roots there") + func verifyCAWithoutACAStillSetsNoRootCertificate() throws { + let verifyCA = connection(type: .postgresql, sslMode: .verifyCa, sslEnabled: true) + let roots = URL(fileURLWithPath: "/tmp/tablepro-trust-test/roots.pem") + + let built = try command(.postgresql, connection: verifyCA, systemTrustAnchorsFile: roots) + + #expect(built.environment["PGSSLMODE"] == "verify-ca") + #expect(built.environment["PGSSLROOTCERT"] == nil) + #expect(!NativeDumpRegistry.postgresNeedsSystemTrustAnchors(dumpRequest(verifyCA), fileExists: { _ in false })) + } + + @Test("pg_dump gets this Mac's roots only for Verify Identity over TCP with no CA file and no root.crt of libpq's own") + func postgresSystemAnchorsFollowTheDriverRule() { + let home = ["HOME": "/Users/test"] + let rootCrt = "/Users/test/.postgresql/root.crt" + func needs(_ connection: DatabaseConnection, rootCrtExists: Bool = false) -> Bool { + NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(connection), + environment: home, + fileExists: { rootCrtExists && $0 == rootCrt } + ) + } + let verifying = connection(type: .postgresql, sslMode: .verifyIdentity, sslEnabled: true) + var withCA = verifying + withCA.sslConfig.caCertificatePath = "/certs/ca.pem" + var socket = verifying + socket.additionalFields[MySQLLocalSocket.fieldKey] = "/tmp/.s.PGSQL.5432" + + #expect(needs(verifying)) + #expect(needs(connection(type: .redshift, sslMode: .verifyIdentity, sslEnabled: true))) + #expect(!needs(verifying, rootCrtExists: true)) + #expect(!needs(withCA)) + #expect(!needs(socket)) + for mode in [SSLMode.disabled, .preferred, .required, .verifyCa] { + #expect(!needs(connection(type: .postgresql, sslMode: mode, sslEnabled: mode != .disabled))) + } + #expect(!NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(verifying), + environment: ["HOME": "/Users/test", "PGSSLROOTCERT": "/etc/corp-ca.pem"], + fileExists: { _ in false } + )) + #expect(NativeDumpRegistry.postgresSSLEnvironment( + verifying.sslConfig, inherited: ["PGSSLROOTCERT": "/etc/corp-ca.pem"] + )["PGSSLROOTCERT"] == "/etc/corp-ca.pem") + #expect(NativeDumpRegistry.postgresDefaultRootCertificatePath(environment: home) == rootCrt) + #expect(NativeDumpRegistry.postgresDefaultRootCertificatePath(environment: [:]) + == NSHomeDirectory() + "/.postgresql/root.crt") + } + + @Test("Only the PostgreSQL tools ask for this Mac's trusted roots", arguments: [ + DatabaseType.mysql, .mariadb, .mongodb, .sqlite, .mssql + ]) + func otherToolsNeverAskForTheSystemAnchors(type: DatabaseType) throws { + let tool = try #require(NativeDumpRegistry.descriptor(for: type)?.commandLineTool) + let verifying = connection(type: type, sslMode: .verifyIdentity, sslEnabled: true) + + #expect(!tool.needsSystemTrustAnchors(dumpRequest(verifying))) + } + + // MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both + // spellings have to be tried before reporting the tool missing. @Test("MySQL tries both the mysql and mariadb tool names") func mysqlTriesBothToolNames() throws { let tool = try #require(NativeDumpRegistry.descriptor(for: .mysql)?.commandLineTool) @@ -786,8 +877,8 @@ struct NativeDumpRegistryTests { #expect(try configurationValue("password", in: contents) == #"p\nuri: mongodb://evil.example.com"#) } - /// A progress bar showing a percentage of a number nobody measured is worse than an - /// indeterminate one, so an engine with no cheap size answer returns nil. + // A progress bar showing a percentage of a number nobody measured is worse than an + // indeterminate one, so an engine with no cheap size answer returns nil. @Test("Only the engines with a cheap size query offer a determinate progress bar") func sizeQueryCoverage() { #expect(NativeDumpService.sizeQuery(for: .postgresql) != nil) diff --git a/apple/TableProTests/Database/ProcessNativeDumpRunnerTests.swift b/apple/TableProTests/Database/ProcessNativeDumpRunnerTests.swift index b65d93bf32..a8047aff9a 100644 --- a/apple/TableProTests/Database/ProcessNativeDumpRunnerTests.swift +++ b/apple/TableProTests/Database/ProcessNativeDumpRunnerTests.swift @@ -8,8 +8,8 @@ import Foundation import Testing @testable import TablePro +import TableProProcess import TableProTestSupport -@testable import TableProUtilities struct ProcessNativeDumpRunnerTests { // A stand-in dump tool: the runner refuses a shell as the executable, as it does for every app-run tool. diff --git a/apple/TableProTests/Extensions/StringHexDumpTests.swift b/apple/TableProTests/Extensions/StringHexDumpTests.swift index c392902461..2adadc35fd 100644 --- a/apple/TableProTests/Extensions/StringHexDumpTests.swift +++ b/apple/TableProTests/Extensions/StringHexDumpTests.swift @@ -179,4 +179,37 @@ struct StringHexDumpTests { } #expect(roundTripped == original) } + + // MARK: - Prefix reads + + @Test("A scalar past U+00FF beyond the shown prefix still makes the whole value UTF-8") + func scalarPastLatin1AfterThePrefix() { + let value = String(repeating: "\u{E9}", count: 100) + "\u{4E2D}" + #expect(value.formattedAsCompactHex(maxBytes: 4) == "0xC3A9C3A9…") + #expect(value.formattedAsEditableHex(maxBytes: 2) == "C3 A9 …") + } + + @Test("A Latin-1 value counts one byte per character, past the prefix too") + func latin1CountsEveryCharacter() throws { + let value = String(repeating: "\u{E9}", count: 40) + #expect(value.formattedAsCompactHex(maxBytes: 40) == "0x" + String(repeating: "E9", count: 40)) + #expect(value.formattedAsCompactHex(maxBytes: 39)?.hasSuffix("…") == true) + let dump = try #require(value.formattedAsHexDump(maxBytes: 16)) + #expect(dump.hasSuffix("… (truncated, \(40.formatted(.number)) bytes total)")) + } + + @Test("Bytes format the same as the Latin-1 string that carries them", arguments: [0, 1, 16, 17, 64, 65, 300]) + func bytesMatchTheirLatin1String(_ count: Int) throws { + let data = Data((0.. PluginQueryResult + private let lock = NSLock() + private var received: [Sent] = [] + private var finished = 0 + + init(hold: AsyncGate? = nil, answer: @escaping @Sendable (String) -> PluginQueryResult = { _ in .empty }) { + self.hold = hold + self.answer = answer + } + + var sent: [Sent] { + lock.withLock { received } + } + + var finishedCount: Int { + lock.withLock { finished } + } + + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + lock.withLock { received.append(Sent(query: query, rowCap: rowCap, context: context)) } + if let hold { + await hold.wait() + } + lock.withLock { finished += 1 } + return answer(query) + } + + static func textRows(column: String, count: Int, bytes: Int, isTruncated: Bool = false) -> PluginQueryResult { + PluginQueryResult( + columns: [column], + columnTypeNames: ["text"], + rows: Array(repeating: [.text(String(repeating: "x", count: bytes))], count: count), + rowsAffected: 0, + executionTime: 0, + isTruncated: isTruncated + ) + } + + // Installed as the live session of a new connection, which the test removes. + @MainActor + static func install(_ driver: AnsweringPluginDriver, connection: DatabaseConnection) -> DatabaseScope { + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.browseDatabase = connection.database + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return DatabaseScope(connectionId: connection.id, database: connection.database, schema: nil) + } +} + +final class RecordingQueryHistory: QueryHistoryRecording, @unchecked Sendable { + private let lock = NSLock() + private var recorded: [QueryHistoryRecordRequest] = [] + + var requests: [QueryHistoryRecordRequest] { + lock.withLock { recorded } + } + + func record(_ request: QueryHistoryRecordRequest) async -> Bool { + lock.withLock { recorded.append(request) } + return true + } +} diff --git a/apple/TableProTests/Helpers/ArrayRowProvider.swift b/apple/TableProTests/Helpers/ArrayRowProvider.swift new file mode 100644 index 0000000000..c7295ae291 --- /dev/null +++ b/apple/TableProTests/Helpers/ArrayRowProvider.swift @@ -0,0 +1,29 @@ +@testable import TablePro + +final class ArrayRowProvider: DataRowProviding { + private let rows: [DataRow] + private let rowLimit: Int? + private let holdsRowPastTheLimit: Bool + private var index = 0 + + init(rows: [DataRow], rowLimit: Int? = nil) { + self.rows = rowLimit.map { Array(rows.prefix($0)) } ?? rows + self.rowLimit = rowLimit + self.holdsRowPastTheLimit = rowLimit.map { rows.count > $0 } ?? false + } + + var endedAtRowLimit: Bool { + guard let rowLimit, index >= rows.count else { return false } + return holdsRowPastTheLimit && rows.count >= rowLimit + } + + func nextRow() async throws -> DataRow? { + guard index < rows.count else { return nil } + defer { index += 1 } + return rows[index] + } + + func drain() async throws { + index = rows.count + } +} diff --git a/apple/TableProTests/Helpers/CopyInRecordingDriver.swift b/apple/TableProTests/Helpers/CopyInRecordingDriver.swift new file mode 100644 index 0000000000..5fd0bd7b0e --- /dev/null +++ b/apple/TableProTests/Helpers/CopyInRecordingDriver.swift @@ -0,0 +1,135 @@ +// +// CopyInRecordingDriver.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit + +// Records each text sent as a statement and each COPY sent through copy-in, with every piece of its data read. +final class CopyInRecordingDriver: DatabaseDriver, @unchecked Sendable { + struct Copy: Equatable { + let statement: String + let data: Data + } + + let connection: DatabaseConnection + var status: ConnectionStatus = .connected + var serverVersion: String? { nil } + + private let copiesIn: Bool + private let lexicalState: PluginSessionLexicalState? + private let lexicalStateAfterFirstStatement: PluginSessionLexicalState? + private let lock = NSLock() + private var statements: [String] = [] + private var copies: [Copy] = [] + + // `lexicalStateAfterFirstStatement` is what the session reads once a statement ran, as after a `SET`. + init( + connection: DatabaseConnection, + copiesIn: Bool, + lexicalState: PluginSessionLexicalState? = nil, + lexicalStateAfterFirstStatement: PluginSessionLexicalState? = nil + ) { + self.connection = connection + self.copiesIn = copiesIn + self.lexicalState = lexicalState + self.lexicalStateAfterFirstStatement = lexicalStateAfterFirstStatement + } + + var sentStatements: [String] { + lock.withLock { statements } + } + + var sentCopies: [Copy] { + lock.withLock { copies } + } + + var sessionLexicalState: PluginSessionLexicalState? { + lock.withLock { + guard let lexicalStateAfterFirstStatement, !(statements.isEmpty && copies.isEmpty) else { return lexicalState } + return lexicalStateAfterFirstStatement + } + } + + var supportsCopyFromStandardInput: Bool { copiesIn } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult { + guard copiesIn else { throw PluginDriverUnsupportedOperation.copyFromStandardInput } + var data = Data() + while let piece = try rows.nextPiece() { + data.append(piece) + } + lock.withLock { copies.append(Copy(statement: statement, data: data)) } + return Self.emptyResult + } + + private static var emptyResult: QueryResult { + QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 0, executionTime: 0, error: nil) + } + + private func record(_ query: String) -> QueryResult { + lock.withLock { statements.append(query) } + return Self.emptyResult + } + + func execute(query: String) async throws -> QueryResult { record(query) } + func executeParameterized(query: String, parameters: [Any?]) async throws -> QueryResult { record(query) } + func executeUserQuery(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryResult { + record(query) + } + + func connect() async throws {} + func disconnect() {} + func testConnection() async throws -> Bool { true } + func ping() async throws {} + func cancelQuery() throws {} + func applyQueryTimeout(_ seconds: Int) async throws {} + + func fetchTables() async throws -> [TableInfo] { [] } + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String) async throws -> [ColumnInfo] { [] } + func fetchAllColumns() async throws -> [String: [ColumnInfo]] { [:] } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } + func fetchApproximateRowCount(table: String) async throws -> Int? { nil } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> DatabaseMetadata { + DatabaseMetadata( + id: database, + name: database, + tableCount: nil, + sizeBytes: nil, + lastAccessed: nil, + isSystemDatabase: false, + icon: "cylinder" + ) + } + + func fetchTableDDL(table: String) async throws -> String { "" } + func fetchTableMetadata(tableName: String) async throws -> TableMetadata { + TableMetadata( + tableName: tableName, + dataSize: nil, + indexSize: nil, + totalSize: nil, + avgRowLength: nil, + rowCount: nil, + comment: nil, + engine: nil, + collation: nil, + createTime: nil, + updateTime: nil + ) + } + + func fetchViewDefinition(view: String) async throws -> String { "" } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} diff --git a/apple/TableProTests/Helpers/CountingOpener.swift b/apple/TableProTests/Helpers/CountingOpener.swift new file mode 100644 index 0000000000..400640dd95 --- /dev/null +++ b/apple/TableProTests/Helpers/CountingOpener.swift @@ -0,0 +1,42 @@ +// +// CountingOpener.swift +// TableProTests +// + +import Foundation +@testable import TablePro + +// Stands in for opening a pooled connection, remembering each driver by the database it was opened for. +@MainActor +final class CountingOpener { + private let connectDelaySeconds: Double + private(set) var opened: [MockDatabaseDriver] = [] + private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] + private(set) var openOrder: [String] = [] + private(set) var peakOpenCount = 0 + var configure: ((MockDatabaseDriver) -> Void)? + var afterOpen: ((DatabaseScope) -> Void)? + var deadOnArrival: Set = [] + + init(connectDelaySeconds: Double = 0) { + self.connectDelaySeconds = connectDelaySeconds + } + + var openCount: Int { + opened.filter { $0.disconnectCallCount == 0 }.count + } + + func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { + let driver = MockDatabaseDriver() + driver.connectDelaySeconds = connectDelaySeconds + driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil + configure?(driver) + opened.append(driver) + openedByDatabase[scope.database] = driver + openOrder.append(scope.database) + peakOpenCount = max(peakOpenCount, openCount) + try await driver.connect() + afterOpen?(scope) + return driver + } +} diff --git a/apple/TableProTests/Helpers/FakeCNAMEResolver.swift b/apple/TableProTests/Helpers/FakeCNAMEResolver.swift new file mode 100644 index 0000000000..7dd8d23819 --- /dev/null +++ b/apple/TableProTests/Helpers/FakeCNAMEResolver.swift @@ -0,0 +1,38 @@ +// +// FakeCNAMEResolver.swift +// TableProTests +// + +import Foundation +import TableProPluginKit + +// Answers from a fixed table and records each name asked, so a test never reaches real DNS. +final class FakeCNAMEResolver: CNAMEResolving, @unchecked Sendable { + private let lock = NSLock() + private var records: [String: String] + private let failing: Set + private var asked: [String] = [] + + init(_ records: [String: String] = [:], failing: Set = []) { + self.records = records + self.failing = failing + } + + var queried: [String] { + lock.withLock { asked } + } + + func point(_ name: String, at target: String?) { + lock.withLock { records[name] = target } + } + + func cnameTarget(of name: String, within timeout: Duration?) async -> CNAMELookup { + lock.withLock { + asked.append(name) + if failing.contains(name) { + return .failed + } + return records[name].map(CNAMELookup.found) ?? .notFound + } + } +} diff --git a/apple/TableProTests/Helpers/FakeMSSQLPlugin.swift b/apple/TableProTests/Helpers/FakeMSSQLPlugin.swift index 9a3a6f9a6c..a21c207a64 100644 --- a/apple/TableProTests/Helpers/FakeMSSQLPlugin.swift +++ b/apple/TableProTests/Helpers/FakeMSSQLPlugin.swift @@ -193,6 +193,9 @@ final class FakeMSSQLPluginDriver: PluginDriverStub, @unchecked Sendable { } func disconnect() { disconnectCallCount += 1 } + // SQL Server moves a live session with USE, so a statement pinned to a database after a reconnect lands. + func switchDatabase(to database: String) async throws {} + func applyQueryTimeout(_ seconds: Int) async throws { queryTimeoutValues.withLock { $0.append(seconds) } } diff --git a/apple/TableProTests/Helpers/MCPFixedApprovalReader.swift b/apple/TableProTests/Helpers/MCPFixedApprovalReader.swift new file mode 100644 index 0000000000..bd34ea2226 --- /dev/null +++ b/apple/TableProTests/Helpers/MCPFixedApprovalReader.swift @@ -0,0 +1,13 @@ +@testable import TablePro + +struct MCPFixedApprovalReader: MCPConnectionApprovalReading { + let approval: MCPConnectionApproval + + init(_ approval: MCPConnectionApproval) { + self.approval = approval + } + + func connectionApproval() async -> MCPConnectionApproval { + approval + } +} diff --git a/apple/TableProTests/Helpers/MCPInMemoryApprovalStore.swift b/apple/TableProTests/Helpers/MCPInMemoryApprovalStore.swift new file mode 100644 index 0000000000..28c495afa9 --- /dev/null +++ b/apple/TableProTests/Helpers/MCPInMemoryApprovalStore.swift @@ -0,0 +1,40 @@ +import Foundation +@testable import TablePro + +actor MCPInMemoryApprovalStore: MCPApprovalStoring { + private var storedGrants: [MCPConnectionGrant] = [] + private var storedDenials: [MCPConnectionDenial] = [] + + init() {} + + func grants() -> [MCPConnectionGrant] { + storedGrants + } + + func record(_ grant: MCPConnectionGrant) { + storedGrants.removeAll { $0.id == grant.id } + storedGrants.append(grant) + } + + func revoke(subject: String, connectionId: UUID) { + storedGrants.removeAll { $0.subject == subject && $0.connectionId == connectionId } + } + + func revokeAll(subject: String) { + storedGrants.removeAll { $0.subject == subject } + } + + func revokeEverything() { + storedGrants.removeAll() + } + + func denial(subject: String, connectionId: UUID, now: Date) -> Date? { + storedDenials.removeAll { $0.expiresAt <= now } + return storedDenials.first { $0.subject == subject && $0.connectionId == connectionId }?.expiresAt + } + + func record(_ denial: MCPConnectionDenial) { + storedDenials.removeAll { $0.subject == denial.subject && $0.connectionId == denial.connectionId } + storedDenials.append(denial) + } +} diff --git a/apple/TableProTests/Helpers/RecordingClipboard.swift b/apple/TableProTests/Helpers/RecordingClipboard.swift index 034d794186..10a41059d7 100644 --- a/apple/TableProTests/Helpers/RecordingClipboard.swift +++ b/apple/TableProTests/Helpers/RecordingClipboard.swift @@ -12,6 +12,7 @@ final class RecordingClipboard: ClipboardProvider { var gridCells: GridCellsClipboardPayload? private(set) var html: String? private(set) var writes: [String] = [] + private(set) var secretWrites: [String] = [] private(set) var images: [NSImage] = [] var hasText: Bool { text != nil } @@ -22,6 +23,11 @@ final class RecordingClipboard: ClipboardProvider { func readGridCells() -> GridCellsClipboardPayload? { gridCells } func writeText(_ text: String) { replace(text: text) } + + func writeSecretText(_ text: String) { + replace(text: text) + secretWrites.append(text) + } func writeCsv(_ csv: String) { replace(text: csv) } func writeImage(_ image: NSImage) { diff --git a/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift b/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift new file mode 100644 index 0000000000..c433bd475c --- /dev/null +++ b/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift @@ -0,0 +1,128 @@ +import Foundation +@testable import TablePro + +// One synced and one local value per key, as the keychain holds them. Only a delete that is not device-only removes +// a synced value. +final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { + private let lock = NSLock() + private var synced: [String: String] = [:] + private var local: [String: String] = [:] + private var unreadableKeys: Set = [] + private var failingKeys: Set = [] + private var failingSyncedRemovals: Set = [] + + func seedSynced(_ value: String, forKey key: String) { + lock.lock() + defer { lock.unlock() } + synced[key] = value + } + + func seedLocal(_ value: String, forKey key: String) { + lock.lock() + defer { lock.unlock() } + local[key] = value + } + + func makeUnreadable(_ key: String) { + lock.lock() + defer { lock.unlock() } + unreadableKeys.insert(key) + } + + func makeReadable(_ key: String) { + lock.lock() + defer { lock.unlock() } + unreadableKeys.remove(key) + } + + func failWrites(to key: String) { + lock.lock() + defer { lock.unlock() } + failingKeys.insert(key) + } + + func failSyncedRemoval(of key: String) { + lock.lock() + defer { lock.unlock() } + failingSyncedRemovals.insert(key) + } + + func allowSyncedRemoval(of key: String) { + lock.lock() + defer { lock.unlock() } + failingSyncedRemovals.remove(key) + } + + func allowWrites(to key: String) { + lock.lock() + defer { lock.unlock() } + failingKeys.remove(key) + } + + func syncedValue(forKey key: String) -> String? { + lock.lock() + defer { lock.unlock() } + return synced[key] + } + + func localValue(forKey key: String) -> String? { + lock.lock() + defer { lock.unlock() } + return local[key] + } + + @discardableResult + func writeString(_ value: String, forKey key: String) -> Bool { + writeString(value, forKey: key, deviceOnly: false) + } + + // Password sync is on in this model, so a write that is not device-only is a synced one. + @discardableResult + func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { + lock.lock() + defer { lock.unlock() } + guard !failingKeys.contains(key) else { return false } + if deviceOnly { + local[key] = value + } else { + synced[key] = value + local[key] = nil + } + return true + } + + func readStringResult(forKey key: String) -> KeychainStringResult { + readStringResult(forKey: key, deviceOnly: false) + } + + func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult { + lock.lock() + defer { lock.unlock() } + guard !unreadableKeys.contains(key) else { return .locked } + let value = deviceOnly ? local[key] ?? synced[key] : synced[key] ?? local[key] + return value.map { .found($0) } ?? .notFound + } + + func delete(forKey key: String) { + delete(forKey: key, deviceOnly: false) + } + + func delete(forKey key: String, deviceOnly: Bool) { + lock.lock() + defer { lock.unlock() } + local[key] = nil + if !deviceOnly { + synced[key] = nil + } + } + + // As the real keychain does, refused while no local value exists, so the secret is never lost on this Mac. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + lock.lock() + defer { lock.unlock() } + guard local[key] != nil, !failingSyncedRemovals.contains(key) else { return false } + synced[key] = nil + return true + } +} diff --git a/apple/TableProTests/Helpers/StubExecutionGates.swift b/apple/TableProTests/Helpers/StubExecutionGates.swift index 83dce282ed..afcfa85d48 100644 --- a/apple/TableProTests/Helpers/StubExecutionGates.swift +++ b/apple/TableProTests/Helpers/StubExecutionGates.swift @@ -27,13 +27,18 @@ internal struct AlwaysAllowGate: ExecutionGate { internal struct AlwaysDenyGate: ExecutionGate { internal let reason: String + internal let cause: OperationDenialCause - internal init(reason: String = "Read-Only connection") { + internal init(reason: String = "Read-Only connection", cause: OperationDenialCause = .policy) { self.reason = reason + self.cause = cause } + // The user answering Cancel in the gate's confirmation. + internal static let cancelling = AlwaysDenyGate(reason: "Operation cancelled by user", cause: .cancelledByUser) + internal func authorize(_ request: OperationRequest) async -> OperationDecision { - .denied(reason: reason) + .denied(reason: reason, cause: cause) } } diff --git a/apple/TableProTests/Helpers/SyncTestEnvironment.swift b/apple/TableProTests/Helpers/SyncTestEnvironment.swift index 2655e6683a..4e743ac45f 100644 --- a/apple/TableProTests/Helpers/SyncTestEnvironment.swift +++ b/apple/TableProTests/Helpers/SyncTestEnvironment.swift @@ -83,7 +83,9 @@ final class SyncTestEnvironment { transport: ScriptedSyncTransport, favorites: SQLFavoriteManager? = nil, makeNetworkMonitor: @escaping () -> SyncNetworkMonitor? = { nil }, - licenseGate: @escaping @MainActor () -> DisableReason? = { nil } + licenseGate: @escaping @MainActor () -> DisableReason? = { nil }, + activationClock: any Clock = ContinuousClock(), + pushRegistrar: SyncPushRegistrar = .inert ) -> SyncCoordinator { let live = AppServices.live let services = AppServices( @@ -137,7 +139,26 @@ final class SyncTestEnvironment { transport: transport, makeNetworkMonitor: makeNetworkMonitor, columnLayouts: self.columnLayouts, - licenseGate: licenseGate + licenseGate: licenseGate, + activationClock: activationClock, + pushRegistrar: pushRegistrar + ) + } +} + +extension SyncPushRegistrar { + static let inert = SyncPushRegistrar(register: {}, unregister: {}) +} + +@MainActor +final class RecordingPushRegistrar { + private(set) var registrations = 0 + private(set) var unregistrations = 0 + + var registrar: SyncPushRegistrar { + SyncPushRegistrar( + register: { [weak self] in self?.registrations += 1 }, + unregister: { [weak self] in self?.unregistrations += 1 } ) } } @@ -169,6 +190,8 @@ actor ScriptedSyncTransport: SyncTransport { private(set) var pullCount = 0 private(set) var pushCount = 0 private(set) var zoneSaveCount = 0 + private(set) var subscriptionSaveCount = 0 + private var subscriptionError: (any Error)? /// What the account reads as, what every pushed item fails with, and what a pull throws. Each /// is changeable mid-test, the way the account or the storage changes under a running app. @@ -264,6 +287,17 @@ actor ScriptedSyncTransport: SyncTransport { zoneSaveCount += 1 } + func failSubscriptionSaves(with error: (any Error)?) { + subscriptionError = error + } + + func saveSubscription() async throws { + subscriptionSaveCount += 1 + if let subscriptionError { + throw subscriptionError + } + } + func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome { pushCount += 1 pushedRecords.append(contentsOf: records) diff --git a/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift b/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift index 01e828927e..04017ad967 100644 --- a/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift +++ b/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift @@ -11,7 +11,34 @@ import Foundation import TableProPluginKit import Testing +private struct ServerError: PluginDriverError { + let pluginErrorMessage: String + var pluginErrorCode: Int? + var pluginSqlState: String? +} + struct ConnectionFailureClassifierTests { + @Test("A refused login reads as one in each engine's own words", arguments: [ + "Login failed for user 'v-token-ro-1'.", + "ORA-01017: invalid username/password; logon denied", + "WRONGPASS invalid username-password pair or user is disabled.", + "Provided username v-token-ro-1 and/or password are incorrect", + "Incorrect username or password was specified.", + "password authentication failed for user \"v-token-ro-1\"", + ]) + func refusedLoginInEachEngine(message: String) { + #expect(ConnectionFailureClassifier.isAuthenticationFailure(ServerError(pluginErrorMessage: message))) + } + + @Test("A SQL Server login refused only while it upgrades, restarts or runs single-user is not a refused login", arguments: [ + "Login failed for user 'v-token-ro-1'. Reason: Server is in script upgrade mode. Only administrator can connect at this time.", + "Login failed for user 'v-token-ro-1'. Reason: Server is in single user mode. Only one administrator can connect at this time.", + "Cannot open database \"inventory\" requested by the login. The login failed.", + ]) + func passingSQLServerRefusals(message: String) { + #expect(!ConnectionFailureClassifier.isAuthenticationFailure(ServerError(pluginErrorMessage: message))) + } + @Test("A Swift cancellation is a cancel, not a failure") func swiftCancellationIsCancelled() { #expect(ConnectionFailureClassifier.isUserCancelled(CancellationError())) diff --git a/apple/TableProTests/Models/DatabaseObjectRefCodingTests.swift b/apple/TableProTests/Models/DatabaseObjectRefCodingTests.swift index 65ee4448c4..c7c843e7b3 100644 --- a/apple/TableProTests/Models/DatabaseObjectRefCodingTests.swift +++ b/apple/TableProTests/Models/DatabaseObjectRefCodingTests.swift @@ -32,4 +32,102 @@ struct DatabaseObjectRefCodingTests { #expect(ref.resolvingDatabase("shop").typeKind == .range) #expect(ref.resolvingDatabase("shop").database == "shop") } + + private var eventKind: SidebarServerObjectKind { + SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + } + + @Test("A ref written before server objects decodes with no server kind") + func legacyRefHasNoServerKind() throws { + let json = """ + {"kind":"trigger","name":"audit","database":"shop","schema":"public","table":"orders","attributes":[]} + """ + let decoded = try JSONDecoder().decode(DatabaseObjectRef.self, from: Data(json.utf8)) + #expect(decoded.serverKind == nil) + #expect(decoded.serverObject == nil) + } + + @Test("A server object ref round-trips its kind and reads back as the object") + func serverObjectRefRoundTrips() throws { + let object = ServerObjectInfo( + kindId: "event", name: "nightly", schema: "shop", detail: "Recurring", + attributes: [ObjectAttribute(label: "Schedule", value: "EVERY 1 DAY")] + ) + let ref = DatabaseObjectRef(serverObject: object, kind: eventKind, database: "shop") + let decoded = try JSONDecoder().decode(DatabaseObjectRef.self, from: JSONEncoder().encode(ref)) + #expect(decoded == ref) + #expect(decoded.kind == .serverObject) + #expect(decoded.serverKind?.id == "event") + #expect(decoded.kindDisplayName == "Event") + #expect(decoded.kindIconName == "calendar.badge.clock") + #expect(decoded.serverObject?.kindId == "event") + #expect(decoded.serverObject?.name == "nightly") + #expect(decoded.displayIdentity == "shop.nightly") + } + + @Test("Resolving the database keeps the server kind, and the tab is titled with the kind") + @MainActor + func serverObjectRefKeepsKind() { + let ref = DatabaseObjectRef(serverObject: ServerObjectInfo(kindId: "event", name: "nightly"), kind: eventKind, database: "") + #expect(ref.resolvingDatabase("shop").serverKind?.id == "event") + #expect(QueryTabManager.objectSourceTitle(for: ref) == String(format: String(localized: "%1$@: %2$@"), "Event", "nightly")) + } + + private var extensionKind: SidebarServerObjectKind { + SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", iconName: "puzzlepiece.extension", + scope: .database + ) + } + + @Test("A database-wide object is titled by its bare name, its schema left to the attributes") + @MainActor + func databaseScopedObjectHasABareName() throws { + let object = ServerObjectInfo(kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384") + let ref = DatabaseObjectRef(serverObject: object, kind: extensionKind, database: "app") + + #expect(ref.isDatabaseScoped) + #expect(ref.displayIdentity == "pgcrypto") + #expect(QueryTabManager.objectSourceTitle(for: ref) == String(format: String(localized: "%1$@: %2$@"), "Extension", "pgcrypto")) + #expect(ref.schema == "public") + + let decoded = try JSONDecoder().decode(DatabaseObjectRef.self, from: JSONEncoder().encode(ref)) + #expect(decoded.serverKind?.scope == .database) + #expect(decoded.displayIdentity == "pgcrypto") + } + + @Test("A container object keeps its schema in its name") + func containerObjectIsQualified() { + let ref = DatabaseObjectRef(serverObject: ServerObjectInfo(kindId: "event", name: "nightly", schema: "shop"), kind: eventKind, database: "shop") + + #expect(!ref.isDatabaseScoped) + #expect(ref.displayIdentity == "shop.nightly") + } + + @Test("A server kind written without its scope decodes, and reads as a container kind") + func serverKindWithoutScopeDecodes() throws { + let json = """ + {"kind":"serverObject","name":"nightly","database":"shop","schema":"shop","attributes":[],\ + "serverKind":{"id":"event","displayName":"Event","iconName":"calendar.badge.clock"}} + """ + let decoded = try JSONDecoder().decode(DatabaseObjectRef.self, from: Data(json.utf8)) + + #expect(decoded.serverKind?.scope == nil) + #expect(!decoded.isDatabaseScoped) + #expect(decoded.displayIdentity == "shop.nightly") + } + + @Test("Two server objects of different kinds are different objects") + func serverKindSeparatesObjects() { + let extensionKind = SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", iconName: "puzzlepiece.extension", + scope: .database + ) + let event = DatabaseObjectRef(serverObject: ServerObjectInfo(kindId: "event", name: "x"), kind: eventKind, database: "shop") + let ext = DatabaseObjectRef(serverObject: ServerObjectInfo(kindId: "extension", name: "x"), kind: extensionKind, database: "shop") + #expect(!event.namesSameObject(as: ext)) + #expect(event.namesSameObject(as: event)) + } } diff --git a/apple/TableProTests/Models/DatabaseTypeTests.swift b/apple/TableProTests/Models/DatabaseTypeTests.swift index 33b8797bea..fa788e1d7a 100644 --- a/apple/TableProTests/Models/DatabaseTypeTests.swift +++ b/apple/TableProTests/Models/DatabaseTypeTests.swift @@ -145,7 +145,7 @@ struct DatabaseTypeTests { func testCACertificateRequirement() { #expect(DatabaseType.trino.requiresCACertificate(for: .verifyIdentity) == false) #expect(DatabaseType.clickhouse.requiresCACertificate(for: .verifyIdentity) == false) - #expect(DatabaseType.postgresql.requiresCACertificate(for: .verifyIdentity) == true) + #expect(DatabaseType.postgresql.requiresCACertificate(for: .verifyIdentity) == false) #expect(DatabaseType.mysql.requiresCACertificate(for: .verifyIdentity) == true) #expect(DatabaseType(rawValue: "FutureDB").requiresCACertificate(for: .verifyIdentity) == true) } diff --git a/apple/TableProTests/Models/ERDiagram/ERDiagramLayoutTests.swift b/apple/TableProTests/Models/ERDiagram/ERDiagramLayoutTests.swift index 30c5594982..c81f5cf4a0 100644 --- a/apple/TableProTests/Models/ERDiagram/ERDiagramLayoutTests.swift +++ b/apple/TableProTests/Models/ERDiagram/ERDiagramLayoutTests.swift @@ -56,39 +56,39 @@ struct ERDiagramLayoutTests { } @Test("An empty graph produces no positions") - func empty() { - let layout = ERDiagramLayout.compute(graph: .empty) + func empty() throws { + let layout = try ERDiagramLayout.compute(graph: .empty) #expect(layout.isEmpty) } @Test("Every node receives a position") - func everyNodePositioned() { + func everyNodePositioned() throws { let graph = makeGraph( tables: ["a", "b", "c", "d", "e"], foreignKeys: [("a", "b"), ("b", "c"), ("d", "e")] ) - let layout = ERDiagramLayout.compute(graph: graph) + let layout = try ERDiagramLayout.compute(graph: graph) #expect(Set(layout.keys) == Set(graph.nodes.map(\.id))) } @Test("Layout is deterministic across runs") - func deterministic() { + func deterministic() throws { let graph = makeGraph( tables: ["orders", "items", "users", "tags", "logs"], foreignKeys: [("items", "orders"), ("orders", "users"), ("tags", "users")] ) - let first = ERDiagramLayout.compute(graph: graph) - let second = ERDiagramLayout.compute(graph: graph) + let first = try ERDiagramLayout.compute(graph: graph) + let second = try ERDiagramLayout.compute(graph: graph) #expect(first == second) } @Test("No two table boxes overlap") - func noOverlap() { + func noOverlap() throws { let graph = makeGraph( tables: ["a", "b", "c", "m", "n", "p", "q"], foreignKeys: [("a", "b"), ("b", "c"), ("m", "n")] ) - let layout = ERDiagramLayout.compute(graph: graph) + let layout = try ERDiagramLayout.compute(graph: graph) let rects = graph.nodes.compactMap { node in layout[node.id].map { rect(for: node, at: $0) } } for i in 0.. CGRect { graph.nodes @@ -117,9 +117,9 @@ struct ERDiagramLayoutTests { } @Test("Isolated tables fill horizontal space instead of stacking vertically") - func isolatedTablesUseWidth() { + func isolatedTablesUseWidth() throws { let graph = makeGraph(tables: (0..<9).map { "t\($0)" }) - let layout = ERDiagramLayout.compute(graph: graph) + let layout = try ERDiagramLayout.compute(graph: graph) let bounds = graph.nodes .compactMap { node in layout[node.id].map { rect(for: node, at: $0) } } .reduce(CGRect.null) { $0.union($1) } @@ -128,18 +128,18 @@ struct ERDiagramLayoutTests { } @Test("A single table is positioned") - func singleTable() { + func singleTable() throws { let graph = makeGraph(tables: ["solo"]) - let layout = ERDiagramLayout.compute(graph: graph) + let layout = try ERDiagramLayout.compute(graph: graph) #expect(layout.count == 1) } @Test("A long foreign-key chain does not stack into a tall narrow column") - func longChainStaysCompact() { + func longChainStaysCompact() throws { let tables = (0..<10).map { "t\($0)" } let chainFks = (0..<9).map { (from: "t\($0)", to: "t\($0 + 1)") } let graph = makeGraph(tables: tables, foreignKeys: chainFks) - let layout = ERDiagramLayout.compute(graph: graph) + let layout = try ERDiagramLayout.compute(graph: graph) let bounds = graph.nodes .compactMap { node in layout[node.id].map { rect(for: node, at: $0) } } .reduce(CGRect.null) { $0.union($1) } @@ -148,4 +148,42 @@ struct ERDiagramLayoutTests { #expect(aspect > 0.7) #expect(aspect < 4.0) } + + // Recorded from the UUID-keyed layout the indexed one replaced, at the default text size, so a + // faster force pass cannot move a single table. The tolerance only absorbs libm differences between + // macOS releases. + @Test("A known graph lays out exactly where the earlier layout put it", .enabled(if: ERDiagramLayout.typeScale == 1)) + func knownGraphKeepsItsPositions() throws { + let graph = makeGraph( + tables: ["a", "b", "c", "d", "e", "z"], + foreignKeys: [("b", "a"), ("c", "a"), ("d", "b"), ("e", "b")] + ) + let expected: [String: CGPoint] = [ + "a": CGPoint(x: 695.23302592634, y: 395.2305267095513), + "b": CGPoint(x: 353.61368040004584, y: 378.72650706900725), + "c": CGPoint(x: 1035.771273795778, y: 405.78940415047737), + "d": CGPoint(x: 150.0, y: 703.8204112685271), + "e": CGPoint(x: 171.4230011204163, y: 91.0), + "z": CGPoint(x: 150.0, y: 885.8204112685271), + ] + let layout = try ERDiagramLayout.compute(graph: graph) + for node in graph.nodes { + let point = try #require(layout[node.id]) + let target = try #require(expected[node.tableName]) + #expect(abs(point.x - target.x) < 1e-6 && abs(point.y - target.y) < 1e-6, "\(node.tableName): \(point)") + } + } + + @Test("A cancelled layout stops and throws instead of running to the end") + func cancelledLayoutStops() async { + let tables = (0..<200).map { "t\($0)" } + let graph = makeGraph(tables: tables, foreignKeys: (1..<200).map { (from: "t\($0)", to: "t\($0 / 2)") }) + let task = Task { + withUnsafeCurrentTask { $0?.cancel() } + return try await ERDiagramLayout.layOut(graph) + } + await #expect(throws: CancellationError.self) { + try await task.value + } + } } diff --git a/apple/TableProTests/Models/ERDiagram/ERDiagramStagedOverlayTests.swift b/apple/TableProTests/Models/ERDiagram/ERDiagramStagedOverlayTests.swift index 1fc9fd2f95..ab99fb6d4c 100644 --- a/apple/TableProTests/Models/ERDiagram/ERDiagramStagedOverlayTests.swift +++ b/apple/TableProTests/Models/ERDiagram/ERDiagramStagedOverlayTests.swift @@ -85,6 +85,26 @@ struct ERDiagramStagedOverlayTests { #expect(dropped.id == saved.id) } + @Test("A staged composite key draws one edge per column pair, each marked added") + func stagedCompositeKeyDrawsEachPair() { + let graph = savedGraph() + let orders = manager(for: "orders", in: graph) + var foreignKey = EditableForeignKeyDefinition.placeholder() + foreignKey.name = "fk_orders_customer" + foreignKey.columns = ["customer_id", "shipper_id"] + foreignKey.referencedTable = "customers" + foreignKey.referencedColumns = ["id", "email"] + orders.addForeignKey(foreignKey) + + let overlaid = ERDiagramStagedOverlay.apply( + ["orders": ERDiagramStagedOverlay.TableEdits(orders)], to: graph + ) + + let edges = overlaid.edges.filter { $0.fkName == "fk_orders_customer" } + #expect(edges.map { "\($0.fromColumn)->\($0.toColumn)" } == ["customer_id->id", "shipper_id->email"]) + #expect(edges.allSatisfy { $0.stagedEdit == .added && $0.toTable == "customers" }) + } + @Test("Added, renamed and dropped columns are marked on their table") func stagedColumnsAreMarked() throws { let graph = savedGraph() diff --git a/apple/TableProTests/Models/MultiRowEditStateTests.swift b/apple/TableProTests/Models/MultiRowEditStateTests.swift index ce162e198a..7733ef8eba 100644 --- a/apple/TableProTests/Models/MultiRowEditStateTests.swift +++ b/apple/TableProTests/Models/MultiRowEditStateTests.swift @@ -95,14 +95,14 @@ struct MultiRowEditStateTests { #expect(field.effectiveValue == nil) } - @Test("effectiveValue returns __DEFAULT__ when isPendingDefault") + @Test("effectiveValue returns the DEFAULT keyword when isPendingDefault") func effectiveValueReturnsDefaultWhenPendingDefault() { let field = FieldEditState( columnIndex: 0, columnName: "id", columnTypeEnum: .text(rawType: nil), isLongText: false, isJson: false, originalValue: "1", hasMultipleValues: false, pendingValue: nil, isPendingNull: false, isPendingDefault: true ) - #expect(field.effectiveValue == "__DEFAULT__") + #expect(field.effectiveValue == "DEFAULT") } @Test("effectiveValue returns nil when no edit is pending") @@ -349,6 +349,59 @@ struct MultiRowEditStateTests { #expect(sut.fields[1].pendingValue == nil) #expect(sut.fields[2].pendingValue == nil) } + + @Test("A new selection keeps each field's identity and gives every editor a new one") + func newSelectionKeepsFieldIdentity() { + let sut = makeSUT(rows: [["1", "Alice", "alice@test.com"]], selectedIndices: [0]) + let fieldIDs = sut.fields.map(\.id) + let editorIDs = sut.fields.map(\.editorID) + + sut.configure( + rowIDs: [.existing(1)], + allRows: [["1", "Bob", "alice@test.com"]], + columns: ["id", "name", "email"], + columnTypes: [.text(rawType: nil), .text(rawType: nil), .text(rawType: nil)] + ) + + #expect(sut.fields.map(\.id) == fieldIDs) + #expect(zip(editorIDs, sut.fields.map(\.editorID)).allSatisfy { $0 != $1 }) + } + + @Test("The same selection and values keep every editor's identity") + func sameSelectionKeepsEditorIdentity() { + let sut = makeSUT(rows: [["1", "Alice", "alice@test.com"]]) + let editorIDs = sut.fields.map(\.editorID) + + sut.configure( + rowIDs: [.existing(0)], + allRows: [["1", "Alicia", "alice@test.com"]], + columns: ["id", "name", "email"], + columnTypes: [.text(rawType: nil), .text(rawType: nil), .text(rawType: nil)] + ) + + let after = sut.fields.map(\.editorID) + #expect(after[0] == editorIDs[0]) + #expect(after[1] != editorIDs[1]) + #expect(after[2] == editorIDs[2]) + } + + @Test("A column renamed in place is a different field") + func renamedColumnIsANewField() { + let sut = makeSUT() + let before = sut.fields.map(\.id) + + sut.configure( + rowIDs: [.existing(0)], + allRows: [["1", "Alice", "alice@test.com"]], + columns: ["id", "full_name", "email"], + columnTypes: [.text(rawType: nil), .text(rawType: nil), .text(rawType: nil)] + ) + + let after = sut.fields.map(\.id) + #expect(after[0] == before[0]) + #expect(after[1] != before[1]) + #expect(after[2] == before[2]) + } } // MARK: - updateField() @@ -644,13 +697,14 @@ struct MultiRowEditStateTests { #expect(edited[0].newValue == nil) } - @Test("Returns __DEFAULT__ for default edit") + @Test("Returns the DEFAULT keyword marked as SQL for a default edit") func returnsDefaultForDefaultEdit() { let sut = makeSUT() sut.setFieldToDefault(at: 0) let edited = sut.getEditedFields() #expect(edited.count == 1) - #expect(edited[0].newValue == "__DEFAULT__") + #expect(edited[0].newValue == "DEFAULT") + #expect(edited[0].isExpression) } @Test("Returns empty array when no edits") @@ -774,18 +828,22 @@ struct MultiRowEditStateTests { #expect(callbackCalls[0].value == nil) } - @Test("setFieldToDefault fires callback with __DEFAULT__") + // Through `onFieldChanged` the keyword would be a literal, which a stored value can also be. + @Test("setFieldToDefault sends the DEFAULT keyword through the expression callback") func setFieldToDefaultFiresCallback() { let sut = makeSUT() - var callbackCalls: [(index: Int, value: String?)] = [] - sut.onFieldChanged = { index, value, _ in - callbackCalls.append((index, value.asText)) + var literals: [PluginCellValue] = [] + var callbackCalls: [(index: Int, sql: String)] = [] + sut.onFieldChanged = { _, value, _ in literals.append(value) } + sut.onFieldExpressionChanged = { index, sql, _ in + callbackCalls.append((index, sql)) } sut.setFieldToDefault(at: 0) + #expect(literals.isEmpty) #expect(callbackCalls.count == 1) #expect(callbackCalls[0].index == 0) - #expect(callbackCalls[0].value == "__DEFAULT__") + #expect(callbackCalls[0].sql == "DEFAULT") } // `onFieldChanged` carries a literal, so an expression sent through it would be saved as quoted text. diff --git a/apple/TableProTests/Models/PaginationStateTests.swift b/apple/TableProTests/Models/PaginationStateTests.swift index 0235101ebe..35e00686ec 100644 --- a/apple/TableProTests/Models/PaginationStateTests.swift +++ b/apple/TableProTests/Models/PaginationStateTests.swift @@ -331,7 +331,7 @@ struct PaginationStateTests { func resetLoadMoreClearsTheQueryAndItsValues() { var state = PaginationState() state.hasMoreRows = true - state.setBaseQueryForMore("SELECT * FROM t WHERE id = ?", parameterValues: ["1"]) + state.setBaseQueryForMore("SELECT * FROM t WHERE id = ?", parameterValues: [TypedParameterValue(text: "1", kind: .text)]) state.resetLoadMore() @@ -404,7 +404,7 @@ struct PaginationStateTests { @Test("Setting the base query clears the previous run's bindings") func setBaseQueryForMoreClearsStaleBindings() { var state = PaginationState() - state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: ["42"]) + state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: [TypedParameterValue(text: "42", kind: .text)]) state.setBaseQueryForMore("SELECT * FROM events", parameterValues: nil) @@ -416,10 +416,11 @@ struct PaginationStateTests { func setBaseQueryForMoreCarriesBindings() { var state = PaginationState() - state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: ["7"]) + let bound = TypedParameterValue(text: "7", kind: .integer) + state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: [bound]) #expect(state.baseQueryForMore == "SELECT * FROM orders WHERE id = $1") - #expect(state.baseQueryParameterValues == ["7"]) + #expect(state.baseQueryParameterValues == [bound]) } @Test("Busy covers every kind of pending work") diff --git a/apple/TableProTests/Models/Query/ResultEditabilityTests.swift b/apple/TableProTests/Models/Query/ResultEditabilityTests.swift index ff496e37f7..a947ea0b1d 100644 --- a/apple/TableProTests/Models/Query/ResultEditabilityTests.swift +++ b/apple/TableProTests/Models/Query/ResultEditabilityTests.swift @@ -16,6 +16,7 @@ struct ResultEditabilityTests { private func origin( tableName: String? = "users", + schemaName: String? = nil, databaseName: String = "app", isEditable: Bool = true, isView: Bool = false, @@ -23,7 +24,7 @@ struct ResultEditabilityTests { ) -> ResultOrigin { ResultOrigin( tableName: tableName, - schemaName: nil, + schemaName: schemaName, databaseName: databaseName, primaryKeyColumns: ["id"], isEditable: isEditable, @@ -36,37 +37,37 @@ struct ResultEditabilityTests { func queryResultOverAViewTakesTheViewGate() { var result = origin() result.viewRowEditing = editable(keys: ["id"]) - #expect(ResultEditability.refusal(for: result, currentDatabaseName: "app") == nil) + #expect(ResultEditability.refusal(for: result, currentDatabaseName: "app", currentSchemaName: nil) == nil) result.primaryKeyColumns = ["name"] - #expect(ResultEditability.refusal(for: result, currentDatabaseName: "app") == .view(nil)) + #expect(ResultEditability.refusal(for: result, currentDatabaseName: "app", currentSchemaName: nil) == .view(nil)) result.viewRowEditing = .readOnly(.couldNotCheck) - #expect(ResultEditability.refusal(for: result, currentDatabaseName: "app") == .view(.couldNotCheck)) + #expect(ResultEditability.refusal(for: result, currentDatabaseName: "app", currentSchemaName: nil) == .view(.couldNotCheck)) } @Test("A result that captured its table is editable") func capturedTableIsEditable() { - #expect(ResultEditability.refusal(for: origin(), currentDatabaseName: "app") == nil) + #expect(ResultEditability.refusal(for: origin(), currentDatabaseName: "app", currentSchemaName: nil) == nil) } @Test("A result with no captured origin refuses instead of borrowing the tab's table") func missingOriginRefuses() { - #expect(ResultEditability.refusal(for: nil, currentDatabaseName: "app") == .unknownTable) + #expect(ResultEditability.refusal(for: nil, currentDatabaseName: "app", currentSchemaName: nil) == .unknownTable) } @Test("A result whose statement resolved no single table refuses") func unresolvedTableRefuses() { #expect( - ResultEditability.refusal(for: origin(tableName: nil, isEditable: false), currentDatabaseName: "app") + ResultEditability.refusal(for: origin(tableName: nil, isEditable: false), currentDatabaseName: "app", currentSchemaName: nil) == .unknownTable ) #expect( - ResultEditability.refusal(for: origin(tableName: ""), currentDatabaseName: "app") == .unknownTable + ResultEditability.refusal(for: origin(tableName: ""), currentDatabaseName: "app", currentSchemaName: nil) == .unknownTable ) } @Test("A view refuses, and says so rather than reporting an unknown table") func viewRefuses() { - #expect(ResultEditability.refusal(for: origin(isView: true), currentDatabaseName: "app") == .view(nil)) + #expect(ResultEditability.refusal(for: origin(isView: true), currentDatabaseName: "app", currentSchemaName: nil) == .view(nil)) } @Test("An updatable view that exposes its table's key is written like a table") @@ -74,7 +75,7 @@ struct ResultEditabilityTests { var view = origin(isView: true) view.objectType = .view view.viewRowEditing = editable(keys: ["id"]) - #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app") == nil) + #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app", currentSchemaName: nil) == nil) } @Test("A view refuses when the keys the edit would use are not the driver's") @@ -82,16 +83,16 @@ struct ResultEditabilityTests { var view = origin(isView: true) view.viewRowEditing = editable(keys: ["id"]) view.primaryKeyColumns = [] - #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app") == .view(nil)) + #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app", currentSchemaName: nil) == .view(nil)) view.primaryKeyColumns = ["code"] - #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app") == .view(nil)) + #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app", currentSchemaName: nil) == .view(nil)) } @Test("A read-only view carries the driver's reason into the refusal") func readOnlyViewSaysWhy() { var view = origin(isView: true) view.viewRowEditing = .readOnly(.keyNotExposed) - #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app") == .view(.keyNotExposed)) + #expect(ResultEditability.refusal(for: view, currentDatabaseName: "app", currentSchemaName: nil) == .view(.keyNotExposed)) } @Test("A materialized view never takes a view's editing answer") @@ -99,27 +100,41 @@ struct ResultEditabilityTests { var matview = origin(isView: true) matview.objectType = .materializedView matview.viewRowEditing = editable(keys: ["id"]) - #expect(ResultEditability.refusal(for: matview, currentDatabaseName: "app") == .view(nil)) + #expect(ResultEditability.refusal(for: matview, currentDatabaseName: "app", currentSchemaName: nil) == .view(nil)) } @Test("A result captured against another database refuses once the tab has moved") func databaseMoveRefuses() { #expect( - ResultEditability.refusal(for: origin(databaseName: "app"), currentDatabaseName: "reporting") + ResultEditability.refusal(for: origin(databaseName: "app"), currentDatabaseName: "reporting", currentSchemaName: nil) == .databaseMoved ) } + @Test("A result read in one schema refuses once the tab's schema picker moved") + func schemaMoveRefuses() { + #expect( + ResultEditability.refusal( + for: origin(schemaName: "staging"), currentDatabaseName: "app", currentSchemaName: "production" + ) == .schemaMoved + ) + #expect( + ResultEditability.refusal( + for: origin(schemaName: "staging"), currentDatabaseName: "app", currentSchemaName: "staging" + ) == nil + ) + } + @Test("An unknown database on either side is not treated as a move") func unknownDatabaseDoesNotRefuse() { - #expect(ResultEditability.refusal(for: origin(databaseName: ""), currentDatabaseName: "app") == nil) - #expect(ResultEditability.refusal(for: origin(databaseName: "app"), currentDatabaseName: "") == nil) + #expect(ResultEditability.refusal(for: origin(databaseName: ""), currentDatabaseName: "app", currentSchemaName: nil) == nil) + #expect(ResultEditability.refusal(for: origin(databaseName: "app"), currentDatabaseName: "", currentSchemaName: nil) == nil) } @Test("A result whose key columns were never looked up refuses") func unresolvedKeysRefuse() { #expect( - ResultEditability.refusal(for: origin(keysResolved: false), currentDatabaseName: "app") + ResultEditability.refusal(for: origin(keysResolved: false), currentDatabaseName: "app", currentSchemaName: nil) == .keysUnresolved ) } @@ -128,7 +143,7 @@ struct ResultEditabilityTests { func resolvedButKeylessTableStaysEditable() { var keyless = origin() keyless.primaryKeyColumns = [] - #expect(ResultEditability.refusal(for: keyless, currentDatabaseName: "app") == nil) + #expect(ResultEditability.refusal(for: keyless, currentDatabaseName: "app", currentSchemaName: nil) == nil) } @Test("Every refusal carries a sentence for the user") @@ -138,7 +153,7 @@ struct ResultEditabilityTests { .baseTableHidden, .baseTableHasNoKey, .keyNotExposed, .couldNotCheck, ] let viewRefusals = reasons.map { ResultEditRefusal.view($0) } - for refusal in [ResultEditRefusal.unknownTable, .databaseMoved, .keysUnresolved] + viewRefusals { + for refusal in [ResultEditRefusal.unknownTable, .databaseMoved, .schemaMoved, .keysUnresolved] + viewRefusals { #expect(!refusal.message.isEmpty) } } diff --git a/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift b/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift index 105d0d2c51..b75d4a25ed 100644 --- a/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift +++ b/apple/TableProTests/Models/Query/ResultSetMenuModelTests.swift @@ -95,6 +95,20 @@ struct ResultSetMenuModelTests { #expect(model.entries.map(model.displayLabel(of:)) == ["users", "orders", "users (2)", "users (3)"]) } + // A 10,000-statement run counted each entry's earlier uses per entry, about 10^8 comparisons to open the chooser. + @Test("Every name of a long run is counted the way one name alone is") + func longRunNamesMatchSingleNameCounting() { + let labels = (0 ..< 3_000).map { index in ["users", "orders", "Result \(index)"][index % 3] } + let model = Self.make(labels: labels, activeIndex: 0) + let expected = labels.indices.map { ResultSetMenuModel.displayLabel(at: $0, in: labels) } + + #expect(model.entries.map { model.displayLabel(of: $0) } == expected) + #expect(ResultSetMenuModel.displayLabels(for: labels) == expected) + #expect(Array(expected.prefix(6)) == ["users", "orders", "Result 2", "users (2)", "orders (2)", "Result 5"]) + #expect(expected.last == "Result 2999") + #expect(model.tabTitle(of: model.entries[2_997]) == "users (1000)") + } + @Test("A long name is cut on a whole character, never inside an emoji") func truncationKeepsWholeCharacters() { let label = "Monthly revenue per region 📈 for Q3" diff --git a/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift b/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift index c13f0553a7..94ab532c4f 100644 --- a/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift +++ b/apple/TableProTests/Models/Query/TabSessionRegistryTests.swift @@ -118,6 +118,32 @@ struct TabSessionRegistryTests { #expect(session.dataRevision > before) } + // A buffer allocated while the old one is still held cannot share its address, so a copy always moves it. + @Test("An edit writes into the session's buffer instead of copying every row") + func editWritesInPlace() { + let registry = TabSessionRegistry() + let session = TabSession() + registry.register(session) + registry.setTableRows(makeRows((0..<1_000).map { "r\($0)" }), for: session.id) + + for row in [0, 499, 999] { + let before = Self.bufferAddress(registry.tableRows(for: session.id)) + registry.updateTableRows(for: session.id) { rows in + rows.edit(row: row, column: 0, value: .text("edited")) + } + #expect(Self.bufferAddress(registry.tableRows(for: session.id)) == before) + } + + let rows = registry.tableRows(for: session.id) + #expect(rows.rows.count == 1_000) + #expect(rows.rows[499].values[0] == .text("edited")) + #expect(rows.rows[500].values[0] == .text("r500")) + } + + private static func bufferAddress(_ rows: TableRows) -> UInt { + rows.rows.withUnsafeBufferPointer { UInt(bitPattern: $0.baseAddress) } + } + // MARK: - rowSetRevision @Test("A cell edit moves dataRevision and leaves rowSetRevision alone") diff --git a/apple/TableProTests/Models/Query/TableAutoRefreshGateTests.swift b/apple/TableProTests/Models/Query/TableAutoRefreshGateTests.swift new file mode 100644 index 0000000000..438f644dbc --- /dev/null +++ b/apple/TableProTests/Models/Query/TableAutoRefreshGateTests.swift @@ -0,0 +1,83 @@ +// +// TableAutoRefreshGateTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct TableAutoRefreshGateTests { + @Test("A visible, idle, loaded table tab refreshes") + func idleTabRefreshes() { + #expect(TableAutoRefreshGate().decision == .refresh) + } + + @Test("Each state that a reload would break waits, with its own reason") + func eachPauseStateWaits() { + let cases: [(String, (inout TableAutoRefreshGate) -> Void, TableAutoRefreshPause)] = [ + ("window hidden", { $0.windowIsVisible = false }, .notVisible), + ("tab not selected", { $0.isSelectedInWindow = false }, .notVisible), + ("disconnected", { $0.isConnected = false }, .disconnected), + ("JSON mode", { $0.viewMode = .json }, .otherViewMode), + ("structure mode", { $0.viewMode = .structure }, .otherViewMode), + ("open transaction", { $0.holdsTransaction = true }, .openTransaction), + ("unsaved edits", { $0.holdsUnsavedWork = true }, .unsavedChanges), + ("sidebar staged drops", { $0.hasSidebarDataWork = true }, .unsavedChanges), + ("open cell editor", { $0.isEditingCell = true }, .editing), + ("grid popover or context menu", { $0.gridHoldsTransientUI = true }, .interacting), + ("inspector pop-out window", { $0.inspectorHoldsValueWindow = true }, .valueWindowOpen), + ("live scroll or drag", { $0.gridIsTrackingInteraction = true }, .interacting), + ("running load", { $0.load = .busy }, .loading), + ("pending count", { $0.isCountPending = true }, .loading), + ("load more", { $0.isLoadingMore = true }, .loading), + ("never loaded", { $0.hasLoaded = false }, .loading), + ("Safe Mode prompt", { $0.showsSafeModePrompt = true }, .safeModePrompt), + ("selection without a key", { gate in + gate.hasSelection = true + gate.hasPrimaryKey = false + }, .unkeyedSelection), + ] + for (name, configure, pause) in cases { + var gate = TableAutoRefreshGate() + configure(&gate) + #expect(gate.decision == .wait(pause), "\(name)") + } + } + + @Test("An open editor names the editor, not the unsaved work it also counts as") + func editorWinsOverUnsavedWork() { + var gate = TableAutoRefreshGate() + gate.isEditingCell = true + gate.holdsUnsavedWork = true + #expect(gate.decision == .wait(.editing)) + } + + @Test("A selection in a keyed table still refreshes, since the key brings it back") + func keyedSelectionRefreshes() { + var gate = TableAutoRefreshGate() + gate.hasSelection = true + #expect(gate.decision == .refresh) + } + + @Test("A failed tick and a tab that is no longer a table stop the schedule") + func failureAndNonTableStop() { + var failed = TableAutoRefreshGate() + failed.previousTickFailed = true + var query = TableAutoRefreshGate() + query.isTableTab = false + + #expect(failed.decision == .stop) + #expect(query.decision == .stop) + } + + @Test("Only the pauses a reader can act on name themselves in the help") + func pauseHelp() { + #expect(TableAutoRefreshPause.unsavedChanges.help == "Auto Refresh waits while the table has unsaved changes.") + #expect(TableAutoRefreshPause.disconnected.help == "Auto Refresh waits until the connection is back.") + #expect(TableAutoRefreshPause.openTransaction.help == "Auto Refresh waits while a transaction is open.") + #expect(TableAutoRefreshPause.valueWindowOpen.help == "Auto Refresh waits while a value is open in its own window.") + #expect(TableAutoRefreshPause.notVisible.help == nil) + #expect(TableAutoRefreshPause.interacting.help == nil) + } +} diff --git a/apple/TableProTests/Models/Query/TableAutoRefreshIntervalTests.swift b/apple/TableProTests/Models/Query/TableAutoRefreshIntervalTests.swift new file mode 100644 index 0000000000..68b519805f --- /dev/null +++ b/apple/TableProTests/Models/Query/TableAutoRefreshIntervalTests.swift @@ -0,0 +1,47 @@ +// +// TableAutoRefreshIntervalTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct TableAutoRefreshIntervalTests { + @Test("The presets run from five seconds to five minutes, after Off") + func presets() { + let seconds = TableAutoRefreshInterval.allCases.map { $0.rawValue } + #expect(seconds == [0, 5, 10, 30, 60, 300]) + #expect(TableAutoRefreshInterval.allCases.filter { !$0.isOn } == [.off]) + #expect(TableAutoRefreshInterval.oneMinute.duration == .seconds(60)) + } + + @Test("Each preset has its own title") + func titles() { + let titles = TableAutoRefreshInterval.allCases.map { $0.title } + #expect(titles == [ + "Off", "Every 5 Seconds", "Every 10 Seconds", "Every 30 Seconds", "Every Minute", "Every 5 Minutes", + ]) + } + + @Test("An auto refresh writes no history and keeps the count; every other load does both") + func autoRefreshTriggerSkipsHistoryAndCount() { + #expect(!TableLoadTrigger.autoRefresh.recordsHistory) + #expect(!TableLoadTrigger.autoRefresh.countsRows) + #expect(TableLoadTrigger.userInitiated.recordsHistory) + #expect(TableLoadTrigger.restore.countsRows) + } + + @Test("A table tab never saves its interval, so a reopened tab starts with it off") + func intervalIsSessionOnly() { + var tab = QueryTab(title: "jobs", query: "SELECT * FROM jobs", tabType: .table, tableName: "jobs") + tab.autoRefreshInterval = .fiveSeconds + + let reopened = QueryTab(from: tab.toPersistedTab(), defaultPageSize: 1_000) + var stopped = tab + stopped.autoRefreshInterval = .off + + #expect(reopened.autoRefreshInterval == .off) + #expect(tab != stopped, "a view keyed on the tab has to see the interval change") + } +} diff --git a/apple/TableProTests/Models/Query/TableRowsPreparedRowsTests.swift b/apple/TableProTests/Models/Query/TableRowsPreparedRowsTests.swift new file mode 100644 index 0000000000..61af8dd4a7 --- /dev/null +++ b/apple/TableProTests/Models/Query/TableRowsPreparedRowsTests.swift @@ -0,0 +1,168 @@ +// +// TableRowsPreparedRowsTests.swift +// TableProTests +// +// A Fetch All or a large first page builds one Row per fetched row. That build runs off the main actor, and the main +// actor only assigns the finished buffer, so a 2,000,000-row result no longer freezes the window while it is copied. +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct TableRowsPreparedRowsTests { + private static let columns = ["id", "email"] + private static let types: [ColumnType] = [.integer(rawType: nil), .text(rawType: nil)] + private static let shape = TableRows.RowShape(columns: columns, masks: .none, exportPolicy: .none, exportTable: nil) + + private static func fetched(_ count: Int) -> [[PluginCellValue]] { + (0 ..< count).map { [.text("\($0)"), .text("user\($0)@example.com")] } + } + + private static func address(of rows: ContiguousArray) -> UnsafeRawPointer? { + rows.withUnsafeBufferPointer { UnsafeRawPointer($0.baseAddress) } + } + + @Test("Preparing a result reads every row off the main actor") + func preparingReadsOffTheMainActor() async { + let source = MainThreadReadCountingRows(base: Self.fetched(500)) + + let prepared = await TableRows.preparing(source, shape: Self.shape) + + #expect(prepared.rows.count == 500) + #expect(source.reads == MainThreadReadCountingRows.Reads(total: 500, onMain: 0)) + } + + @Test("The counting source sees reads made on the main actor") + func countingSourceSeesMainActorReads() { + let source = MainThreadReadCountingRows(base: Self.fetched(50)) + + _ = TableRows.prepare(source, shape: Self.shape) + + #expect(source.reads == MainThreadReadCountingRows.Reads(total: 50, onMain: 50)) + } + + @Test("Installing prepared rows assigns the built buffer and its index instead of building them again") + func installAdoptsTheBuiltBuffer() { + var table = TableRows.from(queryRows: [[.text("0"), .text("old@example.com")]], columns: Self.columns, columnTypes: Self.types) + let prepared = TableRows.prepare( + Self.fetched(3), offset: 10, rowLocators: ["first", nil, "third"], shape: table.rowShape + ) + + #expect(table.install(prepared) == .fullReplace) + #expect(Self.address(of: table.rows) == Self.address(of: prepared.rows)) + #expect(table.rows.map { $0.id } == [.existing(10), .existing(11), .existing(12)]) + #expect(table.index(of: .existing(12)) == 2) + #expect(table.rowLocator(for: .existing(10)) == "first") + #expect(table.rowLocator(for: .existing(11)) == nil) + #expect(table.rowLocator(for: .existing(12)) == "third") + } + + @Test("Rows prepared before a masking rule was saved are refused, and the held rows stay") + func installRefusesRowsBuiltForOtherMasks() { + var table = TableRows.from(queryRows: [[.text("0"), .text("old@example.com")]], columns: Self.columns, columnTypes: Self.types) + let prepared = TableRows.prepare(Self.fetched(3), shape: table.rowShape) + table.addMasks(ColumnMasks(styles: [1: .full])) + + #expect(table.install(prepared) == nil) + #expect(table.rows.map { $0.id } == [.existing(0)]) + #expect(table.value(at: 0, column: 1) == ColumnMasks(styles: [1: .full]).value(.text("old@example.com"), column: 1)) + } + + @Test("Prepared rows install exactly what a replace builds: masks, padding, absent and unloaded cells, documents") + func installMatchesReplace() { + let masks = ColumnMasks(styles: [1: .email]) + let base = TableRows.from( + queryRows: [[.text("0"), .text("old@example.com")]], + columns: Self.columns, + columnTypes: Self.types, + masks: masks, + exportTable: "users" + ) + let incoming: [[PluginCellValue]] = [[.text("1"), .text("jane@example.com")], [.text("2")], [.text("3"), .null, .text("extra")]] + let documents = incoming.indices.map { index in + PluginExportDocument(fields: [PluginExportField( + name: "email", canonicalJSON: "\"doc\(index)@example.com\"", typeName: "string", cellValue: .text("doc\(index)@example.com") + )]) + } + let absent: [Int: Set] = [1: [1]] + let unloaded: [Int: [Int: Int]] = [2: [1: 4_000]] + + var replaced = base + replaced.replace(rows: incoming, offset: 5, exportDocuments: documents, absentCells: absent, unloadedCells: unloaded) + var installed = base + let prepared = TableRows.prepare( + incoming, offset: 5, exportDocuments: documents, absentCells: absent, unloadedCells: unloaded, shape: base.rowShape + ) + + #expect(installed.install(prepared) == .fullReplace) + #expect(installed.rows == replaced.rows) + #expect(installed.index(of: .existing(7)) == replaced.index(of: .existing(7))) + #expect(installed.value(at: 0, column: 1) != .text("jane@example.com")) + } + + // A Stop on a 2,000,000-row Fetch All left the tab on Loading until every row was built. + @Test("A Stop during the off-main build ends it at the next interval, and what it built never installs") + func stopEndsTheBuildWithinAnInterval() async { + let interval = TableRows.cancellationCheckInterval + let source = MainThreadReadCountingRows(base: Self.fetched(interval * 3), cancelsTaskAtRow: interval) + + let prepared = await Task { await TableRows.preparing(source, shape: Self.shape) }.value + var table = TableRows.from(queryRows: [[.text("0"), .text("old@example.com")]], columns: Self.columns, columnTypes: Self.types) + + #expect(source.reads.total == interval * 2) + #expect(!prepared.isComplete) + #expect(prepared.rows.isEmpty) + #expect(table.install(prepared) == nil) + #expect(table.rows.map { $0.id } == [.existing(0)]) + } + + @Test("The build on the caller's actor lands every row even inside a stopped task") + func callerBuildIgnoresAStop() async { + let interval = TableRows.cancellationCheckInterval + let source = MainThreadReadCountingRows(base: Self.fetched(interval * 2), cancelsTaskAtRow: 0) + + let prepared = await Task { TableRows.prepare(source, shape: Self.shape) }.value + + #expect(source.reads.total == interval * 2) + #expect(prepared.isComplete) + #expect(prepared.rows.count == interval * 2) + } +} + +// Counts which thread read each row, so a test can tell where a build ran. +private struct MainThreadReadCountingRows: RandomAccessCollection, Sendable { + struct Reads: Equatable, Sendable { + var total = 0 + var onMain = 0 + } + + let base: [[PluginCellValue]] + // Cancels the task reading the row at this position, the way a Stop lands in the middle of a build. + let cancelsTaskAtRow: Int? + private let counter = OSAllocatedUnfairLock(initialState: Reads()) + + init(base: [[PluginCellValue]], cancelsTaskAtRow: Int? = nil) { + self.base = base + self.cancelsTaskAtRow = cancelsTaskAtRow + } + + var reads: Reads { counter.withLock { $0 } } + var startIndex: Int { base.startIndex } + var endIndex: Int { base.endIndex } + + subscript(position: Int) -> [PluginCellValue] { + let onMain = Thread.isMainThread + counter.withLock { counted in + counted.total += 1 + if onMain { counted.onMain += 1 } + } + if position == cancelsTaskAtRow { + withUnsafeCurrentTask { $0?.cancel() } + } + return base[position] + } +} diff --git a/apple/TableProTests/Models/QueryHistoryEntryTests.swift b/apple/TableProTests/Models/QueryHistoryEntryTests.swift index a1694e6a79..eaacf27606 100644 --- a/apple/TableProTests/Models/QueryHistoryEntryTests.swift +++ b/apple/TableProTests/Models/QueryHistoryEntryTests.swift @@ -1,16 +1,54 @@ -// -// QueryHistoryEntryTests.swift -// TableProTests -// - import Foundation -import TableProPluginKit @testable import TablePro +import TableProPluginKit import Testing +private let previewSamples: [String] = [ + "", + " ", + "\n", + " \t \r\n \u{2028} ", + "SELECT 1", + "SELECT 1;", + " SELECT 1; \n", + "\n\n SELECT 1\n\n\n", + "SELECT id,\r\n name\r\n FROM users\r\n", + "a\u{2028}b\u{2029}c\u{85}d\u{0B}e\u{0C}f\rg", + "\tSELECT\t1\t\n\t\tFROM t\t", + "\u{3000}SELECT\u{A0}1\u{2003}\n\u{A0}", + "abc\n\u{301}def", + "SELECT" + String(repeating: " ", count: 300) + "1", + "SELECT 1\u{37E}", + "SELECT 1\u{600}", + String(repeating: "a", count: 98) + "\u{600}", + String(repeating: "a", count: 99), + String(repeating: "a", count: 99) + ";", + String(repeating: "a", count: 100), + String(repeating: "a", count: 101), + String(repeating: "a", count: 98) + "\u{1F600}", + String(repeating: "a", count: 99) + "\u{1F600}", + String(repeating: "a", count: 100) + "\u{301}", + String(repeating: "a", count: 120) + "\u{37E}", + String(repeating: "a", count: 159), + String(repeating: "a", count: 160), + String(repeating: "a", count: 161), + String(repeating: "a", count: 158) + "\u{1F600}", + String(repeating: "a", count: 159) + "\u{1F600}", + String(repeating: "a", count: 159) + "e\u{301}x", + String(repeating: "a", count: 160) + "\u{301}", + String(repeating: "a", count: 158) + "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}b", + String(repeating: "a", count: 159) + "\u{1F1FA}\u{1F1F8}\u{1F1EC}\u{1F1E7}", + String(repeating: "a", count: 80) + "\n" + String(repeating: "b", count: 79) + "\n\u{301}c", + "a" + String(repeating: "\u{301}", count: 400) + " b\nc", + String(repeating: "a", count: 150) + String(repeating: " ", count: 200), + String(repeating: " SELECT col FROM t\r\n", count: 30), + String(repeating: "word ", count: 40) + "\n\n" + String(repeating: "\t", count: 50), +] + struct QueryHistoryEntryTests { private func makeEntry( query: String, + statementType: QueryHistoryStatementType? = nil, executionTime: TimeInterval = 0.1, rowCount: Int = 10, wasSuccessful: Bool = true @@ -21,6 +59,7 @@ struct QueryHistoryEntryTests { databaseName: "test", databaseType: .postgresql, source: .editor, + statementType: statementType, executionTime: executionTime, rowCount: rowCount, wasSuccessful: wasSuccessful @@ -45,6 +84,69 @@ struct QueryHistoryEntryTests { #expect(entry.singleLinePreview.contains("\n") == false) } + @Test("The previews read as splitting and rejoining the whole query did", arguments: previewSamples) + func previewsMatchTheWholeQueryVersion(query: String) { + let entry = makeEntry(query: query, statementType: .select) + #expect(Array(entry.singleLinePreview.unicodeScalars) == Array(WholeQueryPreview.singleLine(query).unicodeScalars)) + #expect(Array(entry.queryPreview.unicodeScalars) == Array(WholeQueryPreview.query(query).unicodeScalars)) + } + + @Test("The previews read as splitting and rejoining the whole query did on generated queries") + func previewsMatchTheWholeQueryVersionOnGeneratedQueries() { + let pieces = [ + "a", "SELECT", " ", " ", "\t", "\n", "\r", "\r\n", "\u{2028}", "\u{85}", "\u{3000}", "\u{A0}", + "\u{1F600}", "\u{1F468}\u{200D}\u{1F469}", "e\u{301}", "\u{301}", "\u{600}", ";", "\u{37E}", + "\u{1F1FA}\u{1F1F8}", "\u{0B}", + ] + var generator = SeededGenerator(seed: 0x5EED) + var mismatches: [String] = [] + for _ in 0..<500 { + var query = "" + for _ in 0.. 1_000_000) + + var shortPreviews = TimeInterval.infinity + var longPreviews = TimeInterval.infinity + for _ in 0..<5 { + shortPreviews = min(shortPreviews, Self.secondsSpent(previewing: short)) + longPreviews = min(longPreviews, Self.secondsSpent(previewing: long)) + } + + #expect(long.singleLinePreview == short.singleLinePreview) + #expect(long.queryPreview == short.queryPreview) + #expect(longPreviews < shortPreviews * 4) + } + + private static func secondsSpent(previewing entry: QueryHistoryEntry) -> TimeInterval { + let start = ProcessInfo.processInfo.systemUptime + for _ in 0..<100 { + _ = entry.singleLinePreview + _ = entry.queryPreview + } + return ProcessInfo.processInfo.systemUptime - start + } + @Test("statementType is classified from the query when not supplied") func statementTypeIsClassified() { #expect(makeEntry(query: "SELECT 1").statementType == .select) @@ -100,3 +202,30 @@ struct QueryHistoryEntryTests { #expect(entry.databaseDisplayName == "test") } } + +// The previews as they were before they stopped reading at the cut, kept as the reference the new ones are compared +// with. +private enum WholeQueryPreview { + static func singleLine(_ query: String) -> String { + let collapsed = query + .components(separatedBy: .newlines) + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + .joined(separator: " ") + if (collapsed as NSString).length > 160 { + return String(collapsed.prefix(160)) + "…" + } + return collapsed + } + + static func query(_ query: String) -> String { + var trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.hasSuffix(";") { + trimmed += ";" + } + if (trimmed as NSString).length > 100 { + return String(trimmed.prefix(100)) + "…" + } + return trimmed + } +} diff --git a/apple/TableProTests/Models/ResultStatusModelTests.swift b/apple/TableProTests/Models/ResultStatusModelTests.swift index f5a1cc3f23..19304f3637 100644 --- a/apple/TableProTests/Models/ResultStatusModelTests.swift +++ b/apple/TableProTests/Models/ResultStatusModelTests.swift @@ -290,6 +290,16 @@ struct ResultStatusModelTests { #expect(!structure.controls.showsPagination) } + @Test("Auto Refresh is offered on a table tab's data grid only") + func autoRefreshFollowsTheTableGrid() { + let table = makeSnapshot(rowCount: 10) + #expect(model(table, viewMode: .data).controls.showsAutoRefresh) + #expect(!model(table, viewMode: .json).controls.showsAutoRefresh) + #expect(!model(table, viewMode: .structure).controls.showsAutoRefresh) + #expect(!model(makeSnapshot(tabType: .query, rowCount: 10, hasTableName: false)).controls.showsAutoRefresh) + #expect(!model(makeSnapshot(rowCount: 10, hasTableName: false)).controls.showsAutoRefresh) + } + @Test("Highlight rules are offered only where the data grid draws the result") func highlightRulesFollowTheDataGrid() { let table = makeSnapshot(rowCount: 10) diff --git a/apple/TableProTests/Models/ResultStatusPresentationTests.swift b/apple/TableProTests/Models/ResultStatusPresentationTests.swift index f94955df69..2ff0e29653 100644 --- a/apple/TableProTests/Models/ResultStatusPresentationTests.swift +++ b/apple/TableProTests/Models/ResultStatusPresentationTests.swift @@ -35,6 +35,13 @@ struct ResultStatusPresentationTests { #expect(!presentation(.narrow).pageSizeIsInline) } + @Test("Only the narrowest tier leaves Auto Refresh to the Database menu") + func narrowLeavesAutoRefreshToTheMenu() { + #expect(presentation(.regular).showsAutoRefresh) + #expect(presentation(.compact).showsAutoRefresh) + #expect(!presentation(.narrow).showsAutoRefresh) + } + @Test("The narrowest tier swaps the segmented switcher for a pull-down") func narrowUsesAMenuSwitcher() { #expect(!presentation(.narrow).modeSwitcherIsSegmented) diff --git a/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift b/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift index cc67c4e094..cdee26b95c 100644 --- a/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift +++ b/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift @@ -2,9 +2,6 @@ // ClickHousePartStatementsTests.swift // TableProTests // -// The Parts tab's statements name the tab's own database, because ClickHouse resolves an -// unqualified one from a request parameter the app moves whenever any tab runs elsewhere. -// import Foundation @testable import TablePro @@ -14,8 +11,7 @@ struct ClickHousePartStatementsTests { private let quote: (String) -> String = { "`\($0.replacingOccurrences(of: "`", with: "``"))`" } private let escape: (String) -> String = { $0.replacingOccurrences(of: "'", with: "\\'") } - /// The defect: an unqualified DROP PARTITION deleted data from whichever database the shared - /// driver was last pinned to, while the tab reported it was acting on its own. + // An unqualified DROP PARTITION deleted data from whichever database the shared driver was last pinned to. @Test("Every statement names the database it was given") func statementsCarryTheDatabase() { #expect( @@ -24,20 +20,59 @@ struct ClickHousePartStatementsTests { ) #expect( ClickHousePartStatements.dropPartition( - database: "analytics", table: "events", partition: "202401", - quote: quote, escape: escape - ) == "ALTER TABLE `analytics`.`events` DROP PARTITION '202401'" + database: "analytics", table: "events", partitionId: "202401", quote: quote + ) == "ALTER TABLE `analytics`.`events` DROP PARTITION ID '202401'" ) #expect( ClickHousePartStatements.detachPartition( - database: "analytics", table: "events", partition: "202401", - quote: quote, escape: escape - ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION '202401'" + database: "analytics", table: "events", partitionId: "202401", quote: quote + ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION ID '202401'" ) } - /// `currentDatabase()` answers with the request parameter, so the list showed another database's - /// parts under this table's name. + // A tuple key's rendered value `(202401,'eu')` and an unpartitioned table's `tuple()` were each + // wrapped as one string literal, which names no partition. + @Test("Drop and Detach name the partition by its ID for every key shape") + func everyKeyShapeUsesThePartitionId() { + let shapes = [ + "202401-7", + "all", + "4f8b2b7c9a1d3e5f60718293a4b5c6d7", + "1--5", + ] + for id in shapes { + #expect( + ClickHousePartStatements.dropPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == "ALTER TABLE `analytics`.`events` DROP PARTITION ID '\(id)'" + ) + #expect( + ClickHousePartStatements.detachPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION ID '\(id)'" + ) + } + } + + @Test("An ID the server would refuse is never spliced into a statement") + func unusableIdBuildsNothing() { + let unusable = ["", "it's", "a\\'b", "(202401,'eu')", "tuple()", "2024 01", "202401_1", "é"] + for id in unusable { + #expect(!ClickHousePartStatements.isValidPartitionId(id)) + #expect( + ClickHousePartStatements.dropPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == nil + ) + #expect( + ClickHousePartStatements.detachPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == nil + ) + } + } + + // `currentDatabase()` answers with the request parameter, so the list showed another database's parts. @Test("The parts read filters on the named database, never the connection's current one") func partsReadNamesTheDatabase() { let sql = ClickHousePartStatements.parts( @@ -48,9 +83,13 @@ struct ClickHousePartStatementsTests { #expect(!sql.contains("currentDatabase()")) } - /// A connection saved with no database of its own has nothing to qualify against, and an empty - /// qualifier is a statement the server refuses. The parts read has to fall back too: comparing - /// against '' matches nothing, where ClickHouse's own default database is what the user sees. + @Test("The parts read selects the partition ID beside the displayed partition value") + func partsReadSelectsThePartitionId() { + let sql = ClickHousePartStatements.parts(database: "analytics", table: "events", escape: escape) + #expect(sql.contains("SELECT partition, partition_id, name, rows, bytes_on_disk,")) + } + + // With no database the qualifier is refused by the server, and comparing against '' matches nothing. @Test("With no database the name stays unqualified and the read asks the server") func emptyDatabaseLeavesTheNameBare() { #expect( @@ -66,13 +105,11 @@ struct ClickHousePartStatementsTests { #expect(!sql.contains("database = ''")) } - @Test("A quote in a partition value is escaped, and a backtick in a name is quoted") - func valuesAreEscaped() { + @Test("A backtick in a database or table name is quoted") + func namesAreQuoted() { let sql = ClickHousePartStatements.dropPartition( - database: "we`ird", table: "ev`ents", partition: "it's", - quote: quote, escape: escape + database: "we`ird", table: "ev`ents", partitionId: "202401", quote: quote ) - #expect(sql.contains("`we``ird`.`ev``ents`")) - #expect(sql.contains("'it\\'s'")) + #expect(sql == "ALTER TABLE `we``ird`.`ev``ents` DROP PARTITION ID '202401'") } } diff --git a/apple/TableProTests/Models/Settings/SettingsFieldDecodingTests.swift b/apple/TableProTests/Models/Settings/SettingsFieldDecodingTests.swift index 56e576b32f..6e8110f87e 100644 --- a/apple/TableProTests/Models/Settings/SettingsFieldDecodingTests.swift +++ b/apple/TableProTests/Models/Settings/SettingsFieldDecodingTests.swift @@ -66,6 +66,38 @@ struct SettingsFieldDecodingTests { #expect(decoded.rowCopyFormat == .csvWithHeaders) } + @Test("Data Grid settings saved before results kept per run keep every result, and a choice round-trips") + func resultSetsKeptPerRunDefaultsToNoLimit() throws { + let legacy = try decode(DataGridSettings.self, #"{"showRowNumbers":false}"#) + #expect(legacy.resultSetsKeptPerRun == 0) + #expect(legacy.resultSetLimitPerRun == nil) + + var chosen = DataGridSettings.default + chosen.resultSetsKeptPerRun = 50 + let decoded = try decode(DataGridSettings.self, String(decoding: try JSONEncoder().encode(chosen), as: UTF8.self)) + #expect(decoded.resultSetLimitPerRun == 50) + + #expect(try decode(DataGridSettings.self, #"{"resultSetsKeptPerRun":-3}"#).resultSetLimitPerRun == nil) + } + + @Test("Color values by type is off by default and in stored value colors without it, and a choice round-trips") + func valueColorsDefaultOff() throws { + #expect(ValueColorSettings.default.colorsValuesByType == false) + #expect(try decode(ValueColorSettings.self, "{}").colorsValuesByType == false) + #expect(try decode(ValueColorSettings.self, #"{"colorsValuesByType":"yes"}"#).colorsValuesByType == false) + + let chosen = ValueColorSettings(colorsValuesByType: true) + let encoded = String(decoding: try JSONEncoder().encode(chosen), as: UTF8.self) + #expect(try decode(ValueColorSettings.self, encoded) == chosen) + } + + // An older build rewrites the Data Grid blob with only the keys it knows. + @Test("The Data Grid settings no longer carry Color values by type") + func dataGridBlobLeavesValueColorsOut() throws { + let encoded = String(decoding: try JSONEncoder().encode(DataGridSettings.default), as: UTF8.self) + #expect(!encoded.contains("colorsValuesByType")) + } + @Test("A bad MCP value falls back alone and keeps the secure default") func mcpKeepsOtherFields() throws { let settings = try decode( diff --git a/apple/TableProTests/Models/ShortcutUniquenessTests.swift b/apple/TableProTests/Models/ShortcutUniquenessTests.swift index 56c209ebbd..0011b93bda 100644 --- a/apple/TableProTests/Models/ShortcutUniquenessTests.swift +++ b/apple/TableProTests/Models/ShortcutUniquenessTests.swift @@ -60,6 +60,7 @@ struct ShortcutUniquenessTests { .newAgentSession, .openAgentSession, .closeAgentSession, .deleteAgentSession, .newAIConversation, .showPreviousWindowTab, .showNextWindowTab, .toggleBreakpoint, + .commitTransaction, .rollbackTransaction, ] let unbound = Set(ShortcutAction.allCases.filter { KeyboardSettings.defaultShortcuts[$0] == nil }) let unlisted = unbound.subtracting(listed).map(\.rawValue).sorted() diff --git a/apple/TableProTests/Models/Sidebar/SidebarObjectKindTests.swift b/apple/TableProTests/Models/Sidebar/SidebarObjectKindTests.swift index 2915e371b1..5295c6c304 100644 --- a/apple/TableProTests/Models/Sidebar/SidebarObjectKindTests.swift +++ b/apple/TableProTests/Models/Sidebar/SidebarObjectKindTests.swift @@ -69,12 +69,82 @@ struct SidebarObjectKindTests { @Test("Every kind belongs to exactly one category") func categoriesPartitionTheKinds() { - #expect(SidebarObjectKind.allCases.filter { $0.category == .routine } == [.procedure, .function]) - #expect(SidebarObjectKind.allCases.filter { $0.category == .trigger } == [.trigger]) - #expect(SidebarObjectKind.allCases.filter { $0.category == .type } == [.type]) + #expect(SidebarObjectKind.builtIn.filter { $0.category == .routine } == [.procedure, .function]) + #expect(SidebarObjectKind.builtIn.filter { $0.category == .trigger } == [.trigger]) + #expect(SidebarObjectKind.builtIn.filter { $0.category == .type } == [.type]) #expect( - SidebarObjectKind.allCases.filter { $0.category == .table } + SidebarObjectKind.builtIn.filter { $0.category == .table } == [.table, .view, .materializedView, .foreignTable, .sequence] ) + #expect(!SidebarObjectKind.builtIn.contains { $0.category == .server }) + #expect(SidebarObjectKind.server(events).category == .server) + } + + private let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + private let extensions = SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", iconName: "puzzlepiece.extension", + scope: .database + ) + + /// UserDefaults keys, the tree expansion JSON and node ids all spell these, so they never move. + @Test("A built-in kind persists under the name it shipped with") + func builtInPersistenceNamesAreShipped() { + #expect(SidebarObjectKind.builtIn.map { $0.persistenceName } == [ + "table", "view", "materializedView", "foreignTable", "sequence", "procedure", "function", "trigger", "type", + ]) + for kind in SidebarObjectKind.builtIn { + #expect(SidebarObjectKind(persistenceName: kind.persistenceName) == kind) + } + } + + @Test("A declared kind persists by its id and decodes without the driver that declared it") + func declaredKindRoundTripsByIdAlone() { + let kind = SidebarObjectKind.server(events) + #expect(kind.persistenceName == "server.event") + let decoded = SidebarObjectKind(persistenceName: "server.event") + #expect(decoded == kind) + #expect(decoded?.hashValue == kind.hashValue) + #expect(decoded?.serverKind?.id == "event") + } + + @Test("An unknown or empty name decodes to nothing") + func unknownPersistenceNameIsSkipped() { + #expect(SidebarObjectKind(persistenceName: "dictionary") == nil) + #expect(SidebarObjectKind(persistenceName: "server.") == nil) + #expect(SidebarObjectKind(persistenceName: "") == nil) + } + + @Test("Declared kinds follow the built-ins, in the order the driver declared them") + func declaredKindsFollowBuiltIns() { + let ordered = SidebarObjectKind.ordered(declared: [events, extensions]) + #expect(Array(ordered.prefix(SidebarObjectKind.builtIn.count)) == SidebarObjectKind.builtIn) + #expect(Array(ordered.suffix(2)) == [.server(events), .server(extensions)]) + } + + @Test("A declared kind is listed only for its rows or a kept failure, never for being declared") + func declaredKindVisibility() { + let counted: [SidebarObjectKind: Int] = [.trigger: 1, .server(SidebarServerObjectKind(id: "event")): 2] + #expect( + SidebarObjectKind.visible(itemCounts: counted, declared: [events, extensions]) + == [.trigger, .server(events)] + ) + #expect( + SidebarObjectKind.visible(itemCounts: [:], keeping: [.server(extensions)], declared: [events, extensions]) + == [.server(extensions)] + ) + #expect(SidebarObjectKind.visible(itemCounts: counted).isEmpty == false) + #expect(SidebarObjectKind.visible(itemCounts: counted) == [.trigger]) + } + + @Test("A declared kind reads its names and symbol from the driver's descriptor") + func declaredKindDisplay() { + let kind = SidebarObjectKind.server(extensions) + #expect(kind.displayName == "Extension") + #expect(kind.pluralDisplayName == "Extensions") + #expect(kind.iconName == "puzzlepiece.extension") + #expect(kind.title(tableEntityName: "Collections") == "Extensions") + #expect(!kind.isExpandedByDefault) } } diff --git a/apple/TableProTests/Models/UI/MultiRowEditStateExpansionTests.swift b/apple/TableProTests/Models/UI/MultiRowEditStateExpansionTests.swift index 696e460d54..b901e76192 100644 --- a/apple/TableProTests/Models/UI/MultiRowEditStateExpansionTests.swift +++ b/apple/TableProTests/Models/UI/MultiRowEditStateExpansionTests.swift @@ -22,12 +22,12 @@ struct MultiRowEditStateExpansionTests { func expansionSurvivesSelectionChange() { let state = MultiRowEditState() configure(state, row: 0) - let firstID = state.fields[1].id + let firstID = state.fields[1].editorID state.toggleExpansion(ofColumn: 1) configure(state, row: 1) - #expect(state.fields[1].id != firstID) + #expect(state.fields[1].editorID != firstID) #expect(state.expandedColumn == 1) } diff --git a/apple/TableProTests/Models/UI/MultiRowEditStateJsonTests.swift b/apple/TableProTests/Models/UI/MultiRowEditStateJsonTests.swift index fd2de9f139..72af1a5769 100644 --- a/apple/TableProTests/Models/UI/MultiRowEditStateJsonTests.swift +++ b/apple/TableProTests/Models/UI/MultiRowEditStateJsonTests.swift @@ -46,6 +46,17 @@ struct MultiRowEditStateJsonTests { #expect(state.fields[0].pendingValue == "{\"a\":2}") } + @Test("A value stored across lines keeps the typed lines; only a one-line value is compacted") + func formattedValueKeepsItsLayout() { + let stored = "{\n \"retries\": 3,\n \"host\": \"db\"\n}" + let typed = "{\n \"retries\": 5,\n \"host\": \"db\"\n}" + let state = makeState(value: stored, type: .text(rawType: "text"), column: "config") + + state.updateField(at: 0, value: typed) + + #expect(state.fields[0].pendingValue == typed) + } + @Test("Editing back to the original content clears the pending change") func jsonRevertClearsChange() { let original = "{\"a\":1}" diff --git a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift index fdb985927d..871a96b9c0 100644 --- a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift +++ b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift @@ -29,10 +29,11 @@ struct SQLLexicalFeatureMappingTests { .sqlScriptBlocks, .cqlBatches, .hashAndAtInIdentifiers, .angleQuotedIdentifiers, .backslashEscapesInAngleQuotes, .escapeBytesPrefix, .exponentSignsNeedNoDigits, .formFeedEndsLineComments, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, - .withoutDashComments, .escapeStringContinuations, .databendTokens, + .withoutDashComments, .escapeStringContinuations, .databendTokens, .psqlScripts, .nonASCIISpacesEndNames, + .transactionShortForms, .compoundStatementBodies, ] #expect(grammarBits.isDisjoint(with: curatedOnly)) - #expect(grammarBits.union(curatedOnly).rawValue == (1 << 41) - 1) + #expect(grammarBits.union(curatedOnly).rawValue == (1 << 45) - 1) } @Test("SAP HANA's SQLScript blocks stay in the app's curated grammar and never reach the kit") @@ -78,6 +79,7 @@ struct SQLLexicalFeatureMappingTests { SQLLexicalGrammar(pluginFeatures: cockroach.pluginFeatures) == cockroach.subtracting([ .escapeBytesPrefix, .escapeStringContinuations, .atomicRoutineBodies, .bracketedStatementSources, + .psqlScripts, ]) ) } @@ -93,14 +95,16 @@ struct SQLLexicalFeatureMappingTests { @Test("A grammar survives the trip through the kit's features, less the facts only the curated table sets") func roundTrip() { let grammar = DatabaseType.postgresql.lexicalGrammar - let curatedOnly: SQLLexicalGrammar = [.atomicRoutineBodies, .escapeStringContinuations] + let curatedOnly: SQLLexicalGrammar = [.atomicRoutineBodies, .escapeStringContinuations, .psqlScripts] #expect(grammar.isSuperset(of: curatedOnly)) #expect(SQLLexicalGrammar(pluginFeatures: grammar.pluginFeatures) == grammar.subtracting(curatedOnly)) - #expect(SQLLexicalGrammar(pluginFeatures: DatabaseType.mysql.lexicalGrammar.pluginFeatures) == DatabaseType.mysql.lexicalGrammar) + let mySQL = DatabaseType.mysql.lexicalGrammar + #expect(mySQL.contains(.compoundStatementBodies)) + #expect(SQLLexicalGrammar(pluginFeatures: mySQL.pluginFeatures) == mySQL.subtracting(.compoundStatementBodies)) } /// MySQLDriverTests' MySQLLexicalFeatureListTests pins MySQLLexicalFeatures to these same sets. - @Test("The grammars the MySQL plugin keeps are the curated ones") + @Test("The grammars the MySQL plugin keeps are the curated ones, less the routine body only the app reads") func pluginGrammarsMatchTheCuratedTable() { let mySQL: SQLLexicalFeatures = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, @@ -109,7 +113,10 @@ struct SQLLexicalFeatureMappingTests { let databend: SQLLexicalFeatures = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .untaggedDollarQuotes, ] - #expect(SQLLexicalGrammar(pluginFeatures: mySQL) == DatabaseType.mysql.lexicalGrammar) + #expect( + SQLLexicalGrammar(pluginFeatures: mySQL) + == DatabaseType.mysql.lexicalGrammar.subtracting(.compoundStatementBodies) + ) let curatedDatabend = DatabaseType.databend.lexicalGrammar.subtracting([.formFeedEndsLineComments, .databendTokens]) #expect(SQLLexicalGrammar(pluginFeatures: databend) == curatedDatabend) } @@ -121,6 +128,9 @@ struct SQLLexicalFeatureMappingTests { "SELECT $$it's$$; DROP TABLE t; SELECT 'x'", "SELECT $ü$it's$ü$; DROP TABLE t; SELECT 'x'", "SELECT E'\\''; DROP TABLE t; --'", + "SELECT x$e'\\'; DROP TABLE t; --'", + "SELECT x$q'['; DROP TABLE t; --]' FROM dual", + "SELECT x#q'['; DROP TABLE t; --]' FROM dual", "SELECT 1 # '\n; DROP TABLE t; -- '", "SELECT 'a\\'; DROP TABLE t; -- '", "SELECT 1 AS [a]]'b]; DROP TABLE t; SELECT 'x'", diff --git a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift index d33f1b313b..50a341432e 100644 --- a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift +++ b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift @@ -42,12 +42,14 @@ struct RemoteSQLiteAdmissionTests { #expect(token.allSatisfy { $0.isHexDigit }) } - @Test func launcherCommandCarriesNoUserDataAndFitsTheArgumentLimit() { + @Test func launcherCommandCarriesNoUserDataAndFitsThePacketLimit() { let command = RemoteSQLiteAgentSource.launcherCommand() - #expect(command.hasPrefix("sh -c '")) + #expect(command.hasPrefix("sh -c 'b=")) #expect(command.contains(RemoteSQLiteWire.noPythonNotice)) - #expect(command.contains("base64.b64decode")) - // Well under the Linux MAX_ARG_STRLEN of 128 KiB. - #expect(command.utf8.count < 100_000) + #expect(command.contains("zlib.decompress")) + // The exec request is one SSH packet: this command plus SSH_MSG_CHANNEL_REQUEST framing. + // libssh2 rejects a payload at or above MAX_SSH_PACKET_LEN - 0x100 (34744) with -34; 256 B + // covers the framing and still leaves room to spare. + #expect(command.utf8.count + 256 < 34_744) } } diff --git a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift index 2a1b34205a..ad448c1466 100644 --- a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift +++ b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift @@ -174,9 +174,12 @@ struct RemoteSQLiteAgentPipeTests { private var buffer: [UInt8] = [] init(pythonPath: String) throws { - let encoded = Data(RemoteSQLiteAgentSource.python.utf8).base64EncodedString() + // Launch the program the way the launcher ships it: the compressed, base64 blob through + // the same bootstrap, so this suite also covers the decompression round-trip. process.executableURL = URL(fileURLWithPath: pythonPath) - process.arguments = ["-I", "-S", "-c", "import base64,sys;exec(base64.b64decode(sys.argv[1]))", encoded] + process.arguments = [ + "-I", "-S", "-c", RemoteSQLiteAgentSource.interpreterBootstrap, RemoteSQLiteAgentSource.encodedProgram, + ] process.standardInput = toAgent process.standardOutput = fromAgent process.standardError = FileHandle.nullDevice diff --git a/apple/TableProTests/Services/SchemaServiceRefreshTests.swift b/apple/TableProTests/Services/SchemaServiceRefreshTests.swift index baae278511..4a0fca0fde 100644 --- a/apple/TableProTests/Services/SchemaServiceRefreshTests.swift +++ b/apple/TableProTests/Services/SchemaServiceRefreshTests.swift @@ -139,7 +139,7 @@ struct SchemaServiceRefreshTests { } driver.resumeFetchTables() - await reload.value + _ = await reload.value #expect(service.tables(for: connectionId).map(\.name) == ["orders", "invoices"]) #expect(!service.isRefreshing(connectionId: connectionId)) @@ -182,8 +182,8 @@ struct SchemaServiceRefreshTests { } try await waitUntil { driver.tablesCallCount == 2 } driver.resumeFetchTables() - await first.value - await second.value + _ = await first.value + _ = await second.value #expect(driver.tablesCallCount == 2, "Each browse scope must run its own fetch") #expect(service.loadedScope(for: connectionId) == analytics) @@ -241,7 +241,7 @@ struct SchemaServiceRefreshTests { driver.resumeFetchTables() await waiter.value - await reload.value + _ = await reload.value #expect(service.loadedScope(for: connectionId) == secondScope) #expect(service.tables(for: connectionId).map(\.name) == ["events"]) diff --git a/apple/TableProTests/Services/SchemaServiceSideObjectsTests.swift b/apple/TableProTests/Services/SchemaServiceSideObjectsTests.swift index 8b86086c7d..c75a18f94f 100644 --- a/apple/TableProTests/Services/SchemaServiceSideObjectsTests.swift +++ b/apple/TableProTests/Services/SchemaServiceSideObjectsTests.swift @@ -23,6 +23,10 @@ final class SideObjectsMockDriver: DatabaseDriver, @unchecked Sendable { var schemasToReturn: [String] = [] var routineSchemaRequests: [String?] = [] var triggerSchemaRequests: [String?] = [] + var declaredServerKinds: [PluginServerObjectKind] = [] + var serverObjectsByKind: [String: [ServerObjectInfo]] = [:] + var serverObjectsError: Error? + var serverObjectRequests: [String] = [] init(connection: DatabaseConnection) { self.connection = connection @@ -94,6 +98,14 @@ final class SideObjectsMockDriver: DatabaseDriver, @unchecked Sendable { triggerSchemaRequests.append(schema) return triggersBySchema[schema ?? ""] ?? [] } + + var serverObjectKinds: [PluginServerObjectKind] { declaredServerKinds } + + func fetchServerObjects(ofKind kindId: String, schema: String?) async throws -> [ServerObjectInfo] { + serverObjectRequests.append("\(kindId)@\(schema ?? "")") + if let serverObjectsError { throw serverObjectsError } + return serverObjectsByKind[kindId] ?? [] + } } @MainActor @@ -316,4 +328,96 @@ struct SchemaServiceSideObjectsTests { #expect(service.routinesLoadState(for: connectionId, schema: "hr") == .idle) #expect(!service.isSchemaSettled(for: connectionId, schema: "hr")) } + + // MARK: - Server objects + + private var eventKind: PluginServerObjectKind { + PluginServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock" + ) + } + + private func event(_ name: String) -> ServerObjectInfo { + ServerObjectInfo(kindId: "event", name: name, schema: "shop", detail: "Recurring") + } + + @Test("A connection-wide load reads each declared kind beside the routines") + func loadReadsDeclaredKinds() async { + let (connectionId, connection, driver) = makeDriver() + driver.declaredServerKinds = [eventKind] + driver.serverObjectsByKind["event"] = [event("nightly")] + + let service = SchemaService() + await service.load(connectionId: connectionId, driver: driver, connection: connection) + + #expect(service.serverObjectsLoadState(for: connectionId, kindId: "event") == .loaded([event("nightly")])) + #expect(service.serverObjects(for: connectionId, kindId: "event").map { $0.name } == ["nightly"]) + #expect(driver.serverObjectRequests == ["event@"]) + } + + @Test("A driver that declares no kinds is never asked for server objects") + func undeclaredKindsAreNotFetched() async { + let (connectionId, connection, driver) = makeDriver() + + let service = SchemaService() + await service.load(connectionId: connectionId, driver: driver, connection: connection) + + #expect(driver.serverObjectRequests.isEmpty) + #expect(service.serverObjectsLoadState(for: connectionId, kindId: "event") == .idle) + } + + @Test("A server object fetch that fails is reported, and the tables stay loaded") + func failedServerObjectsAreReported() async { + let (connectionId, connection, driver) = makeDriver() + driver.declaredServerKinds = [eventKind] + driver.serverObjectsError = boom + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "orders", schema: "public")] + + let service = SchemaService() + await service.load(connectionId: connectionId, driver: driver, connection: connection) + + #expect(service.serverObjectsLoadState(for: connectionId, kindId: "event") == .failed("boom")) + #expect(service.tables(for: connectionId).map { $0.name } == ["orders"]) + } + + @Test("A failed server object reload keeps the objects already loaded") + func failedServerObjectReloadKeepsObjects() async { + let (connectionId, connection, driver) = makeDriver() + driver.declaredServerKinds = [eventKind] + driver.serverObjectsByKind["event"] = [event("nightly")] + let service = SchemaService() + await service.load(connectionId: connectionId, driver: driver, connection: connection) + + driver.serverObjectsError = boom + let reloaded = await service.reloadServerObjects(connectionId: connectionId, driver: driver, scope: nil, kindId: "event") + + #expect(!reloaded) + #expect(service.serverObjects(for: connectionId, kindId: "event").map { $0.name } == ["nightly"]) + + driver.serverObjectsError = nil + driver.serverObjectsByKind["event"] = [event("nightly"), event("hourly")] + #expect(await service.reloadServerObjects(connectionId: connectionId, driver: driver, scope: nil, kindId: "event")) + #expect(service.serverObjects(for: connectionId, kindId: "event").count == 2) + } + + @Test("Expanding a schema reads its container kinds, and a database kind is left to the database") + func schemaLoadReadsContainerKindsOnly() async { + let (connectionId, _, driver) = makeDriver() + driver.declaredServerKinds = [ + eventKind, + PluginServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", + iconName: "puzzlepiece.extension", scope: .database + ) + ] + driver.serverObjectsByKind["event"] = [event("nightly")] + let service = SchemaService() + + await service.loadSchemaObjects(schema: "hr", in: unnamedDatabase(connectionId), driver: driver) + + #expect(driver.serverObjectRequests == ["event@hr"]) + #expect(service.serverObjects(for: connectionId, schema: "hr", kindId: "event").map { $0.name } == ["nightly"]) + #expect(service.serverObjectsByKind(for: connectionId, schema: "hr").keys.sorted() == ["event"]) + #expect(service.isSchemaSettled(for: connectionId, schema: "hr")) + } } diff --git a/apple/TableProTests/Services/Updates/AppRelauncherTests.swift b/apple/TableProTests/Services/Updates/AppRelauncherTests.swift index b97f4483c7..2c15e63a13 100644 --- a/apple/TableProTests/Services/Updates/AppRelauncherTests.swift +++ b/apple/TableProTests/Services/Updates/AppRelauncherTests.swift @@ -1,5 +1,6 @@ import Foundation @testable import TablePro +import TableProTestSupport import Testing @MainActor @@ -14,6 +15,10 @@ final class AppRelauncherFixture { let suite: EphemeralDefaults var documentsReviewed = true var stoppingWorkApproved = true + var commits: [SessionTransactionTracker.Hold] = [] + var commitSucceeds = true + var commitGate: AsyncGate? + private(set) var commitCalls = 0 private(set) var workQuestions = 0 private(set) var approvedWhenQuitting: [Bool] = [] var hasPendingUpdate = false @@ -21,6 +26,8 @@ final class AppRelauncherFixture { var reopenFails = false private(set) var installs = 0 private(set) var terminations = 0 + // Runs that went through the next run loop turn rather than inside the commit's task. + private(set) var deferredRuns = 0 private(set) var reopens: [(pid: pid_t, bundleURL: URL)] = [] private(set) var relauncher: AppRelauncher! @@ -30,7 +37,12 @@ final class AppRelauncherFixture { defaults: suite.defaults, approveStoppingWork: { [unowned self] in workQuestions += 1 - return stoppingWorkApproved + return stoppingWorkApproved ? QuitPlan(commits: commits, work: []) : nil + }, + commitBeforeQuitting: { [unowned self] _ in + commitCalls += 1 + await commitGate?.wait() + return commitSucceeds }, reviewUnsavedDocuments: { [unowned self] completion in completion(documentsReviewed) }, installPendingUpdate: { [unowned self] in @@ -48,6 +60,10 @@ final class AppRelauncherFixture { relauncher.terminationWasCancelled() } }, + performOutsideTask: { [unowned self] body in + deferredRuns += 1 + body() + }, reopenAfterExit: { [unowned self] pid, url in if reopenFails { throw CocoaError(.executableLoad) } reopens.append((pid, url)) @@ -196,6 +212,101 @@ struct AppRelauncherTests { #expect(!fixture.reopenRequested) } + @Test("With nothing to commit, the relaunch quits at once") + func noCommitQuitsAtOnce() throws { + let fixture = try AppRelauncherFixture() + + fixture.relauncher.relaunch() + + #expect(fixture.commitCalls == 0) + #expect(fixture.relauncher.pendingCommit == nil) + #expect(fixture.deferredRuns == 0) + #expect(fixture.terminations == 1) + } + + // The quit a relaunch starts asks nothing, so a transaction it would end is committed first. + @Test("With an open transaction to commit, the relaunch quits only after the commit, outside its task") + func quitsOnlyAfterTheCommit() async throws { + let fixture = try AppRelauncherFixture() + let gate = AsyncGate() + fixture.commits = [Self.hold()] + fixture.commitGate = gate + + fixture.relauncher.relaunch() + let commit = try #require(fixture.relauncher.pendingCommit) + await gate.arrived(1) + + #expect(fixture.terminations == 0) + #expect(!fixture.relauncher.isRelaunchPending) + + await gate.open() + await commit.value + + #expect(fixture.commitCalls == 1) + #expect(fixture.deferredRuns == 1) + #expect(fixture.terminations == 1) + #expect(fixture.reopens.count == 1) + #expect(fixture.relauncher.pendingCommit == nil) + } + + @Test("Sparkle takes over only after the commit, outside its task") + func sparkleInstallsAfterTheCommit() async throws { + let fixture = try AppRelauncherFixture() + fixture.commits = [Self.hold()] + fixture.hasPendingUpdate = true + + fixture.relauncher.relaunch() + let commit = try #require(fixture.relauncher.pendingCommit) + await commit.value + + #expect(fixture.deferredRuns == 1) + #expect(fixture.installs == 1) + #expect(fixture.relauncher.isRelaunchPending) + } + + @Test("A commit that fails or is cancelled stops the relaunch, and a later quit stays a quit") + func failedCommitStopsTheRelaunch() async throws { + let fixture = try AppRelauncherFixture() + fixture.commits = [Self.hold()] + fixture.commitSucceeds = false + fixture.hasPendingUpdate = true + + fixture.relauncher.relaunch() + let commit = try #require(fixture.relauncher.pendingCommit) + await commit.value + + #expect(fixture.commitCalls == 1) + #expect(fixture.deferredRuns == 0) + #expect(fixture.installs == 0) + #expect(fixture.terminations == 0) + #expect(!fixture.relauncher.isRelaunchPending) + #expect(!fixture.relauncher.hasApprovedRunningWork) + #expect(fixture.relauncher.pendingCommit == nil) + } + + @Test("A second relaunch while the commit runs asks nothing again") + func relaunchDuringTheCommitIsIgnored() async throws { + let fixture = try AppRelauncherFixture() + let gate = AsyncGate() + fixture.commits = [Self.hold()] + fixture.commitGate = gate + + fixture.relauncher.relaunch() + let commit = try #require(fixture.relauncher.pendingCommit) + await gate.arrived(1) + fixture.relauncher.relaunch() + await gate.open() + await commit.value + + #expect(fixture.workQuestions == 1) + #expect(fixture.commitCalls == 1) + #expect(fixture.terminations == 1) + } + + private static func hold() -> SessionTransactionTracker.Hold { + SessionTransactionTracker.Hold(key: .init(connectionId: UUID(), lane: nil), state: .open, revision: 0) + } + @Test("The next launch reads the reopen request once") func reopenRequestIsReadOnce() throws { let fixture = try AppRelauncherFixture() diff --git a/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift b/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift index 21bf318034..393168ff3b 100644 --- a/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift +++ b/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift @@ -7,7 +7,6 @@ // anything, after treating it as untrusted input. // -import AppKit import Foundation import os @testable import TablePro @@ -20,7 +19,6 @@ struct ThemeCatalogSyncTests { @MainActor private final class Harness { let root: URL - let center = NotificationCenter() let suite: EphemeralDefaults let tracker: SyncChangeTracker let fileTrash: ThemeFileTrash @@ -47,7 +45,6 @@ struct ThemeCatalogSyncTests { catalog = ThemeCatalog( userDirectory: userDirectory ?? folders[0], bundleDirectory: nil, - notificationCenter: center, fileTrash: self.fileTrash, syncTracker: tracker ) @@ -64,7 +61,6 @@ struct ThemeCatalogSyncTests { catalog = ThemeCatalog( userDirectory: themes, bundleDirectory: nil, - notificationCenter: center, fileTrash: fileTrash, syncTracker: tracker ) @@ -94,10 +90,21 @@ struct ThemeCatalogSyncTests { ) } + nonisolated private static func valueTheme(_ id: String) throws -> ThemeDefinition { + var values = try theme(id) + values.overrides[.gridString] = try ThemeColor(validating: "#F1FA8C") + values.overrides[.gridNumber] = try ThemeColor(validating: "#BD93F9") + return values + } + nonisolated private static func category(_ id: String) -> String { ThemeCatalog.syncCategory(for: id) } + nonisolated private static func schema(of payload: Data) throws -> Int? { + try (JSONSerialization.jsonObject(with: payload) as? [String: Any])?["schema"] as? Int + } + // MARK: - Local changes @Test("Save, rename, duplicate and import each mark the theme's record dirty") @@ -125,14 +132,13 @@ struct ThemeCatalogSyncTests { #expect(harness.tombstoned.isEmpty) } - @Test("Loading the folder and reloading on activation mark nothing") + @Test("Loading the folder and reloading it mark nothing") func loadsMarkNothing() throws { let harness = try Harness() defer { harness.remove() } try harness.write(try Self.theme("user.existing")) harness.reopen() - harness.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) harness.catalog.reloadSynchronously() #expect(harness.catalog.theme(id: "user.existing") != nil) @@ -378,6 +384,37 @@ struct ThemeCatalogSyncTests { #expect(harness.catalog.theme(id: synced.id) == nil) } + @Test("A synced format 4 theme lands with its value colors and marks nothing") + func remoteFormatFourThemeKeepsItsValueColors() throws { + let harness = try Harness() + defer { harness.remove() } + let synced = try Self.valueTheme("user.values") + let payload = try ThemeFileFormat.encode(synced) + #expect(try Self.schema(of: payload) == 4) + + #expect(harness.catalog.applyRemote(category: Self.category(synced.id), data: payload) == .applied) + + #expect(harness.catalog.storedTheme(id: synced.id) == synced) + #expect(try Data(contentsOf: harness.themes.appendingPathComponent("user.values.json")) == payload) + #expect(harness.dirty.isEmpty) + } + + @Test("A synced format 3 record holding a value color is skipped, since that format cannot hold the key") + func remoteFormatThreeRecordWithAValueColorIsSkipped() throws { + let harness = try Harness() + defer { harness.remove() } + let payload = Data(##"{"schema":3,"id":"user.a","name":"a","appearance":"dark","dataGrid":{"string":"#F1FA8C"}}"##.utf8) + #expect(throws: ThemeLoadError.keysNeedNewerFormat(["dataGrid.string"], format: 4)) { + try ThemeFileFormat.decode(payload) + } + + #expect(harness.catalog.applyRemote(category: Self.category("user.a"), data: payload) == .skipped) + + #expect(try harness.fileNames().isEmpty) + #expect(harness.catalog.storedTheme(id: "user.a") == nil) + #expect(harness.dirty.isEmpty) + } + struct Rejection: CustomTestStringConvertible, Sendable { let label: String let category: String @@ -524,6 +561,24 @@ struct ThemeCatalogSyncTests { #expect(harness.catalog.syncCategories == [Self.category(night.id)]) } + @Test("A theme with a value color syncs as one format 4 record, and a theme without one as format 3") + func payloadFormatFollowsTheKeysSet() throws { + let harness = try Harness() + defer { harness.remove() } + let night = try Self.theme("user.night") + let values = try Self.valueTheme("user.values") + try harness.catalog.save(night) + try harness.catalog.save(values) + + #expect(harness.dirty == [Self.category(night.id), Self.category(values.id)]) + #expect(harness.catalog.syncCategories == [Self.category(night.id), Self.category(values.id)]) + let plain = try #require(harness.catalog.syncPayload(forCategory: Self.category(night.id))) + #expect(try Self.schema(of: plain) == 3) + let colored = try #require(harness.catalog.syncPayload(forCategory: Self.category(values.id))) + #expect(try Self.schema(of: colored) == 4) + #expect(try ThemeFileFormat.decode(colored) == values) + } + // MARK: - Names @Test("Two themes with one name show apart in the list, while their files and records keep the name") @@ -586,7 +641,6 @@ struct ThemeCatalogSyncTests { let catalog = ThemeCatalog( userDirectory: folder, bundleDirectory: nil, - notificationCenter: NotificationCenter(), syncTracker: SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)) ) diff --git a/apple/TableProTests/Theme/ThemeCatalogTests.swift b/apple/TableProTests/Theme/ThemeCatalogTests.swift index 7ee89d9389..02cf1822f0 100644 --- a/apple/TableProTests/Theme/ThemeCatalogTests.swift +++ b/apple/TableProTests/Theme/ThemeCatalogTests.swift @@ -8,9 +8,10 @@ // A file removed outside TablePro is a conflict too: a save must never bring it back. // -import AppKit +import Combine import Foundation @testable import TablePro +import TableProTestSupport import TableProTheme import Testing @@ -18,7 +19,6 @@ import Testing struct ThemeCatalogTests { private struct Sandbox { let root: URL - let center = NotificationCenter() var themes: URL { root.appendingPathComponent("Themes", isDirectory: true) } var incoming: URL { root.appendingPathComponent("Incoming", isDirectory: true) } var trash: URL { root.appendingPathComponent("Trash", isDirectory: true) } @@ -31,7 +31,7 @@ struct ThemeCatalogTests { } } - /// A trash inside the sandbox, so a test never puts a file in the user's own Trash. + // A trash inside the sandbox, so a test never puts a file in the user's own Trash. var fileTrash: ThemeFileTrash { let trash = trash return ThemeFileTrash { url in @@ -43,7 +43,7 @@ struct ThemeCatalogTests { @MainActor func catalog() -> ThemeCatalog { - ThemeCatalog(userDirectory: themes, bundleDirectory: nil, notificationCenter: center, fileTrash: fileTrash) + ThemeCatalog(userDirectory: themes, bundleDirectory: nil, fileTrash: fileTrash) } @discardableResult @@ -133,8 +133,7 @@ struct ThemeCatalogTests { // MARK: - Deleted on disk - /// The file is gone, so writing it again would bring back a theme deleted in Finder or on - /// another Mac. + // Writing the file again would bring back a theme deleted in Finder or on another Mac. @Test("A save of a theme whose file was deleted outside TablePro is refused and recreates nothing") func saveRefusesAFileDeletedOnDisk() throws { let sandbox = try Sandbox() @@ -154,7 +153,7 @@ struct ThemeCatalogTests { #expect(try sandbox.fileNames().isEmpty) } - @Test("After an activation reload drops a deleted theme, a stale save is still refused") + @Test("After a reload drops a deleted theme, a stale save is still refused") func saveAfterReloadRefusesADeletedTheme() throws { let sandbox = try Sandbox() defer { sandbox.remove() } @@ -163,15 +162,14 @@ struct ThemeCatalogTests { let stale = try #require(catalog.theme(id: "user.mine")) try sandbox.delete("Mine.json") - sandbox.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) + catalog.reloadSynchronously() #expect(catalog.theme(id: "user.mine") == nil) #expect(throws: ThemeLoadError.deletedOnDisk("Mine.json")) { try catalog.save(stale) } #expect(try sandbox.fileNames().isEmpty) } - /// A file edited into one that no longer loads keeps its name, so a stale save would otherwise - /// write `-2.json` and leave two files with one id once the first is fixed. + // A stale save would write `-2.json` beside the broken file, leaving two files with one id once it is fixed. @Test("A save of a theme whose file no longer loads is refused, and writes no second file") func saveRefusesAFileThatNoLongerLoads() throws { let sandbox = try Sandbox() @@ -201,22 +199,60 @@ struct ThemeCatalogTests { #expect(catalog.theme(id: "user.mine") == nil) } - // MARK: - Activation + // MARK: - Watching the folder - @Test("The catalog reloads when TablePro becomes active") - func reloadsOnActivation() throws { + @Test("A watching catalog picks up a theme dropped into its folder, edited there and removed from it") + func picksUpChangesInItsFolder() async throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let catalog = sandbox.catalog() + catalog.startWatchingUserDirectory() + defer { catalog.stopWatchingUserDirectory() } + + try sandbox.write(try Self.userTheme(id: "user.dropped", name: "Dropped"), as: "Dropped.json") + try await waitUntil { catalog.theme(id: "user.dropped")?.name == "Dropped" } + + try sandbox.write(try Self.userTheme(id: "user.dropped", name: "Edited"), as: "Dropped.json") + try await waitUntil { catalog.theme(id: "user.dropped")?.name == "Edited" } + + try sandbox.delete("Dropped.json") + try await waitUntil { catalog.theme(id: "user.dropped") == nil } + } + + @Test("A catalog that is not watching keeps its list until it reloads") + func unwatchedCatalogWaitsForAReload() throws { let sandbox = try Sandbox() defer { sandbox.remove() } let catalog = sandbox.catalog() - #expect(catalog.theme(id: "user.dropped") == nil) try sandbox.write(try Self.userTheme(id: "user.dropped", name: "Dropped"), as: "Dropped.json") #expect(catalog.theme(id: "user.dropped") == nil) - sandbox.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) + catalog.reloadSynchronously() #expect(catalog.theme(id: "user.dropped")?.name == "Dropped") } + // The catalog's own save reports a change in its folder too. + @Test("A reload after the catalog's own save publishes nothing") + func reloadAfterOwnSavePublishesNothing() throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let catalog = sandbox.catalog() + var published = 0 + let subscription = catalog.$themes.dropFirst().sink { _ in published += 1 } + defer { subscription.cancel() } + + try catalog.save(try Self.userTheme(id: "user.saved", name: "Saved", overrides: [.editorBackground: "#112233"])) + #expect(published == 1) + let rejected = catalog.rejected + + catalog.reloadSynchronously() + + #expect(published == 1) + #expect(catalog.rejected == rejected) + #expect(catalog.storedTheme(id: "user.saved")?.overrides[.editorBackground]?.hex == "#112233") + } + // MARK: - Trash @Test("Move to Trash moves the file it was loaded from, and restore brings it back") @@ -430,6 +466,8 @@ struct ThemeCatalogTests { let saved = try sandbox.read("legacy.json") #expect(saved.name == String(repeating: "N", count: cap)) #expect(saved.author == "Someone on two lines") + let savedRoot = try JSONSerialization.jsonObject(with: try Data(contentsOf: url)) as? [String: Any] + #expect(savedRoot?["schema"] as? Int == 3) let picked = sandbox.incoming.appendingPathComponent("legacy.json") try bytes.write(to: picked) @@ -446,7 +484,36 @@ struct ThemeCatalogTests { let catalog = sandbox.catalog() #expect(catalog.theme(id: "user.old") == nil) - #expect(catalog.rejected.first?.reason == ThemeLoadError.schemaTooOld(found: 2, supported: 3).localizedDescription) + #expect( + catalog.rejected.first?.reason + == ThemeLoadError.schemaTooOld(found: 2, supported: ThemeSchema.oldestReadable).localizedDescription + ) + } + + @Test("A format 4 file loads its value colors, and a format 3 file holding one is listed with the key") + func valueColorsNeedFormatFour() throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let current: [String: Any] = [ + "schema": 4, "id": "user.current", "name": "Current", "appearance": "dark", + "dataGrid": ["text": "#F8F8F2", "string": "#F1FA8C"], + ] + let mislabeled: [String: Any] = [ + "schema": 3, "id": "user.mislabeled", "name": "Mislabeled", "appearance": "dark", + "dataGrid": ["text": "#F8F8F2", "string": "#F1FA8C"], + ] + try JSONSerialization.data(withJSONObject: current).write(to: sandbox.themes.appendingPathComponent("current.json")) + try JSONSerialization.data(withJSONObject: mislabeled).write(to: sandbox.themes.appendingPathComponent("mislabeled.json")) + + let catalog = sandbox.catalog() + + #expect(catalog.storedTheme(id: "user.current")?.overrides[.gridString]?.hex == "#F1FA8C") + #expect(catalog.theme(id: "user.mislabeled") == nil) + #expect(catalog.rejected.map(\.fileName) == ["mislabeled.json"]) + #expect( + catalog.rejected.first?.reason + == ThemeLoadError.keysNeedNewerFormat(["dataGrid.string"], format: 4).localizedDescription + ) } @Test("The bundled themes load after the two defaults, in the shared order") @@ -456,7 +523,7 @@ struct ThemeCatalogTests { try sandbox.write(try Self.userTheme(id: "user.aardvark", name: "Aardvark"), as: "aardvark.json") let bundle = try #require(Bundle.main.resourceURL) - let catalog = ThemeCatalog(userDirectory: sandbox.themes, bundleDirectory: bundle, notificationCenter: sandbox.center) + let catalog = ThemeCatalog(userDirectory: sandbox.themes, bundleDirectory: bundle) let ids = catalog.themes.map(\.id) #expect(Array(ids.prefix(2)) == [BuiltInThemes.defaultLightId, BuiltInThemes.defaultDarkId]) @@ -466,6 +533,32 @@ struct ThemeCatalogTests { #expect(catalog.rejected.isEmpty) } + @Test("Bundled themes are read once: a reload keeps them as first read, whatever their files hold now") + func bundledThemesAreReadOnce() throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let bundle = sandbox.root.appendingPathComponent("Bundle", isDirectory: true) + try FileManager.default.createDirectory(at: bundle, withIntermediateDirectories: true) + let original = try Self.userTheme(id: "tablepro.fixture", name: "Fixture", overrides: [.editorBackground: "#102030"]) + let file = try sandbox.write(original, as: "tablepro.fixture.json", in: bundle) + + let catalog = ThemeCatalog(userDirectory: sandbox.themes, bundleDirectory: bundle, fileTrash: sandbox.fileTrash) + #expect(catalog.theme(id: "tablepro.fixture") == original) + + try sandbox.write( + try Self.userTheme(id: "tablepro.fixture", name: "Changed", overrides: [.editorBackground: "#FFFFFF"]), + as: "tablepro.fixture.json", + in: bundle + ) + catalog.reloadSynchronously() + #expect(catalog.theme(id: "tablepro.fixture") == original) + + try FileManager.default.removeItem(at: file) + catalog.reloadSynchronously() + #expect(catalog.theme(id: "tablepro.fixture") == original) + #expect(catalog.rejected.isEmpty) + } + // MARK: - Duplicate @Test("Duplicate gives a new user id and a Finder-style name") @@ -568,8 +661,7 @@ struct ThemeCatalogTests { #expect(catalog.theme(id: BuiltInThemes.defaultDarkId) == BuiltInThemes.dark) } - /// The id in a file says nothing about which theme on this Mac it is: the same file imported - /// twice, or on two Macs, would otherwise collide in sync. + // The same file imported twice, or on two Macs, would otherwise collide in sync. @Test("Import mints a new id even when the file's id is free, and keeps a free name") func importMintsAFreshId() throws { let sandbox = try Sandbox() diff --git a/apple/TableProTests/Theme/ThemeContrastTests.swift b/apple/TableProTests/Theme/ThemeContrastTests.swift index b016c80629..494bcf56a4 100644 --- a/apple/TableProTests/Theme/ThemeContrastTests.swift +++ b/apple/TableProTests/Theme/ThemeContrastTests.swift @@ -1,13 +1,3 @@ -// -// ThemeContrastTests.swift -// TableProTests -// -// Sidebar sections, status lines and empty states put the system's secondary label on the window -// background, so a bundled theme that sets one has to keep that label legible. The system's own -// secondary label sits under 4.5:1 on every light window (3.95:1 on white), so a light theme is held -// to the WCAG floor for secondary text, 3:1. -// - import AppKit import Foundation import SwiftUI @@ -23,9 +13,10 @@ struct ThemeContrastTests { .map { try ThemeFileFormat.decode(try Data(contentsOf: $0)) } } + // The system's secondary label is 3.95:1 on white, under 4.5:1 on every light window: WCAG's 3:1 secondary floor. private static let lightSecondaryContrast: CGFloat = 3 - /// `secondaryLabelColor` is translucent, so what the eye reads is the label over the fill. + // secondaryLabelColor is translucent, so what the eye reads is the label over the fill. private static func composite(_ foreground: NSColor, over background: NSColor) -> NSColor { let alpha = foreground.alphaComponent return NSColor( @@ -63,4 +54,126 @@ struct ThemeContrastTests { #expect(ratio >= bar, "\(theme.id): \(ratio), bar \(bar)") } } + + // MARK: - Value colors + + private static let valueSlots = ThemeSlotSection.values.slots.filter { $0 != .gridNullValue } + + // Alternate rows are on by default, and a modified cell or an inserted row washes either row. + private static func rowSurfaces(of theme: ThemeDefinition) throws -> [ThemeColor] { + let background = try #require(theme.overrides[.gridBackground], "\(theme.id)") + let stripe = try #require(theme.value(for: .gridAlternateRow).themedColor, "\(theme.id)") + let rows = [background, stripe.composited(over: background)] + let modified = try #require(theme.overrides[.gridModified], "\(theme.id)") + let inserted = try #require(theme.overrides[.gridInserted], "\(theme.id)") + return rows + [modified, inserted].flatMap { wash in rows.map { wash.composited(over: $0) } } + } + + private static func lowestRatio(of color: ThemeColor, on surfaces: [ThemeColor]) -> Double { + surfaces.map { color.contrastRatio(with: $0) }.min() ?? 0 + } + + @Test("Every bundled value color reaches 4.5:1 on both rows and on the modified and inserted washes over them") + func bundledValueColorsOnEveryRowSurface() throws { + let themes = try Self.bundledThemes() + #expect(themes.count >= 14) + #expect(Self.valueSlots.count == 9) + + for theme in themes { + let surfaces = try Self.rowSurfaces(of: theme) + for slot in Self.valueSlots { + let color = try #require(theme.overrides[slot], "\(theme.id) sets no \(slot.rawValue)") + let ratio = Self.lowestRatio(of: color, on: surfaces) + #expect(ratio >= ThemeContrastNote.minimumRatio, "\(theme.id) \(slot.rawValue) \(color.hex): \(ratio)") + } + #expect(ThemeContrastNote.message(for: .values, in: theme) == nil, "\(theme.id)") + } + } + + @Test("No two value colors in a bundled theme are the same") + func bundledValueColorsAreDistinct() throws { + for theme in try Self.bundledThemes() { + let colors = Self.valueSlots.compactMap { theme.overrides[$0] } + let hexes = colors.map { $0.hex } + #expect(colors.count == Self.valueSlots.count, "\(theme.id)") + #expect(Set(colors).count == colors.count, "\(theme.id): \(hexes)") + } + } + + private static let valuesTheme = ThemeDefinition( + id: "user.aaaa0004", + name: "Ledger", + author: "", + appearance: .dark, + overrides: [ + .gridBackground: ThemeColor(red: 0x28, green: 0x2A, blue: 0x36), + .gridAlternateRow: ThemeColor(red: 0x21, green: 0x22, blue: 0x2C), + .gridModified: ThemeColor(red: 0xFF, green: 0xB8, blue: 0x6C, alpha: 0x26), + .gridInserted: ThemeColor(red: 0x50, green: 0xFA, blue: 0x7B, alpha: 0x26), + .gridText: ThemeColor(red: 0xF8, green: 0xF8, blue: 0xF2), + .gridBoolTrue: ThemeColor(red: 0x50, green: 0xFA, blue: 0x7B), + .gridNumber: ThemeColor(red: 0xD6, green: 0xBC, blue: 0xFF), + ] + ) + + @Test("The values note names the value color with the lowest contrast") + func valuesNoteNamesTheLowest() throws { + var theme = Self.valuesTheme + #expect(ThemeContrastNote.message(for: .values, in: theme) == nil) + + theme.overrides[.gridJSON] = ThemeColor(red: 0x6B, green: 0x5A, blue: 0x30) + theme.overrides[.gridBoolFalse] = ThemeColor(red: 0x80, green: 0x30, blue: 0x30) + let surfaces = try Self.rowSurfaces(of: theme) + let falseRatio = Self.lowestRatio(of: try #require(theme.overrides[.gridBoolFalse]), on: surfaces) + let jsonRatio = Self.lowestRatio(of: try #require(theme.overrides[.gridJSON]), on: surfaces) + #expect(falseRatio < jsonRatio) + #expect(jsonRatio < ThemeContrastNote.minimumRatio) + + let lowest = try #require(ThemeContrastNote.lowest(for: .values, in: theme)) + #expect(lowest.slot == .gridBoolFalse) + #expect(lowest.ratio == falseRatio) + + let formatted = falseRatio.formatted(.number.precision(.fractionLength(1))) + #expect( + ThemeContrastNote.message(for: .values, in: theme) + == "Bool False contrast is \(formatted):1. Aim for at least 4.5:1." + ) + } + + @Test( + "The values note checks the alternate row and the modified and inserted washes over both rows", + arguments: [ + (ThemeSlot.gridAlternateRow, ThemeColor(red: 0x5A, green: 0x5D, blue: 0x70)), + (ThemeSlot.gridModified, ThemeColor(red: 0xFF, green: 0xB8, blue: 0x6C, alpha: 0x80)), + (ThemeSlot.gridInserted, ThemeColor(red: 0x50, green: 0xFA, blue: 0x7B, alpha: 0x80)), + ] + ) + func valuesNoteChecksEveryRowSurface(surface: ThemeSlot, color: ThemeColor) throws { + var theme = Self.valuesTheme + theme.overrides[surface] = color + let background = try #require(theme.overrides[.gridBackground]) + let number = try #require(theme.overrides[.gridNumber]) + let expected = Self.lowestRatio(of: number, on: try Self.rowSurfaces(of: theme)) + #expect(number.contrastRatio(with: background) >= ThemeContrastNote.minimumRatio) + + let lowest = try #require(ThemeContrastNote.lowest(for: .values, in: theme)) + #expect(lowest.slot == .gridNumber) + #expect(lowest.ratio == expected) + #expect(lowest.ratio < ThemeContrastNote.minimumRatio) + } + + @Test("The values note skips NULL and the value colors that follow the grid text") + func valuesNoteSkipsNullAndUnsetColors() { + var theme = Self.valuesTheme + theme.overrides[.gridNullValue] = ThemeColor(red: 0x30, green: 0x32, blue: 0x3E) + #expect(ThemeContrastNote.message(for: .values, in: theme) == nil) + + theme.overrides[.gridText] = ThemeColor(red: 0x44, green: 0x47, blue: 0x5A) + #expect(ThemeContrastNote.message(for: .values, in: theme) == nil) + #expect(ThemeContrastNote.message(for: .dataGrid, in: theme)?.hasPrefix("Text contrast is ") == true) + + theme.overrides[.gridBoolTrue] = nil + theme.overrides[.gridNumber] = nil + #expect(ThemeContrastNote.lowest(for: .values, in: theme) == nil) + } } diff --git a/apple/TableProTests/Theme/ThemeEngineTests.swift b/apple/TableProTests/Theme/ThemeEngineTests.swift index 623bbb49bb..c029caca69 100644 --- a/apple/TableProTests/Theme/ThemeEngineTests.swift +++ b/apple/TableProTests/Theme/ThemeEngineTests.swift @@ -8,7 +8,6 @@ // change subject, so nothing the app draws with moves. // -import AppKit import Combine import Foundation @testable import TablePro @@ -21,7 +20,6 @@ struct ThemeEngineTests { @MainActor private final class Harness { let root: URL - let center = NotificationCenter() let changes = PassthroughSubject() let catalog: ThemeCatalog let engine: ThemeEngine @@ -44,7 +42,6 @@ struct ThemeEngineTests { catalog = ThemeCatalog( userDirectory: folder, bundleDirectory: nil, - notificationCenter: center, fileTrash: ThemeFileTrash { url in let destination = trash.appendingPathComponent(url.lastPathComponent) try FileManager.default.moveItem(at: url, to: destination) @@ -113,17 +110,18 @@ struct ThemeEngineTests { try await waitUntil { harness.engine.pair.dark == night } } - @Test("A file changed on disk repaints once TablePro becomes active") - func activationReloadRepaints() async throws { + @Test("A file changed on disk repaints once the watching catalog sees the change") + func diskChangeRepaints() async throws { let night = try Self.theme("user.night", .dark, background: "#101010") let harness = try Harness(themes: [night]) defer { harness.remove() } harness.engine.select(lightThemeId: BuiltInThemes.defaultLightId, darkThemeId: night.id) + harness.catalog.startWatchingUserDirectory() + defer { harness.catalog.stopWatchingUserDirectory() } var fixed = night fixed.overrides[.editorBackground] = try ThemeColor(validating: "#303030") try ThemeFileFormat.encode(fixed).write(to: harness.themes.appendingPathComponent("\(night.id).json")) - harness.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) try await waitUntil { harness.engine.pair.dark == fixed } } diff --git a/apple/TableProTests/Utilities/RowSortComparatorTests.swift b/apple/TableProTests/Utilities/RowSortComparatorTests.swift index 3283bce86d..e73ab8c16e 100644 --- a/apple/TableProTests/Utilities/RowSortComparatorTests.swift +++ b/apple/TableProTests/Utilities/RowSortComparatorTests.swift @@ -57,6 +57,35 @@ struct RowSortComparatorTests { #expect(result == .orderedSame) } + @Test("A boolean column holding integers orders them as numbers") + func booleanColumnHoldingIntegers() { + let tinyint = ColumnTypeClassifier(family: .mysql).classify(rawTypeName: "TINYINT(1)") + #expect(tinyint.isBooleanType) + #expect(RowSortComparator.compare("-2", "-1", columnType: tinyint) == .orderedAscending) + #expect(RowSortComparator.compare("10", "9", columnType: tinyint) == .orderedDescending) + #expect(RowSortComparator.compare("-128", "127", columnType: tinyint) == .orderedAscending) + #expect(RowSortComparator.compare("0", "1", columnType: tinyint) == .orderedAscending) + #expect(RowSortComparator.compare("1", "1", columnType: tinyint) == .orderedSame) + } + + @Test("A boolean column holding true and false orders false first, whatever the spelling") + func booleanColumnHoldingTruthValues() { + let boolean = ColumnType.boolean(rawType: "BOOLEAN") + #expect(RowSortComparator.compare("false", "true", columnType: boolean) == .orderedAscending) + #expect(RowSortComparator.compare("TRUE", "false", columnType: boolean) == .orderedDescending) + #expect(RowSortComparator.compare("t", "f", columnType: boolean) == .orderedDescending) + #expect(RowSortComparator.compare("t", "true", columnType: boolean) == .orderedSame) + } + + @Test("A boolean column mixing integers and true or false orders them one way") + func booleanColumnMixingIntegersAndTruthValues() { + let boolean = ColumnType.boolean(rawType: "BOOLEAN") + #expect(RowSortComparator.compare("1", "2", columnType: boolean) == .orderedAscending) + #expect(RowSortComparator.compare("2", "true", columnType: boolean) == .orderedDescending) + #expect(RowSortComparator.compare("true", "1", columnType: boolean) == .orderedSame) + #expect(RowSortComparator.compare("false", "-1", columnType: boolean) == .orderedDescending) + } + @Test("text column uses numeric string comparison") func textColumn() { let result = RowSortComparator.compare("file2", "file10", columnType: .text(rawType: "VARCHAR")) diff --git a/apple/TableProTests/ViewModels/AIChatAlwaysAllowTests.swift b/apple/TableProTests/ViewModels/AIChatAlwaysAllowTests.swift new file mode 100644 index 0000000000..edf161e38b --- /dev/null +++ b/apple/TableProTests/ViewModels/AIChatAlwaysAllowTests.swift @@ -0,0 +1,149 @@ +// +// AIChatAlwaysAllowTests.swift +// TableProTests +// + +import Foundation +import TableProMCP +import TableProSyncTransport +@testable import TablePro +import Testing + +@MainActor +struct AIChatAlwaysAllowTests { + private struct WriterTool: ChatTool { + let name = "writer" + let description = "" + let inputSchema: JsonValue = .object(["type": .string("object")]) + let mode: ChatToolMode = .write + + func execute(input: JsonValue, context: ChatToolContext) async throws -> ChatToolResult { + ChatToolResult(content: "1 row changed", isError: false) + } + } + + private let pool = EphemeralDefaultsPool() + private let storage: ConnectionStorage + private let registry: ChatToolRegistry + + init() throws { + let fileURL = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("always-allow-\(UUID().uuidString).json") + try FileManager.default.createDirectory( + at: fileURL.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + let tracker = SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: try pool.make().defaults)) + storage = ConnectionStorage( + fileURL: fileURL, + userDefaults: try pool.make().defaults, + syncTracker: tracker, + appEvents: AppEvents() + ) + registry = ChatToolRegistry() + registry.register(WriterTool()) + } + + private func viewModel(holding connection: DatabaseConnection) -> AIChatViewModel { + let viewModel = AIChatViewModel(services: services()) + viewModel.connection = connection + return viewModel + } + + private func stored(alwaysAllowed tools: Set) throws -> DatabaseConnection { + let connection = DatabaseConnection(name: "Staging", safeModeLevel: .alert, aiAlwaysAllowedTools: tools) + try #require(storage.saveConnections([connection])) + return connection + } + + @Test("A tool forgotten in storage asks again while the pane stays open") + func forgottenToolAsksAgain() throws { + var paneCopy = try stored(alwaysAllowed: []) + paneCopy.aiAlwaysAllowedTools = ["writer"] + + let state = viewModel(holding: paneCopy).computeInitialApprovalState(for: "writer", registry: registry) + + #expect(state == .pending) + } + + @Test("A grant that reached storage after the pane opened skips the card") + func grantInStorageSkipsTheCard() throws { + var paneCopy = try stored(alwaysAllowed: ["writer"]) + paneCopy.aiAlwaysAllowedTools = [] + + let state = viewModel(holding: paneCopy).computeInitialApprovalState(for: "writer", registry: registry) + + #expect(state == .approved) + } + + @Test("A connection with no stored record keeps the pane's own grants") + func unsavedConnectionUsesThePaneCopy() { + let unsaved = DatabaseConnection(name: "Scratch", safeModeLevel: .alert, aiAlwaysAllowedTools: ["writer"]) + + let state = viewModel(holding: unsaved).computeInitialApprovalState(for: "writer", registry: registry) + + #expect(state == .approved) + } + + @Test("Read-Only refuses a granted write tool, and names the level that allows it") + func readOnlyBeatsAGrant() { + let connection = DatabaseConnection(name: "Prod", safeModeLevel: .readOnly, aiAlwaysAllowedTools: ["writer"]) + + let state = viewModel(holding: connection).computeInitialApprovalState(for: "writer", registry: registry) + + guard case .denied(let reason) = state else { + Issue.record("Read-Only must deny a granted write tool") + return + } + #expect(reason.contains(SafeModeLevel.alert.displayName)) + } + + @Test("Always Allow after a revoke is stored again") + func alwaysAllowAfterARevokeIsStored() throws { + var paneCopy = try stored(alwaysAllowed: []) + paneCopy.aiAlwaysAllowedTools = ["writer"] + + viewModel(holding: paneCopy).persistAlwaysAllowed(toolName: "writer") + + #expect(storage.loadConnection(id: paneCopy.id)?.aiAlwaysAllowedTools == ["writer"]) + } + + private func services() -> AppServices { + let live = AppServices.live + return AppServices( + appEvents: live.appEvents, + appSettings: live.appSettings, + appSettingsStorage: live.appSettingsStorage, + connectionStorage: storage, + databaseManager: live.databaseManager, + pluginManager: live.pluginManager, + schemaService: live.schemaService, + schemaRefreshService: live.schemaRefreshService, + schemaProviderRegistry: live.schemaProviderRegistry, + catalogChangeService: live.catalogChangeService, + sqlFavoriteManager: live.sqlFavoriteManager, + favoriteTablesStorage: live.favoriteTablesStorage, + favoriteDatabasesStorage: live.favoriteDatabasesStorage, + tableFolderStorage: live.tableFolderStorage, + aiChatStorage: live.aiChatStorage, + aiKeyStorage: live.aiKeyStorage, + aiAccessApprovals: live.aiAccessApprovals, + groupStorage: live.groupStorage, + tagStorage: live.tagStorage, + sshProfileStorage: live.sshProfileStorage, + credentialProfileStorage: live.credentialProfileStorage, + featureAccess: live.featureAccess, + syncMetadataStorage: live.syncMetadataStorage, + favoritesExpansionState: live.favoritesExpansionState, + linkedFolderWatcher: live.linkedFolderWatcher, + queryHistoryManager: live.queryHistoryManager, + dateFormattingService: live.dateFormattingService, + copilotService: live.copilotService, + mcpServerManager: live.mcpServerManager, + syncTracker: live.syncTracker, + welcomeRouter: live.welcomeRouter, + telemetry: live.telemetry + ) + } +} diff --git a/apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift b/apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift new file mode 100644 index 0000000000..7b9a4447fb --- /dev/null +++ b/apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift @@ -0,0 +1,448 @@ +// +// AIChatSessionReleaseTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +@MainActor +struct AIChatSessionReleaseTests { + private let storage = AIChatStorage.shared + + private func makeStore() -> AgentSessionStore { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("AIChatSessionReleaseTests-\(UUID().uuidString)", isDirectory: true) + return AgentSessionStore(directory: directory) + } + + private static func turns(_ texts: String...) -> [ChatTurn] { + texts.enumerated().map { offset, text in + ChatTurn(role: offset.isMultiple(of: 2) ? .user : .assistant, blocks: [.text(text)]) + } + } + + private static func turns(count: Int) -> [ChatTurn] { + (0.. AIConversation { + var conversation = AIConversation( + messages: [ChatTurnWire(role: .user, blocks: [.text(text)])], + connectionId: connectionId + ) + conversation.updateTitle() + return conversation + } + + // Sinks deliver through `RunLoop.main.perform` and answer on the main actor; both have run once this returns. + private static func afterQueuedMainWork() async { + await withCheckedContinuation { continuation in + RunLoop.main.perform { continuation.resume() } + } + await withCheckedContinuation { continuation in + DispatchQueue.main.async { continuation.resume() } + } + } + + // MARK: - Release + + @Test("A stopped session drops what it holds and keeps its title, draft and conversation") + func stoppedSessionHoldsNothing() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let store = makeStore() + let registry = AgentSessionRegistry(store: store, services: fixture.services) + let session = registry.startSession(for: connection.id) + let vm = session.viewModel + vm.connection = connection + vm.trackTableMentions() + await vm.tableMentionBuild?.value + await Self.afterQueuedMainWork() + vm.messages = Self.turns("Which orders shipped late?", "Twelve.") + vm.persistCurrentConversation() + vm.columnsByTable["orders"] = [TestFixtures.makeColumnInfo()] + vm.foreignKeysByTable["orders"] = [] + vm.queryResults = "12 rows" + vm.inputText = "And yesterday?" + let conversationId = try #require(vm.activeConversationID) + try await waitUntil { session.title == "Which orders shipped late?" } + + registry.stopSession(id: session.id) + await Self.afterQueuedMainWork() + + #expect(vm.messages.isEmpty) + #expect(vm.conversations.isEmpty) + #expect(vm.columnsByTable.isEmpty) + #expect(vm.foreignKeysByTable.isEmpty) + #expect(vm.queryResults == nil) + #expect(vm.tableMentions.names.isEmpty) + #expect(vm.tableMentionSource.isEmpty) + #expect(vm.tableMentionObservation == nil) + #expect(!vm.hasRestoredConversation) + #expect(vm.inputText == "And yesterday?") + #expect(session.conversationId == conversationId) + #expect(session.title == "Which orders shipped late?") + + // Holding nothing, the session has nothing to write over the file on disk. + await vm.lastConversationWrite?.value + registry.persistNow() + let stored = await storage.load(id: conversationId) + #expect(stored?.messages.count == 2) + #expect(store.load().first?.conversationId == conversationId) + + await storage.delete(conversationId) + } + + // The restore waits for the writes the stop booked, so it reads the transcript the stop wrote. + @Test("Reopening a stopped session reads its transcript back from disk") + func reopeningRestoresTheTranscript() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let registry = AgentSessionRegistry(store: makeStore(), services: fixture.services) + let session = registry.startSession(for: connection.id) + let vm = session.viewModel + vm.connection = connection + vm.messages = Self.turns("Which orders shipped late?", "Twelve.") + vm.persistCurrentConversation() + let conversationId = try #require(vm.activeConversationID) + registry.stopSession(id: session.id) + + session.resume() + registry.setDisplayedSession(session.id, for: connection.id) + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + let texts = vm.messages.map { $0.plainText } + #expect(texts == ["Which orders shipped late?", "Twelve."]) + #expect(vm.activeConversationID == conversationId) + #expect(vm.conversations.contains { $0.id == conversationId }) + #expect(vm.tableMentionObservation != nil) + await vm.tableMentionBuild?.value + #expect(vm.tableMentions.names == ["orders"]) + + await storage.delete(conversationId) + } + + @Test("A restore that lands after a release is dropped") + func restoreAfterReleaseIsDropped() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let conversation = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(conversation) + let vm = AIChatViewModel(services: fixture.services, restoringConversation: conversation.id) + vm.connection = connection + + vm.restoreConversationsIfNeeded() + let load = try #require(vm.transcriptLoad) + vm.releaseLoadedState() + await load.value + + #expect(vm.messages.isEmpty) + #expect(vm.conversations.isEmpty) + #expect(vm.activeConversationID == nil) + #expect(vm.conversationToRestore == conversation.id) + + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + #expect(vm.activeConversationID == conversation.id) + #expect(vm.messages.map { $0.plainText } == ["Earlier work"]) + + await storage.delete(conversation.id) + } + + // An engine change on an unread session derived an empty title and renamed it New Session. + @Test("A session restored at launch keeps its title until its transcript is read") + func unopenedSessionKeepsItsTitle() async throws { + let store = makeStore() + let sessionId = UUID() + store.save([ + AgentSessionRecord( + id: sessionId, + connectionId: UUID(), + conversationId: UUID(), + status: .stopped, + title: "Earlier work", + startedAt: Date(timeIntervalSince1970: 1_000), + lastActiveAt: Date(timeIntervalSince1970: 2_000) + ) + ]) + let registry = AgentSessionRegistry(store: store) + let session = try #require(registry.session(id: sessionId)) + + session.viewModel.queryResults = nil + await Self.afterQueuedMainWork() + + #expect(session.title == "Earlier work") + } + + // MARK: - History list + + @Test("The history list holds no transcripts after a restore or a save") + func conversationsHoldSummaries() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + #expect(vm.conversations.contains { $0.id == earlier.id && $0.title == "Earlier work" }) + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + + vm.messages = Self.turns("Which orders shipped late?", "Twelve.") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + + #expect(vm.conversations.first?.id == current) + #expect(vm.conversations.first?.title == "Which orders shipped late?") + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + await vm.lastConversationWrite?.value + let saved = await storage.load(id: current) + #expect(saved?.messages.count == 2) + + await storage.delete(earlier.id) + await storage.delete(current) + } + + // MARK: - Switching + + @Test("Switching reads the chosen transcript and keeps the one it left on disk") + func switchingLoadsTheChosenTranscript() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + vm.messages = Self.turns("Current work") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + + vm.switchConversation(to: earlier.id) + await vm.transcriptLoad?.value + + #expect(vm.activeConversationID == earlier.id) + #expect(vm.messages.map { $0.plainText } == ["Earlier work"]) + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + await vm.lastConversationWrite?.value + let left = await storage.load(id: current) + #expect(left?.messages.map { $0.plainText } == ["Current work"]) + + await storage.delete(earlier.id) + await storage.delete(current) + } + + // The second file is gone, so only a late first load could change the transcript, whichever lands first. + @Test("A switch that lands after a second switch is dropped") + func lateSwitchIsDropped() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let first = Self.stored("First", connectionId: connection.id) + let second = Self.stored("Second", connectionId: connection.id) + await storage.save(first) + await storage.save(second) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + vm.messages = Self.turns("Current work") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + await storage.delete(second.id) + + vm.switchConversation(to: first.id) + let firstLoad = try #require(vm.transcriptLoad) + vm.switchConversation(to: second.id) + let secondLoad = try #require(vm.transcriptLoad) + await firstLoad.value + await secondLoad.value + + #expect(vm.activeConversationID == current) + #expect(vm.messages.map { $0.plainText } == ["Current work"]) + #expect(vm.conversationToRestore == nil) + + await vm.lastConversationWrite?.value + await storage.delete(first.id) + await storage.delete(current) + } + + @Test("A switch that lands after New Conversation is dropped") + func switchAfterNewConversationIsDropped() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + vm.switchConversation(to: earlier.id) + let load = try #require(vm.transcriptLoad) + vm.startNewConversation() + await load.value + + #expect(vm.activeConversationID == nil) + #expect(vm.messages.isEmpty) + + await storage.delete(earlier.id) + } + + // MARK: - Transcripts disk may have trimmed + + @Test( + "A release keeps a transcript longer than disk keeps whole and drops one that fits", + arguments: [AIChatStorage.turnsKeptWhenTrimmed + 10, 10] + ) + func releaseKeepsWhatDiskMayTrim(turnCount: Int) async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + await vm.tableMentionBuild?.value + vm.messages = Self.turns(count: turnCount) + vm.persistCurrentConversation() + vm.columnsByTable["orders"] = [TestFixtures.makeColumnInfo()] + let conversationId = try #require(vm.activeConversationID) + + vm.releaseLoadedState() + + let keeps = turnCount > AIChatStorage.turnsKeptWhenTrimmed + #expect(vm.messages.count == (keeps ? turnCount : 0)) + #expect(vm.activeConversationID == (keeps ? conversationId : nil)) + #expect(vm.conversationToRestore == (keeps ? nil : conversationId)) + #expect(vm.columnsByTable.isEmpty) + #expect(vm.tableMentions.names.isEmpty) + #expect(vm.tableMentionObservation == nil) + + await vm.lastConversationWrite?.value + await storage.delete(conversationId) + } + + // The first file is deleted while the second is shown, so only memory can bring every turn back. + @Test("Switching away from a long transcript and back keeps every turn") + func longTranscriptSurvivesASwitch() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let other = Self.stored("Other work", connectionId: connection.id) + await storage.save(other) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + let turnCount = AIChatStorage.turnsKeptWhenTrimmed + 10 + vm.messages = Self.turns(count: turnCount) + vm.persistCurrentConversation() + let long = try #require(vm.activeConversationID) + + vm.switchConversation(to: other.id) + await vm.transcriptLoad?.value + #expect(vm.activeConversationID == other.id) + #expect(vm.conversations.first { $0.id == long }?.messages.count == turnCount) + #expect(vm.conversations.first { $0.id == other.id }?.messages.isEmpty == true) + await vm.lastConversationWrite?.value + await storage.delete(long) + + vm.switchConversation(to: long) + + #expect(vm.activeConversationID == long) + #expect(vm.messages.count == turnCount) + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + + await vm.lastConversationWrite?.value + await storage.delete(other.id) + await storage.delete(long) + } + + // A read from disk would rebuild every turn, so the same turn objects show nothing was read. + @Test("Picking the conversation on screen keeps its transcript and reads nothing") + func repickingTheActiveConversationReadsNothing() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + vm.messages = Self.turns("Which orders shipped late?", "Twelve.", "And yesterday?") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + let shown = vm.messages + + vm.switchConversation(to: current) + await vm.transcriptLoad?.value + + #expect(vm.messages.count == shown.count) + #expect(zip(vm.messages, shown).allSatisfy { $0 === $1 }) + #expect(vm.activeConversationID == current) + + await vm.lastConversationWrite?.value + await storage.delete(current) + } + + // MARK: - Restore + + // The message is set straight on the engine, standing in for a send whose reply has not come yet. + @Test("A message sent before the restore lands keeps its own conversation") + func messageBeforeRestoreWins() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services, restoringConversation: earlier.id) + vm.connection = connection + + vm.restoreConversationsIfNeeded() + let load = try #require(vm.transcriptLoad) + vm.messages = Self.turns("A new question") + await load.value + + #expect(vm.messages.map { $0.plainText } == ["A new question"]) + #expect(vm.activeConversationID == nil) + #expect(vm.conversationToRestore == nil) + #expect(vm.conversations.contains { $0.id == earlier.id }) + + await storage.delete(earlier.id) + } + + // A restore with no connection would list every connection's conversations. + @Test("A restore with no connection waits for one") + func restoreWaitsForTheConnection() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + let elsewhere = Self.stored("Elsewhere", connectionId: UUID()) + await storage.save(earlier) + await storage.save(elsewhere) + let vm = AIChatViewModel(services: fixture.services, restoringConversation: earlier.id) + + vm.restoreConversationsIfNeeded() + + #expect(vm.transcriptLoad == nil) + #expect(!vm.hasRestoredConversation) + + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + #expect(vm.hasRestoredConversation) + #expect(vm.messages.map { $0.plainText } == ["Earlier work"]) + #expect(vm.conversations.contains { $0.id == earlier.id }) + #expect(!vm.conversations.contains { $0.id == elsewhere.id }) + + await storage.delete(earlier.id) + await storage.delete(elsewhere.id) + } +} diff --git a/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift b/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift index 7c566967fb..f7b8e55054 100644 --- a/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift +++ b/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift @@ -99,6 +99,51 @@ struct ChatToolCallRepairTests { #expect(block.isError) #expect(block.content == ChatToolCallRepair.interruptedContent) } + + private func call(_ id: String, _ state: ToolApprovalState) -> ChatContentBlockWire { + .toolUse(ToolUseBlock(id: id, name: "execute_query", input: .object([:]), approvalState: state)) + } + + private func approvalStates(_ turn: ChatTurnWire) -> [ToolApprovalState] { + turn.blocks.compactMap { block -> ToolApprovalState? in + guard case .toolUse(let call) = block.kind else { return nil } + return call.approvalState + } + } + + @Test("Settling a stored transcript cancels a waiting card and leaves every other state alone") + func settlingCancelsOnlyWaitingCards() { + let turns = [ + ChatTurnWire(role: .assistant, blocks: [ + .text("running it"), + call("a", .pending), + call("b", .approved), + call("c", .denied(reason: "Read-only")), + call("d", .cancelled) + ]), + ChatTurnWire(role: .user, blocks: [result("b")]) + ] + + let settled = ChatToolCallRepair.settlingPendingApprovals(turns) + + #expect(approvalStates(settled[0]) == [.cancelled, .approved, .denied(reason: "Read-only"), .cancelled]) + let blockIds = settled[0].blocks.map { $0.id } + #expect(blockIds == turns[0].blocks.map { $0.id }) + #expect(settled[1] == turns[1]) + } + + @Test("A cancelled or denied call with no result is answered with what happened to it") + func unansweredCancelledAndDeniedCalls() { + let repaired = ChatToolCallRepair.answeringEveryToolCall([ + ChatTurnWire(role: .assistant, blocks: [call("a", .cancelled), call("b", .denied(reason: "Read-only"))]) + ]) + + let contents = (repaired.last?.blocks ?? []).compactMap { block -> String? in + guard case .toolResult(let result) = block.kind else { return nil } + return result.content + } + #expect(contents == [ChatToolCallRepair.stoppedBeforeRunContent, "Read-only"]) + } } @MainActor @@ -215,6 +260,32 @@ struct AIChatStoppedToolRoundTests { } } + private func toolUseStates(in turn: ChatTurn) -> [ToolApprovalState] { + turn.blocks.compactMap { block -> ToolApprovalState? in + guard case .toolUse(let use) = block.kind else { return nil } + return use.approvalState + } + } + + private func toolUseStates(in turn: ChatTurnWire) -> [ToolApprovalState] { + turn.blocks.compactMap { block -> ToolApprovalState? in + guard case .toolUse(let use) = block.kind else { return nil } + return use.approvalState + } + } + + private static func storedWaitingCard(connectionId: UUID? = nil) -> AIConversation { + AIConversation( + messages: [ + ChatTurnWire(role: .user, blocks: [.text("raise every price by ten percent")]), + ChatTurnWire(role: .assistant, blocks: [ + .toolUse(ToolUseBlock(id: "call_7", name: "execute_query", input: .object([:]), approvalState: .pending)) + ]) + ], + connectionId: connectionId + ) + } + @Test("Stop while a tool call waits for approval leaves a result for that call") func stopWhileAwaitingApproval() async throws { let transport = ScriptedTransport(rounds: [Self.toolRound(id: "call_1", name: "writer")]) @@ -287,4 +358,139 @@ struct AIChatStoppedToolRoundTests { } #expect(answered == ["call_9"]) } + + @Test("Stop marks a waiting card cancelled before the transcript is saved") + func stopSettlesAWaitingCardAtOnce() async throws { + let transport = ScriptedTransport(rounds: [Self.toolRound(id: "call_1", name: "writer")]) + let viewModel = AIChatViewModel() + let assistant = start( + viewModel, + transport: transport, + tool: GatedTool(name: "writer", mode: .write, gate: AsyncGate()) + ) + let task = viewModel.streamingTask + try await waitUntil { self.hasToolUse(assistant) } + + viewModel.cancelStream() + + #expect(toolUseStates(in: assistant) == [.cancelled]) + await task?.value + } + + @Test("New Conversation while a card waits stores that card as cancelled") + func newConversationStoresAWaitingCardCancelled() async throws { + let transport = ScriptedTransport(rounds: [Self.toolRound(id: "call_1", name: "writer")]) + let viewModel = AIChatViewModel() + let assistant = start( + viewModel, + transport: transport, + tool: GatedTool(name: "writer", mode: .write, gate: AsyncGate()) + ) + let task = viewModel.streamingTask + try await waitUntil { self.hasToolUse(assistant) } + + viewModel.startNewConversation() + await task?.value + await viewModel.lastConversationWrite?.value + + #expect(viewModel.messages.isEmpty) + let summary = try #require(viewModel.conversations.first) + let stored = await viewModel.chatStorage.load(id: summary.id) + await viewModel.chatStorage.delete(summary.id) + let storedStates = (stored?.messages ?? []).flatMap { toolUseStates(in: $0) } + #expect(storedStates == [.cancelled]) + #expect(!ToolApprovalCenter.shared.hasPending(sessionId: viewModel.sessionId)) + } + + @Test("Switching to a conversation stored with a waiting card shows it cancelled") + func switchingSettlesAStoredWaitingCard() { + let viewModel = AIChatViewModel() + let stored = Self.storedWaitingCard() + viewModel.conversations = [stored] + + viewModel.switchConversation(to: stored.id) + + let states = viewModel.messages.flatMap { toolUseStates(in: $0) } + #expect(states == [.cancelled]) + #expect(!ToolApprovalCenter.shared.isAwaiting(sessionId: viewModel.sessionId, toolUseId: "call_7")) + } + + @Test("A conversation saved with a waiting card by an earlier launch reopens with it cancelled") + func restoreSettlesAStoredWaitingCard() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let stored = Self.storedWaitingCard(connectionId: connection.id) + let viewModel = AIChatViewModel(restoringConversation: stored.id) + await viewModel.chatStorage.save(stored) + viewModel.connection = connection + + viewModel.restoreConversationsIfNeeded() + await viewModel.transcriptLoad?.value + await viewModel.chatStorage.delete(stored.id) + + let states = viewModel.messages.flatMap { toolUseStates(in: $0) } + #expect(states == [.cancelled]) + } + + @Test("A Run click that lands after Stop leaves the card cancelled and runs nothing") + func runClickBehindStopStaysCancelled() async throws { + let gate = AsyncGate() + await gate.open() + let transport = ScriptedTransport(rounds: [Self.toolRound(id: "call_1", name: "writer")]) + let viewModel = AIChatViewModel() + let assistant = start( + viewModel, + transport: transport, + tool: GatedTool(name: "writer", mode: .write, gate: gate) + ) + let task = viewModel.streamingTask + try await waitUntil { self.hasToolUse(assistant) && ToolApprovalCenter.shared.hasPending } + + // Both on the main actor in one turn, so the resumed task sees the click only after Stop. + ToolApprovalCenter.shared.resolve(sessionId: viewModel.sessionId, toolUseId: "call_1", decision: .run) + viewModel.cancelStream() + await task?.value + + #expect(toolUseStates(in: assistant) == [.cancelled]) + let results = toolResults(in: try turnAfter(assistant, in: viewModel)) + let contents = results.map { $0.content } + #expect(contents == ["User cancelled this tool call."]) + } + + private final class ReplyLog: @unchecked Sendable { + private let lock = NSLock() + private var results: [ChatToolResult] = [] + + var contents: [String] { + lock.withLock { results.map { $0.content } } + } + + func record(_ result: ChatToolResult) { + lock.withLock { results.append(result) } + } + } + + // Copilot asks through the shared registry, and its path never checks cancellation. + @Test("A Copilot Run click that lands with New Conversation runs nothing") + func copilotRunBehindNewConversationRunsNothing() async throws { + let gate = AsyncGate() + await gate.open() + let tool = GatedTool(name: "copilot_test_writer", mode: .write, gate: gate) + ChatToolRegistry.shared.register(tool) + defer { ChatToolRegistry.shared.unregister(name: tool.name) } + let replies = ReplyLog() + let call = ToolUseBlock(id: "call_1", name: tool.name, input: .object([:])) + let transport = ScriptedTransport(rounds: [[ + .toolInvocationRequest(block: call, replyToken: ToolReplyToken { replies.record($0) }) + ]]) + let viewModel = AIChatViewModel() + let assistant = start(viewModel, transport: transport, tool: tool) + let task = viewModel.streamingTask + try await waitUntil { self.hasToolUse(assistant) && ToolApprovalCenter.shared.hasPending } + + ToolApprovalCenter.shared.resolve(sessionId: viewModel.sessionId, toolUseId: call.id, decision: .run) + viewModel.startNewConversation() + await task?.value + + #expect(replies.contents == ["User cancelled this tool call."]) + } } diff --git a/apple/TableProTests/ViewModels/AIChatStoppedWriteTests.swift b/apple/TableProTests/ViewModels/AIChatStoppedWriteTests.swift new file mode 100644 index 0000000000..8072ca9ea9 --- /dev/null +++ b/apple/TableProTests/ViewModels/AIChatStoppedWriteTests.swift @@ -0,0 +1,131 @@ +// +// AIChatStoppedWriteTests.swift +// TableProTests +// +// Stop pressed while the assistant's write is in the driver: the write runs to its end, so the transcript must not +// read as a failure the model would retry. +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct AIChatStoppedWriteTests { + private final class OneToolRoundTransport: ChatTransport, @unchecked Sendable { + private let toolName: String + + init(toolName: String) { + self.toolName = toolName + } + + func streamChat( + turns: [ChatTurnWire], + options: ChatTransportOptions + ) -> AsyncThrowingStream { + let events: [ChatStreamEvent] = [ + .toolUseStart(id: "call_1", name: toolName, providerMetadata: nil), + .toolUseDelta(id: "call_1", inputJSONDelta: "{}"), + .toolUseEnd(id: "call_1") + ] + return AsyncThrowingStream { continuation in + for event in events { + continuation.yield(event) + } + continuation.finish() + } + } + + func fetchAvailableModels() async throws -> [AIModelInfo] { [] } + + func testConnection() async throws -> Bool { true } + } + + // The assistant's own execute_query, on a connection the test controls. + private struct WriteThroughExecuteQuery: ChatTool { + let name = "write_rows" + let mode: ChatToolMode = .readOnly + let description = "" + let inputSchema: JsonValue = .object(["type": .string("object")]) + let target: ChatToolContext + + func execute(input: JsonValue, context: ChatToolContext) async throws -> ChatToolResult { + try await ExecuteQueryChatTool().execute( + input: .object(["query": .string("UPDATE orders SET n = n + 1 WHERE id = 1")]), + context: target + ) + } + } + + private static let settings = AISettings( + enabled: true, + includeSchema: false, + includeCurrentQuery: false, + includeQueryResults: false, + maxToolRoundtrips: AISettings.defaultMaxToolRoundtrips, + maxToolRoundtripsEnabled: false, + chatMode: .agent + ) + + @Test("Stop during a write in the driver leaves a result saying it may have been applied, and history keeps the write") + func stopDuringWriteSaysItMayHaveBeenApplied() async throws { + let gate = AsyncGate() + let driver = AnsweringPluginDriver(hold: gate) + let scope = AnsweringPluginDriver.install(driver, connection: TestFixtures.makeConnection(database: "shop", type: .postgresql)) + defer { DatabaseManager.shared.removeSession(for: scope.connectionId) } + let history = RecordingQueryHistory() + let tool = WriteThroughExecuteQuery(target: ChatToolContext( + connectionId: scope.connectionId, + bridge: MCPConnectionBridge(), + authPolicy: MCPAuthPolicy( + connectionResolver: { _ in nil }, + connectionIdsProvider: { [] }, + historyRecorder: history, + executionGate: AlwaysAllowGate() + ), + chatMode: .agent, + policyLookup: { _ in .alwaysAllow } + )) + let registry = ChatToolRegistry() + registry.register(tool) + let viewModel = AIChatViewModel() + let assistant = ChatTurn(role: .assistant, blocks: [], modelId: "test-model", providerId: nil) + viewModel.messages.append(assistant) + viewModel.streamingState = .streaming(assistantID: assistant.id) + viewModel.runStream( + chatMessages: [], + promptContext: nil, + resolved: AIProviderFactory.ResolvedProvider( + provider: OneToolRoundTransport(toolName: tool.name), + model: "test-model", + config: AIProviderConfig(name: "Test", type: .claude) + ), + assistantID: assistant.id, + settings: Self.settings, + registry: registry + ) + let task = viewModel.streamingTask + await gate.arrived(1) + + viewModel.cancelStream() + await task?.value + + let index = try #require(viewModel.messages.firstIndex { $0.id == assistant.id }) + try #require(index + 1 < viewModel.messages.count) + let results = viewModel.messages[index + 1].blocks.compactMap { block -> ToolResultBlock? in + guard case .toolResult(let result) = block.kind else { return nil } + return result + } + #expect(results.map { $0.toolUseId } == ["call_1"]) + #expect(results.first?.isError == true) + #expect(results.first?.content == "Error: \(DatabaseAccessBridge.stoppedWhileRunning.message)") + #expect(history.requests.map { $0.wasSuccessful } == [false]) + #expect(driver.finishedCount == 0) + + await gate.open() + try await waitUntil { driver.finishedCount == 1 } + } +} diff --git a/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift b/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift index 4799cc73f1..16971297b8 100644 --- a/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift +++ b/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift @@ -88,6 +88,44 @@ struct AIChatTableMentionsTests { try await waitUntil { vm.tableMentions.names == ["accounts", "invoices"] } #expect(!vm.hasUnlistedMentionTables) } + + // A cleared source makes any refresh build again, so a build is how a refresh shows. + @Test("Only a schema load on the session's own connection refreshes its index") + func indexFollowsOnlyItsOwnConnection() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.trackTableMentions() + await vm.tableMentionBuild?.value + await Self.afterQueuedMainWork() + await vm.tableMentionBuild?.value + #expect(vm.tableMentions.names == ["orders"]) + vm.tableMentionSource = [] + + let other = TestFixtures.makeConnection(type: .postgresql) + let otherDriver = MockDatabaseDriver(connection: other) + otherDriver.tablesToReturn = [TestFixtures.makeTableInfo(name: "invoices")] + await fixture.schemaService.load(connectionId: other.id, driver: otherDriver, connection: other) + await Self.afterQueuedMainWork() + + #expect(vm.tableMentionSource.isEmpty) + #expect(vm.tableMentionBuild == nil) + + await fixture.schemaService.reload(connectionId: connection.id, driver: fixture.driver, connection: connection) + + try await waitUntil { vm.tableMentionSource.map { $0.name } == ["orders"] } + } + + // Sinks deliver through `RunLoop.main.perform` and answer on the main actor; both have run once this returns. + private static func afterQueuedMainWork() async { + await withCheckedContinuation { continuation in + RunLoop.main.perform { continuation.resume() } + } + await withCheckedContinuation { continuation in + DispatchQueue.main.async { continuation.resume() } + } + } } @MainActor diff --git a/apple/TableProTests/ViewModels/AIChatViewModelProviderSettingsTests.swift b/apple/TableProTests/ViewModels/AIChatViewModelProviderSettingsTests.swift new file mode 100644 index 0000000000..7306942d71 --- /dev/null +++ b/apple/TableProTests/ViewModels/AIChatViewModelProviderSettingsTests.swift @@ -0,0 +1,121 @@ +// +// AIChatViewModelProviderSettingsTests.swift +// TableProTests +// + +import Combine +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +// Claude Agent lists its models without a network call. +private let claudeAgentProvider = AIProviderConfig( + id: UUID(uuidString: "6B1E4F44-31D8-4C47-9B0B-3F0C2A9D1E01") ?? UUID(), + type: .claudeAgent, + model: "sonnet" +) + +// Port 9 refuses the connection at once, so the model list fails without a timeout. +private let unreachableProvider = AIProviderConfig( + id: UUID(uuidString: "6B1E4F44-31D8-4C47-9B0B-3F0C2A9D1E02") ?? UUID(), + type: .custom, + model: "model-a", + endpoint: "http://127.0.0.1:9/v1" +) + +extension GlobalStateSuites { + @Suite(.appSettings { + $0.ai.enabled = true + $0.ai.providers = [claudeAgentProvider] + $0.ai.activeProviderID = claudeAgentProvider.id + }) + @MainActor + struct AIChatViewModelProviderSettingsTests { + private static let agent = claudeAgentProvider + + private func settingsSettle(_ viewModel: AIChatViewModel) async throws { + let revision = viewModel.providerSettingsRevision + AppServices.live.appEvents.aiProviderSaved.send(UUID()) + try await waitUntil { viewModel.providerSettingsRevision > revision } + } + + @Test("A chat panel pick whose provider was removed is cleared when AI settings change") + func removedProviderPickIsCleared() async throws { + let viewModel = AIChatViewModel() + viewModel.selectedProviderId = UUID() + viewModel.selectedModel = "muse-spark-1.3" + + AppServices.live.appEvents.aiSettingsChanged.send(()) + try await settingsSettle(viewModel) + + #expect(viewModel.selectedProviderId == nil) + #expect(viewModel.selectedModel == nil) + } + + @Test("A chat panel pick whose provider is still configured is kept") + func configuredProviderPickIsKept() async throws { + let viewModel = AIChatViewModel() + viewModel.selectedProviderId = Self.agent.id + viewModel.selectedModel = "opus" + + AppServices.live.appEvents.aiSettingsChanged.send(()) + try await settingsSettle(viewModel) + + #expect(viewModel.selectedProviderId == Self.agent.id) + #expect(viewModel.selectedModel == "opus") + } + + /// A key fixed in Settings changes nothing the provider list shows, and the panel kept the + /// list it had fetched with the rejected key. + @Test("Saving a provider makes the chat panel fetch its model list again") + func savedProviderListIsFetchedAgain() async throws { + let viewModel = AIChatViewModel() + await viewModel.loadAvailableModels() + #expect(viewModel.availableModels[Self.agent.id] == ["opus", "sonnet", "haiku"]) + #expect(!viewModel.modelListIsStale(for: Self.agent)) + + let revision = viewModel.providerSettingsRevision + AppServices.live.appEvents.aiProviderSaved.send(Self.agent.id) + try await waitUntil { viewModel.providerSettingsRevision > revision } + #expect(viewModel.modelListIsStale(for: Self.agent)) + + await viewModel.loadAvailableModels() + #expect(!viewModel.modelListIsStale(for: Self.agent)) + } + + /// A server that cannot answer was asked again on every connection switch, and the whole + /// panel waited on it each time. + @Test("A model list that failed to load is not fetched again until the provider changes", .appSettings { + $0.ai.providers = [unreachableProvider] + }) + func failedListIsNotFetchedAgain() async { + let viewModel = AIChatViewModel() + await viewModel.loadAvailableModels() + #expect(viewModel.availableModels[unreachableProvider.id] == []) + #expect(!viewModel.modelListIsStale(for: unreachableProvider)) + + var moved = unreachableProvider + moved.endpoint = "http://127.0.0.1:9/v2" + #expect(viewModel.modelListIsStale(for: moved)) + } + + @Test("A model list depends on the Base URL and the preset, not on the model picked") + func listSourceTracksWhatTheListDependsOn() { + let saved = AIProviderConfig(preset: .meta) + let fetched = AIModelListSource(config: saved) + + var otherModel = saved + otherModel.model = "muse-wonder-1.0" + #expect(AIModelListSource(config: otherModel) == fetched) + + var otherServer = saved + otherServer.endpoint = "https://gateway.example/meta" + #expect(AIModelListSource(config: otherServer) != fetched) + + var noPreset = saved + noPreset.presetID = nil + #expect(AIModelListSource(config: noPreset) != fetched) + } + } +} diff --git a/apple/TableProTests/ViewModels/AIChatViewModelToolLoopTests.swift b/apple/TableProTests/ViewModels/AIChatViewModelToolLoopTests.swift index 0a7cd2519d..22fd5980cb 100644 --- a/apple/TableProTests/ViewModels/AIChatViewModelToolLoopTests.swift +++ b/apple/TableProTests/ViewModels/AIChatViewModelToolLoopTests.swift @@ -46,6 +46,32 @@ struct AIChatViewModelToolLoopTests { } } + private final class ExecutionCounter: @unchecked Sendable { + private let lock = NSLock() + private var count = 0 + + var value: Int { + lock.withLock { count } + } + + func increment() { + lock.withLock { count += 1 } + } + } + + private struct CountingTool: ChatTool { + let name = "counting" + let description = "" + let inputSchema: JsonValue = .object(["type": .string("object")]) + let mode: ChatToolMode = .readOnly + let counter: ExecutionCounter + + func execute(input: JsonValue, context: ChatToolContext) async throws -> ChatToolResult { + counter.increment() + return ChatToolResult(content: "ran", isError: false) + } + } + private static func toolRound(id: String) -> [ChatStreamEvent] { [ .toolUseStart(id: id, name: "noop", providerMetadata: nil), @@ -81,10 +107,13 @@ struct AIChatViewModelToolLoopTests { private func runLoop( viewModel: AIChatViewModel, transport: ChatTransport, - settings: AISettings + settings: AISettings, + tools: [any ChatTool] = [NoopTool()] ) async { let registry = ChatToolRegistry() - registry.register(NoopTool()) + for tool in tools { + registry.register(tool) + } let assistant = ChatTurn(role: .assistant, blocks: [], modelId: "test-model", providerId: nil) viewModel.messages.append(assistant) viewModel.streamingState = .streaming(assistantID: assistant.id) @@ -218,6 +247,44 @@ struct AIChatViewModelToolLoopTests { #expect(viewModel.messages.last?.plainText.contains("finished") == true) } + @Test("A call whose arguments were cut off never runs, and the model gets an error saying why") + func truncatedToolCallNeverRuns() async { + let counter = ExecutionCounter() + let transport = ScriptedTransport(rounds: [ + [ + .toolUseStart(id: "cut", name: "counting", providerMetadata: nil), + .toolUseDelta(id: "cut", inputJSONDelta: #"{"query":"SELECT * FROM orders WH"#), + .toolUseEnd(id: "cut") + ], + [.textDelta("retrying")] + ]) + let viewModel = AIChatViewModel() + + await runLoop( + viewModel: viewModel, + transport: transport, + settings: Self.makeSettings(limit: Self.testLimit, enabled: true), + tools: [CountingTool(counter: counter)] + ) + + #expect(counter.value == 0) + let denial = AIChatViewModel.unparsableToolArgumentsContent + let calls = viewModel.messages.flatMap { $0.blocks }.compactMap { block -> ToolUseBlock? in + guard case .toolUse(let call) = block.kind else { return nil } + return call + } + let callIds = calls.map { $0.id } + #expect(callIds == ["cut"]) + #expect(calls.first?.approvalState == .denied(reason: denial)) + let results = viewModel.messages.flatMap { $0.blocks }.compactMap { block -> ToolResultBlock? in + guard case .toolResult(let result) = block.kind else { return nil } + return result + } + #expect(results == [ToolResultBlock(toolUseId: "cut", content: denial, isError: true)]) + #expect(transport.roundsRequested == 2) + #expect(viewModel.messages.last?.plainText.contains("retrying") == true) + } + @Test("continueToolLoop only acts while paused") func continueIsInertWhenNotPaused() { let viewModel = AIChatViewModel() diff --git a/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift b/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift index 4d0b82aeca..1672d26628 100644 --- a/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift +++ b/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift @@ -76,4 +76,12 @@ struct ConnectionFormChildObservationTests { replacement.type = .postgresql #expect(counter.sends == 1) } + + @Test("a new connection saved Local only has nothing in iCloud Keychain to remove") + func newLocalOnlyConnectionRemovesNothing() { + let coordinator = ConnectionFormCoordinator(connectionId: nil) + coordinator.advanced.localOnly = true + + #expect(!coordinator.removesSyncedSecretsOnSave) + } } diff --git a/apple/TableProTests/ViewModels/ConnectionFormEditsTests.swift b/apple/TableProTests/ViewModels/ConnectionFormEditsTests.swift index aae879f871..2bd6541e68 100644 --- a/apple/TableProTests/ViewModels/ConnectionFormEditsTests.swift +++ b/apple/TableProTests/ViewModels/ConnectionFormEditsTests.swift @@ -19,7 +19,8 @@ struct ConnectionFormEditsTests { completionIdentifierCase: SQLIdentifierCase = .default, additionalFields: [String: String] = [:], ownedAdditionalFieldIDs: Set = [], - maskingRules: [MaskingRule] = [] + maskingRules: [MaskingRule] = [], + revokedAIAlwaysAllowedTools: Set = [] ) -> ConnectionFormEdits { ConnectionFormEdits( name: "Edited", @@ -53,7 +54,8 @@ struct ConnectionFormEditsTests { localOnly: false, additionalFields: additionalFields, ownedAdditionalFieldIDs: ownedAdditionalFieldIDs, - maskingRules: maskingRules + maskingRules: maskingRules, + revokedAIAlwaysAllowedTools: revokedAIAlwaysAllowedTools ) } @@ -111,6 +113,29 @@ struct ConnectionFormEditsTests { #expect(result.passwordSource == .env(variable: "PGPASSWORD")) } + @Test("Forgetting an always-allowed tool removes it and keeps a grant recorded after the form opened") + func forgettingRemovesOnlyTheRevokedTool() { + var stored = populatedConnection() + stored.aiAlwaysAllowedTools = ["execute_query", "list_tables"] + + let result = edits(revokedAIAlwaysAllowedTools: ["execute_query"]).applied(to: stored) + + #expect(result.aiAlwaysAllowedTools == ["list_tables"]) + } + + @Test("The form's edits carry what Forget took off the list") + @MainActor + func coordinatorCarriesForgottenTools() { + let coordinator = ConnectionFormCoordinator(connectionId: nil) + coordinator.advanced.load(from: populatedConnection()) + + coordinator.advanced.forgetAlwaysAllowed("run_query") + let saved = coordinator.buildEdits().applied(to: populatedConnection()) + + #expect(coordinator.buildEdits().revokedAIAlwaysAllowedTools == ["run_query"]) + #expect(saved.aiAlwaysAllowedTools == ["list_tables"]) + } + @Test("Applying edits overwrites every property the form owns") func overwritesOwnedProperties() { let result = edits().applied(to: populatedConnection()) diff --git a/apple/TableProTests/ViewModels/ConnectionFormPortTLSTests.swift b/apple/TableProTests/ViewModels/ConnectionFormPortTLSTests.swift index f4865d2306..a1fbcb95d6 100644 --- a/apple/TableProTests/ViewModels/ConnectionFormPortTLSTests.swift +++ b/apple/TableProTests/ViewModels/ConnectionFormPortTLSTests.swift @@ -118,10 +118,13 @@ struct ConnectionFormPortTLSTests { #expect(coordinator.ssl.validationIssues.isEmpty) } - @Test("PostgreSQL Verify Identity still asks for a CA certificate") - func postgresStillRequiresCA() { + @Test("PostgreSQL Verify Identity saves without a CA certificate, and Verify CA still asks for one") + func postgresNeedsACAOnlyForVerifyCA() { let coordinator = newConnection(.postgresql) coordinator.ssl.select(.verifyIdentity) + #expect(coordinator.ssl.validationIssues.isEmpty) + + coordinator.ssl.select(.verifyCa) #expect(!coordinator.ssl.validationIssues.isEmpty) } } diff --git a/apple/TableProTests/ViewModels/DashboardTileBoardTests.swift b/apple/TableProTests/ViewModels/DashboardTileBoardTests.swift index 9534866282..ca22114b39 100644 --- a/apple/TableProTests/ViewModels/DashboardTileBoardTests.swift +++ b/apple/TableProTests/ViewModels/DashboardTileBoardTests.swift @@ -21,12 +21,14 @@ struct DashboardTileBoardTests { let changes = PassthroughSubject() var catalog: DashboardSavedQueryCatalog var executed: [String] = [] + var executedIn: [DatabaseScope?] = [] var answer: (String) throws -> QueryResult = { _ in Fixture.result(columns: ["count"], rows: [["12"]]) } var clock = Date(timeIntervalSince1970: 1_000_000) var gate: any ExecutionGate = AlwaysAllowGate() var holdExecution: AsyncGate? var loads = 0 var safeModeLevel = SafeModeLevel.silent + var runsIn: DatabaseScope? init(favorites: [SQLFavorite]) throws { suite = try EphemeralDefaults() @@ -51,14 +53,16 @@ struct DashboardTileBoardTests { chartsAvailable: { chartsAvailable }, nullDisplay: { "NULL" }, storage: storage, - now: { self.clock } + now: { self.clock }, + runsIn: { _ in self.runsIn } ) return DashboardTileBoard( connectionId: connectionId, databaseType: databaseType, environment: environment - ) { sql in + ) { sql, scope in self.executed.append(sql) + self.executedIn.append(scope) if let hold = self.holdExecution { await hold.wait() } @@ -177,6 +181,34 @@ struct DashboardTileBoardTests { #expect(await gate.requests.count == 3) } + @Test("On Oracle a tile is asked again when the session's schema moves, and once its catalog read is a minute old") + func oracleTileAsksAgainForItsSchema() async throws { + let saved = favorite() + let fixture = try Fixture(favorites: [saved]) + let gate = CountingGate() + fixture.gate = gate + fixture.runsIn = DatabaseScope(connectionId: fixture.connectionId, database: "FREEPDB1", schema: "HR") + let board = fixture.makeBoard(databaseType: .oracle) + board.add(favoriteId: saved.id) + let tile = try #require(board.tiles.first) + try await waitUntil { board.state(for: tile.id).display != nil } + + await board.refresh(force: false) + #expect(await gate.requests.count == 1) + #expect(await gate.requests.first?.runsIn?.schema == "HR") + #expect(fixture.executedIn.last == fixture.runsIn) + + fixture.runsIn = DatabaseScope(connectionId: fixture.connectionId, database: "FREEPDB1", schema: "APP") + await board.refresh(force: false) + #expect(await gate.requests.count == 2) + #expect(await gate.requests.last?.runsIn?.schema == "APP") + #expect(fixture.executedIn.last == fixture.runsIn) + + fixture.clock.addTimeInterval(61) + await board.refresh(force: false) + #expect(await gate.requests.count == 3) + } + @Test("A slower tile waits for its interval, and Refresh Now runs it anyway") func slowerIntervalWaits() async throws { let saved = favorite() diff --git a/apple/TableProTests/ViewModels/DashboardTileSessionTurnTests.swift b/apple/TableProTests/ViewModels/DashboardTileSessionTurnTests.swift new file mode 100644 index 0000000000..aff3d9d75b --- /dev/null +++ b/apple/TableProTests/ViewModels/DashboardTileSessionTurnTests.swift @@ -0,0 +1,106 @@ +// +// DashboardTileSessionTurnTests.swift +// TableProTests +// + +import Combine +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct DashboardTileSessionTurnTests { + private static let saved = SQLFavorite(name: "Queued jobs", query: "SELECT count(*) FROM jobs") + + private struct Fixture { + let connection: DatabaseConnection + let driver: MockDatabaseDriver + let viewModel: ServerDashboardViewModel + let suite: EphemeralDefaults + } + + // The tile layout is stored before the dashboard opens, so nothing runs until the test asks. + private func makeFixture(checked schema: String) throws -> Fixture { + let connection = TestFixtures.makeConnection(database: "app", type: .mysql) + let driver = MockDatabaseDriver(connection: connection) + driver.currentSchema = schema + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = "app" + DatabaseManager.shared.injectSession(session, for: connection.id) + + let suite = try EphemeralDefaults() + let defaults = suite.defaults + let storage = ServerDashboardTileStorage(defaults: { defaults }) + storage.save(ServerDashboardTileLayout(tiles: [DashboardTile(favoriteId: Self.saved.id)]), for: connection.id) + let checked = DatabaseScope(connectionId: connection.id, database: "app", schema: schema) + let viewModel = ServerDashboardViewModel( + connectionId: connection.id, + databaseType: .mysql, + services: .live, + tileEnvironment: DashboardTileBoard.Environment( + loadCatalog: { _ in DashboardSavedQueryCatalog(favorites: [Self.saved], folders: []) }, + savedQueriesDidChange: Empty().eraseToAnyPublisher(), + gate: AlwaysAllowGate(), + forcesWrite: { _ in false }, + safeModeLevel: { _ in .silent }, + chartsAvailable: { false }, + nullDisplay: { "NULL" }, + storage: storage, + runsIn: { _ in checked } + ) + ) + return Fixture(connection: connection, driver: driver, viewModel: viewModel, suite: suite) + } + + // A tab moves the shared driver to its own schema in its turn and leaves it there. + @Test("A tile waits for the session's turn and runs in the schema the gate was asked about") + func tileRunsInItsTurnPinnedToTheCheckedSchema() async throws { + let fixture = try makeFixture(checked: "hr") + defer { DatabaseManager.shared.removeSession(for: fixture.connection.id) } + let driver = fixture.driver + + let acquired = AsyncGate() + let release = AsyncGate() + let tab = Task { @MainActor in + try await DatabaseManager.shared.sessionDriverGate.withExclusiveAccess( + DatabaseManager.shared.browsedGateKey(for: fixture.connection.id) + ) { + driver.currentSchema = "sales" + await acquired.open() + await release.wait() + } + } + await acquired.wait() + + let refresh = Task { await fixture.viewModel.tileBoard.refresh(force: true) } + try await waitUntil { DatabaseManager.shared.sessionDriverGate.waiterCount(for: fixture.connection.id) >= 1 } + #expect(driver.executedQueries.isEmpty) + + await release.open() + try await tab.value + await refresh.value + + #expect(driver.executedQueries.contains(Self.saved.query)) + #expect(driver.currentSchema == "hr") + } + + @Test("Cancelling a tile's run cancels its statement on the server") + func cancelledTileStopsOnTheServer() async throws { + let fixture = try makeFixture(checked: "hr") + defer { DatabaseManager.shared.removeSession(for: fixture.connection.id) } + let driver = fixture.driver + let held = AsyncGate() + driver.executeGate = held + + let refresh = Task { await fixture.viewModel.tileBoard.refresh(force: true) } + try await waitUntil { driver.executedQueries.contains(Self.saved.query) } + refresh.cancel() + try await waitUntil { driver.cancelQueryCallCount >= 1 } + + await held.open() + await refresh.value + } +} diff --git a/apple/TableProTests/ViewModels/ERDiagramForeignKeyDraftTests.swift b/apple/TableProTests/ViewModels/ERDiagramForeignKeyDraftTests.swift new file mode 100644 index 0000000000..b01bed83a8 --- /dev/null +++ b/apple/TableProTests/ViewModels/ERDiagramForeignKeyDraftTests.swift @@ -0,0 +1,133 @@ +// +// ERDiagramForeignKeyDraftTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct ERDiagramForeignKeyDraftTests { + private typealias Pair = ERDiagramForeignKeyDraft.ColumnPair + + private func draft(_ pairs: [(String, String)]) -> ERDiagramForeignKeyDraft { + ERDiagramForeignKeyDraft( + table: "line_items", + pairs: pairs.map { Pair(column: $0.0, referencedColumn: $0.1) }, + referencedTable: "orders", + name: "fk_line_items_tenant_id" + ) + } + + @Test("The pairs map to the key's two column lists in pair order") + func pairOrderMapsToTheKey() { + var composite = draft([("tenant_id", "tenant_id"), ("order_id", "id")]) + composite.onDelete = .cascade + + let key = composite.foreignKey + + #expect(key.columns == ["tenant_id", "order_id"]) + #expect(key.referencedColumns == ["tenant_id", "id"]) + #expect(key.referencedTable == "orders") + #expect(key.name == "fk_line_items_tenant_id") + #expect(key.onDelete == .cascade) + } + + @Test("A drop on a member of a two-column key offers the whole key, the dragged pair in its place") + func dropOnACompositeMemberOffersTheWholeKey() { + let started = ERDiagramForeignKeyDraft.starting( + table: "line_items", + column: "order_id", + referencedTable: "orders", + keyColumns: ["tenant_id", "id"], + target: "id", + sourceColumns: ["id", "Tenant_ID", "order_id"], + name: "fk_line_items_order_id" + ) + + #expect(started.pairs.map { $0.referencedColumn } == ["tenant_id", "id"]) + #expect(started.pairs.map { $0.column } == ["Tenant_ID", "order_id"]) + #expect(started.issue == nil) + } + + @Test("A drop on a column outside the key keeps that one pair") + func dropOutsideTheKeyIsOnePair() { + let started = ERDiagramForeignKeyDraft.starting( + table: "line_items", + column: "order_code", + referencedTable: "orders", + keyColumns: ["tenant_id", "id"], + target: "code", + sourceColumns: ["order_code"], + name: "fk" + ) + + #expect(started.pairs.map { $0.column } == ["order_code"]) + #expect(started.pairs.map { $0.referencedColumn } == ["code"]) + } + + @Test("Choosing another referenced table rebuilds its side from that table's key and keeps the chosen column") + func referencingRebuildsFromTheKey() { + let single = draft([("order_id", "id")]) + let sources = ["id", "tenant_id", "order_id"] + + let composite = single.referencing("shipments", keyColumns: ["tenant_id", "id"], sourceColumns: sources) + + #expect(composite.referencedTable == "shipments") + #expect(composite.pairs.map { $0.column } == ["tenant_id", "order_id"]) + #expect(composite.pairs.map { $0.referencedColumn } == ["tenant_id", "id"]) + } + + @Test("A table's own id is never taken for the referenced id") + func ownIdIsNotAGuess() { + let single = draft([("order_id", "id")]) + .referencing("orders", keyColumns: ["id"], sourceColumns: ["id", "order_id"]) + + #expect(single.pairs.map { $0.column } == ["order_id"]) + } + + @Test("A column with no namesake stays blank, and the draft says a row is incomplete") + func unguessedColumnIsBlank() { + let composite = draft([("order_id", "id")]) + .referencing("shipments", keyColumns: ["id", "region"], sourceColumns: ["order_id"]) + + #expect(composite.pairs.map { $0.column } == ["order_id", ""]) + #expect(composite.issue == .incompletePair) + } + + @Test("A repeated column on either side is named") + func repeatsAreNamed() { + #expect(draft([("a", "x"), ("a", "y")]).issue == .repeatedColumn("a")) + #expect(draft([("a", "x"), ("b", "x")]).issue == .repeatedReferencedColumn("x")) + #expect(draft([("a", "x"), ("b", "y")]).issue == nil) + } + + @Test("Each whole-value edit changes only the pair it names") + func editsNameTheirPair() throws { + let composite = draft([("a", "x"), ("b", "y")]) + let second = try #require(composite.pairs.last?.id) + + let edited = composite.settingColumn("c", of: second).settingReferencedColumn("z", of: second) + + #expect(edited.pairs.map { $0.column } == ["a", "c"]) + #expect(edited.pairs.map { $0.referencedColumn } == ["x", "z"]) + } + + @Test("Adding, moving and removing pairs keep the key's order, and the last pair stays") + func addMoveRemove() throws { + let composite = draft([("a", "x"), ("b", "y")]).addingPair() + #expect(composite.pairs.count == 3) + #expect(composite.issue == .incompletePair) + + let moved = composite.movingPairs(fromOffsets: IndexSet(integer: 0), toOffset: 2) + #expect(moved.pairs.map { $0.column } == ["b", "a", ""]) + + let first = try #require(moved.pairs.first?.id) + let blank = try #require(moved.pairs.last?.id) + let trimmed = moved.removingPairs([first, blank]) + #expect(trimmed.pairs.map { $0.column } == ["a"]) + + let everything = Set(trimmed.pairs.map { $0.id }) + #expect(trimmed.removingPairs(everything) == trimmed) + } +} diff --git a/apple/TableProTests/ViewModels/ERDiagramSchemaEditsTests.swift b/apple/TableProTests/ViewModels/ERDiagramSchemaEditsTests.swift index e62e4676ea..b52fb16d24 100644 --- a/apple/TableProTests/ViewModels/ERDiagramSchemaEditsTests.swift +++ b/apple/TableProTests/ViewModels/ERDiagramSchemaEditsTests.swift @@ -193,6 +193,34 @@ struct ERDiagramSchemaEditsTests { #expect(diagram == structureTab) } + @Test("A composite foreign key from the diagram composes what the Structure tab composes") + func compositeForeignKeyMatchesTheStructureTab() async throws { + let fixture = try await Self.makeFixture() + defer { fixture.tearDown() } + let draft = ERDiagramForeignKeyDraft( + table: "orders", + pairs: [ + .init(column: "customer_id", referencedColumn: "id"), + .init(column: "shipper_id", referencedColumn: "email") + ], + referencedTable: "customers", + name: "fk_orders_customer", + onDelete: .cascade + ) + + try await fixture.edits.addForeignKey(draft.foreignKey, to: "orders") + let tab = try await Self.structureTabSession(fixture, table: "orders") + tab.changeManager.addForeignKey(draft.foreignKey) + + let diagram = await fixture.edits.previewStatements() + #expect(diagram == [ + "ALTER TABLE \"public\".\"orders\" ADD CONSTRAINT \"fk_orders_customer\" FOREIGN KEY (\"customer_id\", \"shipper_id\") " + + "REFERENCES \"customers\" (\"id\", \"email\") ON DELETE CASCADE;" + ]) + let structureTab = try await Self.statements(of: tab) + #expect(diagram == structureTab) + } + @Test("Dropping a drawn foreign key drops the constraint the Structure tab would drop") func droppedForeignKeyMatchesTheStructureTab() async throws { let fixture = try await Self.makeFixture() diff --git a/apple/TableProTests/ViewModels/ERDiagramViewModelForeignKeyTests.swift b/apple/TableProTests/ViewModels/ERDiagramViewModelForeignKeyTests.swift new file mode 100644 index 0000000000..be40451d62 --- /dev/null +++ b/apple/TableProTests/ViewModels/ERDiagramViewModelForeignKeyTests.swift @@ -0,0 +1,74 @@ +// +// ERDiagramViewModelForeignKeyTests.swift +// TableProTests +// + +import CoreGraphics +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct ERDiagramViewModelForeignKeyTests { + private func loadedFixture() async -> ERDiagramLoadFixture { + let fixture = ERDiagramLoadFixture(tableCount: 1, columnsPerTable: 3) + fixture.driver.addTable("accounts", columns: [ + TestFixtures.makeColumnInfo(name: "tenant_id", isPrimaryKey: true), + TestFixtures.makeColumnInfo(name: "id", isPrimaryKey: true), + TestFixtures.makeColumnInfo(name: "name", dataType: "TEXT", isPrimaryKey: false) + ]) + await fixture.viewModel.loadDiagram() + return fixture + } + + private func draft(of viewModel: ERDiagramViewModel) -> ERDiagramForeignKeyDraft? { + guard case .addForeignKey(let draft) = viewModel.editSheet else { return nil } + return draft + } + + @Test("A drop on a member of a two-column key offers both key columns, the dragged pair in its place") + func dropOnACompositeKeyOffersBothColumns() async throws { + let fixture = await loadedFixture() + defer { fixture.tearDown() } + let target = ERDiagramColumnHit( + nodeId: UUID(), tableName: "accounts", columnName: "id", rowRect: .zero, isDropped: false + ) + + fixture.viewModel.beginForeignKey(from: "table_0", column: "column_1", to: target) + + let started = try #require(draft(of: fixture.viewModel)) + #expect(started.referencedTable == "accounts") + #expect(started.pairs.map { $0.referencedColumn } == ["tenant_id", "id"]) + #expect(started.pairs.last?.column == "column_1") + #expect(started.foreignKey.referencedColumns == ["tenant_id", "id"]) + } + + @Test("Add Foreign Key from a column prefills the referenced table's whole primary key") + func menuPrefillsTheWholeKey() async throws { + let fixture = await loadedFixture() + defer { fixture.tearDown() } + + fixture.viewModel.beginForeignKey(from: "table_0", column: "column_2", to: nil) + + let started = try #require(draft(of: fixture.viewModel)) + #expect(started.referencedTable == "accounts") + #expect(fixture.viewModel.referencedKeyColumns(in: "accounts") == ["tenant_id", "id"]) + #expect(started.pairs.map { $0.referencedColumn } == ["tenant_id", "id"]) + #expect(started.pairs.contains { $0.column == "column_2" }) + } + + @Test("A table with a single-column key still starts with one pair") + func singleKeyIsOnePair() async throws { + let fixture = await loadedFixture() + defer { fixture.tearDown() } + let target = ERDiagramColumnHit( + nodeId: UUID(), tableName: "table_0", columnName: "column_0", rowRect: .zero, isDropped: false + ) + + fixture.viewModel.beginForeignKey(from: "accounts", column: "name", to: target) + + let started = try #require(draft(of: fixture.viewModel)) + #expect(started.pairs.map { $0.column } == ["name"]) + #expect(started.pairs.map { $0.referencedColumn } == ["column_0"]) + } +} diff --git a/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift b/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift index 5a3e12f451..9cff10a01a 100644 --- a/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift +++ b/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift @@ -55,9 +55,13 @@ struct SSLPaneViewModelTests { } private func verifyIdentityIssues(for type: DatabaseType, caCertPath: String = "") -> [String] { + issues(for: type, mode: .verifyIdentity, caCertPath: caCertPath) + } + + private func issues(for type: DatabaseType, mode: SSLMode, caCertPath: String = "") -> [String] { let coordinator = ConnectionFormCoordinator(connectionId: nil) coordinator.network.type = type - coordinator.ssl.select(.verifyIdentity) + coordinator.ssl.select(mode) coordinator.ssl.caCertPath = caCertPath return coordinator.ssl.validationIssues } @@ -72,9 +76,20 @@ struct SSLPaneViewModelTests { @Test("Verify Identity without a CA file is refused for engines that need one") func fileTrustEnginesStillRequireACAFile() { - #expect(!DatabaseType.postgresql.verifiesServerWithSystemTrust) - #expect(verifyIdentityIssues(for: .postgresql).count == 1) - #expect(verifyIdentityIssues(for: .postgresql, caCertPath: "/tmp/ca.pem").isEmpty) + #expect(!DatabaseType.mysql.verifiesServerWithSystemTrust) + #expect(verifyIdentityIssues(for: .mysql).count == 1) + #expect(verifyIdentityIssues(for: .mysql, caCertPath: "/tmp/ca.pem").isEmpty) + } + + @Test( + "The libpq engines save Verify Identity without a CA file and still need one for Verify CA", + arguments: [DatabaseType.postgresql, .redshift, .cockroachdb, .kingbasees] + ) + func libpqEnginesNeedACAFileOnlyForVerifyCA(type: DatabaseType) { + #expect(type.verifiesServerWithSystemTrust) + #expect(verifyIdentityIssues(for: type).isEmpty) + #expect(issues(for: type, mode: .verifyCa).count == 1) + #expect(issues(for: type, mode: .verifyCa, caCertPath: "/tmp/ca.pem").isEmpty) } @Test("A Trino port of 443 escalates the type default to Verify Identity, and another port reverts it") diff --git a/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift b/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift index 1c394ac653..aedc5fa001 100644 --- a/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift +++ b/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift @@ -257,6 +257,41 @@ struct ServerDashboardViewModelTests { #expect(vm.sessionSelection.ids == ["10"]) } + @Test("A hidden, minimized or covered window stops the polling, and showing it again resumes") + func hiddenWindowStopsPolling() { + let vm = makeViewModel(databaseType: .mysql) + vm.startAutoRefresh() + #expect(vm.isPolling) + #expect(vm.tileBoard.isActive) + + vm.windowVisibilityChanged(false) + #expect(!vm.isPolling) + #expect(!vm.tileBoard.isActive) + + vm.windowVisibilityChanged(true) + #expect(vm.isPolling) + #expect(vm.tileBoard.isActive) + vm.stopAutoRefresh() + #expect(!vm.isPolling) + } + + @Test("A window hidden before the dashboard appears keeps it from polling") + func hiddenBeforeAppearStaysIdle() { + let vm = makeViewModel(databaseType: .mysql) + vm.windowVisibilityChanged(false) + vm.startAutoRefresh() + #expect(!vm.isPolling) + #expect(!vm.tileBoard.isActive) + vm.stopAutoRefresh() + } + + @Test("Changing the interval while the dashboard is off screen starts no polling") + func intervalChangeOffScreenStaysIdle() { + let vm = makeViewModel(databaseType: .mysql) + vm.refreshInterval = .tenSeconds + #expect(!vm.isPolling) + } + @Test("stopAutoRefresh clears the refreshing flag") func stopAutoRefreshClearsRefreshingFlag() { let vm = makeViewModel(databaseType: .mysql) diff --git a/apple/TableProTests/ViewModels/SidebarTableOperationWindowTests.swift b/apple/TableProTests/ViewModels/SidebarTableOperationWindowTests.swift new file mode 100644 index 0000000000..946cfa21a0 --- /dev/null +++ b/apple/TableProTests/ViewModels/SidebarTableOperationWindowTests.swift @@ -0,0 +1,89 @@ +import Foundation +import SwiftUI +import Testing +@testable import TablePro + +@MainActor +struct SidebarTableOperationWindowTests { + private final class ConnectionQueues { + var truncates: Set = [] + var deletes: Set = [] + var options: [DatabaseTreeTableRef: TableOperationOptions] = [:] + } + + private let queues = ConnectionQueues() + + private func makeSharedViewModel() -> SidebarViewModel { + let queues = queues + return SidebarViewModel( + pendingTruncates: Binding(get: { queues.truncates }, set: { queues.truncates = $0 }), + pendingDeletes: Binding(get: { queues.deletes }, set: { queues.deletes = $0 }), + tableOperationOptions: Binding(get: { queues.options }, set: { queues.options = $0 }), + databaseType: .mysql, + connectionId: UUID() + ) + } + + private func makeRef(_ name: String) -> DatabaseTreeTableRef { + DatabaseTreeTableRef(database: "shop", schema: nil, table: TestFixtures.makeTableInfo(name: name)) + } + + @Test("A truncate asks only the window it came from, and its confirm writes that window's tables") + func truncateBelongsToTheAskingWindow() throws { + let viewModel = makeSharedViewModel() + let windowA = WindowSidebarState() + let windowB = WindowSidebarState() + let users = makeRef("users") + let orders = makeRef("orders") + windowA.selectTables([users]) + windowB.selectTables([orders]) + + viewModel.batchToggleTruncate(refs: Array(windowA.selectedTables), from: windowA) + + let requestA = try #require(windowA.pendingTableOperation) + #expect(requestA.operation == .truncate) + #expect(requestA.tables == [users]) + #expect(windowB.pendingTableOperation == nil) + + viewModel.batchToggleDelete(refs: Array(windowB.selectedTables), from: windowB) + let requestB = try #require(windowB.pendingTableOperation) + #expect(windowA.pendingTableOperation == requestA) + + let options = TableOperationOptions(ignoreForeignKeys: true) + viewModel.confirmOperation(requestA, options: options) + windowA.resolveTableOperation(requestA) + + #expect(queues.truncates == [users]) + #expect(queues.deletes.isEmpty) + #expect(queues.options == [users: options]) + #expect(windowA.pendingTableOperation == nil) + #expect(windowB.pendingTableOperation == requestB) + } + + @Test("Answering an earlier prompt acts on its own tables, not on a request posted after it") + func confirmActsOnTheRequestItWasShownFor() throws { + let viewModel = makeSharedViewModel() + let window = WindowSidebarState() + let users = makeRef("users") + let orders = makeRef("orders") + + viewModel.batchToggleTruncate(refs: [users], from: window) + let first = try #require(window.pendingTableOperation) + viewModel.batchToggleDelete(refs: [orders], from: window) + let second = try #require(window.pendingTableOperation) + + let options = TableOperationOptions(cascade: true) + viewModel.confirmOperation(first, options: options) + window.resolveTableOperation(first) + + #expect(queues.truncates == [users]) + #expect(queues.deletes.isEmpty) + #expect(queues.options == [users: options]) + #expect(window.pendingTableOperation == second) + + window.resolveTableOperation(second) + + #expect(window.pendingTableOperation == nil) + #expect(queues.deletes.isEmpty) + } +} diff --git a/apple/TableProTests/ViewModels/SidebarViewModelTests.swift b/apple/TableProTests/ViewModels/SidebarViewModelTests.swift index d6689d4181..dd497fbe38 100644 --- a/apple/TableProTests/ViewModels/SidebarViewModelTests.swift +++ b/apple/TableProTests/ViewModels/SidebarViewModelTests.swift @@ -8,34 +8,25 @@ import Testing @MainActor private func makeSUT( - tables: [TableInfo] = [], - selectedTables: Set = [], pendingTruncates: Set = [], pendingDeletes: Set = [], tableOperationOptions: [DatabaseTreeTableRef: TableOperationOptions] = [:], databaseType: DatabaseType = .mysql ) -> ( vm: SidebarViewModel, - tables: Binding<[TableInfo]>, - selectedTables: Binding>, pendingTruncates: Binding>, pendingDeletes: Binding>, tableOperationOptions: Binding<[DatabaseTreeTableRef: TableOperationOptions]> ) { - var tablesState = tables - var selectedState = selectedTables var truncatesState = pendingTruncates var deletesState = pendingDeletes var optionsState = tableOperationOptions - let tablesBinding = Binding(get: { tablesState }, set: { tablesState = $0 }) - let selectedBinding = Binding(get: { selectedState }, set: { selectedState = $0 }) let truncatesBinding = Binding(get: { truncatesState }, set: { truncatesState = $0 }) let deletesBinding = Binding(get: { deletesState }, set: { deletesState = $0 }) let optionsBinding = Binding(get: { optionsState }, set: { optionsState = $0 }) let vm = SidebarViewModel( - selectedTables: selectedBinding, pendingTruncates: truncatesBinding, pendingDeletes: deletesBinding, tableOperationOptions: optionsBinding, @@ -43,7 +34,7 @@ private func makeSUT( connectionId: UUID() ) - return (vm, tablesBinding, selectedBinding, truncatesBinding, deletesBinding, optionsBinding) + return (vm, truncatesBinding, deletesBinding, optionsBinding) } @MainActor @@ -61,74 +52,75 @@ struct SidebarViewModelTests { // MARK: - Batch Toggle Truncate - @Test("batchToggleTruncate shows dialog for new tables") + @Test("batchToggleTruncate asks the window for new tables") @MainActor func batchToggleTruncateShowsDialog() { let table = makeRef("users") - let (vm, _, _, _, _, _) = makeSUT(selectedTables: [table]) + let (vm, _, _, _) = makeSUT() + let window = WindowSidebarState() - vm.batchToggleTruncate() + vm.batchToggleTruncate(refs: [table], from: window) - #expect(vm.showOperationDialog) - #expect(vm.pendingOperationType == .truncate) - #expect(vm.pendingOperationTables == [table]) + #expect(window.pendingTableOperation?.operation == .truncate) + #expect(window.pendingTableOperation?.tables == [table]) } - @Test("batchToggleTruncate cancels when all already pending") + @Test("batchToggleTruncate unstages when all already pending") @MainActor func batchToggleTruncateCancels() { let table = makeRef("users") - let (vm, _, _, truncatesBinding, _, optionsBinding) = makeSUT( - selectedTables: [table], + let (vm, truncatesBinding, _, optionsBinding) = makeSUT( pendingTruncates: [table], tableOperationOptions: [table: TableOperationOptions()] ) + let window = WindowSidebarState() - vm.batchToggleTruncate() + vm.batchToggleTruncate(refs: [table], from: window) - #expect(!vm.showOperationDialog) + #expect(window.pendingTableOperation == nil) #expect(!truncatesBinding.wrappedValue.contains(table)) #expect(optionsBinding.wrappedValue[table] == nil) } - @Test("batchToggleTruncate does nothing when no selection") + @Test("batchToggleTruncate does nothing for no tables") @MainActor func batchToggleTruncateNoSelection() { - let (vm, _, _, _, _, _) = makeSUT() + let (vm, _, _, _) = makeSUT() + let window = WindowSidebarState() - vm.batchToggleTruncate() + vm.batchToggleTruncate(refs: [], from: window) - #expect(!vm.showOperationDialog) + #expect(window.pendingTableOperation == nil) } // MARK: - Batch Toggle Delete - @Test("batchToggleDelete shows dialog for new tables") + @Test("batchToggleDelete asks the window for new tables") @MainActor func batchToggleDeleteShowsDialog() { let table = makeRef("orders") - let (vm, _, _, _, _, _) = makeSUT(selectedTables: [table]) + let (vm, _, _, _) = makeSUT() + let window = WindowSidebarState() - vm.batchToggleDelete() + vm.batchToggleDelete(refs: [table], from: window) - #expect(vm.showOperationDialog) - #expect(vm.pendingOperationType == .drop) - #expect(vm.pendingOperationTables == [table]) + #expect(window.pendingTableOperation?.operation == .drop) + #expect(window.pendingTableOperation?.tables == [table]) } - @Test("batchToggleDelete cancels when all already pending") + @Test("batchToggleDelete unstages when all already pending") @MainActor func batchToggleDeleteCancels() { let table = makeRef("orders") - let (vm, _, _, _, deletesBinding, optionsBinding) = makeSUT( - selectedTables: [table], + let (vm, _, deletesBinding, optionsBinding) = makeSUT( pendingDeletes: [table], tableOperationOptions: [table: TableOperationOptions()] ) + let window = WindowSidebarState() - vm.batchToggleDelete() + vm.batchToggleDelete(refs: [table], from: window) - #expect(!vm.showOperationDialog) + #expect(window.pendingTableOperation == nil) #expect(!deletesBinding.wrappedValue.contains(table)) #expect(optionsBinding.wrappedValue[table] == nil) } @@ -139,16 +131,10 @@ struct SidebarViewModelTests { @MainActor func confirmTruncateMovesFromDeletes() { let table = makeRef("users") - let (vm, _, _, truncatesBinding, deletesBinding, optionsBinding) = makeSUT( - selectedTables: [table], - pendingDeletes: [table] - ) - - vm.pendingOperationType = .truncate - vm.pendingOperationTables = [table] + let (vm, truncatesBinding, deletesBinding, optionsBinding) = makeSUT(pendingDeletes: [table]) let options = TableOperationOptions(ignoreForeignKeys: true) - vm.confirmOperation(options: options) + vm.confirmOperation(PendingTableOperation(operation: .truncate, tables: [table]), options: options) #expect(truncatesBinding.wrappedValue.contains(table)) #expect(!deletesBinding.wrappedValue.contains(table)) @@ -159,16 +145,10 @@ struct SidebarViewModelTests { @MainActor func confirmDropMovesFromTruncates() { let table = makeRef("users") - let (vm, _, _, truncatesBinding, deletesBinding, optionsBinding) = makeSUT( - selectedTables: [table], - pendingTruncates: [table] - ) - - vm.pendingOperationType = .drop - vm.pendingOperationTables = [table] + let (vm, truncatesBinding, deletesBinding, optionsBinding) = makeSUT(pendingTruncates: [table]) let options = TableOperationOptions(cascade: true) - vm.confirmOperation(options: options) + vm.confirmOperation(PendingTableOperation(operation: .drop, tables: [table]), options: options) #expect(!truncatesBinding.wrappedValue.contains(table)) #expect(deletesBinding.wrappedValue.contains(table)) @@ -180,58 +160,34 @@ struct SidebarViewModelTests { func confirmOperationStoresOptions() { let t1 = makeRef("t1") let t2 = makeRef("t2") - let (vm, _, _, _, _, optionsBinding) = makeSUT(selectedTables: [t1, t2]) - - vm.pendingOperationType = .truncate - vm.pendingOperationTables = [t1, t2] + let (vm, _, _, optionsBinding) = makeSUT() let options = TableOperationOptions(ignoreForeignKeys: true, cascade: true) - vm.confirmOperation(options: options) + vm.confirmOperation(PendingTableOperation(operation: .truncate, tables: [t1, t2]), options: options) #expect(optionsBinding.wrappedValue[t1] == options) #expect(optionsBinding.wrappedValue[t2] == options) } - - @Test("confirmOperation resets dialog state after confirm") - @MainActor - func confirmOperationResetsDialogState() { - let table = makeRef("users") - let (vm, _, _, _, _, _) = makeSUT(selectedTables: [table]) - - vm.pendingOperationType = .truncate - vm.pendingOperationTables = [table] - vm.showOperationDialog = true - - vm.confirmOperation(options: TableOperationOptions()) - - #expect(vm.pendingOperationType == nil) - #expect(vm.pendingOperationTables.isEmpty) - } - } extension GlobalStateSuites { @Suite(.recordingClipboard) @MainActor struct SidebarViewModelCopyTests { - @Test("copySelectedTableNames copies sorted comma-separated names") + @Test("copyTableNames copies sorted comma-separated names") func copyTableNames() throws { let clipboard = try #require(RecordingClipboard.current) - let t1 = makeRef("zebra") - let t2 = makeRef("alpha") - let (vm, _, _, _, _, _) = makeSUT(selectedTables: [t1, t2]) - vm.copySelectedTableNames() + SidebarViewModel.copyTableNames(of: [makeRef("zebra"), makeRef("alpha")]) #expect(clipboard.text == "alpha,zebra") } - @Test("copySelectedTableNames does nothing when no selection") + @Test("copyTableNames does nothing for no tables") func copyTableNamesNoSelection() throws { let clipboard = try #require(RecordingClipboard.current) - let (vm, _, _, _, _, _) = makeSUT() - vm.copySelectedTableNames() + SidebarViewModel.copyTableNames(of: []) #expect(clipboard.text == nil) } @@ -247,16 +203,13 @@ private func makeViewModel( defaults: UserDefaults, clock: any Clock = ContinuousClock() ) -> SidebarViewModel { - var selectedState: Set = [] var truncates: Set = [] var deletes: Set = [] var options: [DatabaseTreeTableRef: TableOperationOptions] = [:] - let selectedBinding = Binding(get: { selectedState }, set: { selectedState = $0 }) let truncatesBinding = Binding(get: { truncates }, set: { truncates = $0 }) let deletesBinding = Binding(get: { deletes }, set: { deletes = $0 }) let optionsBinding = Binding(get: { options }, set: { options = $0 }) return SidebarViewModel( - selectedTables: selectedBinding, pendingTruncates: truncatesBinding, pendingDeletes: deletesBinding, tableOperationOptions: optionsBinding, @@ -434,6 +387,56 @@ struct SidebarViewModelMultiSectionTests { #expect(defaults.bool(forKey: key) == true) } + @Test("A declared kind's flat expansion persists under the one per-connection key and restores by id") + func declaredKindExpansionPersistsUnderOneKey() { + let defaults = suite.defaults + let connectionId = UUID() + let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + let vm = makeViewModel(connectionId: connectionId, databaseType: .mysql, defaults: defaults) + #expect(vm.expanded[.server(events)] == false) + + vm.expanded[.server(events)] = true + + let key = SidebarPersistenceKey.serverKindsExpanded(connectionId: connectionId) + #expect(defaults.dictionary(forKey: key)?["event"] as? Bool == true) + #expect(defaults.object(forKey: "sidebar.\(connectionId.uuidString).server.event.expanded") == nil) + #expect(SidebarPersistenceKey.all(connectionId: connectionId).contains(key)) + + let restored = makeViewModel(connectionId: connectionId, databaseType: .mysql, defaults: defaults) + #expect(restored.expanded[.server(SidebarServerObjectKind(id: "event"))] == true) + + SidebarPersistenceKey.removeAll(connectionId: connectionId, defaults: defaults) + #expect(defaults.object(forKey: key) == nil) + } + + @Test("Every built-in kind's expansion key is listed for cleanup") + func builtInExpansionKeysAreListed() { + let connectionId = UUID() + let all = SidebarPersistenceKey.all(connectionId: connectionId) + for kind in SidebarObjectKind.builtIn { + #expect(all.contains(SidebarPersistenceKey.expanded(connectionId: connectionId, kind: kind))) + } + #expect(SidebarPersistenceKey.expanded(connectionId: connectionId, kind: .materializedView) + == "sidebar.\(connectionId.uuidString).materializedView.expanded") + } + + @Test("filteredServerObjects narrows by the search and follows each kind on its own") + func filteredServerObjectsBySearch() { + let vm = makeViewModel(databaseType: .mysql, defaults: suite.defaults) + let events = [ + ServerObjectInfo(kindId: "event", name: "nightly_cleanup", schema: "shop"), + ServerObjectInfo(kindId: "event", name: "hourly_rollup", schema: "shop") + ] + #expect(vm.filteredServerObjects(kindId: "event", from: events).count == 2) + + vm.searchText = "nightly" + + #expect(vm.filteredServerObjects(kindId: "event", from: events).map { $0.name } == ["nightly_cleanup"]) + #expect(vm.filteredServerObjects(kindId: "extension", from: []).isEmpty) + } + @Test("recents expansion defaults to expanded and persists across init") func recentsExpansionPersists() { let defaults = suite.defaults diff --git a/apple/TableProTests/ViewModels/VersionHistoryViewModelTests.swift b/apple/TableProTests/ViewModels/VersionHistoryViewModelTests.swift index 89a29599c9..c0f502b491 100644 --- a/apple/TableProTests/ViewModels/VersionHistoryViewModelTests.swift +++ b/apple/TableProTests/ViewModels/VersionHistoryViewModelTests.swift @@ -236,6 +236,19 @@ struct VersionHistoryViewModelTests { #expect(viewModel.page.entries.isEmpty) #expect(viewModel.listState == .failed(VersionHistoryError.subjectNotFound.localizedDescription)) } + + @Test("A query tab that closed clears the list and says how to get it back") + func closedTabClearsTheList() async throws { + let provider = makeProvider() + let viewModel = makeViewModel(provider) + await viewModel.loadList() + await viewModel.detailTask?.value + + await provider.setLoadError(.tabClosed) + await viewModel.loadList() + #expect(viewModel.page.entries.isEmpty) + #expect(viewModel.listState == .failed(VersionHistoryError.tabClosed.localizedDescription)) + } } struct VersionComparisonTests { diff --git a/apple/TableProTests/ViewModels/WelcomeViewModelTeamLibraryTests.swift b/apple/TableProTests/ViewModels/WelcomeViewModelTeamLibraryTests.swift index 69fcdadaa9..9b310390fe 100644 --- a/apple/TableProTests/ViewModels/WelcomeViewModelTeamLibraryTests.swift +++ b/apple/TableProTests/ViewModels/WelcomeViewModelTeamLibraryTests.swift @@ -52,6 +52,30 @@ struct WelcomeViewModelTeamLibraryTests { #expect(fixture.sync.content.connections.map(\.settings.name) == ["Prod", "Staging"]) } + @Test("A published connection carries neither its tunnel command nor its startup commands") + func publishLeavesCommandsOut() async throws { + let fixture = try await TeamLibraryFixture() + var prod = DatabaseConnection(name: "Prod", type: .postgresql) + prod.tunnelCommandMode = .inline(TunnelCommandConfiguration( + method: .custom, + command: "cloudflared access tcp --url localhost:{port} --service-token-secret S3cret" + )) + prod.startupCommands = "SET app.token = 'S3cret';" + + let outcome = await WelcomeViewModel.publishConnections( + [prod], + to: TeamLibraryFixture.team, + through: fixture.sync, + confirm: { _ in true } + ) + + #expect(outcome == .finished(.published(1))) + let data = try #require(await fixture.pushes().first?.first?["data"] as? [String: Any]) + #expect(data["name"] as? String == "Prod") + #expect(data["tunnelCommand"] == nil) + #expect(data["startupCommands"] == nil) + } + @Test("A member who only reads the library gets told so, and nothing is pushed") func readerCannotPublish() async throws { let fixture = try await TeamLibraryFixture(role: "member") diff --git a/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift b/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift index e4afcb5559..6404c8dd2c 100644 --- a/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift +++ b/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift @@ -332,6 +332,42 @@ final class WelcomeViewModelTests { #expect(viewModel.listState == .noFilterMatch) } + @Test func testExcludingFromSyncAsksFirst() throws { + let prod = DatabaseConnection(name: "Prod", type: .postgresql) + save([prod]) + viewModel.loadConnections() + + viewModel.perform(.setIncludedInSync([prod.id], false)) + + #expect(viewModel.showExcludeFromSyncConfirmation) + #expect(viewModel.connectionsToExcludeFromSync.map(\.id) == [prod.id]) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly == false) + } + + @Test func testConfirmingExcludeFromSyncMarksTheConnectionLocalOnly() throws { + let prod = DatabaseConnection(name: "Prod", type: .postgresql) + save([prod]) + viewModel.loadConnections() + viewModel.perform(.setIncludedInSync([prod.id], false)) + + viewModel.confirmExcludeFromSync() + + #expect(viewModel.connectionsToExcludeFromSync.isEmpty) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly) + } + + @Test func testIncludingInSyncNeedsNoConfirmation() throws { + var prod = DatabaseConnection(name: "Prod", type: .postgresql) + prod.localOnly = true + save([prod]) + viewModel.loadConnections() + + viewModel.perform(.setIncludedInSync([prod.id], true)) + + #expect(!viewModel.showExcludeFromSyncConfirmation) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly == false) + } + @Test func testDeletingTheLastConnectionClearsTheSearchItCanNoLongerRun() { let only = DatabaseConnection(name: "Only", type: .mysql, sortOrder: 0) save([only]) diff --git a/apple/TableProTests/ViewModels/WindowSidebarStateTests.swift b/apple/TableProTests/ViewModels/WindowSidebarStateTests.swift index 131a852862..3b72dee2d0 100644 --- a/apple/TableProTests/ViewModels/WindowSidebarStateTests.swift +++ b/apple/TableProTests/ViewModels/WindowSidebarStateTests.swift @@ -214,6 +214,41 @@ struct WindowSidebarStateTests { ) } + @Test("A declared kind's group expansion persists by id and restores before any driver exists") + func declaredKindGroupExpansionRoundTrips() throws { + let defaults = try makeDefaults() + let connectionId = UUID() + let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + let group = DatabaseTreeObjectGroup(database: "shop", schema: nil, kind: .server(events)) + + let state = WindowSidebarState(connectionId: connectionId, defaults: defaults) + #expect(!state.isTreeObjectGroupExpanded(group)) + state.setTreeObjectGroup(group, expanded: true) + + let stored = try #require(defaults.data(forKey: "com.TablePro.sidebar.treeExpansion.\(connectionId.uuidString)")) + #expect(String(decoding: stored, as: UTF8.self).contains("\"server.event\"")) + + let restored = WindowSidebarState(connectionId: connectionId, defaults: defaults) + #expect(restored.isTreeObjectGroupExpanded(group)) + let decodedOnly = DatabaseTreeObjectGroup( + database: "shop", schema: nil, kind: .server(SidebarServerObjectKind(id: "event")) + ) + #expect(restored.isTreeObjectGroupExpanded(decodedOnly)) + } + + @Test("A built-in group still persists under the kind name it shipped with") + func builtInGroupKeepsShippedName() throws { + let defaults = try makeDefaults() + let connectionId = UUID() + let state = WindowSidebarState(connectionId: connectionId, defaults: defaults) + state.setTreeObjectGroup(DatabaseTreeObjectGroup(database: "shop", schema: "public", kind: .materializedView), expanded: true) + + let stored = try #require(defaults.data(forKey: "com.TablePro.sidebar.treeExpansion.\(connectionId.uuidString)")) + #expect(String(decoding: stored, as: UTF8.self).contains("\"materializedView\"")) + } + @Test("A window without a connection does not persist") func nilConnectionDoesNotPersist() throws { let defaults = try makeDefaults() diff --git a/apple/TableProTests/Views/Compare/CompareRowGridModelTests.swift b/apple/TableProTests/Views/Compare/CompareRowGridModelTests.swift index 4ac04fa01c..8ee9d8f4d4 100644 --- a/apple/TableProTests/Views/Compare/CompareRowGridModelTests.swift +++ b/apple/TableProTests/Views/Compare/CompareRowGridModelTests.swift @@ -725,6 +725,7 @@ struct CompareRowGridModelTests { placeholderText: .secondaryLabelColor, booleanTrueText: .labelColor, booleanFalseText: .labelColor, + valueColors: DataGridValueColors(all: .labelColor), rowNumberText: .secondaryLabelColor, deletedRowText: .systemRed, modifiedColumnTint: .systemYellow, diff --git a/apple/TableProTests/Views/Components/BrandIconTests.swift b/apple/TableProTests/Views/Components/BrandIconTests.swift new file mode 100644 index 0000000000..697ee755a5 --- /dev/null +++ b/apple/TableProTests/Views/Components/BrandIconTests.swift @@ -0,0 +1,112 @@ +// +// BrandIconTests.swift +// TableProTests +// + +import AppKit +import Foundation +@testable import TablePro +import Testing + +struct BrandIconTests { + private static func catalogIconSlugs() throws -> [String] { + let url = try #require(Bundle.main.url(forResource: "clients", withExtension: "json")) + let object = try JSONSerialization.jsonObject(with: Data(contentsOf: url)) + let catalog = try #require(object as? [String: Any]) + let clients = try #require(catalog["clients"] as? [[String: Any]]) + return try clients.map { client in + try #require(client["icon"] as? String, "\(client["id"] ?? "a client") has no icon") + } + } + + private static var providerIcons: [(name: String, icon: BrandIcon)] { + AIProviderType.allCases.map { ($0.rawValue, $0.icon) } + AIProviderPreset.all.map { ($0.id, $0.icon) } + } + + private static func visiblePixelCount(of image: NSImage, side: Int) throws -> Int { + let rep = try #require(NSBitmapImageRep( + bitmapDataPlanes: nil, pixelsWide: side, pixelsHigh: side, bitsPerSample: 8, samplesPerPixel: 4, + hasAlpha: true, isPlanar: false, colorSpaceName: .deviceRGB, bytesPerRow: 0, bitsPerPixel: 0 + )) + NSGraphicsContext.saveGraphicsState() + NSGraphicsContext.current = NSGraphicsContext(bitmapImageRep: rep) + image.draw(in: NSRect(x: 0, y: 0, width: side, height: side)) + NSGraphicsContext.restoreGraphicsState() + var count = 0 + for y in 0.. 0 { + count += 1 + } + } + return count + } + + @Test("Every client icon in the bundled catalog is an asset in the app") + func catalogIconsResolve() throws { + let slugs = try Self.catalogIconSlugs() + #expect(!slugs.isEmpty) + for slug in Set(slugs) { + #expect(NSImage(named: BrandIcon.assetName(for: slug)) != nil, "\(slug)-icon is missing") + } + } + + @Test("Every AI provider and preset icon is an asset, and its fallback is an SF Symbol") + func providerIconsResolve() { + for (name, icon) in Self.providerIcons { + if let slug = icon.slug { + #expect(icon.assetImage != nil, "\(name): \(slug)-icon is missing") + } + #expect( + NSImage(systemSymbolName: icon.fallbackSystemImage, accessibilityDescription: nil) != nil, + "\(name): \(icon.fallbackSystemImage) is not an SF Symbol" + ) + } + } + + @Test("A custom provider has no brand and draws its SF Symbol") + func customProviderFallsBack() { + let icon = AIProviderType.custom.icon + #expect(icon.slug == nil) + #expect(icon.assetImage == nil) + #expect(icon.nsImage() != nil) + } + + @Test("A slug with no asset draws the fallback symbol") + func missingAssetFallsBack() throws { + let icon = BrandIcon(slug: "no-such-brand", fallbackSystemImage: "server.rack") + #expect(icon.assetImage == nil) + let image = try #require(icon.nsImage()) + #expect(image.isTemplate) + } + + @Test("The menu copy is the asked square and keeps the asset's template flag") + func menuCopyKeepsTemplateFlag() throws { + let template = try #require(BrandIcon(slug: "claude", fallbackSystemImage: "brain").nsImage(pointSize: 16)) + #expect(template.size == NSSize(width: 16, height: 16)) + #expect(template.isTemplate) + + let original = try #require(BrandIcon(slug: "vscode", fallbackSystemImage: "curlybraces").nsImage()) + #expect(original.size == NSSize(width: 16, height: 16)) + #expect(!original.isTemplate) + } + + // A one-color official mark is a template; a multicolor one keeps its colors, which VS Code's brand terms require. + @Test("Only the brands whose official logo has several colors skip template rendering") + func onlyMulticolorLogosAreOriginal() throws { + let slugs = try Set(Self.catalogIconSlugs() + Self.providerIcons.compactMap { $0.icon.slug }) + let original = slugs.filter { slug in + NSImage(named: BrandIcon.assetName(for: slug))?.isTemplate == false + } + #expect(original == ["kiro", "mlx", "requesty", "vscode"]) + } + + @Test("Every brand asset draws something at menu size") + @MainActor + func everyAssetDraws() throws { + let slugs = try Set(Self.catalogIconSlugs() + Self.providerIcons.compactMap { $0.icon.slug }) + for slug in slugs { + let image = try #require(BrandIcon(slug: slug, fallbackSystemImage: "questionmark").nsImage(pointSize: 16)) + #expect(try Self.visiblePixelCount(of: image, side: 32) > 0, "\(slug)-icon draws nothing") + } + } +} diff --git a/apple/TableProTests/Views/Components/TransferProgressTests.swift b/apple/TableProTests/Views/Components/TransferProgressTests.swift index 168f4dcac1..df84675e92 100644 --- a/apple/TableProTests/Views/Components/TransferProgressTests.swift +++ b/apple/TableProTests/Views/Components/TransferProgressTests.swift @@ -47,6 +47,19 @@ struct TransferProgressTests { #expect(progress.amountText == "\(grouped(5_000)) of about \(grouped(4_000)) statements") } + @Test("A file fraction drives the bar, short of the end, while the amount still counts statements") + func fileFractionDrivesTheBar() { + let reading = TransferProgress( + job: .dataImport, completed: 1, total: 40, unit: .statements, isEstimate: true, fileFraction: 0.6 + ) + let past = TransferProgress(job: .dataImport, completed: 40, total: 40, unit: .statements, fileFraction: 1.4) + + #expect(reading.fraction == 0.6) + #expect(reading.amountText == "1 of about 40 statements") + #expect(past.fraction == 0.99) + #expect(past.amountText == "40 of 40 statements") + } + @Test("Counts pick the singular where the number is one") func singular() { #expect(TransferProgress(job: .export, completed: 1, unit: .rows).amountText == "1 row") diff --git a/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift b/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift index a99dbec0aa..1b6250f752 100644 --- a/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift +++ b/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift @@ -22,13 +22,18 @@ struct PasswordSourceTestModelTests { @Test("A refused test reports why and never runs the source") func refusalSkipsTheRun() async { let runs = RunCounter() - let model = PasswordSourceTestModel { _, _ in + let model = PasswordSourceTestModel { _, _, _ in runs.increment() return "secret" } let source = PasswordSource.command(shell: "echo secret") - await model.run(source, context: context, refusal: PasswordSourceResolver.ResolutionError.storeNotTrusted) + await model.run( + source, + context: context, + leaseOwner: context.connectionId, + refusal: PasswordSourceResolver.ResolutionError.storeNotTrusted + ) #expect(runs.value == 0) #expect(model.outcome(for: source, context: context) == .failed( @@ -41,7 +46,7 @@ struct PasswordSourceTestModelTests { let model = PasswordSourceTestModel() let source = PasswordSource.command(shell: "echo hunter2") - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: source, context: context) == .returned("hunter2")) #expect(!model.isRunning) @@ -52,7 +57,7 @@ struct PasswordSourceTestModelTests { let model = PasswordSourceTestModel() let source = PasswordSource.command(shell: "echo 'op: not signed in' >&2; exit 3") - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) guard case .failed(let message) = model.outcome(for: source, context: context) else { Issue.record("Expected a failure") @@ -64,10 +69,10 @@ struct PasswordSourceTestModelTests { @Test("A result is shown only for the source that produced it") func outcomeBelongsToItsSource() async { - let model = PasswordSourceTestModel { _, _ in "secret" } + let model = PasswordSourceTestModel { _, _, _ in "secret" } let tested = PasswordSource.env(variable: "A") - await model.run(tested, context: context, refusal: nil) + await model.run(tested, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: tested, context: context) == .returned("secret")) #expect(model.outcome(for: .env(variable: "B"), context: context) == nil) @@ -75,13 +80,13 @@ struct PasswordSourceTestModelTests { @Test("Editing the connection's details hides a result that used the old ones") func outcomeBelongsToItsConnectionDetails() async { - let model = PasswordSourceTestModel { _, context in context.host } + let model = PasswordSourceTestModel { _, context, _ in context.host } let source = PasswordSource.command(shell: "echo {host}") let moved = ConnectionScriptContext( connectionId: context.connectionId, connectionName: "c", host: "other", port: 1, user: "u", database: "d" ) - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: source, context: context) == .returned("h")) #expect(model.outcome(for: source, context: moved) == nil) @@ -89,13 +94,13 @@ struct PasswordSourceTestModelTests { @Test("A new connection's details match from one render to the next") func outcomeIgnoresTheThrowawayId() async { - let model = PasswordSourceTestModel { _, _ in "secret" } + let model = PasswordSourceTestModel { _, _, _ in "secret" } let source = PasswordSource.env(variable: "A") let rerendered = ConnectionScriptContext( connectionId: UUID(), connectionName: "c", host: "h", port: 1, user: "u", database: "d" ) - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: source, context: rerendered) == .returned("secret")) } diff --git a/apple/TableProTests/Views/ConnectionForm/ViewModels/AdvancedPaneViewModelTests.swift b/apple/TableProTests/Views/ConnectionForm/ViewModels/AdvancedPaneViewModelTests.swift index cb75ba406b..bf5abddb6b 100644 --- a/apple/TableProTests/Views/ConnectionForm/ViewModels/AdvancedPaneViewModelTests.swift +++ b/apple/TableProTests/Views/ConnectionForm/ViewModels/AdvancedPaneViewModelTests.swift @@ -31,6 +31,34 @@ struct AdvancedPaneViewModelTests { #expect(fields["externalAccess"] == nil) } + @Test("Lists the connection's always-allowed tools in name order with nothing forgotten") + func loadsAlwaysAllowedTools() { + let connection = DatabaseConnection(name: "Staging", aiAlwaysAllowedTools: ["list_tables", "execute_query"]) + let viewModel = AdvancedPaneViewModel() + + viewModel.load(from: connection) + + #expect(viewModel.aiAlwaysAllowedTools == ["execute_query", "list_tables"]) + #expect(viewModel.revokedAIAlwaysAllowedTools.isEmpty) + } + + @Test("Forget takes a tool off the list and into the revoked set, and loading again drops it") + func forgetMovesAToolIntoTheRevokedSet() { + let connection = DatabaseConnection(name: "Staging", aiAlwaysAllowedTools: ["list_tables", "execute_query"]) + let viewModel = AdvancedPaneViewModel() + viewModel.load(from: connection) + + viewModel.forgetAlwaysAllowed("execute_query") + + #expect(viewModel.aiAlwaysAllowedTools == ["list_tables"]) + #expect(viewModel.revokedAIAlwaysAllowedTools == ["execute_query"]) + + viewModel.load(from: connection) + + #expect(viewModel.aiAlwaysAllowedTools == ["execute_query", "list_tables"]) + #expect(viewModel.revokedAIAlwaysAllowedTools.isEmpty) + } + @Test("Shows a Redis database index saved as db4 as the number the stepper takes") func loadsDbPrefixedRedisIndexAsANumber() throws { try #require( diff --git a/apple/TableProTests/Views/DataGrid/DataGridCellAppearanceTests.swift b/apple/TableProTests/Views/DataGrid/DataGridCellAppearanceTests.swift index c8ea6aa420..7b8523d2e1 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridCellAppearanceTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridCellAppearanceTests.swift @@ -6,6 +6,7 @@ import AppKit @testable import TablePro import TableProPluginKit +import TableProTestSupport import TableProTheme import Testing @@ -19,6 +20,7 @@ struct DataGridCellAppearanceTests { placeholderText: .secondaryLabelColor, booleanTrueText: .labelColor, booleanFalseText: .labelColor, + valueColors: DataGridValueColors(all: .labelColor), rowNumberText: .secondaryLabelColor, deletedRowText: .systemRed, modifiedColumnTint: .systemYellow, @@ -35,6 +37,7 @@ struct DataGridCellAppearanceTests { placeholderText: .systemBrown, booleanTrueText: .systemGreen, booleanFalseText: .systemPink, + valueColors: DataGridValueColors(all: .systemPurple), rowNumberText: .systemTeal, deletedRowText: .systemRed, modifiedColumnTint: .systemYellow, @@ -55,6 +58,7 @@ struct DataGridCellAppearanceTests { onEmphasizedSelection: Bool = false, checkboxMark: DataGridCheckboxMark? = nil, textTone: NSColor? = nil, + valueTone: DataGridValueTone = .plain, palette: DataGridCellPalette? = nil ) -> DataGridCellAppearance { DataGridCellAppearance.resolve( @@ -71,7 +75,8 @@ struct DataGridCellAppearanceTests { nullDisplayString: "NULL", onEmphasizedSelection: onEmphasizedSelection, checkboxMark: checkboxMark, - textTone: textTone + textTone: textTone, + valueTone: valueTone ) } @@ -397,6 +402,390 @@ struct DataGridCellAppearanceTests { #expect(found.textColor == themedPalette.findMatchText) } + // MARK: - Value tones + + private let tonedPalette = DataGridCellPalette( + regularFont: .systemFont(ofSize: 13), + italicFont: .systemFont(ofSize: 13), + mediumFont: .systemFont(ofSize: 13, weight: .medium), + text: .systemPurple, + placeholderText: .systemBrown, + booleanTrueText: .systemGreen, + booleanFalseText: .systemPink, + valueColors: DataGridValueColors( + string: NSColor(srgbRed: 0.80, green: 0.20, blue: 0.10, alpha: 1), + number: NSColor(srgbRed: 0.10, green: 0.40, blue: 0.80, alpha: 1), + dateTime: NSColor(srgbRed: 0.50, green: 0.20, blue: 0.70, alpha: 1), + binary: NSColor(srgbRed: 0.40, green: 0.40, blue: 0.40, alpha: 1), + json: NSColor(srgbRed: 0.10, green: 0.50, blue: 0.30, alpha: 1), + uuid: NSColor(srgbRed: 0.60, green: 0.40, blue: 0.10, alpha: 1), + enumeration: NSColor(srgbRed: 0.70, green: 0.10, blue: 0.40, alpha: 1) + ), + rowNumberText: .systemTeal, + deletedRowText: .systemRed, + modifiedColumnTint: .systemYellow, + findMatchTint: .systemOrange, + selectionText: .systemIndigo, + focusBorder: .systemMint + ) + + @Test("Each value tone draws its theme color, a plain value the text color") + func eachToneDrawsItsColor() { + let colors = tonedPalette.valueColors + let expected: [(DataGridValueTone, NSColor)] = [ + (.string, colors.string), + (.number, colors.number), + (.dateTime, colors.dateTime), + (.binary, colors.binary), + (.json, colors.json), + (.uuid, colors.uuid), + (.enumeration, colors.enumeration), + (.plain, .systemPurple), + ] + let distinctColors = Set(expected.map { ThemeTestFixture.hex($0.1) }) + #expect(distinctColors.count == expected.count) + for (tone, color) in expected { + #expect(resolve(valueTone: tone, palette: tonedPalette).textColor == color, "\(tone)") + } + } + + @Test("A find match, an emphasized selection, a deleted row and an owner tone outrank the value color") + func stateOutranksTheValueColor() { + let found = resolve(isCurrentFindMatch: true, valueTone: .number, palette: tonedPalette) + let selected = resolve(onEmphasizedSelection: true, valueTone: .number, palette: tonedPalette) + let deleted = resolve( + visualState: RowVisualState(isDeleted: true, isInserted: false, modifiedColumns: []), + valueTone: .number, + palette: tonedPalette + ) + let toned = resolve(textTone: .systemGreen, valueTone: .number, palette: tonedPalette) + let modified = resolve( + visualState: RowVisualState(isDeleted: false, isInserted: false, modifiedColumns: [0]), + valueTone: .number, + palette: tonedPalette + ) + + #expect(found.textColor == tonedPalette.findMatchText) + #expect(selected.textColor == .systemIndigo) + #expect(deleted.textColor == .systemRed) + #expect(toned.textColor == .systemGreen) + #expect(modified.textColor == tonedPalette.valueColors.number) + #expect(modified.backgroundTint == tonedPalette.modifiedColumnTint) + } + + @Test("A placeholder, a pending DEFAULT and a pending expression keep their own colors over a value tone") + func placeholdersAndPendingValuesIgnoreTheTone() { + let null = resolve(text: "", rawValue: nil, placeholder: .null, valueTone: .number, palette: tonedPalette) + let pendingDefault = resolve(text: "", placeholder: .defaultMarker, valueTone: .number, palette: tonedPalette) + let expression = resolve( + text: "now()", + rawValue: "now()", + visualState: RowVisualState(isDeleted: false, isInserted: false, modifiedColumns: [0], expressionColumns: [0]), + valueTone: .dateTime, + palette: tonedPalette + ) + + #expect(null.textColor == .systemBrown) + #expect(pendingDefault.textColor == .systemBlue) + #expect(expression.textColor == .systemBlue) + } + + @Test("A boolean keeps its true and false colors whatever the tone, and an unreadable one draws plain") + func booleanBranchComesFirst() { + let isTrue = resolve(kind: .boolean, text: "1", rawValue: "1", valueTone: .number, palette: tonedPalette) + let isFalse = resolve(kind: .boolean, text: "f", rawValue: "f", palette: tonedPalette) + let unreadable = resolve(kind: .boolean, text: "maybe", rawValue: "maybe", palette: tonedPalette) + + #expect(isTrue.textColor == .systemGreen) + #expect(isFalse.textColor == .systemPink) + #expect(unreadable.textColor == .systemPurple) + } + + @Test("A theme's value colors reach the grid's palette, and unset ones draw the grid text") + func themeReachesValueColors() throws { + let aqua = try #require(NSAppearance(named: .aqua)) + let themed = ThemeScope.fixed(try ThemeTestFixture.theme(.light, [ + .gridString: "#AA1100", + .gridNumber: "#0044AA", + .gridDateTime: "#7722AA", + .gridBinary: "#555555", + .gridJSON: "#117744", + .gridUUID: "#996611", + .gridEnum: "#AA1166", + ])).dataGridCellPalette(in: aqua) + let unset = ThemeScope.fixed(BuiltInThemes.light).dataGridCellPalette(in: aqua) + + #expect(ThemeTestFixture.hex(themed.valueColors.string) == "#AA1100") + #expect(ThemeTestFixture.hex(themed.valueColors.number) == "#0044AA") + #expect(ThemeTestFixture.hex(themed.valueColors.dateTime) == "#7722AA") + #expect(ThemeTestFixture.hex(themed.valueColors.binary) == "#555555") + #expect(ThemeTestFixture.hex(themed.valueColors.json) == "#117744") + #expect(ThemeTestFixture.hex(themed.valueColors.uuid) == "#996611") + #expect(ThemeTestFixture.hex(themed.valueColors.enumeration) == "#AA1166") + #expect(unset.valueColors == DataGridValueColors(all: unset.text)) + } + + private static let typedTheme: [ThemeSlot: String] = [ + .gridText: "#111111", + .gridString: "#AA1100", + .gridNumber: "#0044AA", + .gridDateTime: "#7722AA", + .gridBinary: "#555555", + .gridJSON: "#117744", + .gridUUID: "#996611", + .gridEnum: "#AA1166", + .gridBoolTrue: "#118811", + .gridBoolFalse: "#881111", + .gridNullValue: "#777777", + ] + + @MainActor + private struct TypedGrid { + let owner: GridTestOwner + let model: DataGridModel + let presenter: DataGridCellPresenter + + init( + values: [PluginCellValue], + type: ColumnType, + colorsValuesByType: Bool, + settingIsOn: Bool = true, + overridesSetting: Bool = false, + isForeignKey: Bool = false, + format: ValueDisplayFormat? + ) throws { + owner = GridTestOwner(TableRows.from( + queryRows: values.map { [$0] }, + columns: ["value"], + columnTypes: [type], + columnForeignKeys: isForeignKey + ? ["value": ForeignKeyInfo(name: "fk_value", column: "value", referencedTable: "targets", referencedColumn: "id")] + : [:] + )) + var configuration = DataGridConfiguration() + configuration.colorsValuesByType = colorsValuesByType + configuration.overridesValueColorSetting = overridesSetting + model = DataGridModel( + configuration: configuration, + themeScope: .fixed(try ThemeTestFixture.theme(.light, DataGridCellAppearanceTests.typedTheme)) + ) + model.dataSource = owner + model.reloadRows() + if let format { model.store.displayFormats = [format] } + presenter = DataGridCellPresenter(model: model, valueColors: ValueColorSettings(colorsValuesByType: settingIsOn)) + presenter.refreshColumnPresentations() + } + + func colors() throws -> [String?] { + let aqua = try #require(NSAppearance(named: .aqua)) + let pass = try #require(presenter.makePass(in: aqua)) + let column = try #require(model.columns.column(at: 0)) + return try (0.. [String?] { + let grid = try TypedGrid( + values: values, + type: type, + colorsValuesByType: colorsValuesByType, + settingIsOn: settingIsOn, + overridesSetting: overridesSetting, + format: format + ) + return try withExtendedLifetime(grid.owner) { try grid.colors() } + } + + private static let uuidBytes = Data([ + 0xAF, 0x49, 0x45, 0x3B, 0x7F, 0x2F, 0xFB, 0x58, + 0xFC, 0xD3, 0x2B, 0xD3, 0x99, 0x59, 0x9F, 0xA5, + ]) + + @Test("A grid that does not opt in draws every value in the grid text") + func gridOffDrawsPlain() throws { + #expect(try presentedColors(of: [.text("42")], type: .integer(rawType: "INT"), colorsValuesByType: false) == ["#111111"]) + #expect(try presentedColors(of: [.text("ok")], type: .text(rawType: "VARCHAR(8)"), colorsValuesByType: false) == ["#111111"]) + #expect(try presentedColors( + of: [.text("1700000000")], + type: .integer(rawType: "BIGINT"), + colorsValuesByType: false, + format: .unixTimestamp + ) == ["#111111"]) + } + + private static let everyTypedColumn: [(value: PluginCellValue, type: ColumnType, format: ValueDisplayFormat?, hex: String)] = [ + (.text("42"), .integer(rawType: "INT"), nil, "#0044AA"), + (.text("ok"), .text(rawType: "VARCHAR(8)"), nil, "#AA1100"), + (.text("2024-01-08"), .date(rawType: "DATE"), nil, "#7722AA"), + (.bytes(Data([0x9F, 0x3A])), .blob(rawType: "BLOB"), nil, "#555555"), + (.text(#"["ops"]"#), .json(rawType: "JSON"), nil, "#117744"), + (.text("8f14e45f-ceea-467f-a8f5-2e1c6a3d9b07"), .text(rawType: "UUID"), nil, "#996611"), + (.text("admin"), .enumType(rawType: "ENUM", values: ["admin", "staff"]), nil, "#AA1166"), + (.text("1700000000"), .integer(rawType: "BIGINT"), .unixTimestamp, "#7722AA"), + ] + + @Test("With Color values by type off, an opted-in grid draws every typed value in the grid text") + func settingOffDrawsPlain() throws { + for column in Self.everyTypedColumn { + let on = try presentedColors(of: [column.value], type: column.type, format: column.format) + let off = try presentedColors(of: [column.value], type: column.type, settingIsOn: false, format: column.format) + #expect(on == [column.hex], "\(column.type)") + #expect(off == ["#111111"], "\(column.type)") + } + } + + @Test("Booleans and NULL keep their colors whether Color values by type is on or off") + func booleansAndNullIgnoreTheSetting() throws { + let values: [PluginCellValue] = [.text("true"), .text("false"), .null] + let type = ColumnType.boolean(rawType: "BOOLEAN") + let expected: [String?] = ["#118811", "#881111", "#777777"] + + #expect(try presentedColors(of: values, type: type, settingIsOn: true) == expected) + #expect(try presentedColors(of: values, type: type, settingIsOn: false) == expected) + #expect(try presentedColors(of: [.null], type: .integer(rawType: "INT"), settingIsOn: false) == ["#777777"]) + } + + @Test("A grid that overrides the setting colors values by type while it is off, and only if it opted in") + func overrideIgnoresTheSetting() throws { + let value: [PluginCellValue] = [.text("42")] + let type = ColumnType.integer(rawType: "INT") + + #expect(try presentedColors(of: value, type: type, settingIsOn: false, overridesSetting: true) == ["#0044AA"]) + #expect(try presentedColors( + of: value, + type: type, + colorsValuesByType: false, + settingIsOn: false, + overridesSetting: true + ) == ["#111111"]) + } + + @Test("A foreign key cell keeps its link and takes its type's color") + func foreignKeyTakesItsTypeColor() throws { + let number = try TypedGrid( + values: [.text("7")], + type: .integer(rawType: "INT"), + colorsValuesByType: true, + isForeignKey: true, + format: nil + ) + let string = try TypedGrid( + values: [.text("ops")], + type: .text(rawType: "VARCHAR(8)"), + colorsValuesByType: true, + isForeignKey: true, + format: nil + ) + + #expect(number.presenter.presentation(forDataIndex: 0).kind == .foreignKey) + #expect(try withExtendedLifetime(number.owner) { try number.colors() } == ["#0044AA"]) + #expect(string.presenter.presentation(forDataIndex: 0).kind == .foreignKey) + #expect(try withExtendedLifetime(string.owner) { try string.colors() } == ["#AA1100"]) + } + + @Test("An opted-in grid colors a value by its column's type and by the format it is shown in") + func gridOnColorsByTypeAndFormat() throws { + #expect(try presentedColors(of: [.text("42")], type: .integer(rawType: "INT")) == ["#0044AA"]) + #expect(try presentedColors(of: [.text("ok")], type: .text(rawType: "VARCHAR(8)")) == ["#AA1100"]) + #expect(try presentedColors(of: [.text("10.0.0.1")], type: .text(rawType: "INET")) == ["#111111"]) + #expect(try presentedColors(of: [.text("1700000000")], type: .integer(rawType: "BIGINT"), format: .unixTimestamp) == ["#7722AA"]) + #expect(try presentedColors(of: [.text("1700000000")], type: .integer(rawType: "BIGINT"), format: .json) == ["#0044AA"]) + } + + @Test("Under a UUID format a 16-byte value takes the UUID color and an 8-byte one in the same column stays binary") + func uuidFormatColorsOnlyTheValuesItRenders() throws { + let colors = try presentedColors( + of: [.bytes(Self.uuidBytes), .bytes(Self.uuidBytes.prefix(8))], + type: .blob(rawType: "VARBINARY(16)"), + format: .uuid + ) + + #expect(colors == ["#996611", "#555555"]) + } + + @Test("A value its decoder cannot read keeps the column's color, one it reads takes the JSON color") + func failedDecoderKeepsTheBaseColor() throws { + let colors = try presentedColors( + of: [.bytes(Data([0x82, 0xA7, 0x63, 0x6F, 0x6D, 0x70])), .bytes(Data([0x81, 0xA1, 0x61, 0x01]))], + type: .blob(rawType: "BLOB"), + format: .messagePack + ) + + #expect(colors == ["#555555", "#117744"]) + } + + @Test("A text column under a UUID format keeps the string color for a value that is not a UUID") + func textThatIsNotAUUIDStaysAString() throws { + let colors = try presentedColors( + of: [.text("not a uuid"), .text("AF49453B7F2FFB58FCD32BD399599FA5")], + type: .text(rawType: "VARCHAR(40)"), + format: .uuid + ) + + #expect(colors == ["#AA1100", "#996611"]) + } + + @Test("The prewarmed text carries the same color as text formatted at draw time") + func prewarmedTextKeepsTheFormattedFlag() throws { + let grid = try TypedGrid( + values: [.bytes(Self.uuidBytes), .bytes(Self.uuidBytes.prefix(8))], + type: .blob(rawType: "VARBINARY(16)"), + colorsValuesByType: true, + format: .uuid + ) + for displayRow in 0.. String? { + let aqua = try #require(NSAppearance(named: .aqua)) + let presenter = grid.controller.presenter + let pass = try #require(presenter.makePass(in: aqua)) + let row = try #require(grid.model.row(at: 0)) + let column = try #require(grid.model.columns.column(at: 0)) + let appearance = try #require(presenter.appearance( + of: row, + displayRow: 0, + column: column, + visualState: .empty, + pass: pass, + onEmphasizedSelection: false + )) + return ThemeTestFixture.hex(appearance.textColor) + } + + @Test("Turning Color values by type on or off reaches a mounted grid's cells, with no reload of its rows") + func settingChangeRecolorsWithoutReload() async throws { + let settings = AppSettingsManager.shared + let original = settings.valueColors + defer { settings.valueColors = original } + settings.valueColors = ValueColorSettings(colorsValuesByType: false) + var configuration = DataGridConfiguration() + configuration.colorsValuesByType = true + let scope = ThemeScope.fixed(try ThemeTestFixture.theme(.light, [.gridText: "#111111", .gridNumber: "#0044AA"])) + let grid = HostedDataGrid( + rows: TableRows.from(queryRows: [[.text("42")]], columns: ["value"], columnTypes: [.integer(rawType: "INT")]), + configuration: configuration, + prepare: { _, model in model.themeScope = scope } + ) + defer { grid.close() } + let rowView = try #require(grid.main.rowView(atRow: 0, makeIfNecessary: true)) + let rowID = try #require(grid.model.row(at: 0)).id + let off = try valueColor(in: grid) + let cached = try #require(grid.model.display.cache.box(forID: rowID)) + + settings.valueColors = ValueColorSettings(colorsValuesByType: true) + try await waitUntil { grid.controller.presenter.registry.colorsValuesByType } + let on = try valueColor(in: grid) + + settings.valueColors = ValueColorSettings(colorsValuesByType: false) + try await waitUntil { !grid.controller.presenter.registry.colorsValuesByType } + let offAgain = try valueColor(in: grid) + + #expect(off == "#111111") + #expect(on == "#0044AA") + #expect(offAgain == "#111111") + #expect(grid.model.display.cache.box(forID: rowID) === cached) + #expect(grid.main.rowView(atRow: 0, makeIfNecessary: false) === rowView) + } + } +} diff --git a/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift b/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift index f2f53cd346..565e4eba21 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift @@ -798,6 +798,67 @@ struct DataGridHeaderMenuTests { ]) } + @Test("The list Filter Values builds off the main actor is the one the main actor builds") + func distinctValuesOffTheMainActor() async throws { + let rows = TableRows.from( + queryRows: [ + [.text("2024-03-15 09:30:00.123456"), .text("1700000000"), .bytes(Data("signup".utf8)), .text("b")], + [.text("2024-03-15 09:30:00.123457"), .text("1700000000"), .bytes(Data([0x89, 0x50])), .null], + [.text("2024-03-15 09:30:00.123456"), .null, .bytes(Data("signup".utf8)), .text("a\rb")], + ], + columns: ["created", "stamp", "payload", "note"], + columnTypes: [ + .datetime(rawType: "DATETIME(6)"), + .integer(rawType: "BIGINT"), + .blob(rawType: "VARBINARY(255)"), + .text(rawType: "TEXT"), + ] + ) + let grid = CommandGrid(rows) + grid.model.setDisplayFormat(.unixTimestamp, for: try #require(grid.model.columns.column(at: 1))) + grid.model.setDisplayFormat(.text, for: try #require(grid.model.columns.column(at: 2))) + let created = try #require(grid.model.columns.column(at: 0)) + #expect(grid.commands.distinctValues(for: created).map { $0.count } == [2, 1]) + + for column in grid.model.columns.columns { + let offMain = await ColumnValueTally.distinctValuesOffTheMainActor( + grid.model.rows, + dataIndex: column.dataIndex, + columnType: column.type, + displayFormat: grid.model.displayFormat(for: column), + databaseType: grid.model.configuration.traits.databaseType, + dateFormat: DateFormattingService.shared.currentFormat + ) + #expect(offMain == grid.commands.distinctValues(for: column), "\(column.name)") + } + } + + @Test("Filter Values asks whether to stop every 4,096 rows and gives up when told to") + func valueTallyChecksForCancellation() { + let rows = TableRows.from( + queryRows: (0..<10_000).map { [PluginCellValue.text("\($0 % 7)")] }, + columns: ["v"], + columnTypes: [.text(rawType: "TEXT")] + ) + let dates = DateDisplayRenderer(format: .iso8601) + var checks = 0 + + let counted = ColumnValueTally.tally( + rows, dataIndex: 0, columnType: .text(rawType: "TEXT"), displayFormat: nil, databaseType: nil, dates: dates + ) { + checks += 1 + return false + } + let stopped = ColumnValueTally.tally( + rows, dataIndex: 0, columnType: .text(rawType: "TEXT"), displayFormat: nil, databaseType: nil, dates: dates, + isCancelled: { true } + ) + + #expect(counted?.order.count == 7) + #expect(checks == 3) + #expect(stopped == nil) + } + @Test("Fill Column is offered on a writable column that is not a key") func fillColumnOffer() { var configuration = CommandGrid.editable() diff --git a/apple/TableProTests/Views/DataGrid/DataGridConfigurationTests.swift b/apple/TableProTests/Views/DataGrid/DataGridConfigurationTests.swift index b975c6ba27..8f4b9d71d8 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridConfigurationTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridConfigurationTests.swift @@ -4,8 +4,10 @@ // import AppKit +import SwiftUI @testable import TablePro import TableProPluginKit +import TableProTestSupport import Testing @MainActor @@ -34,7 +36,7 @@ struct DataGridConfigurationTests { return store } - @Test("A default grid offers value filters and column sort, and edits nothing") + @Test("A default grid offers value filters and column sort, edits nothing and colors no value by type") func defaults() { let configuration = DataGridConfiguration() @@ -43,6 +45,94 @@ struct DataGridConfigurationTests { #expect(configuration.showRowNumbers) #expect(!configuration.isEditable) #expect(configuration.commands.isEmpty) + #expect(!configuration.colorsValuesByType) + #expect(!configuration.overridesValueColorSetting) + } + + // MARK: - Value colors + + @Test("The data tab, routine debug results and both comparison grids color values by type") + func resultGridsColorValuesByType() async throws { + let fixture = CoordinatorFixture() + let tab = fixture.addTab(QueryTab(title: "Query 1", query: "SELECT 1", tabType: .query)) + let result = PluginQueryResult( + columns: ["total"], + columnTypeNames: ["integer"], + rows: [[.text("7")]], + rowsAffected: 0, + timing: PluginQueryTiming(total: 0) + ) + let (routine, _) = RoutineDebugFixtures.controller(session: FakeDebugSession(performAnswer: .finished(result))) + routine.start() + try await waitUntil { routine.state.pausedLocation != nil && !routine.variables.isEmpty } + routine.perform(.continueExecution) + try await waitUntil { routine.hasEnded } + let debugGrid = try #require(routine.resultGrid) + + #expect(fixture.coordinator.gridConfiguration(for: tab).colorsValuesByType) + #expect(debugGrid.model.configuration.colorsValuesByType) + #expect(CompareRowGridModel().model.configuration.colorsValuesByType) + #expect(ResultComparisonGridModel().model.configuration.colorsValuesByType) + } + + @Test("Table Listing draws its catalog rows in the grid text") + func tableListingStaysPlain() { + #expect(!TableListingTabView.gridConfiguration(databaseType: .mysql).colorsValuesByType) + } + + @Test("Structure lists, Structure sections and data files draw values in the grid text") + func definitionAndFileGridsStayPlain() throws { + let list = StructureListGridModel() + list.load( + columns: ["Name"], + rows: [StructureListRow(id: "a", values: ["alpha"])], + defaultWidths: [], + minimumWidths: [], + databaseType: .mysql, + sortState: nil + ) + let suite = try EphemeralDefaults() + let editor = StructureDefinitionEditor( + manager: StructureChangeManager(), + context: .existingTable( + connection: TestFixtures.makeConnection(type: .mysql), + databaseName: "db", + schemaName: nil, + tableName: "t", + objectKind: .table, + isSessionBound: false + ), + defaults: suite.defaults + ) + editor.refresh() + + #expect(!list.model.configuration.colorsValuesByType) + #expect(!editor.adapter(for: .columns).model.configuration.colorsValuesByType) + #expect(!DataFileContentView.gridConfiguration(for: DataFileController()).colorsValuesByType) + } + + @Test("An agent session's rows draw values in the grid text") + func agentRowsStayPlain() async throws { + let run = AgentQueryRun(id: "1", sql: "select total from orders", resultJSON: #"{"columns":["total"],"rows":[["7"]]}"#) + let window = ThemeTestFixture.host( + AgentResultRowsView(runs: [run], artifacts: AgentArtifactCache(), connection: nil), + size: NSSize(width: 600, height: 300) + ) + defer { window.contentView = nil } + + try await waitUntil { + window.contentView?.layoutSubtreeIfNeeded() + return Self.mainTable(in: window.contentView)?.controller?.model.rowCount == 1 + } + let model = try #require(Self.mainTable(in: window.contentView)?.controller?.model) + + #expect(!model.configuration.colorsValuesByType) + } + + private static func mainTable(in view: NSView?) -> DataGridTableView? { + guard let view else { return nil } + if let table = view as? DataGridTableView, table.paneKind == .main { return table } + return view.subviews.lazy.compactMap { mainTable(in: $0) }.first } // MARK: - Sort diff --git a/apple/TableProTests/Views/DataGrid/DataGridContextMenuTransientTests.swift b/apple/TableProTests/Views/DataGrid/DataGridContextMenuTransientTests.swift new file mode 100644 index 0000000000..4196cd69e4 --- /dev/null +++ b/apple/TableProTests/Views/DataGrid/DataGridContextMenuTransientTests.swift @@ -0,0 +1,43 @@ +// +// DataGridContextMenuTransientTests.swift +// TableProTests +// + +import AppKit +@testable import TablePro +import Testing + +@MainActor +struct DataGridContextMenuTransientTests { + @Test("An open row menu holds the grid until it closes") + func rowMenuHoldsTheGrid() throws { + let grid = HostedDataGrid(rows: GridFixtureRows.people) + defer { grid.close() } + let menu = try #require(grid.main.menu) + + #expect(!grid.model.holdsTransientUI) + grid.controller.menuWillOpen(menu) + #expect(grid.model.holdsTransientUI) + grid.controller.menuDidClose(menu) + #expect(!grid.model.holdsTransientUI) + } + + @Test("The header menu reports its opening to the grid and keeps the items it was built with") + func headerMenuHoldsTheGrid() throws { + let grid = HostedDataGrid(rows: GridFixtureRows.people) + defer { grid.close() } + let column = try #require(grid.tableColumn(named: "name")) + let menu = try #require(grid.controller.headerMenu(forTableColumn: column, in: .main)) + let items = menu.items.count + #expect(items > 0) + #expect(menu.delegate === grid.controller) + + grid.controller.menuNeedsUpdate(menu) + #expect(menu.items.count == items) + + grid.controller.menuWillOpen(menu) + #expect(grid.model.holdsTransientUI) + grid.controller.menuDidClose(menu) + #expect(!grid.model.holdsTransientUI) + } +} diff --git a/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift b/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift index e423818a3e..abcad752eb 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift @@ -222,7 +222,7 @@ struct DataGridControllerTests { // MARK: - Find @Test("A find match moves the active cell to the match's column, so the arrow keys continue from it") - func findMatchSetsActiveColumn() { + func findMatchSetsActiveColumn() async { let grid = HostedDataGrid(rows: GridFixtureRows.people) defer { grid.close() } grid.model.setSelection(cell(0, 0)) @@ -230,6 +230,7 @@ struct DataGridControllerTests { grid.model.showFind() grid.model.find.term = "dee" grid.model.findQueryDidChange() + await grid.model.findSearchSettled() #expect(grid.model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) #expect(grid.main.selectedRowIndexes == IndexSet(integer: 3)) diff --git a/apple/TableProTests/Views/DataGrid/DataGridInvisibleCharacterTests.swift b/apple/TableProTests/Views/DataGrid/DataGridInvisibleCharacterTests.swift index d7407746f6..c57045fbde 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridInvisibleCharacterTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridInvisibleCharacterTests.swift @@ -18,6 +18,7 @@ struct DataGridInvisibleCharacterTests { placeholderText: .secondaryLabelColor, booleanTrueText: .black, booleanFalseText: .black, + valueColors: DataGridValueColors(all: .black), rowNumberText: .secondaryLabelColor, deletedRowText: .systemRed, modifiedColumnTint: .systemYellow, diff --git a/apple/TableProTests/Views/DataGrid/DataGridPendingChangeMarkTests.swift b/apple/TableProTests/Views/DataGrid/DataGridPendingChangeMarkTests.swift index 5f2c5c43ae..9ae1140a94 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridPendingChangeMarkTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridPendingChangeMarkTests.swift @@ -18,6 +18,7 @@ struct DataGridPendingChangeMarkTests { placeholderText: .secondaryLabelColor, booleanTrueText: .labelColor, booleanFalseText: .labelColor, + valueColors: DataGridValueColors(all: .labelColor), rowNumberText: .secondaryLabelColor, deletedRowText: .systemRed, modifiedColumnTint: .systemYellow, diff --git a/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift b/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift index 4a313e942b..3d0242a892 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridValueFilterTests.swift @@ -343,6 +343,52 @@ struct DataGridValueFilterTests { #expect(grid.owner.displayOrderRequests == 2) } + // The click returns before the list is sorted, so the main actor never waits on the comparisons. + @Test("Filter Values sorts its list away from the click, and a reload while it sorts opens nothing") + func valueListSortsBeforeThePopoverOpens() async throws { + var configuration = DataGridConfiguration() + configuration.commands = [.valueFilter] + let grid = HostedDataGrid(rows: Self.statuses, configuration: configuration) + defer { grid.close() } + let status = try #require(grid.model.columns.columns.first { $0.name == "status" }) + + grid.controller.presentValueFilter(for: status) + + let listing = try #require(grid.controller.menuState.valueFilterListing) + #expect(grid.controller.menuState.valueFilterPopover == nil) + + grid.model.reloadRows() + await listing.value + + #expect(grid.controller.menuState.valueFilterListing == nil) + #expect(grid.controller.menuState.valueFilterPopover == nil) + } + + // A connection switch unparents the grid without dismantling it, and NSPopover raises on a view with no window. + @Test("A list that finishes after the grid left its window, or while the window is off screen, opens nothing") + func valueListOpensOnlyInAVisibleWindow() async throws { + var configuration = DataGridConfiguration() + configuration.commands = [.valueFilter] + let grid = HostedDataGrid(rows: Self.statuses, configuration: configuration) + defer { grid.close() } + let status = try #require(grid.model.columns.columns.first { $0.name == "status" }) + #expect(!grid.window.isVisible) + + grid.controller.presentValueFilter(for: status) + let offScreen = try #require(grid.controller.menuState.valueFilterListing) + await offScreen.value + #expect(grid.controller.menuState.valueFilterPopover == nil) + + grid.controller.presentValueFilter(for: status) + let unparented = try #require(grid.controller.menuState.valueFilterListing) + grid.controller.view.removeFromSuperview() + await unparented.value + + #expect(grid.controller.view.window == nil) + #expect(grid.controller.menuState.valueFilterListing == nil) + #expect(grid.controller.menuState.valueFilterPopover == nil) + } + @Test("Replacing the rows closes an open value filter, whose values were the old rows'") func replacingTheRowsClosesTheValueFilter() { let grid = HostedDataGrid(rows: Self.statuses) diff --git a/apple/TableProTests/Views/DataGrid/DataGridValueToneTests.swift b/apple/TableProTests/Views/DataGrid/DataGridValueToneTests.swift new file mode 100644 index 0000000000..67485c798b --- /dev/null +++ b/apple/TableProTests/Views/DataGrid/DataGridValueToneTests.swift @@ -0,0 +1,225 @@ +// +// DataGridValueToneTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct DataGridValueToneTests { + private static let baseTones: [(ColumnType?, DataGridValueTone)] = [ + (.integer(rawType: "BIGINT"), .number), + (.decimal(rawType: "NUMERIC(10,2)"), .number), + (.date(rawType: "DATE"), .dateTime), + (.timestamp(rawType: "TIMESTAMPTZ"), .dateTime), + (.datetime(rawType: "DATETIME"), .dateTime), + (.blob(rawType: "BYTEA"), .binary), + (.json(rawType: "JSONB"), .json), + (.enumType(rawType: "ENUM('a','b')", values: nil), .enumeration), + (.set(rawType: "SET('a','b')", values: ["a", "b"]), .enumeration), + (.text(rawType: "VARCHAR(255)"), .string), + (.text(rawType: "character varying"), .string), + (.text(rawType: "TEXT"), .string), + (.text(rawType: "CITEXT"), .string), + (.text(rawType: "String"), .string), + (.text(rawType: "Nullable(String)"), .string), + (.text(rawType: "LowCardinality(Nullable(String))"), .string), + (.text(rawType: "keyword"), .string), + (.text(rawType: "wildcard"), .string), + (.text(rawType: "constant_keyword"), .string), + (.text(rawType: "ascii"), .string), + (.text(rawType: "long"), .string), + (.text(rawType: "sysname"), .string), + (.integer(rawType: "long"), .number), + (.blob(rawType: "long raw"), .binary), + (.text(rawType: "UUID"), .uuid), + (.text(rawType: "Nullable(UUID)"), .uuid), + (.text(rawType: "UNIQUEIDENTIFIER"), .uuid), + (.text(rawType: "INET"), .plain), + (.text(rawType: "INTERVAL"), .plain), + (.text(rawType: "XML"), .plain), + (.text(rawType: "ObjectId"), .plain), + (.text(rawType: nil), .plain), + (.text(rawType: ""), .plain), + (.boolean(rawType: "BOOLEAN"), .plain), + (.spatial(rawType: "GEOMETRY"), .plain), + (.vector(rawType: "VECTOR(3)"), .plain), + (.array(rawType: "TEXT[]", element: .text(rawType: "TEXT")), .plain), + (nil, .plain), + ] + + @Test("A column's type gives its tone", arguments: baseTones) + func baseToneFollowsTheType(type: ColumnType?, expected: DataGridValueTone) { + #expect(DataGridValueTone.base(of: type, declaresEnumLabels: false) == expected) + } + + @Test("Character types only the tone knows leave a filter's LIKE operand cast", arguments: [ + "keyword", "wildcard", "constant_keyword", "ascii", "long", "sysname", + ]) + func toneOnlyCharacterTypesKeepTheirQuoting(rawType: String) { + #expect(DataGridValueTone.base(of: .text(rawType: rawType), declaresEnumLabels: false) == .string) + #expect(!ColumnTypeSQLQuoting.isCharacterType(.text(rawType: rawType))) + } + + @Test("Declared labels make a text column an enum, never an array, vector or number") + func declaredLabelsMarkTextOnly() { + #expect(DataGridValueTone.base(of: .text(rawType: "mood"), declaresEnumLabels: true) == .enumeration) + #expect(DataGridValueTone.base(of: .text(rawType: "mood"), declaresEnumLabels: false) == .plain) + #expect(DataGridValueTone.base(of: .text(rawType: "VARCHAR(8)"), declaresEnumLabels: true) == .enumeration) + let enumArray = ColumnType.array(rawType: "mood[]", element: .enumType(rawType: "mood", values: ["ok"])) + #expect(DataGridValueTone.base(of: enumArray, declaresEnumLabels: true) == .plain) + #expect(DataGridValueTone.base(of: .array(rawType: "INT[]", element: .integer(rawType: "INT")), declaresEnumLabels: false) == .plain) + #expect(DataGridValueTone.base(of: .vector(rawType: "VECTOR(3)"), declaresEnumLabels: true) == .plain) + #expect(DataGridValueTone.base(of: .integer(rawType: "INT"), declaresEnumLabels: true) == .number) + } + + @Test("A format the formatter rendered gives the tone of what it shows, a fallback keeps the column's") + func renderedFormatRemapsTheTone() { + let column = DataGridColumnTone(columnType: .integer(rawType: "BIGINT"), declaresEnumLabels: false) + + #expect(column.tone(displayedAs: .unixTimestamp, isFormatted: true) == .dateTime) + #expect(column.tone(displayedAs: .unixTimestamp, isFormatted: false) == .number) + #expect(column.tone(displayedAs: .raw, isFormatted: true) == .number) + #expect(column.tone(displayedAs: nil, isFormatted: false) == .number) + } + + @Test("A plain column stays plain under every format") + func plainColumnIgnoresFormats() { + for format in ValueDisplayFormat.presetFormats { + #expect(DataGridColumnTone.plain.tone(displayedAs: format, isFormatted: true) == .plain, "\(format.rawValue)") + } + } + + @Test("Each format names the tone of what it shows") + func eachFormatNamesItsTone() { + let connection = UUID() + let expected: [(ValueDisplayFormat, DataGridValueTone?)] = [ + (.raw, nil), + (.text, .string), + (.uuid, .uuid), + (.uuidSwappedTime, .uuid), + (.unixTimestamp, .dateTime), + (.unixTimestampMillis, .dateTime), + (.json, .json), + (.phpSerialized, .json), + (.messagePack, .json), + (.protobuf, .json), + (.igbinary, .json), + (.avro(schemaID: "ab12"), .json), + (.protobufSchema(schemaID: "ab12", message: "shop.Order"), .json), + (.schemaRegistry(connectionID: connection), .json), + ] + for (format, tone) in expected { + #expect(DataGridValueTone.shown(by: format) == tone, "\(format.rawValue)") + } + } + + private struct FormatCase { + let type: ColumnType + let value: PluginCellValue + let format: ValueDisplayFormat + let tone: DataGridValueTone + + init(_ type: ColumnType, _ value: PluginCellValue, _ format: ValueDisplayFormat, _ tone: DataGridValueTone) { + self.type = type + self.value = value + self.format = format + self.tone = tone + } + + var description: String { "\(format.rawValue) on \(type.rawType ?? "untyped") \(type.displayName)" } + } + + private static let uuidBytes = Data([ + 0xAF, 0x49, 0x45, 0x3B, 0x7F, 0x2F, 0xFB, 0x58, + 0xFC, 0xD3, 0x2B, 0xD3, 0x99, 0x59, 0x9F, 0xA5, + ]) + private static let messagePackMap = Data([0x81, 0xA1, 0x61, 0x01]) + private static let truncatedMessagePack = Data([0x82, 0xA7, 0x63, 0x6F, 0x6D, 0x70]) + + @Test("A cell takes the tone of what the formatter shows for it") + @MainActor + func cellToneFollowsTheFormatter() { + let uuidBytes = Self.uuidBytes + let messagePackMap = Self.messagePackMap + let truncatedMessagePack = Self.truncatedMessagePack + let cases: [FormatCase] = [ + FormatCase(.integer(rawType: "BIGINT"), .text("1700000000"), .unixTimestamp, .dateTime), + FormatCase(.integer(rawType: "BIGINT"), .text("1700000000000"), .unixTimestampMillis, .dateTime), + FormatCase(.integer(rawType: "BIGINT"), .text("soon"), .unixTimestamp, .number), + FormatCase(.blob(rawType: "BINARY(16)"), .bytes(uuidBytes), .uuid, .uuid), + FormatCase(.blob(rawType: "BINARY(16)"), .bytes(uuidBytes), .uuidSwappedTime, .uuid), + FormatCase(.blob(rawType: "BINARY(16)"), .bytes(uuidBytes.prefix(8)), .uuid, .binary), + FormatCase(.blob(rawType: "BLOB"), .bytes(Data("hello".utf8)), .text, .string), + FormatCase(.blob(rawType: "BLOB"), .bytes(messagePackMap), .messagePack, .json), + FormatCase(.blob(rawType: "BLOB"), .bytes(truncatedMessagePack), .messagePack, .binary), + FormatCase(.text(rawType: "TEXT"), .text(#"{"a":1}"#), .json, .json), + FormatCase(.text(rawType: "TEXT"), .text("a:0:{}"), .phpSerialized, .json), + FormatCase(.text(rawType: "VARCHAR(40)"), .text("pending"), .json, .string), + FormatCase(.text(rawType: "VARCHAR(40)"), .text(#"{"a":"#), .json, .string), + FormatCase(.text(rawType: "VARCHAR(40)"), .text("pending"), .phpSerialized, .string), + FormatCase(.text(rawType: "CHAR(36)"), .text("AF49453B7F2FFB58FCD32BD399599FA5"), .uuid, .uuid), + FormatCase(.text(rawType: "CHAR(36)"), .text("af49453b-7f2f-fb58-fcd3-2bd399599fa5"), .uuid, .uuid), + FormatCase(.text(rawType: "VARCHAR(40)"), .text("not a uuid"), .uuid, .string), + FormatCase(.text(rawType: nil), .text("AF49453B7F2FFB58FCD32BD399599FA5"), .uuid, .uuid), + FormatCase(.integer(rawType: "INT"), .text("42"), .uuid, .number), + FormatCase(.integer(rawType: "INT"), .text("42"), .json, .number), + FormatCase(.decimal(rawType: "DOUBLE"), .text("1700000000"), .unixTimestamp, .number), + FormatCase(.text(rawType: "TEXT"), .text("1700000000"), .unixTimestamp, .string), + FormatCase(.text(rawType: "TEXT"), .text("hello"), .text, .string), + FormatCase(.blob(rawType: "BLOB"), .bytes(messagePackMap), .json, .binary), + FormatCase(.blob(rawType: "BLOB"), .bytes(uuidBytes), .raw, .binary), + FormatCase(.text(rawType: "INET"), .text("10.0.0.1"), .uuid, .plain), + ] + for testCase in cases { + let shown = CellDisplayFormatter.display( + testCase.value, + columnType: testCase.type, + displayFormat: testCase.format, + databaseType: nil + ) + let column = DataGridColumnTone(columnType: testCase.type, declaresEnumLabels: false) + + #expect(column.tone(displayedAs: testCase.format, isFormatted: shown.isFormatted) == testCase.tone, "\(testCase.description)") + } + } + + @Test("A MongoDB binary value under a UUID format stays binary, as the formatter shows it") + @MainActor + func mongoBinaryRefusesUUID() { + let column = DataGridColumnTone(columnType: .blob(rawType: "BinData"), declaresEnumLabels: false) + for format in [ValueDisplayFormat.uuid, .uuidSwappedTime] { + let shown = CellDisplayFormatter.display( + .bytes(Self.uuidBytes), + columnType: .blob(rawType: "BinData"), + displayFormat: format, + databaseType: .mongodb + ) + #expect(column.tone(displayedAs: format, isFormatted: shown.isFormatted) == .binary, "\(format.rawValue)") + } + } + + @Test("The formatter only reports a format it applies to the column") + @MainActor + func formattedOnlyWhereTheFormatApplies() { + let types: [ColumnType] = [ + .text(rawType: "TEXT"), .integer(rawType: "INT"), .decimal(rawType: "DECIMAL"), .blob(rawType: "BLOB"), + .json(rawType: "JSON"), .date(rawType: "DATE"), .boolean(rawType: "BOOL"), + ] + let values: [PluginCellValue] = [ + .text("1700000000"), .text("AF49453B7F2FFB58FCD32BD399599FA5"), .text(#"{"a":1}"#), .bytes(Self.uuidBytes), + .bytes(Self.messagePackMap), .null, + ] + for type in types { + for format in ValueDisplayFormat.presetFormats { + for value in values { + let shown = CellDisplayFormatter.display(value, columnType: type, displayFormat: format, databaseType: nil) + let applies = format != .raw && format.isApplicable(to: type) + #expect(!shown.isFormatted || applies, "\(format.rawValue) on \(type.displayName)") + } + } + } + } +} diff --git a/apple/TableProTests/Views/DataGrid/DataGridViewportTests.swift b/apple/TableProTests/Views/DataGrid/DataGridViewportTests.swift index b5ada9e63f..3494536405 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridViewportTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridViewportTests.swift @@ -126,6 +126,26 @@ struct DataGridViewportTests { #expect(NSLocationInRange(250, grid.main.rows(in: grid.main.visibleRect))) } + @Test("A placement keeps a cell cursor on its column rather than selecting the whole row on column 0") + func placedCursorKeepsItsColumn() async throws { + let grid = makeGrid() + defer { grid.close() } + let column = DataGridColumnID(name: "c3", occurrence: 0) + let cell = GridSelectionSnapshot.Cell(rowID: .existing(40), columnID: column) + var cursor = GridSelectionSnapshot(rowIDs: [.existing(40)], rectangles: [], activeCell: cell, anchor: cell) + cursor.mode = .cursor + + grid.model.store.viewportAnchor = GridViewportPlacement( + firstVisibleRow: .existing(30), firstVisibleOffset: 0, selection: cursor, revealsSelection: false + ) + grid.controller.applyStagedViewport() + + try await waitUntil { grid.model.selection.mode == .cursor } + let position = try #require(grid.model.columns.displayPosition(of: column)) + #expect(grid.model.selection.activeCell == GridCoord(row: 40, displayColumn: position)) + #expect(grid.model.selectedRowIDs == [.existing(40)]) + } + @Test("Revealing a row scrolls the frozen pane with it") func revealScrollsBothPanes() async throws { let grid = makeGrid() diff --git a/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift b/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift index 395c5de217..5d8ba4999e 100644 --- a/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift +++ b/apple/TableProTests/Views/DataGrid/RowVisualIndexTests.swift @@ -9,6 +9,120 @@ import TableProPluginKit import TableProTheme import Testing +@MainActor +private final class CountingChangeManager: ChangeManaging { + var changes: [RowChange] = [] + private(set) var scans = 0 + private(set) var lookups = 0 + + var hasChanges: Bool { !changes.isEmpty } + var reloadVersion: Int { 0 } + var canRedo: Bool { false } + var insertedRowIDs: Set { [] } + + var rowChanges: [RowChange] { + scans += 1 + return changes + } + + func pendingChange(forRow rowID: RowID) -> RowChange? { + lookups += 1 + return changes.first { $0.rowID == rowID } + } + + func isRowDeleted(_ rowID: RowID) -> Bool { false } + + func recordCellChange( + rowID: RowID, + columnIndex: Int, + columnName: String, + oldValue: PluginCellValue, + newValue: PluginCellValue, + originalRow: [PluginCellValue]? + ) {} + + func undoRowDeletion(rowID: RowID) {} +} + +@MainActor +private final class RepaintRecordingView: DataGridMountedView { + private(set) var changes: [DataGridChange] = [] + + func apply(_ change: DataGridChange) { + changes.append(change) + } + + func selectionDidChange() {} + func configurationDidChange() {} + func reveal(row: Int, displayColumn: Int?) {} + func revealColumn(_ displayColumn: Int) {} + func sortDidChange() {} + func summaryRulesDidChange() {} + func viewportSample() -> (row: Int, offset: CGFloat)? { nil } + func beginEditing(row: Int, displayColumn: Int) -> Bool { false } + var isEditing: Bool { false } + func commitEditing() -> Bool { true } + func discardEditing() {} + func focus() -> Bool { false } + func anchor(forColumn displayColumn: Int) -> (view: NSView, rect: NSRect)? { nil } +} + +@MainActor +struct UndoStepRepaintTests { + private let fixture = CoordinatorFixture() + + // Undoing a command over every row runs one step per cell, so a step that repaints every row costs rows squared. + @Test("Undoing a step repaints the row it names and no other") + func undoRepaintsOnlyItsRow() { + let rows = TableRows.from( + queryRows: (0..<40).map { row -> [PluginCellValue] in [.text("\(row)"), .text("b")] }, + columns: ["id", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)], + hasAuthoritativeSchema: true + ) + let tab = fixture.addTab( + QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users"), + rows: rows + ) + let model = fixture.coordinator.gridModel(forTab: tab.id) + let view = RepaintRecordingView() + model.attach(view) + + fixture.coordinator.handleUndoResult(UndoResult( + action: .cellEdit( + rowID: .existing(7), columnIndex: 1, columnName: "name", + previousValue: "a", newValue: "b", originalRow: ["7", "a"] + ), + needsRowRemoval: false, + needsRowRestore: false, + restoreRow: nil + )) + + var repainted = IndexSet() + var marksRepainted = false + for change in view.changes { + guard case .rowsUpdated(let indexes, let columns) = change else { continue } + repainted.formUnion(indexes) + if columns == nil { marksRepainted = true } + } + #expect(repainted == IndexSet(integer: 7)) + #expect(marksRepainted) + #expect(fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id).rows[7].values[1] == "a") + } + + @Test("A batch step names every row it holds") + func batchStepNamesItsRows() { + let deletion = UndoAction.batchRowDeletion(rows: [(rowID: .existing(1), originalRow: []), (rowID: .existing(3), originalRow: [])]) + let insertion = UndoAction.batchRowInsertion( + rows: [InsertedRowLocation(rowID: .existing(5), storageIndex: 5)], + rowValues: [[]] + ) + + #expect(RowEditingCoordinator.rowIDs(touchedBy: deletion) == [.existing(1), .existing(3)]) + #expect(RowEditingCoordinator.rowIDs(touchedBy: insertion) == [.existing(5)]) + } +} + @MainActor struct RowVisualIndexTests { private func makeManager() -> DataChangeManager { @@ -110,6 +224,55 @@ struct RowVisualIndexTests { #expect(index.visualState(for: inserted).expressionColumns == [1]) } + @Test("Updating rows looks each one's change up by its id, never scanning every pending change") + func updateRowLooksTheChangeUp() { + let manager = CountingChangeManager() + manager.changes = (0..<2_000).map { row in + RowChange( + rowID: .existing(row), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "name", oldValue: "a", newValue: "b")] + ) + } + let changeManager = AnyChangeManager(manager) + let index = RowVisualIndex() + + for row in 0..<2_000 { + index.updateRow(.existing(row), from: changeManager) + } + + #expect(manager.scans == 0) + #expect(manager.lookups == 2_000) + #expect(index.visualState(for: .existing(1_999)).modifiedColumns == [1]) + } + + @Test("A row's pending change is the one a scan of every change finds, after removals reorder them") + func pendingChangeMatchesAScan() { + let manager = makeManager() + for row in 0..<6 { + manager.recordCellChange( + rowID: .existing(row), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "b", originalRow: [.text("\(row)"), "a"] + ) + } + let inserted = RowID.inserted(UUID()) + manager.recordRowInsertion(rowID: inserted, values: [.null, .null]) + manager.recordRowDeletion(rowID: .existing(2), originalRow: ["2", "b"]) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "b", newValue: "a", originalRow: ["0", "b"] + ) + manager.undoRowDeletion(rowID: .existing(2)) + manager.recordRowDeletion(rowID: .existing(4), originalRow: ["4", "b"]) + + for rowID in (0..<8).map(RowID.existing) + [inserted] { + let scanned = manager.rowChanges.first { $0.rowID == rowID } + #expect(manager.pendingChange(forRow: rowID) == scanned) + } + #expect(manager.pendingChange(forRow: .existing(4))?.type == .delete) + #expect(manager.pendingChange(forRow: .existing(0)) == nil) + } + @Test("A deleted or new row tints the whole row from its theme slot, a highlighted one with the rule's wash") func rowTintFollowsTheState() { let scope = ThemeScope.live diff --git a/apple/TableProTests/Views/ERDiagram/ERDiagramAccessibilityTests.swift b/apple/TableProTests/Views/ERDiagram/ERDiagramAccessibilityTests.swift index a0f20aa78b..920262e0cd 100644 --- a/apple/TableProTests/Views/ERDiagram/ERDiagramAccessibilityTests.swift +++ b/apple/TableProTests/Views/ERDiagram/ERDiagramAccessibilityTests.swift @@ -243,6 +243,41 @@ struct ERDiagramAccessibilityTests { #expect(held.accessibilityFrame() == NSAccessibility.screenRect(fromView: view, rect: destination)) } + @Test("A table whose columns change in place, with its rect unchanged, reports the new columns") + func columnChangeWithoutAMoveReachesTheElement() { + var scene = makeScene() + let (view, _) = makeView(scene) + let held = (view.accessibilityChildren() as? [ERDiagramNodeElement] ?? []) + .first { $0.accessibilityLabel() == "employees" } + #expect(held?.accessibilityValue() as? String == "2 columns") + + guard let index = scene.nodes.firstIndex(where: { $0.tableName == "employees" }) else { + Issue.record("no node") + return + } + scene.nodes[index].displayColumns.append(column("name")) + view.scene = scene + + #expect(held?.accessibilityValue() as? String == "3 columns") + #expect((held?.accessibilityChildren() ?? []).count == 3) + } + + @Test("A table added without moving the others gets an element") + func addedTableGetsAnElement() { + var scene = makeScene() + let (view, _) = makeView(scene) + #expect((view.accessibilityChildren() ?? []).count == 2) + + let added = node("invoices", columns: [column("id", primaryKey: true)]) + scene.nodes.append(added) + scene.nodeIndex["invoices"] = added.id + scene.nodeRects[added.id] = CGRect(x: 600, y: 40, width: ERDiagramLayout.nodeWidth, height: 60) + view.scene = scene + + let labels = (view.accessibilityChildren() as? [ERDiagramNodeElement] ?? []).compactMap { $0.accessibilityLabel() } + #expect(labels.contains("invoices")) + } + @Test("A held element reports a new selection without asking for children again") func heldElementFollowsSelection() { var scene = makeScene() diff --git a/apple/TableProTests/Views/ERDiagram/ERDiagramBitmapExportTests.swift b/apple/TableProTests/Views/ERDiagram/ERDiagramBitmapExportTests.swift new file mode 100644 index 0000000000..e734ccc66a --- /dev/null +++ b/apple/TableProTests/Views/ERDiagram/ERDiagramBitmapExportTests.swift @@ -0,0 +1,75 @@ +import AppKit +@testable import TablePro +import Testing + +@MainActor +struct ERDiagramBitmapExportTests { + private func scene(tablesAt origins: [CGPoint]) -> ERDiagramScene { + let columns = [ERColumnDisplay(id: "id", name: "id", dataType: "integer", isPrimaryKey: true, isForeignKey: false, isNullable: false)] + var nodes: [ERTableNode] = [] + var rects: [UUID: CGRect] = [:] + var index: [String: UUID] = [:] + for (offset, origin) in origins.enumerated() { + let node = ERTableNode(id: UUID(), tableName: "t\(offset)", columns: columns, displayColumns: columns, clusterId: 0) + nodes.append(node) + rects[node.id] = CGRect( + origin: origin, + size: CGSize(width: ERDiagramLayout.nodeWidth, height: ERDiagramLayout.estimateHeight(columnCount: 1)) + ) + index[node.tableName] = node.id + } + return ERDiagramScene(nodes: nodes, edges: [], nodeRects: rects, nodeIndex: index) + } + + private func pixels(of image: NSImage?) throws -> (wide: Int, high: Int) { + let rep = try #require(image?.representations.first as? NSBitmapImageRep) + return (rep.pixelsWide, rep.pixelsHigh) + } + + private func appearance() throws -> NSAppearance { + try #require(NSAppearance(named: .aqua)) + } + + @Test("A small diagram still renders at twice its size") + func smallDiagramKeepsTheRequestedScale() throws { + let small = scene(tablesAt: [.zero, CGPoint(x: 400, y: 0)]) + let image = ERDiagramSceneRenderer.image(small, appearance: try appearance(), scale: 2) + let size = try #require(image?.size) + + let rendered = try pixels(of: image) + #expect(rendered.wide == Int((size.width * 2).rounded(.down))) + #expect(rendered.high == Int((size.height * 2).rounded(.down))) + } + + @Test("A diagram wider than the side limit renders within it") + func wideDiagramStaysWithinTheSideLimit() throws { + let wide = scene(tablesAt: [.zero, CGPoint(x: 20_000, y: 0)]) + + let rendered = try pixels(of: ERDiagramSceneRenderer.image(wide, appearance: try appearance(), scale: 2)) + + #expect(CGFloat(rendered.wide) <= ERDiagramSceneRenderer.maxBitmapSide) + #expect(rendered.wide > 16_000) + } + + @Test("A diagram of hundreds of tables is scaled to the pixel budget, never past the requested scale") + func largeDiagramStaysWithinThePixelBudget() { + let size = CGSize(width: 7_500, height: 7_600) + let scale = ERDiagramSceneRenderer.bitmapScale(for: size, requested: 2) + + let wide = (size.width * scale).rounded(.down) + let high = (size.height * scale).rounded(.down) + #expect(scale < 2) + #expect(wide * high <= ERDiagramSceneRenderer.maxBitmapPixels) + #expect(max(wide, high) <= ERDiagramSceneRenderer.maxBitmapSide) + #expect(ERDiagramSceneRenderer.bitmapScale(for: CGSize(width: 800, height: 600), requested: 2) == 2) + } + + @Test("PNG export encodes the capped bitmap") + func pngExportIsCapped() throws { + let wide = scene(tablesAt: [.zero, CGPoint(x: 20_000, y: 0)]) + let data = try #require(ERDiagramSceneRenderer.pngData(wide, appearance: try appearance(), scale: 2)) + let rep = try #require(NSBitmapImageRep(data: data)) + + #expect(CGFloat(rep.pixelsWide) <= ERDiagramSceneRenderer.maxBitmapSide) + } +} diff --git a/apple/TableProTests/Views/ERDiagram/ERDiagramEdgeCullingTests.swift b/apple/TableProTests/Views/ERDiagram/ERDiagramEdgeCullingTests.swift new file mode 100644 index 0000000000..1eeb3f32ec --- /dev/null +++ b/apple/TableProTests/Views/ERDiagram/ERDiagramEdgeCullingTests.swift @@ -0,0 +1,95 @@ +import AppKit +@testable import TablePro +import Testing + +@MainActor +struct ERDiagramEdgeCullingTests { + @MainActor + private final class StrokeRecorder: ERDiagramCanvas { + private(set) var strokedBounds: [CGRect] = [] + + func fill(_ path: CGPath, color: CGColor) {} + + func stroke(_ path: CGPath, color: CGColor, lineWidth: CGFloat, rounded: Bool) { + strokedBounds.append(path.boundingBoxOfPath) + } + + func clip(to path: CGPath, _ draw: () -> Void) { + draw() + } + + func drawText( + _ text: String, + font: NSFont, + color: NSColor, + at point: CGPoint, + anchor: ERDiagramTextAnchor, + maxWidth: CGFloat + ) {} + + func drawSymbol(named name: String, pointSize: CGFloat, color: NSColor, at point: CGPoint, anchor: ERDiagramTextAnchor) {} + } + + private static let colors = ERDiagramEdgeRenderer.Colors( + stroke: NSColor.gray.cgColor, + selection: NSColor.blue.cgColor, + dropped: NSColor.red.cgColor + ) + + private static func edge(_ from: String, _ to: String) -> EREdge { + EREdge( + id: UUID(), + fkName: "fk_\(from)_\(to)", + fromTable: from, + fromColumn: "\(to)_id", + toTable: to, + toColumn: "id", + cardinality: .manyToOne + ) + } + + private static let rects: [String: CGRect] = [ + "a": CGRect(x: 40, y: 40, width: 220, height: 80), + "b": CGRect(x: 40, y: 300, width: 220, height: 80), + "c": CGRect(x: 3_000, y: 40, width: 220, height: 80), + "d": CGRect(x: 3_000, y: 300, width: 220, height: 80), + ] + + private func strokes(of edges: [EREdge], in dirtyRect: CGRect) -> [CGRect] { + var index: [String: UUID] = [:] + var nodeRects: [UUID: CGRect] = [:] + for (name, rect) in Self.rects { + let id = UUID() + index[name] = id + nodeRects[id] = rect + } + let recorder = StrokeRecorder() + ERDiagramEdgeRenderer.drawRoutes( + ERDiagramEdgeRenderer.routes(edges: edges, nodeRects: nodeRects, nodeIndex: index), + colors: Self.colors, + selectedEdgeId: nil, + dirtyRect: dirtyRect, + on: recorder + ) + return recorder.strokedBounds + } + + @Test("Only the edges that cross the drawn rect are stroked") + func strokesOnlyEdgesInTheDirtyRect() { + let edges = [Self.edge("a", "b"), Self.edge("c", "d")] + let everything = strokes(of: edges, in: .infinite) + let left = CGRect(x: 0, y: 0, width: 1_000, height: 1_000) + let culled = strokes(of: edges, in: left) + + #expect(!culled.isEmpty) + #expect(culled.count * 2 == everything.count) + #expect(culled.allSatisfy { $0.insetBy(dx: -3, dy: -3).intersects(left) }) + } + + @Test("An edge whose tables are both outside the drawn rect still draws where it crosses it") + func strokesAnEdgePassingThrough() { + let middle = CGRect(x: 1_000, y: 0, width: 400, height: 1_000) + #expect(!strokes(of: [Self.edge("a", "c")], in: middle).isEmpty) + #expect(strokes(of: [Self.edge("a", "b")], in: middle).isEmpty) + } +} diff --git a/apple/TableProTests/Views/Editor/SQLEditorTabHistoryTests.swift b/apple/TableProTests/Views/Editor/SQLEditorTabHistoryTests.swift new file mode 100644 index 0000000000..35c6ca944a --- /dev/null +++ b/apple/TableProTests/Views/Editor/SQLEditorTabHistoryTests.swift @@ -0,0 +1,138 @@ +// One SQL editor serves every query tab of a window. Handing it another tab's text, or text a command wrote to the +// tab (Clear Query, a history entry, an AI suggestion), went through `setText`, which cleared the undo history. + +import AppKit +import SwiftUI +import TableProEditorKit +import TableProTestSupport +import TableProTextEngine +import Testing + +@testable import TablePro + +@MainActor +struct SQLEditorTabHistoryTests { + private final class Tabs { + var texts: [UUID: String] = [:] + + func binding(_ id: UUID) -> Binding { + Binding(get: { self.texts[id] ?? "" }, set: { self.texts[id] = $0 }) + } + } + + private func input(_ tabs: Tabs, tab id: UUID, histories: EditorUndoHistories) -> SQLEditorInput { + SQLEditorInput( + text: tabs.binding(id), + cursorPositions: .constant([]), + databaseType: .sqlite, + tabID: id, + undoHistories: histories + ) + } + + // The tab's next render after the edit wrote its text, as the query tab runs it. + private func type(_ string: String, at location: Int, in editor: SQLEditorController, input: SQLEditorInput) { + editor.textViewController.textView.replaceCharacters(in: NSRange(location: location, length: 0), with: string) + editor.apply(input) + } + + @Test("Undo after switching away and back takes back the tab's last edit") + func undoSurvivesATabSwitch() throws { + let tabs = Tabs() + let histories = EditorUndoHistories() + let first = UUID() + let second = UUID() + tabs.texts[first] = "SELECT 1" + tabs.texts[second] = "SELECT 2" + let editor = SQLEditorController(input: input(tabs, tab: first, histories: histories)) + defer { editor.teardown() } + let textView = try #require(editor.textViewController.textView) + type("0", at: 8, in: editor, input: input(tabs, tab: first, histories: histories)) + try #require(tabs.texts[first] == "SELECT 10") + + editor.apply(input(tabs, tab: second, histories: histories)) + #expect(textView.string == "SELECT 2") + #expect(textView.undoManager?.canUndo == false) + + editor.apply(input(tabs, tab: first, histories: histories)) + #expect(textView.string == "SELECT 10") + textView.undoManager?.undo() + #expect(textView.string == "SELECT 1") + #expect(tabs.texts[first] == "SELECT 1") + } + + @Test("Text a command writes to the tab, as Clear Query does, can be undone") + func clearQueryCanBeUndone() throws { + let tabs = Tabs() + let histories = EditorUndoHistories() + let tab = UUID() + tabs.texts[tab] = "SELECT 1" + let editor = SQLEditorController(input: input(tabs, tab: tab, histories: histories)) + defer { editor.teardown() } + let textView = try #require(editor.textViewController.textView) + + tabs.texts[tab] = "" + editor.apply(input(tabs, tab: tab, histories: histories)) + #expect(textView.string == "") + + textView.undoManager?.undo() + #expect(textView.string == "SELECT 1") + #expect(tabs.texts[tab] == "SELECT 1") + } + + @Test("A tab whose text changed while it was away comes back with no history") + func changedTabTextDropsItsHistory() throws { + let tabs = Tabs() + let histories = EditorUndoHistories() + let first = UUID() + let second = UUID() + tabs.texts[first] = "SELECT 1" + tabs.texts[second] = "SELECT 2" + let editor = SQLEditorController(input: input(tabs, tab: first, histories: histories)) + defer { editor.teardown() } + let textView = try #require(editor.textViewController.textView) + type("0", at: 8, in: editor, input: input(tabs, tab: first, histories: histories)) + + editor.apply(input(tabs, tab: second, histories: histories)) + tabs.texts[first] = "SELECT 3" + editor.apply(input(tabs, tab: first, histories: histories)) + + #expect(textView.string == "SELECT 3") + #expect(textView.undoManager?.canUndo == false) + } + + @Test("A new editor for the tab, as after a table tab was shown, gets the tab's history back") + func historyOutlivesTheEditor() throws { + let tabs = Tabs() + let histories = EditorUndoHistories() + let tab = UUID() + tabs.texts[tab] = "SELECT 1" + let editor = SQLEditorController(input: input(tabs, tab: tab, histories: histories)) + type("0", at: 8, in: editor, input: input(tabs, tab: tab, histories: histories)) + editor.teardown() + + let next = SQLEditorController(input: input(tabs, tab: tab, histories: histories)) + defer { next.teardown() } + let textView = try #require(next.textViewController.textView) + #expect(textView.string == "SELECT 10") + textView.undoManager?.undo() + + #expect(textView.string == "SELECT 1") + } + + @Test("Vim marks restored before Vim is installed wait for it, and a replaced document drops them") + func pendingVimMarksFollowTheDocument() { + let coordinator = SQLEditorCoordinator(diagnosticsClock: ManualClock()) + coordinator.databaseType = .mysql + let controller = EditorControllerFixture.make(string: "SELECT 1", coordinators: [coordinator]) + defer { coordinator.destroy() } + var marks = VimDocumentMarks() + marks.marks["a"] = 3 + + coordinator.restoreVimDocumentMarks(marks) + #expect(coordinator.vimDocumentMarks?.marks["a"] == 3) + + controller.setText("SELECT 2") + #expect(coordinator.vimDocumentMarks == nil) + } +} diff --git a/apple/TableProTests/Views/Editor/SQLFoldScannerPathTests.swift b/apple/TableProTests/Views/Editor/SQLFoldScannerPathTests.swift new file mode 100644 index 0000000000..1534746570 --- /dev/null +++ b/apple/TableProTests/Views/Editor/SQLFoldScannerPathTests.swift @@ -0,0 +1,193 @@ +import AppKit +import Combine +import Darwin +import Foundation +@testable import TablePro +@testable import TableProEditorKit +import TableProGrammars +import TableProSQLGrammar +import TableProTestSupport +import TableProTextEngine +import TableProUtilities +import Testing + +// SQL folding runs `SQLFoldScanner` off the main thread on a copy of the text. These hold that path to the folds the +// per-line walk gives, and the main thread to a copy per keystroke instead of a scan. +// Serialized: the keystroke case reads the main thread's CPU time, which a case running beside it would add to. +@MainActor +@Suite(.serialized) +struct SQLFoldScannerPathTests { + @MainActor + private final class LineWalkOnly: LineFoldProvider { + private let base: SQLLineFoldProvider + + init(grammar: SQLLexicalGrammar) { + base = SQLLineFoldProvider(grammar: grammar) + } + + func foldLevelAtLine( + lineNumber: Int, + lineRange: NSRange, + previousDepth: Int, + controller: TextViewController + ) -> [LineFoldProviderLineInfo] { + base.foldLevelAtLine( + lineNumber: lineNumber, + lineRange: lineRange, + previousDepth: previousDepth, + controller: controller + ) + } + } + + private struct Fold: Equatable { + let depth: Int + let range: Range + } + + private static let mixedScript = """ + CREATE TABLE users ( + id INT PRIMARY KEY, + name TEXT, + CHECK ( + id > 0 + ) + ); + /* a comment + over lines */ + SELECT id, + CASE + WHEN id > 1 THEN 'a' + ELSE 'b' + END AS label + FROM users + WHERE id IN ( + SELECT id FROM users + ); + DO $$ + BEGIN + PERFORM 1; + END + $$; + SELECT 1; SELECT (2); + INSERT INTO users VALUES + (1, 'a'), + (2, 'b'); + """ + + private func makeController(text: String, provider: LineFoldProvider) -> TextViewController { + let controller = TextViewController( + string: text, + language: .sql, + configuration: SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: .init(foldingSizeLimit: StatementRunController.defaultSizeLimit) + ), + cursorPositions: [], + highlightProviders: [], + foldProvider: provider + ) + controller.loadView() + controller.view.frame = NSRect(x: 0, y: 0, width: 1_000, height: 1_000) + controller.view.layoutSubtreeIfNeeded() + return controller + } + + private func folds(in controller: TextViewController) -> [Fold] { + controller.foldModel? + .getFolds(in: 0.. Double { + var info = thread_basic_info() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let thread = pthread_mach_thread_np(pthread_self()) + let result = withUnsafeMutablePointer(to: &info) { pointer in + pointer.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + thread_info(thread, thread_flavor_t(THREAD_BASIC_INFO), $0, &count) + } + } + guard result == KERN_SUCCESS else { return 0 } + return Double(info.user_time.seconds + info.system_time.seconds) * 1_000 + + Double(info.user_time.microseconds + info.system_time.microseconds) / 1_000 + } + + @Test("The scanner path folds a script the way the per-line walk does", arguments: ["\n", "\r\n"]) + func scannerMatchesLineWalk(lineEnding: String) async throws { + let text = Self.mixedScript.replacingOccurrences(of: "\n", with: lineEnding) + let scanned = makeController(text: text, provider: SQLLineFoldProvider(grammar: TestGrammar.postgres)) + let walked = makeController(text: text, provider: LineWalkOnly(grammar: TestGrammar.postgres)) + + try await waitUntil(timeout: .seconds(30)) { !folds(in: scanned).isEmpty && !folds(in: walked).isEmpty } + + #expect(folds(in: walked).count >= 6) + #expect(folds(in: scanned) == folds(in: walked)) + } + + // A pass used to walk every line through the provider on the main thread, the scan included; now the main thread + // only copies the text and takes the result. Both are timed on the same document in the same run, so a slow runner + // moves both sides. Long single-line statements keep the fold count, and the Debug fold store's fill time, low. + @Test("A fold pass costs the main thread a fraction of the per-line walk") + func passCostsAFractionOfTheWalk() async throws { + let statement = "SELECT '" + String(repeating: "x", count: 180) + "' FROM t;\n" + let block = "CREATE TABLE t (\n id INT\n);\n" + String(repeating: statement, count: 20) + let text = String(repeating: block, count: 300_000 / (block as NSString).length) + #expect((text as NSString).length > LineFoldModel.immediateRecalculationLimit) + + let walked = TextViewController(string: text, language: .sql, configuration: configuration, cursorPositions: []) + let walker = SQLLineFoldProvider(grammar: TestGrammar.postgres) + var walkCosts: [Double] = [] + for _ in 0..<3 { + var lines = walked.textView.layoutManager.lineStorage.makeIterator() + let start = Self.mainThreadCPUMilliseconds() + while let line = lines.next() { + _ = walker.foldLevelAtLine( + lineNumber: line.index, lineRange: line.range, previousDepth: 0, controller: walked + ) + } + walkCosts.append(Self.mainThreadCPUMilliseconds() - start) + } + + let scanned = TextViewController( + string: text, + language: .sql, + configuration: configuration, + cursorPositions: [], + foldProvider: SQLLineFoldProvider(grammar: TestGrammar.postgres) + ) + let model = LineFoldModel(controller: scanned, foldView: NSView()) + defer { model.destroy() } + try await waitUntil(timeout: .seconds(30)) { !model.getFolds(in: 0..<(text as NSString).length).isEmpty } + // Awaited, not polled: polling while the pass runs would bill the main thread for the wait. + let (published, publish) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + let subscription = model.$foldCache.dropFirst().sink { _ in publish.yield() } + defer { subscription.cancel() } + var passCosts: [Double] = [] + for _ in 0..<3 { + let start = Self.mainThreadCPUMilliseconds() + model.refresh() + for await _ in published { break } + passCosts.append(Self.mainThreadCPUMilliseconds() - start) + } + + let walk = walkCosts.sorted()[1] + let pass = passCosts.sorted()[1] + #expect(pass < walk / 4, "a pass took \(pass) ms of main thread, the walk \(walk) ms") + } + + private var configuration: SourceEditorConfiguration { + SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: .init(foldingSizeLimit: StatementRunController.defaultSizeLimit) + ) + } +} diff --git a/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift b/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift new file mode 100644 index 0000000000..ba5ea78102 --- /dev/null +++ b/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift @@ -0,0 +1,109 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProEditorKit +import TableProGrammars +import TableProSQLGrammar +import TableProTextEngine +import Testing + +private let caretTableMongoScript = """ +db.users.find({ name: "a;b" }); +// note + +for (var i = 0; i < 3; i++) { + print(i); +} +db.orders.insertOne({ total: 1 }) +db.orders.find() +// trailing +""" + +private let caretTableRedisScript = "GET a; GET b\n\nSET k \"a\nb\"\nGET c \n " + +@MainActor +struct StatementCaretTableTests { + private struct Resolved: Equatable { + let offset: Int + let sql: String + let hasContent: Bool + let contentRange: NSRange + + init(_ statement: SQLStatementScanner.LocatedStatement) { + offset = statement.offset + sql = statement.sql + hasContent = statement.hasContent + contentRange = statement.contentRange + } + } + + @Test("Every caret resolves to what a scan of the whole document resolves to", arguments: [ + (QueryStatementModel.javascript, caretTableMongoScript), + (QueryStatementModel.redis, caretTableRedisScript), + (QueryStatementModel.javascript, ""), + (QueryStatementModel.redis, ""), + ]) + func everyCaretMatchesAFullScan(model: QueryStatementModel, text: String) throws { + let table = try #require(QueryStatementCaretTable(text: text, model: model)) + for caret in -1...((text as NSString).length + 1) { + let fresh = QueryStatementScanner.locatedStatementAtCursor( + in: text, cursorPosition: caret, model: model, grammar: .ansi + ) + #expect(Resolved(table.statement(atCursor: caret)) == Resolved(fresh), "caret \(caret)") + } + } + + @Test("A SQL document has no table, since it resumes through its own index") + func sqlHasNoTable() { + #expect(QueryStatementCaretTable(text: "SELECT 1;", model: .sql) == nil) + } + + private func makeController(text: String) -> TextViewController { + let configuration = SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: EditorPeripherals.editor(lineNumbers: true, folding: true, statementRunControls: true) + ) + let controller = TextViewController(string: text, language: .sql, configuration: configuration, cursorPositions: []) + controller.loadView() + return controller + } + + private func band(at caret: Int, in controller: TextViewController, subject: StatementRunController) -> NSRange? { + controller.setCursorPositions([CursorPosition(range: NSRange(location: caret, length: 0))]) + subject.refreshHighlight(in: controller) + return controller.highlightedStatementRange + } + + private func freshBand(at caret: Int, in controller: TextViewController) -> NSRange? { + let statement = QueryStatementScanner.locatedStatementAtCursor( + in: controller.textView.string, cursorPosition: caret, model: .javascript, grammar: .ansi + ) + guard statement.hasContent, statement.contentRange.length > 0 else { return nil } + return statement.contentRange + } + + @Test("Caret moves through a MongoDB script read it once, and an edit reads it again") + func caretMovesReadTheScriptOnce() { + let script = String(repeating: "db.orders.find({ total: 1 });\n", count: 2_000) + let controller = makeController(text: script) + let subject = StatementRunController() + subject.statementModel = .javascript + subject.install(on: controller) + let readsAtInstall = subject.caretTableReads + + for caret in stride(from: 59_000, to: 58_000, by: -7) { + #expect(band(at: caret, in: controller, subject: subject) == freshBand(at: caret, in: controller)) + } + #expect(subject.caretTableReads <= readsAtInstall + 1) + + controller.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "db.a.find();\n") + let afterEdit = subject.caretTableReads + #expect(band(at: 100, in: controller, subject: subject) == freshBand(at: 100, in: controller)) + #expect(band(at: 58_013, in: controller, subject: subject) == freshBand(at: 58_013, in: controller)) + #expect(subject.caretTableReads == afterEdit + 1) + } +} diff --git a/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift b/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift index 249dff0232..70fe90613f 100644 --- a/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift +++ b/apple/TableProTests/Views/Editor/StatementRunControllerTests.swift @@ -174,6 +174,81 @@ struct StatementRunControllerTests { #expect(controller.highlightedStatementRange == nil) } + // MARK: - The band across edits + + // Long enough that the band's index records boundaries, which an edit near the top has to drop. + private let indexedScript = String(repeating: "SELECT 1;\n", count: 1_000) + + private func freshBand(in controller: TextViewController, caret: Int) -> NSRange? { + let statement = SQLStatementScanner.locatedStatementAtCursor(in: controller.textView.string, cursorPosition: caret) + guard statement.hasContent, statement.contentRange.length > 0 else { return nil } + return statement.contentRange + } + + private func band(at caret: Int, in controller: TextViewController, subject: StatementRunController) -> NSRange? { + controller.setCursorPositions([CursorPosition(range: NSRange(location: caret, length: 0))]) + subject.refreshHighlight(in: controller) + return controller.highlightedStatementRange + } + + @Test("An edit typed into the editor reaches a band that already read the script") + func typedEditReachesTheBand() throws { + let controller = makeController(text: indexedScript) + let subject = makeSubject() + subject.install(on: controller) + #expect(band(at: 9_995, in: controller, subject: subject) == NSRange(location: 9_990, length: 9)) + + // An open comment, because the editor pairs a typed quote, and it swallows every boundary the band had read. + controller.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "/*") + try #require(controller.textView.string.hasPrefix("/*SELECT")) + + #expect(freshBand(in: controller, caret: 9_995) == nil) + #expect(band(at: 9_995, in: controller, subject: subject) == nil) + } + + @Test("Text set on the text view without the editor's edit path reaches the band") + func textSetDirectlyReachesTheBand() { + let controller = makeController(text: indexedScript) + let subject = makeSubject() + subject.install(on: controller) + #expect(band(at: 9_995, in: controller, subject: subject) == NSRange(location: 9_990, length: 9)) + + controller.textView.string = "'" + indexedScript + + #expect(band(at: 9_995, in: controller, subject: subject) == freshBand(in: controller, caret: 9_995)) + } + + @Test("A replaced document gets a band read from the new text") + func replacedDocumentReachesTheBand() { + let controller = makeController(text: indexedScript) + let subject = makeSubject() + subject.install(on: controller) + #expect(band(at: 9_995, in: controller, subject: subject) == NSRange(location: 9_990, length: 9)) + + controller.setText("'" + indexedScript) + + #expect(band(at: 9_995, in: controller, subject: subject) == freshBand(in: controller, caret: 9_995)) + } + + // 5,000 statements: resolving from the top walks about 50,000 units per caret move. + @Test("Moving the caret through a long script rescans only near the caret") + func caretMovesRescanOnlyNearTheCaret() { + let controller = makeController(text: String(repeating: "SELECT 1;\n", count: 5_000)) + let subject = makeSubject() + subject.install(on: controller) + let start = 49_995 + _ = band(at: start, in: controller, subject: subject) + let warmed = subject.caretIndexUnitsScanned + + let moves = 500 + for step in 1...moves { + _ = band(at: start - step, in: controller, subject: subject) + } + + #expect(controller.highlightedStatementRange == freshBand(in: controller, caret: start - moves)) + #expect(subject.caretIndexUnitsScanned - warmed <= moves * (SQLStatementCursorIndex.defaultCheckpointSpacing + 20)) + } + // MARK: - Callback @Test("Pressing a control reports that control's own statement, not the caret's") diff --git a/apple/TableProTests/Views/Editor/VimDocumentMarksTests.swift b/apple/TableProTests/Views/Editor/VimDocumentMarksTests.swift new file mode 100644 index 0000000000..5056ec1b91 --- /dev/null +++ b/apple/TableProTests/Views/Editor/VimDocumentMarksTests.swift @@ -0,0 +1,55 @@ +// Marks are offsets into one document. The SQL editor hands one engine every query tab's text, so a tab's marks are +// saved and restored with its text and dropped when another document comes in. + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct VimDocumentMarksTests { + private let buffer: VimTextBufferMock + private let engine: VimEngine + + init() { + buffer = VimTextBufferMock(text: "hello world\nsecond line\nthird line\n") + engine = VimEngine(buffer: buffer) + } + + private func keys(_ chars: String) { + for char in chars { _ = engine.process(char, shift: false) } + } + + private var pos: Int { buffer.selectedRange().location } + + @Test("Marks saved with a document come back with it") + func marksRoundTrip() { + buffer.setSelectedRange(NSRange(location: 7, length: 0)) + keys("ma") + let saved = engine.documentMarks + + engine.documentDidReplace() + #expect(engine.documentMarks.isEmpty) + buffer.setSelectedRange(NSRange(location: 20, length: 0)) + keys("`a") + #expect(pos == 20) + + engine.restoreDocumentMarks(saved) + keys("`a") + #expect(pos == 7) + } + + @Test("A replaced document drops the last visual range and the insert point") + func replacedDocumentDropsTheOtherOffsets() { + engine.lastVisualStart = 0 + engine.lastVisualEnd = 4 + engine.lastVisualLinewise = true + engine.lastInsertOffset = 12 + + engine.documentDidReplace() + + #expect(engine.lastVisualStart == nil) + #expect(engine.lastVisualEnd == nil) + #expect(!engine.lastVisualLinewise) + #expect(engine.lastInsertOffset == nil) + } +} diff --git a/apple/TableProTests/Views/Import/ImportStateTransferProgressTests.swift b/apple/TableProTests/Views/Import/ImportStateTransferProgressTests.swift new file mode 100644 index 0000000000..d42279ebae --- /dev/null +++ b/apple/TableProTests/Views/Import/ImportStateTransferProgressTests.swift @@ -0,0 +1,53 @@ +// +// ImportStateTransferProgressTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct ImportStateTransferProgressTests { + @Test("An SQL import's bar follows the file's bytes read, and its amount still counts statements") + func sqlImportFollowsBytesRead() { + var state = ImportState( + isImporting: true, processedStatements: 3, estimatedTotalStatements: 100, fileName: "dump.sql" + ) + state.fileBytes = 1_000 + state.fileBytesRead = 250 + + let progress = state.transferProgress + + #expect(progress.fraction == 0.25) + #expect(progress.amountText == "3 of about 100 statements") + } + + @Test("Before the file's size is known, an SQL import's bar is statements done of the total") + func noFileSizeFallsBackToStatements() { + var state = ImportState( + isImporting: true, processedStatements: 25, estimatedTotalStatements: 100, fileName: "dump.sql" + ) + state.totalIsEstimate = false + + let progress = state.transferProgress + + #expect(progress.fraction == 0.25) + #expect(progress.amountText == "25 of 100 statements") + } + + @Test("A row import's bar ignores the file's bytes read") + func rowImportIgnoresBytes() { + var state = ImportState( + isImporting: true, processedStatements: 30, estimatedTotalStatements: 120, fileName: "people.csv" + ) + state.countsRows = true + state.totalIsEstimate = false + state.fileBytes = 1_000 + state.fileBytesRead = 900 + + let progress = state.transferProgress + + #expect(progress.fraction == 0.25) + #expect(progress.amountText == "30 of 120 rows") + } +} diff --git a/apple/TableProTests/Views/Import/ImportStatementCountTests.swift b/apple/TableProTests/Views/Import/ImportStatementCountTests.swift new file mode 100644 index 0000000000..9bb6aabc11 --- /dev/null +++ b/apple/TableProTests/Views/Import/ImportStatementCountTests.swift @@ -0,0 +1,41 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +struct ImportStatementCountTests { + private func script(statements: Int) throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("count-\(UUID().uuidString).sql") + try Data(String(repeating: "SELECT 1;\n", count: statements).utf8).write(to: url) + return url + } + + @Test("A count reaches every statement in the file") + func countsEveryStatement() async throws { + let url = try script(statements: 1_000) + defer { try? FileManager.default.removeItem(at: url) } + + let count = try await ImportStatementCount.count(url, encoding: .utf8, grammar: TestGrammar.postgres) + + #expect(count == 1_000) + } + + @Test("Cancelling the dialog's count stops the scan before the end of the file", .timeLimit(.minutes(1))) + func cancelReachesTheScan() async throws { + let total = 2_000_000 + let url = try script(statements: total) + defer { try? FileManager.default.removeItem(at: url) } + + let counting = Task { + try await ImportStatementCount.count(url, encoding: .utf8, grammar: TestGrammar.postgres) + } + counting.cancel() + + switch await counting.result { + case .success(let count): + #expect(count < total) + case .failure(let error): + #expect(error is CancellationError) + } + } +} diff --git a/apple/TableProTests/Views/Main/FetchAllQueryTaskTests.swift b/apple/TableProTests/Views/Main/FetchAllQueryTaskTests.swift index 4e18a0aadb..5409b0869d 100644 --- a/apple/TableProTests/Views/Main/FetchAllQueryTaskTests.swift +++ b/apple/TableProTests/Views/Main/FetchAllQueryTaskTests.swift @@ -54,6 +54,42 @@ struct FetchAllQueryTaskTests { #expect(tabManager.tabs.first { $0.id == tab.id }?.pagination.isLoadingMore == false) } + @Test("Fetch All asks the gate again, and a refusal reaches no driver") + func refusedFetchAllRunsNothing() async { + let connection = TestFixtures.makeConnection() + let driver = GatedQueryDriver(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let tabManager = QueryTabManager() + let coordinator = MainContentCoordinator( + connection: connection, + tabManager: tabManager, + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState() + ) + coordinator.executionGate = AlwaysDenyGate() + // Open, so a fetch that skipped the gate would finish and be counted rather than hang. + await driver.release() + let tab = QueryTab(title: "Query", query: "SELECT purge_orders()", tabType: .query) + tabManager.tabs.append(tab) + tabManager.selectedTabId = tab.id + + coordinator.paginationCoordinator.performFetchAll( + tabId: tab.id, + baseQuery: "SELECT purge_orders()", + scope: DatabaseScope(connectionId: connection.id, database: connection.database, schema: nil) + ) + await coordinator.queryTasks.task(for: tab.id)?.value + + #expect(driver.queryCount == 0) + #expect(tabManager.tabs.first { $0.id == tab.id }?.execution.errorMessage != nil) + #expect(tabManager.tabs.first { $0.id == tab.id }?.pagination.isLoadingMore == false) + } + @Test("Fetch All on a query tab shows what its own run printed, not the first run's or anything before it") func fetchAllReplacesTheOutput() async { let firstRun = PluginServerOutput(lines: ["NOTICE: page 1"], isTruncated: false) @@ -107,6 +143,12 @@ private final class GatedQueryDriver: DatabaseDriver, @unchecked Sendable { private let answer = AsyncGate() var serverOutputs: [PluginServerOutput] = [] + private let lock = NSLock() + private var queries = 0 + + var queryCount: Int { + lock.withLock { queries } + } init(connection: DatabaseConnection) { self.connection = connection @@ -123,6 +165,7 @@ private final class GatedQueryDriver: DatabaseDriver, @unchecked Sendable { /// The gate ignores cancellation, so the answer still comes back after the task is cancelled. func executeUserQuery(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryResult { + lock.withLock { queries += 1 } await answer.wait() return Self.emptyResult } diff --git a/apple/TableProTests/Views/Main/GridReloadViewportTests.swift b/apple/TableProTests/Views/Main/GridReloadViewportTests.swift index 7de3db2103..c85b15082f 100644 --- a/apple/TableProTests/Views/Main/GridReloadViewportTests.swift +++ b/apple/TableProTests/Views/Main/GridReloadViewportTests.swift @@ -147,6 +147,27 @@ struct GridReloadViewportTests { withExtendedLifetime(fixture) {} } + @Test("Reading the same table again keeps a cell cursor a cursor, on its own column") + func sameSourceKeepsTheCursorAndItsColumn() throws { + let fixture = makeFixture() + defer { fixture.coordinator.teardown() } + fixture.model.selectCell(GridCoord(row: 2, displayColumn: 1)) + let column = try #require(fixture.model.activeCell?.column.id) + + fixture.coordinator.setActiveTableRows( + Self.rows(ids: Array((1 ... 10).reversed())), + for: fixture.tabId, + viewport: .keepPlace, + source: .sameSource + ) + + let selection = try #require(fixture.takePlacement()?.selection) + #expect(selection.mode == .cursor) + #expect(selection.rowIDs == [.existing(7)]) + #expect(selection.activeCell == GridSelectionSnapshot.Cell(rowID: .existing(7), columnID: column)) + withExtendedLifetime(fixture) {} + } + @Test("A new result does not carry the selection over") func newSourceRestoresNoSelection() { let fixture = makeFixture() diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorAutoRefreshTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorAutoRefreshTests.swift new file mode 100644 index 0000000000..e314bd5c04 --- /dev/null +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorAutoRefreshTests.swift @@ -0,0 +1,449 @@ +// +// MainContentCoordinatorAutoRefreshTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct MainContentCoordinatorAutoRefreshTests { + @Test("A tick re-reads the selected table tab and writes no history row") + func tickRereadsWithoutHistory() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + + await harness.tickOnce() + + #expect(harness.tableReads == 1) + let tab = try #require(harness.tab) + #expect(tab.execution.errorMessage == nil) + #expect(tab.autoRefreshInterval == .fiveSeconds) + await harness.expectOnlyTheMarkerIsRecorded() + } + + @Test("A manual load records its row, so the marker check would see a tick's") + func manualLoadRecordsHistory() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + + harness.coordinator.executeTableTabQueryDirectly(viewport: .keepPlace) + await harness.coordinator.queryTasks.task(for: harness.tabId)?.value + await harness.recordMarker() + + #expect(harness.recordAttempts == 2) + try await waitUntil { await harness.historyCount == 2 } + } + + @Test("A tick keeps the derived count rather than counting again") + func tickKeepsTheCount() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + harness.coordinator.tabManager.mutate(tabId: harness.tabId) { tab in + tab.pagination.applyDerivedRowCount(42, isApproximate: false) + } + + await harness.tickOnce() + + let tab = try #require(harness.tab) + #expect(harness.tableReads == 1) + #expect(tab.pagination.totalRowCount == 42) + #expect(!tab.pagination.isCountPending) + } + + @Test("An open cell editor holds the tick: no query, no commit, and the bar says why") + func openEditorPausesWithoutCommitting() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + let model = harness.coordinator.gridModel(forTab: harness.tabId) + let mount = DataTabGridMountStub() + mount.isEditing = true + model.attach(mount) + + await harness.tickOnce() + + #expect(harness.tableReads == 0) + #expect(mount.commitCount == 0) + #expect(harness.tab?.autoRefreshPause == .editing) + #expect(harness.tab?.autoRefreshInterval == .fiveSeconds) + withExtendedLifetime(mount) {} + } + + @Test("An editable inspector pop-out holds the tick until its window goes") + func inspectorPopOutPauses() async { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + let paneState = TrailingPaneState() + harness.coordinator.trailingPaneState = paneState + var popOut: JSONViewerWindowController? = JSONViewerWindowController() + if let popOut { InspectorPopOutRegistry.record(popOut, openedFrom: paneState) } + + await harness.tickOnce() + + #expect(harness.tableReads == 0) + #expect(harness.tab?.autoRefreshPause == .valueWindowOpen) + + popOut = nil + #expect(!harness.coordinator.autoRefreshGate(forTab: harness.tabId).inspectorHoldsValueWindow) + withExtendedLifetime(paneState) {} + } + + @Test("A tick's rows are dropped when a value window opens during its read, and the next tick applies its own") + func tickRowsWaitForAValueWindowOpenedMidRead() async { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + let paneState = TrailingPaneState() + harness.coordinator.trailingPaneState = paneState + let read = AsyncGate() + harness.driver.holdReads(on: read) + + harness.start() + await harness.clock.waitUntilSleeping() + harness.clock.advance(by: .seconds(5)) + await read.arrived(1) + var popOut: JSONViewerWindowController? = JSONViewerWindowController() + if let popOut { InspectorPopOutRegistry.record(popOut, openedFrom: paneState) } + await read.open() + await harness.clock.waitUntilSleeping() + + #expect(harness.tableReads == 1) + #expect(harness.loadedRowCount == 1, "the held tick's two rows must not replace the one on screen") + #expect(harness.tab?.autoRefreshInterval == .fiveSeconds) + + popOut = nil + harness.clock.advance(by: .seconds(5)) + await harness.clock.waitUntilSleeping() + + #expect(harness.tableReads == 2) + #expect(harness.loadedRowCount == 2, "with the window gone the next tick's rows land") + withExtendedLifetime(paneState) {} + } + + @Test("A tick waits while the tab's database holds an open transaction") + func openTransactionPauses() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + let tracker = DatabaseManager.shared.sessionTransactions + let key = SessionTransactionTracker.Key(connectionId: harness.connection.id, lane: nil) + tracker.record(.inTransaction, on: harness.driver, for: key) + defer { tracker.record(.idle, on: harness.driver, for: key) } + + await harness.tickOnce() + + #expect(harness.tableReads == 0) + #expect(harness.tab?.autoRefreshPause == .openTransaction) + } + + @Test("A disconnected session runs nothing and waits for the connection") + func disconnectedSessionWaits() async throws { + let harness = AutoRefreshHarness(status: .disconnected) + defer { harness.tearDown() } + + await harness.tickOnce() + + #expect(harness.tableReads == 0) + #expect(harness.tab?.autoRefreshPause == .disconnected) + } + + @Test("A failed tick turns Auto Refresh off and leaves the tab's error on screen") + func failedTickStops() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + harness.driver.failReads() + + harness.start() + await harness.clock.waitUntilSleeping() + harness.clock.advance(by: .seconds(5)) + try await waitUntil { harness.tab?.autoRefreshInterval == .off } + + #expect(harness.tab?.execution.errorMessage != nil) + #expect(harness.clock.pendingSleeperCount == 0) + await harness.expectOnlyTheMarkerIsRecorded() + } + + @Test("A tab that is not selected never ticks") + func backgroundTabNeverRuns() async throws { + let harness = AutoRefreshHarness() + defer { harness.tearDown() } + harness.coordinator.observeSelectionForAutoRefresh() + harness.start() + await harness.clock.waitUntilSleeping() + + let other = QueryTab(title: "Query", query: "SELECT 1", tabType: .query) + harness.coordinator.tabManager.tabs.append(other) + harness.coordinator.tabManager.selectedTabId = other.id + try await waitUntil { !harness.coordinator.autoRefresh.isRunning } + harness.clock.advance(by: .seconds(5)) + + #expect(harness.tableReads == 0) + #expect(harness.clock.pendingSleeperCount == 0) + } +} + +// MARK: - Harness + +@MainActor +private final class AutoRefreshHarness { + let connection: DatabaseConnection + let driver: AutoRefreshDriver + let coordinator: MainContentCoordinator + let clock = ManualClock() + let tabId: UUID + private let history: QueryHistoryManager + private let attempts: RecordAttemptCounter + + init(status: ConnectionStatus = .connected) { + connection = DatabaseConnection(name: "Auto Refresh", database: "app", type: .postgresql) + driver = AutoRefreshDriver(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = status + DatabaseManager.shared.injectSession(session, for: connection.id) + + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("auto-refresh-\(UUID().uuidString).db") + // Asked once per write, first thing, on the task that records it. + let attempts = RecordAttemptCounter() + self.attempts = attempts + history = QueryHistoryManager( + storage: QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true), + isCapturePaused: { + attempts.increment() + return false + } + ) + coordinator = MainContentCoordinator( + connection: connection, + tabManager: QueryTabManager(), + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState(), + services: AppServices.live.replacingQueryHistory(history) + ) + coordinator.executionGate = AlwaysAllowGate() + coordinator.autoRefresh.clock = clock + coordinator.autoRefresh.visibility = { _, _ in .onScreen } + + var tab = QueryTab(title: "jobs", query: "SELECT * FROM jobs LIMIT 1000", tabType: .table, tableName: "jobs") + tab.tableContext.primaryKeyColumns = ["id"] + tab.execution.lastExecutedAt = Date() + coordinator.tabManager.tabs.append(tab) + coordinator.tabManager.selectedTabId = tab.id + tabId = tab.id + coordinator.setActiveTableRows( + TableRows.from( + queryRows: [[.text("1"), .text("first")]], + columns: ["id", "name"], + columnTypes: [.text(rawType: "INTEGER"), .text(rawType: "TEXT")] + ), + for: tab.id + ) + } + + var tab: QueryTab? { + coordinator.tabManager.tabs.first { $0.id == tabId } + } + + var tableReads: Int { + driver.sentStatements.filter { $0.contains("jobs") }.count + } + + var loadedRowCount: Int { + coordinator.tabSessionRegistry.tableRows(for: tabId).rows.count + } + + var historyCount: Int { + get async { await history.count(scope: .connection(connection.id)) } + } + + var recordAttempts: Int { attempts.value } + + // Through the same manager and store after the load, so a row the load recorded is written before it. + func recordMarker() async { + await history.record(QueryHistoryRecordRequest( + query: "SELECT 'marker'", + connectionId: connection.id, + databaseName: connection.database, + databaseType: connection.type, + source: .editor, + executionTime: 0, + rowCount: 0, + wasSuccessful: true + )) + } + + func expectOnlyTheMarkerIsRecorded(sourceLocation: SourceLocation = #_sourceLocation) async { + await recordMarker() + #expect(recordAttempts == 1, "the tick asked to record \(recordAttempts - 1) rows", sourceLocation: sourceLocation) + #expect(await historyCount == 1, sourceLocation: sourceLocation) + } + + func start() { + coordinator.setAutoRefreshInterval(.fiveSeconds, forTab: tabId) + } + + // One interval passes, and the schedule is back to waiting once the tick it ran has settled. + func tickOnce() async { + start() + await clock.waitUntilSleeping() + clock.advance(by: .seconds(5)) + await clock.waitUntilSleeping() + } + + func tearDown() { + coordinator.stopAutoRefresh() + coordinator.cancelAllQueryTasks() + DatabaseManager.shared.removeSession(for: connection.id) + coordinator.teardown() + } +} + +private final class RecordAttemptCounter: Sendable { + private let count = OSAllocatedUnfairLock(initialState: 0) + + var value: Int { count.withLock { $0 } } + + func increment() { + count.withLock { $0 += 1 } + } +} + +private struct AutoRefreshReadFailure: LocalizedError { + var errorDescription: String? { "relation \"jobs\" does not exist" } +} + +private final class AutoRefreshDriver: DatabaseDriver, @unchecked Sendable { + let connection: DatabaseConnection + var status: ConnectionStatus = .connected + var serverVersion: String? { nil } + + private let lock = NSLock() + private var statements: [String] = [] + private var fails = false + private var readGate: AsyncGate? + + init(connection: DatabaseConnection) { + self.connection = connection + } + + var sentStatements: [String] { + lock.withLock { statements } + } + + func failReads() { + lock.withLock { fails = true } + } + + // Every later read of the table waits on the gate until the test opens it. + func holdReads(on gate: AsyncGate) { + lock.withLock { readGate = gate } + } + + private func answer(_ query: String) async throws -> QueryResult { + let (failing, gate) = lock.withLock { () -> (Bool, AsyncGate?) in + statements.append(query) + let readsTable = query.contains("jobs") + return (fails && readsTable, readsTable ? readGate : nil) + } + if let gate { await gate.wait() } + if failing { throw AutoRefreshReadFailure() } + return ScriptAnsweringDriver.resultSet(columns: ["id", "name"], rows: [["1", "first"], ["2", "second"]]) + } + + func execute(query: String) async throws -> QueryResult { try await answer(query) } + func executeParameterized(query: String, parameters: [Any?]) async throws -> QueryResult { try await answer(query) } + func executeUserQuery(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryResult { + try await answer(query) + } + + func connect() async throws {} + func disconnect() {} + func testConnection() async throws -> Bool { true } + func ping() async throws {} + func cancelQuery() throws {} + func applyQueryTimeout(_ seconds: Int) async throws {} + + func fetchTables() async throws -> [TableInfo] { [] } + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String) async throws -> [ColumnInfo] { [] } + func fetchAllColumns() async throws -> [String: [ColumnInfo]] { [:] } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } + func fetchApproximateRowCount(table: String) async throws -> Int? { nil } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> DatabaseMetadata { + DatabaseMetadata( + id: database, + name: database, + tableCount: nil, + sizeBytes: nil, + lastAccessed: nil, + isSystemDatabase: false, + icon: "cylinder" + ) + } + + func fetchTableDDL(table: String) async throws -> String { "" } + func fetchTableMetadata(tableName: String) async throws -> TableMetadata { + TableMetadata( + tableName: tableName, + dataSize: nil, + indexSize: nil, + totalSize: nil, + avgRowLength: nil, + rowCount: nil, + comment: nil, + engine: nil, + collation: nil, + createTime: nil, + updateTime: nil + ) + } + + func fetchViewDefinition(view: String) async throws -> String { "" } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} + +@MainActor +private extension AppServices { + func replacingQueryHistory(_ manager: QueryHistoryManager) -> AppServices { + AppServices( + appEvents: appEvents, + appSettings: appSettings, + appSettingsStorage: appSettingsStorage, + connectionStorage: connectionStorage, + databaseManager: databaseManager, + pluginManager: pluginManager, + schemaService: schemaService, + schemaRefreshService: schemaRefreshService, + schemaProviderRegistry: schemaProviderRegistry, + catalogChangeService: catalogChangeService, + sqlFavoriteManager: sqlFavoriteManager, + favoriteTablesStorage: favoriteTablesStorage, + favoriteDatabasesStorage: favoriteDatabasesStorage, + tableFolderStorage: tableFolderStorage, + aiChatStorage: aiChatStorage, + aiKeyStorage: aiKeyStorage, + aiAccessApprovals: aiAccessApprovals, + groupStorage: groupStorage, + tagStorage: tagStorage, + sshProfileStorage: sshProfileStorage, + credentialProfileStorage: credentialProfileStorage, + featureAccess: featureAccess, + syncMetadataStorage: syncMetadataStorage, + favoritesExpansionState: favoritesExpansionState, + linkedFolderWatcher: linkedFolderWatcher, + queryHistoryManager: manager, + dateFormattingService: dateFormattingService, + copilotService: copilotService, + mcpServerManager: mcpServerManager, + syncTracker: syncTracker, + welcomeRouter: welcomeRouter, + telemetry: telemetry + ) + } +} diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorCaretPublishTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorCaretPublishTests.swift new file mode 100644 index 0000000000..8be11ca00a --- /dev/null +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorCaretPublishTests.swift @@ -0,0 +1,27 @@ +import Combine +import Foundation +import SwiftUI +@testable import TablePro +import TableProEditorKit +import Testing + +// The query editor binds its caret to `$coordinator.cursorPositions`. Every view in the main window observes the +// coordinator, so a publish per caret move re-rendered the window on each arrow key. +@MainActor +struct MainContentCoordinatorCaretPublishTests { + @Test("Moving the caret writes through the editor's binding without publishing the coordinator") + func caretMoveDoesNotPublish() { + let fixture = CoordinatorFixture() + var publishes = 0 + let subscription = fixture.coordinator.objectWillChange.sink { _ in publishes += 1 } + defer { subscription.cancel() } + + let binding = ObservedObject(wrappedValue: fixture.coordinator).projectedValue.cursorPositions + for offset in 0..<20 { + binding.wrappedValue = [CursorPosition(range: NSRange(location: offset, length: 0))] + } + + #expect(publishes == 0) + #expect(fixture.coordinator.cursorPositions.map { $0.range } == [NSRange(location: 19, length: 0)]) + } +} diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift index 47cd3e727d..14cee2efb9 100644 --- a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift @@ -528,7 +528,7 @@ struct MainContentCoordinatorSortTests { let idx = try #require(tabManager.tabs.firstIndex(where: { $0.id == tabId })) let result = ResultSet(label: "audit", tableRows: rows) result.baseQuery = "SELECT msg FROM audit WHERE msg = ?" - result.baseQueryParameterValues = ["hello"] + result.baseQueryParameterValues = [TypedParameterValue(text: "hello", kind: .text)] tabManager.mutate(at: idx) { $0.display.replaceUnpinnedResults(with: [result]) } coordinator.setActiveTableRows(rows, for: tabId) @@ -894,7 +894,7 @@ private final class StatementRecordingDriver: DatabaseDriver, @unchecked Sendabl } private func record(_ query: String, parameters: [Any?]?) -> QueryResult { - let bound = parameters.map { values in values.map { $0 as? String } } + let bound = parameters.map { values in values.map { ($0 as? TypedParameterValue)?.text ?? $0 as? String } } lock.withLock { statements.append(SentStatement(sql: query, parameters: bound)) } guard query.uppercased().hasPrefix("SELECT") else { return QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 1, executionTime: 0, error: nil) diff --git a/apple/TableProTests/Views/Main/QueryContainerChangeTests.swift b/apple/TableProTests/Views/Main/QueryContainerChangeTests.swift new file mode 100644 index 0000000000..f107a28891 --- /dev/null +++ b/apple/TableProTests/Views/Main/QueryContainerChangeTests.swift @@ -0,0 +1,109 @@ +// +// QueryContainerChangeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct QueryContainerChangeTests { + private func queryTab(database: String, schema: String? = nil) -> QueryTab { + var tab = QueryTab(title: "Query 1", query: "SELECT * FROM users", tabType: .query) + tab.tableContext.databaseName = database + tab.tableContext.schemaName = schema + return tab + } + + // Rows read from `users`, the way a finished run leaves them: editable, keyed, with more to load. + private func showRows(on tab: inout QueryTab, database: String, schema: String?) { + let rows = ResultSet( + label: "users", + tableRows: TableRows.from(queryRows: [[.text("42")]], columns: ["id"], columnTypes: [.text(rawType: "INTEGER")]) + ) + rows.origin = ResultOrigin( + tableName: "users", + schemaName: schema, + databaseName: database, + primaryKeyColumns: ["id"], + isEditable: true, + keysResolved: true + ) + tab.display.resultSets = [rows] + tab.tableContext.tableName = "users" + tab.tableContext.isEditable = true + tab.pagination.hasMoreRows = true + tab.pagination.baseQueryForMore = "SELECT * FROM users" + } + + @Test("The picker does not move a tab while its query runs") + func runningTabKeepsItsDatabase() { + let fixture = CoordinatorFixture() + let tab = fixture.addTab(queryTab(database: "staging")) + let running = fixture.coordinator.tabExecution.claim(tab.id) + + fixture.coordinator.changeQueryContainer(of: tab.id, to: "production", target: .database) + + #expect(fixture.coordinator.tabExecution.isCurrent(running)) + #expect(fixture.tabManager.tabs.first { $0.id == tab.id }?.tableContext.databaseName == "staging") + } + + @Test("Rows still on screen from the old database cannot be edited or paged from the new one") + func movedTabLeavesTheOldRowsReadOnly() { + let fixture = CoordinatorFixture() + fixture.coordinator.executionGate = AlwaysDenyGate.cancelling + var tab = queryTab(database: "staging") + showRows(on: &tab, database: "staging", schema: nil) + fixture.addTab(tab) + #expect(fixture.coordinator.activeResultEditRefusal == nil) + + fixture.coordinator.changeQueryContainer(of: tab.id, to: "production", target: .database) + + let moved = fixture.tabManager.tabs.first { $0.id == tab.id } + #expect(moved?.tableContext.databaseName == "production") + #expect(moved?.pagination.hasMoreRows == false) + #expect(moved?.pagination.baseQueryForMore == nil) + #expect(fixture.coordinator.activeResultEditRefusal == .databaseMoved) + } + + @Test("Rows left from the old schema after the schema picker moved the tab cannot be edited") + func schemaPickerLeavesTheOldRowsReadOnly() { + let fixture = CoordinatorFixture() + fixture.coordinator.executionGate = AlwaysDenyGate.cancelling + var tab = queryTab(database: "app", schema: "staging") + showRows(on: &tab, database: "app", schema: "staging") + fixture.addTab(tab) + #expect(fixture.coordinator.activeResultEditRefusal == nil) + + fixture.coordinator.changeQueryContainer(of: tab.id, to: "production", target: .schema) + + #expect(fixture.tabManager.tabs.first { $0.id == tab.id }?.tableContext.schemaName == "production") + #expect(fixture.coordinator.activeResultEditRefusal == .schemaMoved) + } + + @Test("Showing a pinned result read in another schema leaves the tab where the picker put it, and offers no Fetch All") + func resultSwitchKeepsTheSchemaBinding() throws { + let fixture = CoordinatorFixture() + var tab = queryTab(database: "app", schema: "staging") + showRows(on: &tab, database: "app", schema: "staging") + let stagingResult = try #require(tab.display.resultSets.first) + stagingResult.isTruncated = true + stagingResult.baseQuery = "SELECT * FROM users" + let productionResult = ResultSet(label: "Result 2") + productionResult.origin = ResultOrigin(tableName: nil, schemaName: "production", databaseName: "app") + tab.display.resultSets.append(productionResult) + tab.display.activeResultSetId = productionResult.id + tab.tableContext.schemaName = "production" + fixture.addTab(tab) + + fixture.coordinator.applyResultSetSwitch(to: stagingResult.id, in: tab.id) + + let switched = fixture.tabManager.tabs.first { $0.id == tab.id } + #expect(switched?.tableContext.schemaName == "production") + #expect(switched?.pagination.hasMoreRows == false) + #expect(switched?.pagination.baseQueryForMore == nil) + #expect(fixture.coordinator.activeResultEditRefusal == .schemaMoved) + } +} diff --git a/apple/TableProTests/Views/Main/QueryTabVersionTriggerTests.swift b/apple/TableProTests/Views/Main/QueryTabVersionTriggerTests.swift new file mode 100644 index 0000000000..1a0301d4ba --- /dev/null +++ b/apple/TableProTests/Views/Main/QueryTabVersionTriggerTests.swift @@ -0,0 +1,165 @@ +// +// QueryTabVersionTriggerTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +@MainActor +private final class CaptureSwitch { + var isPaused = false +} + +@MainActor +struct QueryTabVersionTriggerTests { + private let storage = QueryTabVersionStorage( + databaseURL: FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("query_tab_triggers_\(UUID().uuidString).db"), + announce: { _ in }, + removeDatabaseOnDeinit: true + ) + private let capture = CaptureSwitch() + + // The draft-save clock never moves unless a case advances it, so no idle version lands mid-case. + private func makeFixture( + query: String = "SELECT 1", + connection: DatabaseConnection = TestFixtures.makeConnection() + ) -> (CoordinatorFixture, QueryTab, ManualClock) { + let fixture = CoordinatorFixture(connection: connection) + let clock = ManualClock() + fixture.coordinator.draftSaveClock = clock + fixture.coordinator.executionGate = AlwaysDenyGate() + fixture.coordinator.openTabInNewWindow = { _ in } + let capture = capture + fixture.coordinator.tabVersions = QueryTabVersionRecorder( + storage: storage, + isCapturePaused: { capture.isPaused } + ) + let tab = fixture.addTab(QueryTab(title: "Query 1", query: query, tabType: .query)) + return (fixture, tab, clock) + } + + private func versions(of tabId: UUID) -> [(reason: QueryTabVersionReason, text: String?)] { + storage.summaries(tabId: tabId).map { ($0.reason, storage.version(id: $0.id, tabId: tabId)?.query) } + } + + @Test("Running a query tab records its text") + func runRecordsTheText() async throws { + let (fixture, tab, _) = makeFixture(query: "SELECT 1") + + fixture.coordinator.runQuery(viewport: .firstRow) + + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + let recorded = versions(of: tab.id) + #expect(recorded.map { $0.reason } == [.run]) + #expect(recorded.first?.text == "SELECT 1") + } + + @Test("Run All records the tab's text") + func runAllRecordsTheText() async throws { + let (fixture, tab, _) = makeFixture(query: "SELECT 1; SELECT 2") + + fixture.coordinator.runAllStatements() + + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + #expect(versions(of: tab.id).first?.text == "SELECT 1; SELECT 2") + } + + @Test("Clear Query keeps the text it removes as a version") + func clearQueryRecordsTheReplacedText() async throws { + let (fixture, tab, _) = makeFixture(query: "SELECT long_draft FROM t") + + fixture.coordinator.clearSelectedQuery() + + #expect(fixture.tabManager.selectedTab?.content.query == "") + #expect(!fixture.coordinator.toolbarState.hasQueryText) + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + let recorded = versions(of: tab.id) + #expect(recorded.first?.reason == .beforeReplace) + #expect(recorded.first?.text == "SELECT long_draft FROM t") + } + + @Test("Loading a query into the current tab keeps the text it replaces as a version") + func loadIntoEditorRecordsTheReplacedText() async throws { + let (fixture, tab, _) = makeFixture(query: "SELECT typed") + + fixture.coordinator.loadQueryIntoEditor("SELECT * FROM users") + + #expect(fixture.tabManager.selectedTab?.content.query == "SELECT * FROM users") + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + #expect(versions(of: tab.id).first?.reason == .beforeReplace) + #expect(versions(of: tab.id).first?.text == "SELECT typed") + } + + @Test("Inserting a saved query appends it and keeps the text before it") + func insertFavoriteAppendsAndRecords() async throws { + let (fixture, tab, _) = makeFixture(query: "SELECT 1") + + fixture.coordinator.insertFavorite(SQLFavorite(name: "Users", query: "SELECT * FROM users")) + + #expect(fixture.tabManager.selectedTab?.content.query == "SELECT 1\n\nSELECT * FROM users") + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + #expect(versions(of: tab.id).first?.text == "SELECT 1") + } + + // Any window may record and save: the old first-coordinator check left a second window's typing unsaved for 30s. + @Test("An idle pause records a version from a window that is not the connection's first") + func idlePauseRecordsFromAnyWindow() async throws { + let (fixture, tab, clock) = makeFixture(query: "SELECT draft") + let firstWindow = CoordinatorFixture(connection: fixture.connection) + firstWindow.coordinator.registerEagerly() + #expect(MainContentCoordinator.activeCoordinators[fixture.coordinator.instanceId] == nil) + + fixture.coordinator.scheduleDraftSave(editedTab: tab.id) + await clock.waitUntilSleeping() + clock.advance(by: MainContentCoordinator.draftSaveDebounce) + + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + #expect(versions(of: tab.id).first?.reason == .idle) + #expect(versions(of: tab.id).first?.text == "SELECT draft") + } + + @Test("With Query History capture paused, nothing a tab runs or replaces is kept") + func pausedCaptureRecordsNothing() async throws { + let (fixture, tab, clock) = makeFixture(query: "SELECT secret") + capture.isPaused = true + + fixture.coordinator.runQuery(viewport: .firstRow) + fixture.coordinator.loadQueryIntoEditor("SELECT other") + #expect(fixture.coordinator.tabVersions.record(tab, connectionId: fixture.connection.id, reason: .run) == nil) + + // A version recorded after the pause lifts is the only one, so nothing earlier was queued. + capture.isPaused = false + fixture.coordinator.scheduleDraftSave(editedTab: tab.id) + await clock.waitUntilSleeping() + clock.advance(by: MainContentCoordinator.draftSaveDebounce) + try await waitUntil { !storage.summaries(tabId: tab.id).isEmpty } + #expect(versions(of: tab.id).map { $0.reason } == [.idle]) + #expect(versions(of: tab.id).first?.text == "SELECT other") + } + + @Test("Revert To writes the chosen version into the tab and keeps the text it replaced") + func revertRestoresAndRecords() async throws { + let (fixture, tab, _) = makeFixture(query: "SELECT first") + await storage.record(tabId: tab.id, connectionId: fixture.connection.id, text: "SELECT first", reason: .run) + let version = try #require(storage.summaries(tabId: tab.id).first) + fixture.tabManager.mutate(tabId: tab.id) { $0.content.query = "SELECT second" } + + fixture.coordinator.revertQueryTab(tab.id, toVersion: version.id) + + try await waitUntil { fixture.tabManager.selectedTab?.content.query == "SELECT first" } + #expect(versions(of: tab.id).first?.reason == .beforeRestore) + #expect(versions(of: tab.id).first?.text == "SELECT second") + } + + @Test("A table tab records nothing") + func tableTabRecordsNothing() { + let (fixture, _, _) = makeFixture() + let table = QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users") + #expect(fixture.coordinator.tabVersions.record(table, connectionId: fixture.connection.id, reason: .run) == nil) + } +} diff --git a/apple/TableProTests/Views/Main/ReadOnlySafeModeEntryPointTests.swift b/apple/TableProTests/Views/Main/ReadOnlySafeModeEntryPointTests.swift new file mode 100644 index 0000000000..0032a8c583 --- /dev/null +++ b/apple/TableProTests/Views/Main/ReadOnlySafeModeEntryPointTests.swift @@ -0,0 +1,800 @@ +// +// ReadOnlySafeModeEntryPointTests.swift +// TableProTests +// +// At Read-Only every read the app runs for the user reaches a driver that declares `.enforcesReadOnlyMode` with +// `enforcesReadOnly` set, so `SELECT purge_orders()`, or a view that calls it, runs in the engine's read-only mode. +// + +import Combine +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +private final class ReadOnlyRecordingDriver: PluginDriverStub, @unchecked Sendable { + struct Call: Equatable { + let entry: String + let query: String + let context: PluginStatementContext? + } + + private let enforcesReadOnlyMode: Bool + + init(enforcesReadOnlyMode: Bool = true) { + self.enforcesReadOnlyMode = enforcesReadOnlyMode + } + + var capabilities: PluginCapabilities { + enforcesReadOnlyMode ? [.readOnlyStatements, .enforcesReadOnlyMode] : [.readOnlyStatements] + } + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { true } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + private let lock = NSLock() + private var recorded: [Call] = [] + + var calls: [Call] { + lock.withLock { recorded } + } + + func call(containing text: String) -> Call? { + calls.first { $0.query.contains(text) } + } + + var entries: [String] { + calls.map(\.entry) + } + + private static let answer = PluginQueryResult( + columns: ["n"], columnTypeNames: ["INT8"], rows: [[.text("1")]], rowsAffected: 0, executionTime: 0 + ) + + private func record(_ entry: String, _ query: String, _ context: PluginStatementContext?) -> PluginQueryResult { + lock.withLock { recorded.append(Call(entry: entry, query: query, context: context)) } + return Self.answer + } + + func execute(query: String) async throws -> PluginQueryResult { + record("execute", query, nil) + } + + func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { + record("execute", query, context) + } + + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + record("executeParameterized", query, context) + } + + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + record("executeUserQuery", query, context) + } + + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + record("executeBoundedQuery", query, nil) + } + + func executeBoundedQuery( + query: String, + rowCap: Int, + context: PluginStatementContext + ) async throws -> PluginQueryResult? { + record("executeBoundedQuery", query, context) + } + + func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { + _ = record("streamRows", query, context) + return AsyncThrowingStream { $0.finish() } + } + + func beginTransaction() async throws { + _ = record("BEGIN", "", nil) + } + + func beginTransaction(mode: PluginTransactionAccessMode) async throws { + _ = record("BEGIN", "", nil) + } + + func commitTransaction() async throws { + _ = record("COMMIT", "", nil) + } + + func rollbackTransaction() async throws { + _ = record("ROLLBACK", "", nil) + } + + func sessionTransactionState() async -> PluginSessionTransactionState { .idle } + + // DuckDB switches databases in place, so its session route pins the scope's database before every read. + func switchDatabase(to database: String) async throws {} +} + +private final class ReadingExportFormat: ExportFormatPlugin, @unchecked Sendable { + static let pluginName = "Reading" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Reads every table's rows and writes an empty file" + static let formatId = "read-only-reading" + static let formatDisplayName = "Reading" + static let defaultFileExtension = "txt" + static let iconName = "doc" + + private(set) var wasAskedToWrite = false + + required init() {} + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + wasAskedToWrite = true + for table in tables { + for try await _ in dataSource.streamExportRows(for: table) {} + } + try Data().write(to: destination) + return ExportFormatResult() + } +} + +@MainActor +private final class FixedDriverProvider: ScopedMetadataProviding { + private let drivers: [UUID: DatabaseDriver] + + init(_ drivers: [UUID: DatabaseDriver]) { + self.drivers = drivers + } + + func withMetadataDriver( + scope: DatabaseScope, + workload: MetadataConnectionPool.Workload, + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T + ) async throws -> T { + guard let driver = drivers[scope.connectionId] else { + throw ObjectCopyError.refused("No driver for \(scope.connectionId)") + } + return try await body(driver) + } + + func browseScope(for connectionId: UUID) -> DatabaseScope? { nil } +} + +@MainActor +struct ReadOnlySafeModeEntryPointTests { + private static let purge = "purge_orders()" + + private func makeConnection(_ level: SafeModeLevel, type: DatabaseType = .postgresql) -> DatabaseConnection { + DatabaseConnection(name: "Shop", database: "app", type: type, safeModeLevel: level) + } + + private func makeFixture( + _ level: SafeModeLevel, + type: DatabaseType = .postgresql, + enforcesReadOnlyMode: Bool = true + ) -> (CoordinatorFixture, ReadOnlyRecordingDriver) { + let driver = ReadOnlyRecordingDriver(enforcesReadOnlyMode: enforcesReadOnlyMode) + let connection = makeConnection(level, type: type) + let fixture = CoordinatorFixture( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + fixture.coordinator.toolbarState.safeModeLevel = level + fixture.coordinator.executionGate = AlwaysAllowGate() + return (fixture, driver) + } + + private func rawFilterTableTab() -> QueryTab { + var tab = QueryTab( + title: "orders", + query: "SELECT * FROM orders WHERE (purge_orders() IS NULL) LIMIT 100", + tabType: .table, + tableName: "orders" + ) + let filter = TableFilter.rawCondition("purge_orders() IS NULL") + tab.filterState.filters = [filter] + tab.filterState.commit = .all + tab.filterState.executedFilters = [filter] + tab.filterState.executedLogicMode = .and + return tab + } + + // MARK: - Run + + @Test("Run at Read-Only sends the statement in the engine's read-only mode") + func runCarriesTheMode() async throws { + let (fixture, driver) = makeFixture(.readOnly) + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(QueryTab(title: "Query", query: "SELECT purge_orders()", tabType: .query)) + + #expect(fixture.coordinator.runStatement("SELECT purge_orders()")) + try await waitUntil { driver.call(containing: Self.purge) != nil } + + #expect(driver.call(containing: Self.purge)?.context?.enforcesReadOnly == true) + } + + @Test("Run at Silent sends the statement as it always did") + func runAtSilentIsUnchanged() async throws { + let (fixture, driver) = makeFixture(.silent) + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(QueryTab(title: "Query", query: "SELECT purge_orders()", tabType: .query)) + + #expect(fixture.coordinator.runStatement("SELECT purge_orders()")) + try await waitUntil { driver.call(containing: Self.purge) != nil } + + #expect(driver.call(containing: Self.purge)?.context?.enforcesReadOnly != true) + } + + // MARK: - Run All + + @Test("Run All at Read-Only opens no transaction and sends each statement in the read-only mode") + func runAllRunsInAutocommit() async throws { + let (fixture, driver) = makeFixture(.readOnly) + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(QueryTab(title: "Query", query: "SELECT purge_orders();\nSELECT 42", tabType: .query)) + + fixture.coordinator.queryExecutionCoordinator.runAllStatements() + try await waitUntil { driver.call(containing: "42") != nil } + + let statements = driver.calls.filter { $0.query.contains(Self.purge) || $0.query.contains("42") } + #expect(!driver.entries.contains("BEGIN")) + #expect(statements.count == 2) + #expect(statements.allSatisfy { $0.context?.enforcesReadOnly == true }) + } + + @Test("Run All at Silent still wraps the batch in the app's transaction") + func runAllAtSilentWraps() async throws { + let (fixture, driver) = makeFixture(.silent) + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(QueryTab(title: "Query", query: "SELECT purge_orders();\nSELECT 42", tabType: .query)) + + fixture.coordinator.queryExecutionCoordinator.runAllStatements() + try await waitUntil { driver.entries.contains("COMMIT") } + + let entries = driver.entries + let begin = try #require(entries.firstIndex(of: "BEGIN")) + let first = try #require(driver.calls.firstIndex { $0.query.contains(Self.purge) }) + #expect(begin < first) + #expect(driver.calls.allSatisfy { $0.context?.enforcesReadOnly != true }) + } + + // MARK: - Table browse + + @Test("A raw filter that calls a function runs in the read-only mode at Read-Only, without the gate, on a driver with it") + func rawFilterBrowseCarriesTheMode() async throws { + let (fixture, driver) = makeFixture(.readOnly) + let gate = RequestRecordingGate(allows: true) + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(rawFilterTableTab()) + + #expect(fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) == nil) + try await waitUntil { driver.call(containing: Self.purge) != nil } + + #expect(await gate.requests.isEmpty) + #expect(driver.call(containing: Self.purge)?.context?.enforcesReadOnly == true) + } + + @Test("On a driver without the mode, every load under the raw filter goes to the gate") + func rawFilterIsGatedOnEveryRun() async throws { + let (fixture, driver) = makeFixture(.readOnly, enforcesReadOnlyMode: false) + let gate = RequestRecordingGate(allows: true) + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + let tab = fixture.addTab(rawFilterTableTab()) + + let first = fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) + #expect(first != nil) + await first?.value + try await waitUntil { driver.call(containing: Self.purge) != nil } + let second = fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) + #expect(second != nil) + await second?.value + + let browses = await gate.requests.filter { $0.sql == tab.content.query } + #expect(browses.count == 2) + } + + // Oracle and Dameng call a function named bare, which only the catalog the gate reads can tell from a column. + @Test("On Oracle and Dameng every raw filter goes to the gate at Read-Only, parentheses or not", arguments: [ + DatabaseType.oracle, .dameng, + ]) + func bareNameRawFilterGoesToTheGate(type: DatabaseType) { + let (fixture, _) = makeFixture(.readOnly, type: type, enforcesReadOnlyMode: false) + let (postgres, _) = makeFixture(.readOnly, enforcesReadOnlyMode: false) + let filters = [TableFilter.rawCondition("purge_orders IS NULL")] + + #expect(fixture.coordinator.rawFilterNeedsReadOnlyGate(filters)) + #expect(!postgres.coordinator.rawFilterNeedsReadOnlyGate(filters)) + } + + // A view, or a row security policy, can call a writing function with no call in the browse's text. + @Test("A table browse with no raw filter runs in the read-only mode at Read-Only without asking") + func plainBrowseCarriesTheMode() async throws { + let (fixture, driver) = makeFixture(.readOnly) + let gate = RequestRecordingGate(allows: true) + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(QueryTab(title: "order_cleanup", query: "SELECT * FROM order_cleanup LIMIT 100", tabType: .table, tableName: "order_cleanup")) + + #expect(fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) == nil) + try await waitUntil { driver.call(containing: "FROM order_cleanup") != nil } + + #expect(await gate.requests.isEmpty) + #expect(driver.call(containing: "FROM order_cleanup")?.context?.enforcesReadOnly == true) + } + + // MARK: - Filtered count + + @Test("The host's COUNT(*) under a raw filter runs in the read-only mode at Read-Only") + func hostCountCarriesTheMode() async throws { + let driver = ReadOnlyRecordingDriver() + let adapter = PluginDriverAdapter(connection: makeConnection(.readOnly), pluginDriver: driver) + let filters = [TableFilter.rawCondition("purge_orders() IS NULL")] + let countSQL = "SELECT COUNT(*) FROM orders WHERE (purge_orders() IS NULL)" + + _ = try await ExactRowCounter.count( + on: adapter, table: "orders", filters: filters, logicMode: .and, countSQL: countSQL, safeModeIsReadOnly: true + ) + _ = try await ExactRowCounter.count( + on: adapter, table: "orders", filters: filters, logicMode: .and, countSQL: countSQL + ) + + #expect(driver.calls.map { $0.context?.enforcesReadOnly == true } == [true, false]) + } + + // DuckDB counts on its session driver, which this test installs; a pooled engine would open a connection of its own. + @Test("Count Exactly with a raw filter that calls a function runs in the read-only mode without the gate") + func countExactlyCarriesTheMode() async throws { + let (fixture, driver) = makeFixture(.readOnly, type: .duckdb) + let gate = RequestRecordingGate(allows: true) + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(rawFilterTableTab()) + + fixture.coordinator.paginationCoordinator.requestExactRowCount() + try await waitUntil { driver.call(containing: "COUNT") != nil } + + #expect(await !gate.requests.contains { $0.sql?.contains("COUNT") == true }) + #expect(driver.call(containing: "COUNT")?.context?.enforcesReadOnly == true) + } + + @Test("Count Exactly with a raw filter that calls a function goes to the gate on a driver without the mode") + func countExactlyIsGatedWithoutTheMode() async throws { + let (fixture, driver) = makeFixture(.readOnly, type: .mysql, enforcesReadOnlyMode: false) + let gate = RequestRecordingGate(allows: false) + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + let tab = fixture.addTab(rawFilterTableTab()) + + fixture.coordinator.paginationCoordinator.requestExactRowCount() + try await waitUntil { + fixture.tabManager.tabs.first { $0.id == tab.id }?.pagination.isCountingExact == false + } + + #expect(await gate.requests.contains { $0.sql?.contains("COUNT") == true }) + #expect(driver.call(containing: "COUNT") == nil) + } + + // MARK: - Export + + @Test("Every export read runs in the read-only mode at Read-Only, filtered or not, and Silent changes nothing") + func exportRowScopeCarriesTheMode() async throws { + let driver = ReadOnlyRecordingDriver() + let adapter = PluginDriverAdapter(connection: makeConnection(.readOnly), pluginDriver: driver) + let filtered = PluginExportTable( + name: "orders", databaseName: "", tableType: "table", schema: nil, kind: .table, + rowScope: PluginExportRowScope(filter: "purge_orders() IS NULL") + ) + let limited = PluginExportTable( + name: "orders", databaseName: "", tableType: "table", schema: nil, kind: .table, + rowScope: PluginExportRowScope(rowLimit: 5) + ) + + let atReadOnly = ExportDataSourceAdapter(driver: adapter, databaseType: .postgresql, safeModeIsReadOnly: true) + for try await _ in atReadOnly.streamExportRows(for: filtered) {} + for try await _ in atReadOnly.streamExportRows(for: limited) {} + let atSilent = ExportDataSourceAdapter(driver: adapter, databaseType: .postgresql) + for try await _ in atSilent.streamExportRows(for: filtered) {} + + #expect(driver.calls.map { $0.context?.enforcesReadOnly == true } == [true, true, false]) + #expect(driver.calls.first?.query.contains(Self.purge) == true) + } + + @Test("At Read-Only an export's snapshot transaction is not opened, and its other statements run read-only") + func exportSnapshotIsSkippedAtReadOnly() async throws { + let driver = ReadOnlyRecordingDriver() + let adapter = PluginDriverAdapter(connection: makeConnection(.readOnly), pluginDriver: driver) + let atReadOnly = ExportDataSourceAdapter(driver: adapter, databaseType: .postgresql, safeModeIsReadOnly: true) + + _ = try await atReadOnly.execute(query: "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + _ = try await atReadOnly.execute(query: "SELECT 1") + _ = try await atReadOnly.execute(query: "COMMIT") + + #expect(driver.calls.map { $0.query } == ["SELECT 1"]) + #expect(driver.calls.first?.context?.enforcesReadOnly == true) + } + + private func installReadOnlySession(type: DatabaseType = .postgresql) -> (DatabaseConnection, ReadOnlyRecordingDriver, PluginDriverAdapter) { + installSession(.readOnly, type: type) + } + + private func installSession( + _ level: SafeModeLevel, + type: DatabaseType = .postgresql, + id: UUID = UUID() + ) -> (DatabaseConnection, ReadOnlyRecordingDriver, PluginDriverAdapter) { + let driver = ReadOnlyRecordingDriver() + let connection = DatabaseConnection(id: id, name: "Shop", database: "app", type: type, safeModeLevel: level) + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + var session = ConnectionSession(connection: connection, driver: adapter) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return (connection, driver, adapter) + } + + private func exportURL() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).txt") + } + + @Test("A filtered export at Read-Only asks the gate with its filter before any count or row is read") + func filteredExportAsksTheGateFirst() async throws { + let (connection, driver, adapter) = installReadOnlySession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let format = ReadingExportFormat() + let service = ExportService(driver: adapter, databaseType: .postgresql) { _ in format } + let gate = RequestRecordingGate(allows: false) + service.executionGate = gate + var item = ExportObjectItem(name: "orders", kind: .table) + item.rowScope = PluginExportRowScope(filter: "purge_orders() IS NULL") + var config = ExportConfiguration() + config.formatId = ReadingExportFormat.formatId + let url = exportURL() + defer { try? FileManager.default.removeItem(at: url) } + + await #expect(throws: ExecutionGateError.self) { + try await service.export(objects: [item], config: config, to: url) + } + + let requests = await gate.requests + #expect(requests.count == 1) + #expect(requests.first?.connectionId == connection.id) + #expect(requests.first?.sql?.contains(Self.purge) == true) + #expect(driver.calls.isEmpty) + #expect(!format.wasAskedToWrite) + } + + @Test("An export at Read-Only counts and reads its rows in the read-only mode") + func exportCountAndRowsCarryTheMode() async throws { + let (connection, driver, adapter) = installReadOnlySession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let format = ReadingExportFormat() + let service = ExportService(driver: adapter, databaseType: .postgresql) { _ in format } + service.executionGate = AlwaysAllowGate() + var config = ExportConfiguration() + config.formatId = ReadingExportFormat.formatId + let url = exportURL() + defer { try? FileManager.default.removeItem(at: url) } + + try await service.export(objects: [ExportObjectItem(name: "order_cleanup", kind: .table)], config: config, to: url) + + let count = try #require(driver.call(containing: "COUNT(")) + let rows = try #require(driver.calls.first { $0.entry == "streamRows" }) + #expect(count.context?.enforcesReadOnly == true) + #expect(rows.context?.enforcesReadOnly == true) + } + + @Test("Export Query Results at Read-Only asks the gate with its query before it runs") + func queryExportAsksTheGateFirst() async throws { + let (connection, driver, adapter) = installReadOnlySession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let format = ReadingExportFormat() + let service = ExportService(driver: adapter, databaseType: .postgresql) { _ in format } + let gate = RequestRecordingGate(allows: false) + service.executionGate = gate + var config = ExportConfiguration() + config.formatId = ReadingExportFormat.formatId + let url = exportURL() + defer { try? FileManager.default.removeItem(at: url) } + + await #expect(throws: ExecutionGateError.self) { + try await service.exportStreamingQuery(query: "SELECT purge_orders()", config: config, to: url) + } + + #expect(await gate.requests.map { $0.sql } == ["SELECT purge_orders()"]) + #expect(driver.calls.isEmpty) + #expect(!format.wasAskedToWrite) + } + + @Test("Transfer To at Read-Only asks the gate with the source filter before anything else") + func transferFilterAsksTheGateFirst() async throws { + let (connection, _, _) = installReadOnlySession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let gate = RequestRecordingGate(allows: false) + var item = ExportObjectItem(name: "orders", kind: .table) + item.rowScope = PluginExportRowScope(filter: "purge_orders() IS NULL") + let request = TableTransferService.Request(objects: [item], sourceType: .postgresql, destinationType: .mysql) + + await #expect(throws: ExecutionGateError.self) { + try await TableTransferService().run( + request: request, + source: DatabaseScope(connectionId: connection.id, database: "app", schema: nil), + destination: DatabaseScope(connectionId: UUID(), database: "archive", schema: nil), + operationDescription: "Transfer 1 table into archive?", + confirmedByCaller: false, + gate: gate + ) + } + + let requests = await gate.requests + #expect(requests.count == 1) + #expect(requests.first?.connectionId == connection.id) + #expect(requests.first?.sql?.contains(Self.purge) == true) + } + + // The gate's own decision on a fallback engine held at Read-Only: the export is refused without a question. + @Test("A held Read-Only refuses an export filter that calls a function on a driver without the mode") + func heldReadOnlyRefusesAnExportFilter() async throws { + let (connection, driver, adapter) = installReadOnlySession(type: .oracle) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let format = ReadingExportFormat() + let service = ExportService(driver: adapter, databaseType: .oracle) { _ in format } + let confirm = StubConfirming(answer: true) + service.executionGate = DefaultExecutionGate( + confirming: confirm, + authenticating: StubAuthenticating(answer: true), + safeModeLevelResolver: { _ in .readOnly }, + forcesWriteResolver: { _ in false }, + readOnlyFunctionPolicyResolver: { _, _ in ReadOnlyFunctionPolicy(functionsCanWrite: true, floor: .readOnly) } + ) + var item = ExportObjectItem(name: "ORDERS", kind: .table) + item.rowScope = PluginExportRowScope(filter: "purge_orders() = 1") + var config = ExportConfiguration() + config.formatId = ReadingExportFormat.formatId + let url = exportURL() + defer { try? FileManager.default.removeItem(at: url) } + + await #expect(throws: ExecutionGateError.self) { + try await service.export(objects: [item], config: config, to: url) + } + + #expect(confirm.callCount == 0) + #expect(driver.calls.isEmpty) + } + + // MARK: - Dashboard + + @Test("A dashboard tile runs its saved query in the read-only mode at Read-Only") + func dashboardTileCarriesTheMode() async throws { + let driver = ReadOnlyRecordingDriver() + let connection = makeConnection(.readOnly) + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let suite = try EphemeralDefaults() + let defaults = suite.defaults + let saved = SQLFavorite(name: "Purge", query: "SELECT purge_orders()") + let viewModel = ServerDashboardViewModel( + connectionId: connection.id, + databaseType: .postgresql, + services: .live, + tileEnvironment: DashboardTileBoard.Environment( + loadCatalog: { _ in DashboardSavedQueryCatalog(favorites: [saved], folders: []) }, + savedQueriesDidChange: Empty().eraseToAnyPublisher(), + gate: AlwaysAllowGate(), + forcesWrite: { _ in false }, + safeModeLevel: { _ in .readOnly }, + chartsAvailable: { false }, + nullDisplay: { "NULL" }, + storage: ServerDashboardTileStorage(defaults: { defaults }) + ) + ) + + viewModel.tileBoard.add(favoriteId: saved.id) + try await waitUntil { driver.call(containing: Self.purge) != nil } + + let call = driver.call(containing: Self.purge) + #expect(call?.entry == "executeBoundedQuery") + #expect(call?.context?.enforcesReadOnly == true) + } + + // MARK: - Driver without the mode + + @Test("A raw filter that calls a function is refused at Read-Only on a driver without the mode, with no question") + func rawFilterIsRefusedWithoutTheMode() async throws { + let (fixture, driver) = makeFixture(.readOnly, enforcesReadOnlyMode: false) + let confirm = StubConfirming(answer: true) + fixture.coordinator.executionGate = DefaultExecutionGate( + confirming: confirm, + authenticating: StubAuthenticating(answer: true), + safeModeLevelResolver: { _ in .readOnly }, + forcesWriteResolver: { _ in false }, + readOnlyFunctionPolicyResolver: { _, _ in ReadOnlyFunctionPolicy(functionsCanWrite: true) } + ) + defer { fixture.coordinator.cancelAllQueryTasks() } + fixture.addTab(rawFilterTableTab()) + + let refusal = fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) + #expect(refusal != nil) + await refusal?.value + + #expect(confirm.callCount == 0) + #expect(driver.call(containing: Self.purge) == nil) + } + + // MARK: - Data Compare + + private func compareEndpoint(_ connection: DatabaseConnection) -> DatabaseEndpoint { + DatabaseEndpoint( + scope: DatabaseScope(connectionId: connection.id, database: connection.database, schema: nil), + connectionName: connection.name, + databaseType: connection.type, + safeModeLevel: connection.safeModeLevel, + color: .blue + ) + } + + private func comparePlan(sourceFilter: String = "") -> DataComparePlan { + DataComparePlan( + table: "orders", + schema: nil, + columns: [CompareColumn(name: "id", sourceType: "INTEGER")], + scope: DataTableScope(keyColumns: ["id"], sourceFilter: sourceFilter), + isEnabled: true + ) + } + + // DuckDB reads both sides on the session drivers this test installs; a pooled engine would open connections of its own. + @Test("Data Compare reads the Read-Only side in the read-only mode, and the other side as before") + func compareReadsTheReadOnlySideReadOnly() async throws { + let (source, sourceDriver, _) = installSession(.readOnly, type: .duckdb) + let (target, targetDriver, _) = installSession(.silent, type: .duckdb) + defer { + DatabaseManager.shared.removeSession(for: source.id) + DatabaseManager.shared.removeSession(for: target.id) + } + let service = CompareRowService(manager: .shared, gate: AlwaysAllowGate()) + + _ = try? await service.compare( + plan: comparePlan(), + source: compareEndpoint(source), sourceConnection: source, + target: compareEndpoint(target), targetConnection: target, + options: DataCompareOptions() + ) + + let read = try #require(sourceDriver.call(containing: "orders")) + #expect(read.context?.enforcesReadOnly == true) + #expect(targetDriver.calls.allSatisfy { $0.context?.enforcesReadOnly != true }) + } + + @Test("A Data Compare filter on a Read-Only connection asks the gate before either side is read") + func compareFilterAsksTheGateFirst() async throws { + let (source, sourceDriver, _) = installSession(.readOnly, type: .duckdb) + let (target, targetDriver, _) = installSession(.silent, type: .duckdb) + defer { + DatabaseManager.shared.removeSession(for: source.id) + DatabaseManager.shared.removeSession(for: target.id) + } + let gate = RequestRecordingGate(allows: false) + let service = CompareRowService(manager: .shared, gate: gate) + + await #expect(throws: ExecutionGateError.self) { + _ = try await service.compare( + plan: self.comparePlan(sourceFilter: "purge_orders() IS NULL"), + source: self.compareEndpoint(source), sourceConnection: source, + target: self.compareEndpoint(target), targetConnection: target, + options: DataCompareOptions() + ) + } + + let requests = await gate.requests + #expect(requests.count == 1) + #expect(requests.first?.connectionId == source.id) + #expect(requests.first?.sql?.contains(Self.purge) == true) + #expect(sourceDriver.call(containing: "orders") == nil) + #expect(targetDriver.call(containing: "orders") == nil) + } + + // MARK: - Copy Objects + + private func copyRequest(source: DatabaseConnection, target: DatabaseConnection) -> ObjectCopyRequest { + ObjectCopyRequest( + source: compareEndpoint(source), + destination: .existing(compareEndpoint(target)), + objects: [ObjectCopySelection(kind: .table, name: "order_cleanup", schema: nil)], + content: .data, + existingPolicy: .replace, + errorHandling: .stopAndRollback, + wrapEachTableInTransaction: true + ) + } + + private func copyPlan(source: DatabaseConnection, target: DatabaseConnection, query: String, filtered: Bool) -> ObjectCopyPlan { + let step = ObjectCopyTableStep( + selection: ObjectCopySelection(kind: .table, name: "order_cleanup", schema: nil), + dropStatements: [], + sequenceStatements: [], + createStatements: [], + truncateStatements: [], + columns: ["id"], + primaryKeyColumns: ["id"], + sourceQuery: query, + targetTable: "order_cleanup", + targetSchema: nil, + estimatedRows: nil, + copiesData: true, + copiesIdentityColumn: false, + hasRowFilter: filtered, + note: nil + ) + return ObjectCopyPlan( + request: copyRequest(source: source, target: target), + createsDatabase: false, + tableSteps: [step], + definitionSteps: [], + keysInside: ObjectCopyKeysInside(created: true, rows: true) + ) + } + + @Test("Copy Objects reads a Read-Only source, a view among them, in the read-only mode") + func objectCopyReadsTheSourceReadOnly() async throws { + let (source, sourceDriver, sourceAdapter) = installSession(.readOnly) + let target = DatabaseConnection(name: "Archive", database: "archive", type: .mysql) + let targetAdapter = PluginDriverAdapter(connection: target, pluginDriver: ReadOnlyRecordingDriver()) + defer { DatabaseManager.shared.removeSession(for: source.id) } + let runner = ObjectCopyRunner( + manager: FixedDriverProvider([source.id: sourceAdapter, target.id: targetAdapter]), + gate: AlwaysAllowGate() + ) + + _ = try await runner.run( + copyPlan(source: source, target: target, query: "SELECT id FROM order_cleanup", filtered: false), + progress: ObjectCopyProgress(progress: Progress()) + ) + + let read = try #require(sourceDriver.call(containing: "order_cleanup")) + #expect(read.context?.enforcesReadOnly == true) + } + + @Test("A Copy Objects row filter on a Read-Only source asks the gate before anything runs") + func objectCopyFilterAsksTheGateFirst() async throws { + let (source, sourceDriver, sourceAdapter) = installSession(.readOnly) + let target = DatabaseConnection(name: "Archive", database: "archive", type: .mysql) + let targetAdapter = PluginDriverAdapter(connection: target, pluginDriver: ReadOnlyRecordingDriver()) + defer { DatabaseManager.shared.removeSession(for: source.id) } + let gate = RequestRecordingGate(allows: false) + let runner = ObjectCopyRunner( + manager: FixedDriverProvider([source.id: sourceAdapter, target.id: targetAdapter]), + gate: gate + ) + let query = "SELECT id FROM order_cleanup WHERE purge_orders() IS NULL" + + await #expect(throws: ExecutionGateError.self) { + _ = try await runner.run( + self.copyPlan(source: source, target: target, query: query, filtered: true), + progress: ObjectCopyProgress(progress: Progress()) + ) + } + + #expect(await gate.requests.map { $0.sql } == [query]) + #expect(await gate.requests.first?.connectionId == source.id) + #expect(sourceDriver.call(containing: "order_cleanup") == nil) + } +} diff --git a/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift new file mode 100644 index 0000000000..ce095016eb --- /dev/null +++ b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift @@ -0,0 +1,212 @@ +// +// ResultRowsBuildOffMainTests.swift +// TableProTests +// +// A query's rows are built off the main actor, under the shape the tab will install them with. The main actor then +// assigns the buffer, and only builds again when that shape changed while the rows were built. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@MainActor +struct ResultRowsBuildOffMainTests { + private static let columns = ["id", "email"] + private static let types: [ColumnType] = [.integer(rawType: "int"), .text(rawType: "varchar")] + private static let fetched: [[PluginCellValue]] = [[.text("1"), .text("jane@example.com")], [.text("2"), .text("john@example.com")]] + + private let fixture: CoordinatorFixture = { + var connection = TestFixtures.makeConnection(database: "shop") + connection.maskingRules = [MaskingRule(pattern: "*.email", style: .email)] + return CoordinatorFixture(connection: connection) + }() + + private static func address(of rows: ContiguousArray) -> UnsafeRawPointer? { + rows.withUnsafeBufferPointer { UnsafeRawPointer($0.baseAddress) } + } + + private func addUsersTab() -> QueryTab { + fixture.addTab(QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users")) + } + + private func load(_ prepared: TableRows.PreparedRows, into tab: QueryTab) -> TableRows { + fixture.coordinator.applyPhase1Result( + tabId: tab.id, + columns: Self.columns, + columnTypes: Self.types, + rows: Self.fetched, + executionTime: 0, + rowsAffected: 0, + statusMessage: nil, + tableName: "users", + isEditable: true, + metadata: nil, + hasSchema: false, + read: TableFreshness.Read(startedAt: .now, includesDefinition: false), + sql: "SELECT * FROM users", + connection: fixture.connection, + prepared: prepared + ) + return fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id) + } + + @Test("A first page built off the main actor is installed without another pass over its rows") + func firstPageAdoptsThePreparedRows() async { + let tab = addUsersTab() + let shape = fixture.coordinator.phase1RowShape(tabId: tab.id, tableName: "users", columns: Self.columns, columnTypes: Self.types) + let prepared = await TableRows.preparing(Self.fetched, shape: shape) + + let installed = load(prepared, into: tab) + + #expect(Self.address(of: installed.rows) == Self.address(of: prepared.rows)) + #expect(installed.value(at: 0, column: 1) == .text("j•••@example.com")) + #expect(installed.masks.isMasked(1)) + } + + @Test("A first page built before a masking rule was saved is built again under the rule") + func firstPageRebuildsRowsBuiltForOtherMasks() async { + let unmasked = TableRows.RowShape(columns: Self.columns, masks: .none, exportPolicy: .none, exportTable: "users") + let prepared = await TableRows.preparing(Self.fetched, shape: unmasked) + + let installed = load(prepared, into: addUsersTab()) + + #expect(Self.address(of: installed.rows) != Self.address(of: prepared.rows)) + #expect(installed.value(at: 0, column: 1) == .text("j•••@example.com")) + #expect(installed.value(at: 1, column: 1) == .text("j•••@example.com")) + } + + @Test("A Run All statement's rows built off the main actor become its result without another pass") + func runAllStatementAdoptsBuiltRows() async throws { + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let runner: QueryExecutionCoordinator = fixture.coordinator.queryExecutionCoordinator + let result = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"], ["2", "john@example.com"]]) + + let build = await runner.buildStatementRows([(sql: "SELECT * FROM users", result: result)], tabId: tab.id) + let built = try #require(build.statement(at: 0, readBy: fixture.coordinator.tableNameReading)) + let resultSet = runner.makeStatementResultSet( + result: result, sql: "SELECT * FROM users", index: 0, baseQuery: nil, tabId: tab.id, built: built + ) + + #expect(built.tableName == fixture.coordinator.extractTableName(from: "SELECT * FROM users")) + #expect(Self.address(of: resultSet.tableRows.rows) == Self.address(of: built.prepared.rows)) + #expect(resultSet.tableRows.value(at: 1, column: 1) == .text("j•••@example.com")) + #expect(resultSet.tableRows.masks.isMasked(1)) + } + + @Test("A Run All statement built before a masking rule applied is built again under it") + func runAllStatementRebuildsRowsBuiltForOtherMasks() async throws { + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let runner: QueryExecutionCoordinator = fixture.coordinator.queryExecutionCoordinator + let result = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"]]) + let reading = fixture.coordinator.tableNameReading + + let build = await QueryExecutionCoordinator.statementRows( + [(sql: "SELECT * FROM users", result: result)], reading: reading, policy: .none + ) + let built = try #require(build.statement(at: 0, readBy: reading)) + let resultSet = runner.makeStatementResultSet( + result: result, sql: "SELECT * FROM users", index: 0, baseQuery: nil, tabId: tab.id, built: built + ) + + #expect(Self.address(of: resultSet.tableRows.rows) != Self.address(of: built.prepared.rows)) + #expect(resultSet.tableRows.value(at: 0, column: 1) == .text("j•••@example.com")) + } + + @Test("Table names read under another session schema are not used") + func statementNamesNeedTheSameReading() async { + let reading = TableNameReading(dialect: .generic, readings: .single(.ansi), browseSchema: "public") + let moved = TableNameReading(dialect: .generic, readings: .single(.ansi), browseSchema: "sales") + let result = ScriptAnsweringDriver.resultSet(columns: ["n"], rows: [["1"]]) + + let build = await QueryExecutionCoordinator.statementRows( + [(sql: "SELECT * FROM public.users", result: result)], reading: reading, policy: .none + ) + + #expect(build.statement(at: 0, readBy: reading)?.tableName == "users") + #expect(build.statement(at: 0, readBy: moved) == nil) + #expect(build.statement(at: 1, readBy: reading) == nil) + } + + private func runBatch(answeredWith resultSets: [QueryResult]) async throws -> [ResultSet] { + let connection = fixture.connection + let driver = ScriptAnsweringDriver(connection: connection, answer: { _ in ScriptAnsweringDriver.batch(resultSets) }) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let runner: QueryExecutionCoordinator = fixture.coordinator.queryExecutionCoordinator + runner.executeBatches(runner.executionBatches(in: "SELECT * FROM users"), parameters: []) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + await handle.value + return try #require(fixture.tabManager.tabs.first { $0.id == tab.id }).display.resultSets + } + + @Test("A batch's result set that maps to its query is named after its table and masked") + func batchResultMappedToItsQuery() async throws { + let users = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"], ["2", "john@example.com"]]) + + let results = try await runBatch(answeredWith: [users]) + + let result = try #require(results.first) + #expect(results.count == 1) + #expect(result.label == "users") + #expect(result.tableRows.rows.count == 2) + #expect(result.tableRows.value(at: 1, column: 1) == .text("j•••@example.com")) + } + + @Test("A batch's numbered result sets are each masked and keep their order") + func batchNumberedResultsAreMasked() async throws { + let first = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"]]) + let second = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["2", "john@example.com"], ["3", "ann@example.com"]]) + + let results = try await runBatch(answeredWith: [first, second]) + + #expect(results.map { $0.tableRows.rows.count } == [1, 2]) + #expect(results.map { $0.tableRows.value(at: 0, column: 0) } == [.text("1"), .text("2")]) + #expect(results.allSatisfy { $0.tableRows.value(at: 0, column: 1) != .text("jane@example.com") }) + #expect(results.last?.tableRows.value(at: 1, column: 1) == .text("a•••@example.com")) + } + + @Test("Fetch All lands every row, masked the way the page it replaces is") + func fetchAllMasksLikeTheHeldPage() async throws { + let connection = fixture.connection + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: { _ in + ScriptAnsweringDriver.resultSet( + columns: ["id", "email"], + rows: [["1", "jane@example.com"], ["2", "john@example.com"], ["3", "ann@example.com"]] + ) + }) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let pageMasks = ColumnMasks(styles: [1: .full]) + let tab = fixture.addTab( + QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query), + rows: TableRows.from(queryRows: [Self.fetched[0]], columns: Self.columns, columnTypes: Self.types, masks: pageMasks) + ) + + fixture.coordinator.paginationCoordinator.performFetchAll( + tabId: tab.id, + baseQuery: "SELECT * FROM users", + scope: DatabaseScope(connectionId: connection.id, database: connection.database, schema: nil) + ) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + await handle.value + + let rows = fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id) + let hidden = pageMasks.value(.text("jane@example.com"), column: 1) + #expect(rows.rows.map { $0.id } == [.existing(0), .existing(1), .existing(2)]) + #expect(rows.rows.map { $0.values[0] } == [.text("1"), .text("2"), .text("3")]) + #expect(rows.rows.allSatisfy { $0.values[1] == hidden }) + #expect(rows.masks == pageMasks) + } +} diff --git a/apple/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift b/apple/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift index 6afd854e6a..02d5af2e59 100644 --- a/apple/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift +++ b/apple/TableProTests/Views/Main/ResultStatusBarLayoutTests.swift @@ -29,7 +29,8 @@ struct ResultStatusBarLayoutTests { pagination: PaginationState = PaginationState(), statusMessage: String? = nil, selectionSummary: SelectionSummaryState? = nil, - lastTiming: PluginQueryTiming? = nil + lastTiming: PluginQueryTiming? = nil, + autoRefresh: StatusBarAutoRefreshState = .unavailable ) -> ResultStatusBar { let snapshot = StatusBarSnapshot( tabId: UUID(), @@ -92,7 +93,8 @@ struct ResultStatusBarLayoutTests { resultSetMenu: ResultSetMenuModel(entries: [], activeOrdinal: 0, total: 0), resultSetActions: .none, onToggleFilters: {}, - onFetchAll: {} + onFetchAll: {}, + autoRefresh: autoRefresh ) } @@ -629,6 +631,47 @@ struct ResultStatusBarLayoutTests { expectFills(bar, at: pane) } + private static var autoRefreshOn: StatusBarAutoRefreshState { + StatusBarAutoRefreshState(interval: .fiveSeconds, pause: nil, onChange: { _ in }) + } + + @Test("A table tab with Auto Refresh on is never wider than its host", arguments: statusBarHostWidths) + func autoRefreshBarNeverExceedsItsHost(width: CGFloat) { + let bar = makeBar( + rowCount: 1_000, + hasColumns: true, + tabType: .table, + viewMode: .data, + pagination: PaginationState(totalRowCount: 5_000, pageSize: 1_000), + autoRefresh: Self.autoRefreshOn + ) + #expect(bar.model.controls.showsAutoRefresh) + expectFills(bar, at: width) + } + + @Test("The Auto Refresh glyph keeps its own color only while an interval runs") + func autoRefreshGlyphShowsTheOnState() throws { + let on = try #require(ResultStatusBar.autoRefreshSymbol(isOn: true)) + let off = try #require(ResultStatusBar.autoRefreshSymbol(isOn: false)) + + #expect(!on.isTemplate, "a template image draws in the control's color and hides the on state") + #expect(off.isTemplate) + #expect(on.size == off.size, "the glyph must not change width when the interval turns on") + } + + @Test("The narrowest tier still holds 280pt with Auto Refresh on the bar") + func narrowestTierHoldsWithAutoRefresh() { + let bar = makeBar( + rowCount: 1_000, + hasColumns: true, + tabType: .table, + viewMode: .data, + pagination: PaginationState(totalRowCount: 5_000, pageSize: 1_000), + autoRefresh: Self.autoRefreshOn + ) + expectFills(bar, at: 280) + } + /// The grid shares a `VStack` with the bar, closed by a `.frame(maxWidth: .infinity)` that can /// never report less than its widest child, which is how the bar's width reached the grid at /// all. Measuring the column rather than the bar alone is what pins that path. diff --git a/apple/TableProTests/Views/Main/SaveAttributionTests.swift b/apple/TableProTests/Views/Main/SaveAttributionTests.swift index 57aea277e9..c46cdf7b6f 100644 --- a/apple/TableProTests/Views/Main/SaveAttributionTests.swift +++ b/apple/TableProTests/Views/Main/SaveAttributionTests.swift @@ -68,7 +68,7 @@ struct SaveAttributionTests { #expect(fixture.changeManager.hasChanges) #expect(fixture.tabManager.tabs.first { $0.id == other.id }?.execution.errorMessage == nil) // The save is over: a second one is not refused as already running. - #expect(fixture.coordinator.rowEditingCoordinator.beginSaveInFlight()) + #expect(fixture.coordinator.rowEditingCoordinator.beginSaveInFlight(tabId: nil)) fixture.coordinator.rowEditingCoordinator.endSaveInFlight() } diff --git a/apple/TableProTests/Views/Main/SaveInFlightEditGateTests.swift b/apple/TableProTests/Views/Main/SaveInFlightEditGateTests.swift new file mode 100644 index 0000000000..107cce1726 --- /dev/null +++ b/apple/TableProTests/Views/Main/SaveInFlightEditGateTests.swift @@ -0,0 +1,75 @@ +// +// SaveInFlightEditGateTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +// A landing save clears the tab's staged edits and reloads its rows, so an edit typed during the write was lost. +@MainActor +struct SaveInFlightEditGateTests { + private func makeCoordinator() throws -> MainContentCoordinator { + let coordinator = MainContentCoordinator( + connection: TestFixtures.makeConnection(database: "shop", type: .postgresql), + tabManager: QueryTabManager(), + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState() + ) + coordinator.openTableTab(TableInfo(name: "orders", type: .table, rowCount: nil, schema: "public")) + let index = try #require(coordinator.tabManager.selectedTabIndex) + coordinator.tabManager.tabs[index].tableContext.isEditable = true + coordinator.tabManager.tabs[index].display.resultsViewMode = .data + coordinator.setActiveTableRows( + TableRows.from( + queryRows: [[.text("1"), .text("open")]], + columns: ["id", "status"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)], + hasAuthoritativeSchema: true + ), + for: coordinator.tabManager.tabs[index].id + ) + coordinator.changeManager.configureForTable( + tableName: "orders", + columns: ["id", "status"], + primaryKeyColumns: ["id"], + databaseType: .postgresql, + generatedColumns: [] + ) + return coordinator + } + + @Test("The saving tab takes no edit until the save ends") + func savingTabIsReadOnly() throws { + let coordinator = try makeCoordinator() + defer { coordinator.teardown() } + let tabId = try #require(coordinator.tabManager.selectedTabId) + #expect(coordinator.canEditActiveResult) + + #expect(coordinator.rowEditingCoordinator.beginSaveInFlight(tabId: tabId)) + #expect(!coordinator.canEditActiveResult) + #expect(coordinator.isSavingSelectedTab) + coordinator.deleteRows([.existing(0)]) + coordinator.stageInspectorFieldEdit( + columnIndex: 1, value: .text("closed"), rowIDs: [.existing(0)], continuity: .discrete + ) + #expect(!coordinator.changeManager.hasChanges) + + coordinator.rowEditingCoordinator.endSaveInFlight() + #expect(coordinator.canEditActiveResult) + #expect(!coordinator.isSavingSelectedTab) + } + + @Test("A save running for another tab leaves this one editable") + func otherTabStaysEditable() throws { + let coordinator = try makeCoordinator() + defer { coordinator.teardown() } + + #expect(coordinator.rowEditingCoordinator.beginSaveInFlight(tabId: UUID())) + #expect(coordinator.canEditActiveResult) + #expect(!coordinator.isSavingSelectedTab) + coordinator.rowEditingCoordinator.endSaveInFlight() + } +} diff --git a/apple/TableProTests/Views/Main/SchemaChangeTokenGateTests.swift b/apple/TableProTests/Views/Main/SchemaChangeTokenGateTests.swift new file mode 100644 index 0000000000..6856d43f14 --- /dev/null +++ b/apple/TableProTests/Views/Main/SchemaChangeTokenGateTests.swift @@ -0,0 +1,31 @@ +@testable import TablePro +import Testing + +struct SchemaChangeTokenGateTests { + private static func admissions(_ tokens: [String?], forgettingAfter forgetIndex: Int? = nil) -> [Bool] { + var gate = SchemaChangeTokenGate() + var admitted: [Bool] = [] + for (index, token) in tokens.enumerated() { + admitted.append(gate.admits(token)) + if index == forgetIndex { + gate.forget() + } + } + return admitted + } + + @Test("A write that leaves the schema token alone reloads nothing after the first reload") + func unchangedTokenIsSkipped() { + #expect(Self.admissions(["12", "12", "12"]) == [true, false, false]) + } + + @Test("A moved token reloads, and so does every event of an engine with no token") + func movedOrMissingTokenReloads() { + #expect(Self.admissions(["12", "13", nil, nil, "13"]) == [true, true, true, true, true]) + } + + @Test("A failed reload forgets the token, so the next write reloads again") + func forgottenTokenReloads() { + #expect(Self.admissions(["12", "12"], forgettingAfter: 0) == [true, true]) + } +} diff --git a/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift b/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift new file mode 100644 index 0000000000..b75f6881d9 --- /dev/null +++ b/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift @@ -0,0 +1,40 @@ +// +// StopRollbackNoticeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct StopRollbackNoticeTests { + private func fixture() -> (CoordinatorFixture, UUID) { + let fixture = CoordinatorFixture(connection: TestFixtures.makeConnection(type: .postgresql)) + let tab = fixture.addTab(QueryTab(title: "Q1", query: "UPDATE t SET a = 1", tabType: .query)) + return (fixture, tab.id) + } + + private func errorMessage(_ fixture: CoordinatorFixture, _ tabId: UUID) -> String? { + fixture.tabManager.tabs.first { $0.id == tabId }?.execution.errorMessage + } + + @Test("A rollback the server did for a Stop is shown on the stopped tab") + func rollbackIsShown() { + let (fixture, tabId) = fixture() + + fixture.coordinator.reportStopRollback(PluginStopRolledBackTransaction(code: 3_110), tabId: tabId) + + #expect(errorMessage(fixture, tabId)?.contains("rolled back the transaction") == true) + } + + @Test("A plain cancel leaves the tab as Stop left it") + func plainCancelShowsNothing() { + let (fixture, tabId) = fixture() + + fixture.coordinator.reportStopRollback(CancellationError(), tabId: tabId) + + #expect(errorMessage(fixture, tabId) == nil) + } +} diff --git a/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift b/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift new file mode 100644 index 0000000000..920544485f --- /dev/null +++ b/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift @@ -0,0 +1,40 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct TableLoadAfterSafeModePromptTests { + @Test("A table load turned away while a query waits for approval runs once that approval closes") + func turnedAwayLoadRunsAfterThePrompt() async throws { + let connection = DatabaseConnection( + name: "Confirm", database: "app", type: .postgresql, safeModeLevel: .alertFull + ) + let fixture = CoordinatorFixture( + connection: connection, + driver: ScriptAnsweringDriver(connection: connection, sendsBatchesWhole: false) + ) + fixture.coordinator.toolbarState.safeModeLevel = .alertFull + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + + fixture.addTab(QueryTab(title: "Query", query: "SELECT 1", tabType: .query)) + #expect(fixture.coordinator.runStatement("SELECT 1")) + try await waitUntil { gate.askedCount == 1 } + + var table = QueryTab(title: "users", query: "SELECT * FROM users LIMIT 1000", tabType: .table, tableName: "users") + table.pagination.isLoading = true + fixture.addTab(table) + #expect(fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) == nil) + #expect(fixture.tabManager.tabs.first { $0.id == table.id }?.pagination.isLoading == false) + + await gate.answer(allowing: true) + try await waitUntil { gate.askedCount == 2 } + await gate.answer(allowing: false) + try await waitUntil { + fixture.tabManager.tabs.first { $0.id == table.id }?.execution.errorMessage == "Refused by the test" + } + } +} diff --git a/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift b/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift new file mode 100644 index 0000000000..bb0a30c859 --- /dev/null +++ b/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift @@ -0,0 +1,129 @@ +import AppKit +import Foundation +import SwiftUI +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct WindowCloseSaveEveryConnectionTests { + private func actions( + for coordinator: MainContentCoordinator, + in window: NSWindow, + attach: Bool = true + ) -> MainContentCommandActions { + var selectedTables: Set = [] + var pendingTruncates: Set = [] + var pendingDeletes: Set = [] + var tableOperationOptions: [DatabaseTreeTableRef: TableOperationOptions] = [:] + let actions = MainContentCommandActions( + coordinator: coordinator, + connection: coordinator.connection, + selectedTables: Binding(get: { selectedTables }, set: { selectedTables = $0 }), + pendingTruncates: Binding(get: { pendingTruncates }, set: { pendingTruncates = $0 }), + pendingDeletes: Binding(get: { pendingDeletes }, set: { pendingDeletes = $0 }), + tableOperationOptions: Binding( + get: { tableOperationOptions }, + set: { tableOperationOptions = $0 } + ), + trailingPaneState: TrailingPaneState() + ) + actions.window = window + if attach { coordinator.commandActions = actions } + return actions + } + + @Test("Save on a window close keeps the window while another connection holds edits only their tab can save") + func saveKeepsTheWindowForAnotherConnectionsEdits() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + let frontActions = actions(for: front, in: harness.window) + let backActions = actions(for: back, in: harness.window) + try harness.addSelectedTab(to: harness.foreground) + try harness.addSelectedTab(to: harness.background) + + var edited = QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users") + edited.pendingChanges.insertedRowIDs = [.inserted(UUID())] + edited.pendingChanges.columns = ["id", "name"] + edited.pendingChanges.primaryKeyColumns = ["id"] + back.tabManager.tabs.append(edited) + frontActions.confirmSaveChanges = { _, _ in .save } + + let outcome = await frontActions.closeWindowAwaiting() + + #expect(outcome == .cancelled) + #expect(back.tabManager.tabs.contains { $0.id == edited.id }) + withExtendedLifetime(backActions) {} + } + + // Its actions arrive only once its pane mounts, after Save has shown it. + @Test("Save waits for a connection that has not shown its pane yet, then saves it and closes") + func saveReachesAConnectionNotShownYet() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + front.contentWindow = harness.window + let frontActions = actions(for: front, in: harness.window) + let backActions = actions(for: back, in: harness.window, attach: false) + back.commandActions = nil + try harness.addSelectedTab(to: harness.foreground) + back.toolbarState.hasCreateTablePending = true + frontActions.confirmSaveChanges = { _, _ in + Task { @MainActor in back.commandActions = backActions } + return .save + } + + let outcome = await frontActions.closeWindowAwaiting() + + #expect(outcome == .closed) + } + + @Test("A close cancelled while it waits for a connection's pane stops after that one wait and keeps the window") + func cancelledCloseStopsWaiting() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + front.contentWindow = harness.window + let frontActions = actions(for: front, in: harness.window) + back.commandActions = nil + try harness.addSelectedTab(to: harness.foreground) + back.toolbarState.hasCreateTablePending = true + let clock = CountingClock() + frontActions.paneWaitClock = clock + frontActions.confirmSaveChanges = { _, _ in .save } + + let close = Task { await frontActions.closeWindowAwaiting() } + await clock.firstSleep.wait() + close.cancel() + let outcome = await close.value + + #expect(outcome == .cancelled) + #expect(clock.sleepCount == 1) + } +} + +// Time stands still, so a sleep ends only by cancellation; the first one opens a gate the test waits on. +private final class CountingClock: Clock, @unchecked Sendable { + let firstSleep = AsyncGate() + private let base = ManualClock() + private let lock = NSLock() + private var count = 0 + + var sleepCount: Int { lock.withLock { count } } + var now: ManualClock.Instant { base.now } + var minimumResolution: Duration { base.minimumResolution } + + func sleep(until deadline: ManualClock.Instant, tolerance: Duration?) async throws { + let isFirst = lock.withLock { + count += 1 + return count == 1 + } + if isFirst { await firstSleep.open() } + try await base.sleep(until: deadline, tolerance: tolerance) + } +} diff --git a/apple/TableProTests/Views/ObjectSource/ObjectSourceLoaderTests.swift b/apple/TableProTests/Views/ObjectSource/ObjectSourceLoaderTests.swift index a9c3151c1f..5e2029f58a 100644 --- a/apple/TableProTests/Views/ObjectSource/ObjectSourceLoaderTests.swift +++ b/apple/TableProTests/Views/ObjectSource/ObjectSourceLoaderTests.swift @@ -117,6 +117,68 @@ struct ObjectSourceLoaderTests { #expect(loader.source == definition) #expect(driver.recordedCalls == [fetchCall]) } + + @Test("A server object reads its definition and fresh attributes through the driver") + func serverObjectLoadsThroughDriver() async { + let driver = ServerObjectSourceDriver() + driver.reread = PluginServerObjectInfo( + kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384", + definition: "CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;", + attributes: [PluginObjectAttribute(label: "Default Version", value: "1.3")] + ) + let kind = SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", + iconName: "puzzlepiece.extension", scope: .database + ) + let object = ServerObjectInfo(kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384") + let ref = DatabaseObjectRef(serverObject: object, kind: kind, database: "shop") + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: driver) + let metadata = ScriptedSourceMetadata(driver: adapter) + let loader = ObjectSourceLoader(connectionId: UUID(), objectRef: ref, metadata: metadata) + + await loader.load() + + // A database-wide object is read with no schema: `public` is only where its members live. + #expect(metadata.scopes.map { $0.schema } == [nil]) + #expect(metadata.scopes.map { $0.database } == ["shop"]) + #expect(loader.source == "CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;") + #expect(loader.attributes == [ObjectAttribute(label: "Default Version", value: "1.3")]) + let requested = driver.requested + #expect(requested.map { $0.kindId } == ["extension"]) + #expect(requested.map { $0.identity } == ["16384"]) + } + + @Test("A server object the driver cannot read reports the failure") + func serverObjectWithoutDefinitionFails() async { + let driver = ServerObjectSourceDriver() + let kind = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + let ref = DatabaseObjectRef(serverObject: ServerObjectInfo(kindId: "event", name: "nightly"), kind: kind, database: "shop") + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .mysql), pluginDriver: driver) + let loader = ObjectSourceLoader(connectionId: UUID(), objectRef: ref, metadata: ScriptedSourceMetadata(driver: adapter)) + + await loader.load() + + guard case .failed = loader.state else { + Issue.record("expected the load to fail") + return + } + } +} + +private final class ServerObjectSourceDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var log: [PluginServerObjectInfo] = [] + var reread: PluginServerObjectInfo? + + var requested: [PluginServerObjectInfo] { lock.withLock { log } } + + func fetchServerObject(_ object: PluginServerObjectInfo) async throws -> PluginServerObjectInfo { + lock.withLock { log.append(object) } + guard let reread else { throw PluginObjectSourceError.unsupported(object.name) } + return reread + } } /// Throws `CancellationError` once on request, the way a metadata read does when the task that @@ -125,6 +187,7 @@ struct ObjectSourceLoaderTests { private final class ScriptedSourceMetadata: ScopedMetadataProviding { private let driver: DatabaseDriver var cancelsNextRead = false + private(set) var scopes: [DatabaseScope] = [] init(driver: DatabaseDriver) { self.driver = driver @@ -135,6 +198,7 @@ private final class ScriptedSourceMetadata: ScopedMetadataProviding { workload: MetadataConnectionPool.Workload, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { + scopes.append(scope) if cancelsNextRead { cancelsNextRead = false throw CancellationError() diff --git a/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift b/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift index c766504cd9..3a41435a3b 100644 --- a/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift +++ b/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift @@ -290,6 +290,44 @@ extension GlobalStateSuites { #expect(grid.commands.pasteAnchor(.current(grid.model.selection), block: single) == GridCoord(row: 0, displayColumn: Fixture.note)) } + @Test("A binary value copies whole and pastes back as the same bytes") + func binaryValueRoundTrips() throws { + let clipboard = try #require(RecordingClipboard.current) + let grid = Fixture.grid() + let bytes = Data((0..<100).map { UInt8($0) }) + _ = grid.owner.tableRows.edit(row: 0, column: Fixture.payload, value: .bytes(bytes)) + grid.model.reloadRows() + grid.selectCell(0, Fixture.payload) + grid.controller.copy(nil) + #expect(clipboard.text == "0x" + bytes.map { String(format: "%02X", $0) }.joined()) + + grid.selectCell(1, Fixture.payload) + grid.controller.paste(nil) + + #expect(grid.owner.staged(1, Fixture.payload)?.value == .bytes(bytes)) + } + + @Test("Hex from another app pastes into a binary column as bytes, and a cut-short copy is refused") + func foreignHexIntoBinaryColumn() throws { + let clipboard = try #require(RecordingClipboard.current) + let grid = Fixture.grid() + + clipboard.text = "0x4142" + grid.selectCell(1, Fixture.payload) + grid.controller.paste(nil) + #expect(grid.owner.staged(1, Fixture.payload)?.value == .bytes(Data([0x41, 0x42]))) + + clipboard.text = "0x41…" + grid.selectCell(2, Fixture.payload) + grid.controller.paste(nil) + #expect(grid.owner.staged(2, Fixture.payload) == nil) + + clipboard.text = "0x41" + grid.selectCell(2, Fixture.note) + grid.controller.paste(nil) + #expect(grid.owner.staged(2, Fixture.note)?.value == .text("0x41")) + } + @Test("Paste is enabled over a cell selection with no cursor when one value is on the clipboard") func pasteIsEnabledWithoutACursor() throws { try #require(RecordingClipboard.current).text = "archived" diff --git a/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift b/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift index 511e3343f5..9dbd8a9d6c 100644 --- a/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift +++ b/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift @@ -39,23 +39,23 @@ extension GlobalStateSuites { "Find matches a lone CR as the space the cell shows while marks are off", .appSettings { $0.dataGrid.showInvisibleCharacters = false } ) - func findReadsTheUnmarkedText() { + func findReadsTheUnmarkedText() async { let (model, owner) = makeModel(values: ["a\rb"]) defer { withExtendedLifetime(owner) {} } - #expect(find("a b", in: model).count == 1) + #expect(await find("a b", in: model).count == 1) } @Test( "Find does not read a marked CR as a space", .appSettings { $0.dataGrid.showInvisibleCharacters = true } ) - func findReadsTheMarkedText() { + func findReadsTheMarkedText() async { let (model, owner) = makeModel(values: ["a\rb"]) defer { withExtendedLifetime(owner) {} } - #expect(find("a b", in: model).isEmpty) - #expect(find("b", in: model).count == 1) + #expect(await find("a b", in: model).isEmpty) + #expect(await find("b", in: model).count == 1) } // The model holds its owner weakly, so the caller keeps the owner alive. @@ -74,10 +74,11 @@ extension GlobalStateSuites { return (model, owner) } - private func find(_ term: String, in model: DataGridModel) -> [GridFindState.Match] { + private func find(_ term: String, in model: DataGridModel) async -> [GridFindState.Match] { model.showFind() model.find.term = term model.findQueryDidChange() + await model.findSearchSettled() return model.find.matches } } diff --git a/apple/TableProTests/Views/Results/JSONViewerWindowCommitTests.swift b/apple/TableProTests/Views/Results/JSONViewerWindowCommitTests.swift index d5a4ab4112..9db90b2b44 100644 --- a/apple/TableProTests/Views/Results/JSONViewerWindowCommitTests.swift +++ b/apple/TableProTests/Views/Results/JSONViewerWindowCommitTests.swift @@ -15,7 +15,7 @@ struct JSONViewerWindowCommitTests { private let edited = #"{"qty":25,"sku":"A-1"}"# private func commit(saving displayText: String, baseline: String?) -> String? { - JSONViewerWindowController.valueToCommit(saved: JsonReindenter.normalize(displayText), baseline: baseline) + JSONViewerWindowController.valueToCommit(saved: displayText, baseline: baseline) } @Test("Save commits an edit the popover made before the window opened") @@ -60,6 +60,16 @@ struct JSONViewerWindowCommitTests { #expect(commit(saving: "{}", baseline: nil) == "{}") } + @Test("A document stored across lines keeps its lines when edited, one stored on one line stays compact") + func editKeepsTheStoredLayout() { + let formatted = JsonReindenter.reindent(stored) + let windowText = JsonReindenter.reindent(edited) + + #expect(commit(saving: windowText, baseline: formatted) == windowText) + #expect(commit(saving: windowText, baseline: stored) == edited) + #expect(commit(saving: windowText, baseline: stored + "\n") == edited) + } + @Test("text that is not JSON is compared as written") func invalidTextIsComparedAsWritten() { #expect(commit(saving: "{oops", baseline: "{oops") == nil) diff --git a/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift b/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift index d64d2975d0..167773b1fa 100644 --- a/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift +++ b/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift @@ -128,6 +128,30 @@ struct ResultsJsonViewTests { } } + @Test("A cell a pending edit set to the default renders as null, and stored text that reads DEFAULT stays text") + func pendingDefaultRendersAsNull() { + let inserted = RowID.inserted(UUID()) + let tableRows = TableRows( + rows: [ + Row(id: .existing(0), values: [.text("DEFAULT"), .text("a")]), + Row(id: inserted, values: [.columnDefault, .text("b")]), + ], + columns: ["status", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] + ) + + let result = ResultsJsonView.computeJson( + tableRows: tableRows, + displayIDs: nil, + selectedIndices: [], + defaultedCells: [inserted: [0]], + columnLayout: ColumnLayoutState() + ) + + #expect(result.json.contains("\"status\": \"DEFAULT\"")) + #expect(result.json.contains("\"status\": null")) + } + @Test("an empty result renders an empty array") func emptyResultRendersEmptyArray() { let result = ResultsJsonView.computeJson( diff --git a/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift b/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift index 26110af586..926bf1f31f 100644 --- a/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift +++ b/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift @@ -176,7 +176,7 @@ struct SelectionSummaryTrackerTests { defer { grid.close() } grid.model.setSelection(Self.block(rows: 0...1, columns: 0...0)) - grid.edit(row: 1, to: .value(.text(PluginCellValue.defaultMarkerText), isExpression: false)) + grid.edit(row: 1, to: .value(.columnDefault, isExpression: true)) await grid.tracker.pendingTask?.value #expect(state.summary?.emptyCount == 1) diff --git a/apple/TableProTests/Views/RowInspector/BooleanPickerViewTests.swift b/apple/TableProTests/Views/RowInspector/BooleanPickerViewTests.swift new file mode 100644 index 0000000000..f9955a4a2c --- /dev/null +++ b/apple/TableProTests/Views/RowInspector/BooleanPickerViewTests.swift @@ -0,0 +1,33 @@ +// +// BooleanPickerViewTests.swift +// TableProTests +// + +@testable import TablePro +import Testing + +struct BooleanPickerViewTests { + @Test("Stored booleans select true or false") + func storedBooleansSelectAChoice() { + for text in ["1", "true", "t", "yes"] { + #expect(BooleanPickerView.normalized(text) == "1", "\(text)") + } + for text in ["0", "false", "f", "no"] { + #expect(BooleanPickerView.normalized(text) == "0", "\(text)") + } + #expect(BooleanPickerView.unrecognizedValue(of: .value("1")) == nil) + } + + @Test("A TINYINT(1) holding 2 shows 2, never false") + func otherIntegerShowsItself() { + #expect(BooleanPickerView.normalized("2") == "2") + #expect(BooleanPickerView.unrecognizedValue(of: .value("2")) == "2") + } + + @Test("NULL and an empty value add no extra choice") + func statesAddNoChoice() { + #expect(BooleanPickerView.unrecognizedValue(of: .null) == nil) + #expect(BooleanPickerView.unrecognizedValue(of: .value("")) == nil) + #expect(BooleanPickerView.unrecognizedValue(of: .multipleValues) == nil) + } +} diff --git a/apple/TableProTests/Views/RowInspector/GeometryFieldModeTests.swift b/apple/TableProTests/Views/RowInspector/GeometryFieldModeTests.swift index 118c5262df..72042bc4d2 100644 --- a/apple/TableProTests/Views/RowInspector/GeometryFieldModeTests.swift +++ b/apple/TableProTests/Views/RowInspector/GeometryFieldModeTests.swift @@ -108,6 +108,16 @@ struct GeometryFieldModeTests { #expect(latch.shown(preferred: .text, preview: drawable) == .map) } + @Test("A mode latched for one row is undecided for the next") + func latchBelongsToItsRow() { + let first = UUID() + var latch = ModeLatch().scoped(to: first) + latch.offer(preferred: .map, preview: .unavailable(.null)) + + #expect(latch.scoped(to: first).mode == .text) + #expect(latch.scoped(to: UUID()).mode == nil) + } + @Test("A large value shows Map while it is first read, then follows the read") func aLargeValueWaitsForItsFirstRead() { var latch = ModeLatch() diff --git a/apple/TableProTests/Views/RowInspector/GeometryFieldPreviewStoreTests.swift b/apple/TableProTests/Views/RowInspector/GeometryFieldPreviewStoreTests.swift index ab3c5e4ab1..54dceb1ead 100644 --- a/apple/TableProTests/Views/RowInspector/GeometryFieldPreviewStoreTests.swift +++ b/apple/TableProTests/Views/RowInspector/GeometryFieldPreviewStoreTests.swift @@ -44,6 +44,20 @@ struct GeometryFieldPreviewStoreTests { #expect(store.token == token) } + @Test("Another row drops the last row's preview while its own large value reads") + func anotherRowDropsThePreview() { + let store = GeometryFieldPreviewStore() + let first = UUID() + _ = store.resolve(request("POINT(1 2)"), source: .spatialColumn, documentID: first) + + let sameRow = store.resolve(request(largeLine), source: .spatialColumn, documentID: first) + let nextRow = store.resolve(request(largeLine), source: .spatialColumn, documentID: UUID()) + + #expect(isDrawable(sameRow.preview)) + #expect(nextRow.preview == nil) + #expect(nextRow.pending == request(largeLine)) + } + @Test("A new value is read and moves the token") func newValueMovesTheToken() { let store = GeometryFieldPreviewStore() diff --git a/apple/TableProTests/Views/RowInspector/InspectorFocusTraversalTests.swift b/apple/TableProTests/Views/RowInspector/InspectorFocusTraversalTests.swift index bf9e994cd5..dad6703337 100644 --- a/apple/TableProTests/Views/RowInspector/InspectorFocusTraversalTests.swift +++ b/apple/TableProTests/Views/RowInspector/InspectorFocusTraversalTests.swift @@ -83,7 +83,7 @@ struct InspectorFocusTraversalTests { #expect(registry.fieldsWithoutTabStop.isEmpty) } - /// Field ids are reissued on every selection change, so the registry would otherwise grow. + // Editor ids are reissued on every selection change, so the registry would otherwise grow. @Test("Ids the list no longer shows are forgotten") func reissuedIdsAreForgotten() { let registry = InspectorTabStopRegistry() diff --git a/apple/TableProTests/Views/RowInspector/InspectorRowChangeTests.swift b/apple/TableProTests/Views/RowInspector/InspectorRowChangeTests.swift new file mode 100644 index 0000000000..de86e6b977 --- /dev/null +++ b/apple/TableProTests/Views/RowInspector/InspectorRowChangeTests.swift @@ -0,0 +1,134 @@ +import AppKit +import SwiftUI +@testable import TablePro +import TableProTestSupport +import Testing + +// Selecting another row updates the list in place: each field keeps its control and shows the new value, and an +// editor that seeds state from the value starts over inside it. +@MainActor +struct InspectorRowChangeTests { + private static let columns = ["id", "name", "note", "meta"] + private static let types: [ColumnType] = [ + .integer(rawType: "INT"), .text(rawType: "VARCHAR(40)"), .text(rawType: "TEXT"), .json(rawType: "JSON"), + ] + + private static func descendants(of root: NSView, as type: View.Type) -> [View] { + let own = (root as? View).map { [$0] } ?? [] + return own + root.subviews.flatMap { descendants(of: $0, as: type) } + } + + private static func select(_ row: Int, in state: MultiRowEditState) { + state.configure( + rowIDs: [.existing(row)], + allRows: [["\(row)", "name \(row)", "note \(row)", #"{"row": \#(row)}"#]], + absentCells: [[]], + columns: columns, + columnTypes: types + ) + } + + private static func host(_ state: MultiRowEditState) -> NSWindow { + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 320, height: 600), + styleMask: [.titled], + backing: .buffered, + defer: false + ) + window.isReleasedWhenClosed = false + window.contentView = NSHostingView(rootView: InspectorFieldListView( + editState: state, + isEditable: true, + databaseType: .mysql + )) + window.layoutIfNeeded() + return window + } + + private static func singleLineValues(in window: NSWindow) -> [String] { + guard let root = window.contentView else { return [] } + return descendants(of: root, as: ValueTextFieldView.self).map(\.stringValue).sorted() + } + + @Test("A single-line field keeps its control and shows the newly selected row's value") + func singleLineFieldFollowsTheRowInPlace() async throws { + let state = MultiRowEditState() + Self.select(0, in: state) + let window = Self.host(state) + defer { window.orderOut(nil) } + let root = try #require(window.contentView) + try await waitUntil { Self.singleLineValues(in: window) == ["0", "name 0"] } + let fields = Self.descendants(of: root, as: ValueTextFieldView.self) + + Self.select(1, in: state) + try await waitUntil { + root.layoutSubtreeIfNeeded() + return Self.singleLineValues(in: window) == ["1", "name 1"] + } + + let after = Self.descendants(of: root, as: ValueTextFieldView.self) + #expect(after.count == fields.count) + #expect(after.allSatisfy { field in fields.contains { $0 === field } }) + } + + @Test("A multi-line field keeps its text view, shows the new row from the start and drops the old row's typing undo") + func multiLineFieldStartsOverInPlace() async throws { + let state = MultiRowEditState() + Self.select(0, in: state) + let window = Self.host(state) + defer { window.orderOut(nil) } + let root = try #require(window.contentView) + func noteViews() -> [NSTextView] { + Self.descendants(of: root, as: NSTextView.self).filter { $0.string.hasPrefix("note ") } + } + try await waitUntil { noteViews().map(\.string) == ["note 0"] } + let before = try #require(noteViews().first) + let undo = try #require(window.undoManager) + // The test host sends no events, so the group the end of the typing event would close is closed here. + undo.groupsByEvent = false + defer { undo.groupsByEvent = true } + window.makeFirstResponder(before) + before.setSelectedRange(NSRange(location: 6, length: 0)) + undo.beginUndoGrouping() + before.insertText(" typed", replacementRange: NSRange(location: NSNotFound, length: 0)) + undo.endUndoGrouping() + #expect(undo.canUndo) + + Self.select(1, in: state) + try await waitUntil { + root.layoutSubtreeIfNeeded() + return noteViews().map(\.string) == ["note 1"] + } + + #expect(noteViews().first === before) + #expect(before.selectedRange() == NSRange(location: 0, length: 0)) + #expect(before.undoManager === undo) + #expect(!undo.canUndo) + } + + @Test("A JSON field keeps its editor and shows the newly selected row's document") + func jsonFieldFollowsTheRowInPlace() async throws { + let state = MultiRowEditState() + Self.select(0, in: state) + let window = Self.host(state) + defer { window.orderOut(nil) } + let root = try #require(window.contentView) + func jsonEditors() -> [NSView] { + Self.descendants(of: root, as: NSView.self).filter { $0.accessibilityIdentifier() == "inspector-json-field" } + } + func shows(_ row: Int) -> Bool { + jsonEditors().contains { ($0.accessibilityValue() as? String)?.contains("\"row\": \(row)") == true } + } + try await waitUntil { shows(0) } + let before = try #require(jsonEditors().first) + + Self.select(1, in: state) + try await waitUntil { + root.layoutSubtreeIfNeeded() + return shows(1) + } + + #expect(jsonEditors().count == 1) + #expect(jsonEditors().first === before) + } +} diff --git a/apple/TableProTests/Views/Settings/ThemeEditorWindowTests.swift b/apple/TableProTests/Views/Settings/ThemeEditorWindowTests.swift index 578fff0877..45cd70f5df 100644 --- a/apple/TableProTests/Views/Settings/ThemeEditorWindowTests.swift +++ b/apple/TableProTests/Views/Settings/ThemeEditorWindowTests.swift @@ -1,6 +1,7 @@ import AppKit import Foundation @testable import TablePro +import TableProPluginKit import TableProTestSupport import TableProTheme import Testing @@ -46,8 +47,7 @@ struct ThemeEditorWindowTests { #expect(try fixture.onDisk("Midnight.json").name == "Night Owl") } - /// The preview's editor and grid keep undo histories of their own, so while either held focus - /// Edit > Undo did nothing to the theme. + // The preview's editor and grid have undo histories of their own, so while either held focus Edit > Undo missed the theme. @Test("The preview takes no clicks and gives up focus, so Undo reaches the theme") func previewNeverHoldsFocus() throws { let fixture = try Fixture() @@ -71,8 +71,7 @@ struct ThemeEditorWindowTests { windows.closeEditor(for: "user.aaaa0002") } - /// AppKit's key view loop reached the preview's editor and grid, and Undo then went to their empty - /// histories instead of the theme's. + // AppKit's key view loop reached the preview's editor and grid, and Undo then went to their empty histories. @Test("Tab and Shift-Tab pass over the preview") func tabPassesOverThePreview() throws { let fixture = try Fixture() @@ -117,7 +116,7 @@ struct ThemeEditorWindowTests { return view.isInsideThemePreview } - /// Quitting closes no window, so the field's text reached no commit. + // Quitting closes no window, so the field's text reached no commit. @Test("Quitting writes the name still being typed") func quitWritesTheTypedName() async throws { let fixture = try Fixture() @@ -198,8 +197,7 @@ struct ThemeEditorWindowTests { #expect(try fixture.onDisk("Midnight.json").name == "Night Owl") } - /// The close button takes no focus, so the rename was still pending, refused, and dropped as the - /// window closed in the same event. + // The close button takes no focus, so the pending rename was refused and dropped as the window closed in the same event. @Test("Closing with a taken name keeps the window open with the reason") func closeWithATakenNameKeepsTheWindow() throws { let fixture = try Fixture() @@ -287,4 +285,123 @@ struct ThemeEditorWindowTests { #expect(controller.window?.representedURL?.lastPathComponent == "\(newId).json") windows.closeEditor(for: newId) } + + // MARK: - Preview grid + + @Test("The preview grid has a column for every value color, both booleans and a NULL") + func previewSampleShowsEveryValueColor() throws { + let rows = ThemePreviewGridSample.tableRows + let tones = Set(rows.columnTypes.map { DataGridValueTone.base(of: $0, declaresEnumLabels: false) }) + #expect(tones == [.plain, .string, .number, .dateTime, .binary, .json, .uuid, .enumeration]) + #expect(ThemePreviewGridSample.configuration.colorsValuesByType) + #expect(ThemePreviewGridSample.configuration.overridesValueColorSetting) + + let boolean = try #require(rows.columnTypes.firstIndex { $0.isBooleanType }) + #expect(Set(rows.rows.map { $0.values[boolean] }) == [.text("true"), .text("false")]) + #expect(rows.rows.contains { $0.values.contains(.null) }) + + let marked = [ + ThemePreviewGridSample.selectedRow, + ThemePreviewGridSample.modifiedRow, + ThemePreviewGridSample.deletedRow, + ThemePreviewGridSample.insertedRow, + ] + #expect(Set(marked).count == marked.count) + #expect(marked.allSatisfy { rows.rows.indices.contains($0) }) + let modified = rows.rows[ThemePreviewGridSample.modifiedRow].values[ThemePreviewGridSample.modifiedColumn] + #expect(modified != .null) + } + + // The preview shows the theme being edited, so it never waits for the grid setting to be turned on. + @Test("The preview grid draws the theme's value colors while Color values by type is off") + func previewColorsValuesWithTheSettingOff() throws { + let owner = GridTestOwner(ThemePreviewGridSample.tableRows) + let model = DataGridModel( + configuration: ThemePreviewGridSample.configuration, + themeScope: .fixed(try ThemeTestFixture.theme(.light, [ + .gridText: "#111111", + .gridString: "#AA1100", + .gridNumber: "#0044AA", + ])) + ) + model.dataSource = owner + model.reloadRows() + let presenter = DataGridCellPresenter(model: model, valueColors: ValueColorSettings(colorsValuesByType: false)) + presenter.refreshColumnPresentations() + let aqua = try #require(NSAppearance(named: .aqua)) + let pass = try #require(presenter.makePass(in: aqua)) + let row = try #require(model.row(at: 0)) + + let colors = try ["id", "name"].map { name in + let index = try #require(ThemePreviewGridSample.columns.firstIndex { $0.name == name }) + let column = try #require(model.columns.column(at: index)) + let appearance = try #require(presenter.appearance( + of: row, + displayRow: 0, + column: column, + visualState: .empty, + pass: pass, + onEmphasizedSelection: false + )) + return ThemeTestFixture.hex(appearance.textColor) + } + withExtendedLifetime(owner) {} + + #expect(colors == ["#0044AA", "#AA1100"]) + } + + @Test("The Values section says where to turn value colors on, only while Color values by type is off") + func valuesNoteFollowsTheSetting() { + for section in ThemeSlotSection.allCases { + #expect(ThemeValueColorsNote.message(for: section, colorsValuesByType: true) == nil, "\(section)") + let off = ThemeValueColorsNote.message(for: section, colorsValuesByType: false) + #expect((off != nil) == (section == .values), "\(section)") + } + let note = ThemeValueColorsNote.message(for: .values, colorsValuesByType: false) + #expect(note?.hasSuffix("Settings > Data.") == true) + } + + // A UUID is wider than any column the window has room for, so it alone ends in an ellipsis. + @Test("Every sample value but the UUID fits its column at the default grid font") + func previewSampleValuesFitTheirColumns() { + let font = DataGridFontCache(from: .default).regular + let null = DataGridSettings.default.nullDisplay + let rows = ThemePreviewGridSample.tableRows + for (index, column) in ThemePreviewGridSample.columns.enumerated() where !column.type.isUUIDType { + for row in rows.rows { + let shown = CellDisplayFormatter.format(row.values[index], columnType: column.type) ?? null + let width = (shown as NSString).size(withAttributes: [.font: font]).width + + 2 * DataGridMetrics.cellHorizontalInset + #expect(width <= column.width, "\(column.name) \(shown): \(width)") + } + } + } + + // The preview takes no scroll events, so a column past the window's edge never shows. + @Test("The preview grid shows every sample column inside the editor window") + func previewGridFitsTheWindow() async throws { + let fixture = try Fixture() + try fixture.install(Fixture.theme("user.aaaa0002", "Midnight"), as: "Midnight.json") + let windows = makeWindows(fixture) + windows.openEditor(for: "user.aaaa0002") + let controller = try #require(windows.controller(for: "user.aaaa0002")) + defer { windows.closeEditor(for: "user.aaaa0002") } + let content = try #require(controller.window?.contentView) + + try await waitUntil { + content.layoutSubtreeIfNeeded() + return Self.previewGrid(in: content) != nil + } + let grid = try #require(Self.previewGrid(in: content)) + let clip = try #require(grid.enclosingScrollView?.contentView) + #expect(grid.rect(ofColumn: grid.numberOfColumns - 1).maxX <= clip.bounds.width) + } + + // The main pane holds the row numbers and every column; the frozen pane only the row numbers. + private static func previewGrid(in root: NSView) -> NSTableView? { + let columnCount = ThemePreviewGridSample.columns.count + 1 + return subviews(of: root) + .compactMap { $0 as? NSTableView } + .first { $0.isInsideThemePreview && $0.numberOfColumns == columnCount } + } } diff --git a/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift b/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift index e109e82e9a..81d11ac545 100644 --- a/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift +++ b/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift @@ -38,7 +38,6 @@ final class ThemeLibraryTestFixture { catalog = ThemeCatalog( userDirectory: folders[0], bundleDirectory: nil, - notificationCenter: center, fileTrash: .folder(folders[1]), syncTracker: tracker ) diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeAttachedSchemaTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeAttachedSchemaTests.swift index 08da1f5199..b605b1f234 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeAttachedSchemaTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeAttachedSchemaTests.swift @@ -76,7 +76,6 @@ struct DatabaseTreeAttachedSchemaTests { let sidebarState = SharedSidebarState.forConnection(connection.id) sidebarState.sidebarLayout = .flat let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterMemoTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterMemoTests.swift index b06d1d8826..ce7331b5b3 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterMemoTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterMemoTests.swift @@ -172,7 +172,6 @@ struct DatabaseTreeFilterMemoTests { let sidebarState = SharedSidebarState.forConnection(connection.id) sidebarState.sidebarLayout = .flat let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterTests.swift index 21e52a2e21..433a1c4796 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeFilterTests.swift @@ -132,9 +132,67 @@ struct DatabaseTreeFilterTests { #expect(SidebarObjectKind.visible(itemCounts: buckets.itemCounts) == [.table]) } - @Test("Types is the last kind listed") + @Test("Types is the last built-in kind listed, and declared kinds follow it") func typesIsTheLastKind() { - #expect(SidebarObjectKind.allCases.last == .type) + #expect(SidebarObjectKind.builtIn.last == .type) + let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + #expect(SidebarObjectKind.ordered(declared: [events]).suffix(2) == [.type, .server(events)]) + } + + private var eventKind: SidebarServerObjectKind { + SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + } + + private func event(_ name: String, schema: String = "shop") -> ServerObjectInfo { + ServerObjectInfo(kindId: "event", name: name, schema: schema, detail: "Recurring") + } + + @Test("Server objects are filtered by the search and counted under their declared kind") + func serverObjectsFilteredAndCounted() { + let buckets = DatabaseTreeFilter.objectBuckets( + tables: [table("users")], + routines: [], + triggers: [], + serverObjects: ["event": [event("nightly_cleanup"), event("hourly_rollup")]], + searchText: "nightly" + ) + #expect(buckets.serverObjects["event"]?.map { $0.name } == ["nightly_cleanup"]) + #expect(buckets.itemCounts[.server(eventKind)] == 1) + #expect(SidebarObjectKind.visible(itemCounts: buckets.itemCounts, declared: [eventKind]) == [.server(eventKind)]) + #expect(!buckets.isEmpty) + } + + @Test("A search matching no server object leaves its kind with no section") + func serverObjectsWithoutMatchHaveNoSection() { + let buckets = DatabaseTreeFilter.objectBuckets( + tables: [table("users")], + routines: [], + triggers: [], + serverObjects: ["event": [event("nightly_cleanup")]], + searchText: "users" + ) + #expect(buckets.serverObjects["event"]?.isEmpty == true) + #expect(SidebarObjectKind.visible(itemCounts: buckets.itemCounts, declared: [eventKind]) == [.table]) + } + + @Test("Only server objects keep a container from reading as empty") + func serverObjectsAloneAreContent() { + let buckets = DatabaseTreeFilter.objectBuckets( + tables: [], routines: [], triggers: [], serverObjects: ["event": [event("nightly")]], searchText: "" + ) + #expect(!buckets.isEmpty) + let none = DatabaseTreeFilter.objectBuckets(tables: [], routines: [], triggers: [], serverObjects: ["event": []], searchText: "") + #expect(none.isEmpty) + } + + @Test("One server object listed twice is drawn once") + func serverObjectsDeduplicated() { + let objects = [event("nightly"), event("nightly"), event("nightly", schema: "audit")] + #expect(DatabaseTreeFilter.filteredServerObjects(objects, searchText: "").count == 2) } @Test("visibleSchemas drops system schemas and deduplicates") diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeFlatRootTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeFlatRootTests.swift index bc8187afbd..98ee8abed3 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeFlatRootTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeFlatRootTests.swift @@ -26,10 +26,17 @@ struct DatabaseTreeFlatRootTests { private let boom = NSError(domain: "test", code: 1, userInfo: [NSLocalizedDescriptionKey: "permission denied"]) - private func makeFixture(_ configure: (SideObjectsMockDriver) -> Void) async throws -> Fixture { + private func makeFixture( + injectsSession: Bool = false, + _ configure: (SideObjectsMockDriver) -> Void + ) async throws -> Fixture { let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) let driver = SideObjectsMockDriver(connection: connection) configure(driver) + // The outline asks the session's driver which kinds it declares. + if injectsSession { + DatabaseManager.shared.injectSession(ConnectionSession(connection: connection, driver: driver), for: connection.id) + } await SchemaService.shared.load(connectionId: connection.id, driver: driver, connection: connection) let scrollView = SidebarOutlineScaffold.makeScrollView( @@ -45,7 +52,6 @@ struct DatabaseTreeFlatRootTests { let sidebarState = SharedSidebarState.forConnection(connection.id) sidebarState.sidebarLayout = .flat let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), @@ -69,6 +75,9 @@ struct DatabaseTreeFlatRootTests { } private func tearDown(_ fixture: Fixture) async { + if DatabaseManager.shared.session(for: fixture.connectionId) != nil { + DatabaseManager.shared.removeSession(for: fixture.connectionId) + } await SchemaService.shared.invalidate(connectionId: fixture.connectionId) SharedSidebarState.removeConnection(fixture.connectionId) _ = fixture.scrollView @@ -142,4 +151,86 @@ struct DatabaseTreeFlatRootTests { #expect(rootStatuses(fixture).isEmpty) await tearDown(fixture) } + + private var extensionKind: PluginServerObjectKind { + PluginServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", + iconName: "puzzlepiece.extension", scope: .database + ) + } + + private var extensions: SidebarObjectKind { + .server(SidebarServerObjectKind(extensionKind)) + } + + @Test("A declared kind with objects gets a section after the built-in kinds") + func declaredKindIsListedAfterBuiltIns() async throws { + let fixture = try await makeFixture(injectsSession: true) { driver in + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "orders", schema: "public")] + driver.declaredServerKinds = [extensionKind] + driver.serverObjectsByKind["extension"] = [ + ServerObjectInfo(kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384", detail: "1.3") + ] + } + + #expect(rootSections(fixture) == [.table, extensions]) + let section = try #require(section(fixture, extensions)) + let rows = fixture.coordinator.resolvedChildren(of: section) + guard case .serverObject(let ref) = rows.first?.kind else { + Issue.record("The Extensions section lists no extension row") + await tearDown(fixture) + return + } + #expect(rows.count == 1) + #expect(ref.object.name == "pgcrypto") + #expect(ref.kind.pluralDisplayName == "Extensions") + #expect(ref.objectRef.kind == .serverObject) + await tearDown(fixture) + } + + @Test("A declared kind that returned nothing gets no section") + func emptyDeclaredKindIsOmitted() async throws { + let fixture = try await makeFixture(injectsSession: true) { driver in + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "orders", schema: "public")] + driver.declaredServerKinds = [extensionKind] + } + + #expect(rootSections(fixture) == [.table]) + await tearDown(fixture) + } + + @Test("A declared kind whose fetch failed keeps its section, showing the error") + func failedDeclaredKindKeepsItsSection() async throws { + let fixture = try await makeFixture(injectsSession: true) { driver in + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "orders", schema: "public")] + driver.declaredServerKinds = [extensionKind] + driver.serverObjectsError = boom + } + + #expect(rootSections(fixture) == [.table, extensions]) + let section = try #require(section(fixture, extensions)) + let rows = fixture.coordinator.resolvedChildren(of: section) + guard case .status(let status) = rows.first?.kind else { + Issue.record("The failed section shows no status row") + await tearDown(fixture) + return + } + #expect(status == .error("permission denied")) + await tearDown(fixture) + } + + @Test("A search that matches only a server object keeps its section") + func searchMatchingOnlyAServerObject() async throws { + let fixture = try await makeFixture(injectsSession: true) { driver in + driver.tablesToReturn = [TestFixtures.makeTableInfo(name: "orders", schema: "public")] + driver.declaredServerKinds = [extensionKind] + driver.serverObjectsByKind["extension"] = [ServerObjectInfo(kindId: "extension", name: "pgcrypto", schema: "public")] + } + fixture.viewModel.searchText = "crypto" + fixture.coordinator.searchText = "crypto" + fixture.coordinator.refresh() + + #expect(rootSections(fixture) == [extensions]) + await tearDown(fixture) + } } diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeMenuSpecTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeMenuSpecTests.swift index 37660fb554..7e0d7ea310 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeMenuSpecTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeMenuSpecTests.swift @@ -1041,6 +1041,55 @@ struct DatabaseTreeMenuSpecTests { } } + // MARK: - Server objects + + private var eventKind: SidebarServerObjectKind { + SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + } + + private func serverObjectRef(_ name: String) -> DatabaseTreeServerObjectRef { + DatabaseTreeServerObjectRef( + database: "shop", schema: "shop", kind: eventKind, + object: ServerObjectInfo(kindId: "event", name: name, schema: "shop", detail: "Recurring") + ) + } + + @Test("A server object row copies its name and shows its DDL, and offers nothing that writes") + func serverObjectRowItems() { + let ref = serverObjectRef("nightly_cleanup") + let sections = DatabaseTreeMenuSpec.sections(for: context(clicked: .serverObject(ref))) + + #expect(commands(sections) == [.copyText("nightly_cleanup"), .showObjectSource(ref.objectRef)]) + #expect(titles(sections) == [String(localized: "Copy Name"), String(localized: "Show DDL")]) + #expect(ref.objectRef.kind == .serverObject) + #expect(ref.objectRef.serverKind?.id == "event") + } + + @Test("A server object kind's group offers Refresh and no New Folder or New Type") + func serverObjectGroupItems() { + let group = DatabaseTreeObjectGroup(database: "shop", schema: nil, kind: .server(eventKind)) + let sections = DatabaseTreeMenuSpec.sections(for: context(clicked: .containerObjectKindSection(group))) + + #expect(commands(sections) == [.refreshContainerObjectKind(group)]) + } + + @Test("A flat server object section offers Refresh alone") + func flatServerObjectSectionItems() { + let sections = DatabaseTreeMenuSpec.sections(for: context(clicked: .objectKindSection(.server(eventKind)))) + + #expect(commands(sections) == [.refreshObjectKind(.server(eventKind))]) + } + + @Test("A read-only connection keeps the server object row's menu") + func serverObjectRowReadOnly() { + let ref = serverObjectRef("nightly_cleanup") + let sections = DatabaseTreeMenuSpec.sections(for: context(clicked: .serverObject(ref), isReadOnly: true)) + + #expect(commands(sections) == [.copyText("nightly_cleanup"), .showObjectSource(ref.objectRef)]) + } + // MARK: - Types private func userTypeRef(_ name: String, schema: String? = "public") -> DatabaseTreeUserTypeRef { diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeNodeTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeNodeTests.swift index bd0cb34067..427f6f3421 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeNodeTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeNodeTests.swift @@ -333,4 +333,87 @@ struct DatabaseTreeNodeTests { #expect(groups.map(\.kind) == [.procedure]) #expect(groups.allSatisfy { $0.schema == "HR" }) } + + // MARK: - Server objects + + private let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + private let extensions = SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", iconName: "puzzlepiece.extension", + scope: .database + ) + + @Test("A container kind's group follows Types in a container and carries the declared descriptor") + func containerKindGroupFollowsTypes() { + let groups = DatabaseTreeObjectGroupResolver.groups( + database: "shop", + schema: nil, + itemCounts: [.table: 2, .trigger: 1, .type: 1, .server(SidebarServerObjectKind(id: "event")): 3], + declared: [events] + ) + + #expect(groups.map { $0.kind } == [.table, .trigger, .type, .server(events)]) + #expect(groups.last?.kind.pluralDisplayName == "Events") + } + + @Test("A database kind's group lists beside the schemas only when it holds rows or failed") + func databaseKindGroupNeedsRowsOrFailure() { + let empty = DatabaseTreeObjectGroupResolver.groups(database: "app", schema: nil, itemCounts: [:], declared: [extensions]) + #expect(empty.isEmpty) + + let phases = DatabaseTreeSidePhases( + routines: .idle, triggers: nil, types: nil, serverObjects: ["extension": .failed("permission denied")] + ) + let failed = DatabaseTreeObjectGroupResolver.groups( + database: "app", + schema: nil, + itemCounts: [:], + keeping: phases.failedKinds(among: [.server(extensions)]), + declared: [extensions] + ) + #expect(failed.map { $0.kind } == [.server(extensions)]) + #expect(phases.placeholder(for: .server(extensions)) == .error("permission denied")) + } + + @Test("A server kind's phase is its own fetch's, and an undeclared kind reads as loaded") + func serverKindPhases() { + let phases = DatabaseTreeSidePhases( + routines: .loaded, triggers: .loaded, types: nil, serverObjects: ["event": .loading] + ) + + #expect(phases.phase(for: .server(events)) == .loading) + #expect(phases.phase(for: .server(extensions)) == .loaded) + #expect(phases.all == [.loaded, .loaded, .loading]) + #expect(DatabaseTreeNode.Status.emptyContainer(sideStates: phases.all) == .loading) + #expect(phases.placeholder(for: .server(events)) == nil) + } + + @Test("A server object row is a selectable leaf with an id of its own") + func serverObjectRowIdentity() { + let object = ServerObjectInfo(kindId: "event", name: "nightly", schema: "shop") + let ref = DatabaseTreeServerObjectRef(database: "shop", schema: "shop", kind: events, object: object) + let node = DatabaseTreeNode(id: DatabaseTreeNode.serverObjectId(ref), kind: .serverObject(ref)) + let trigger = DatabaseTreeTriggerRef( + database: "shop", schema: "shop", + trigger: TriggerInfo(name: "nightly", timing: "AFTER", event: "INSERT", statement: "") + ) + + #expect(!node.isExpandable) + #expect(!node.isGroupRow) + #expect(!node.isContainer) + #expect(DatabaseTreeSelection.isSelectable(node.kind)) + #expect(DatabaseTreeNode.serverObjectId(ref) != DatabaseTreeNode.triggerId(trigger)) + #expect(DatabaseTreeDoubleClickResolver.resolve(node: node) == .openObjectSource(ref.objectRef)) + } + + @Test("A declared kind's group id spells its persistence name, and built-in ids are unchanged") + func serverGroupIdsUsePersistenceName() { + #expect(DatabaseTreeNode.objectKindSectionId(.table) == "kindSection\u{1}table") + #expect( + DatabaseTreeNode.containerObjectKindSectionId(objectGroup(kind: .materializedView)) + == "containerKindSection\u{1}shop\u{1}public\u{1}materializedView" + ) + #expect(DatabaseTreeNode.objectKindSectionId(.server(events)) == "kindSection\u{1}server.event") + } } diff --git a/apple/TableProTests/Views/Sidebar/DatabaseTreeReconcileTests.swift b/apple/TableProTests/Views/Sidebar/DatabaseTreeReconcileTests.swift index 7446b88e1c..d28f57b3b2 100644 --- a/apple/TableProTests/Views/Sidebar/DatabaseTreeReconcileTests.swift +++ b/apple/TableProTests/Views/Sidebar/DatabaseTreeReconcileTests.swift @@ -53,7 +53,6 @@ struct DatabaseTreeReconcileTests { let sidebarState = SharedSidebarState.forConnection(connection.id) sidebarState.sidebarLayout = .flat let viewModel = SidebarViewModel( - selectedTables: .constant([]), pendingTruncates: .constant([]), pendingDeletes: .constant([]), tableOperationOptions: .constant([:]), diff --git a/apple/TableProTests/Views/Sidebar/ServerObjectRowLogicTests.swift b/apple/TableProTests/Views/Sidebar/ServerObjectRowLogicTests.swift new file mode 100644 index 0000000000..f25dc8f89e --- /dev/null +++ b/apple/TableProTests/Views/Sidebar/ServerObjectRowLogicTests.swift @@ -0,0 +1,67 @@ +// +// ServerObjectRowLogicTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct ServerObjectRowLogicTests { + private let events = SidebarServerObjectKind( + id: "event", displayName: "Event", pluralDisplayName: "Events", iconName: "calendar.badge.clock", scope: .container + ) + private let extensions = SidebarServerObjectKind( + id: "extension", displayName: "Extension", pluralDisplayName: "Extensions", iconName: "puzzlepiece.extension", + scope: .database + ) + + private func event(enabled: Bool?, status: String) -> ServerObjectInfo { + ServerObjectInfo( + kindId: "event", name: "nightly", schema: "shop", detail: "Recurring", isEnabled: enabled, + attributes: [ObjectAttribute(label: "Status", value: status), ObjectAttribute(label: "Schedule", value: "EVERY 1 DAY")] + ) + } + + @Test("The tooltip names the state once, in the driver's own words") + func tooltipNamesStateOnce() { + let tooltip = ServerObjectRowLogic.tooltip(for: event(enabled: false, status: "Disabled on Replica"), kind: events) + + #expect(tooltip == "shop.nightly\nRecurring\nStatus: Disabled on Replica\nSchedule: EVERY 1 DAY") + } + + @Test("A database-wide object's tooltip opens with its bare name") + func databaseScopedTooltipIsBare() { + let object = ServerObjectInfo( + kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384", detail: "1.3", + attributes: [ObjectAttribute(label: "Schema", value: "public")] + ) + + #expect(ServerObjectRowLogic.tooltip(for: object, kind: extensions) == "pgcrypto\n1.3\nSchema: public") + #expect(ServerObjectRowLogic.displayName(for: object, kind: extensions) == "pgcrypto") + #expect(ServerObjectRowLogic.displayName(for: event(enabled: true, status: "Enabled"), kind: events) == "shop.nightly") + } + + @Test("The accessibility label reads kind, name and detail") + func accessibilityLabel() { + let label = ServerObjectRowLogic.accessibilityLabel(for: event(enabled: true, status: "Enabled"), kind: events) + #expect(label == String(format: String(localized: "%1$@: %2$@"), "Event", "nightly") + ", Recurring") + } + + // `ServerObjectInfo` is equal by id, so a row holding it would not redraw a new version or state. + @Test("A refreshed object with the same id hands the row new drawn values") + func refreshedObjectChangesTheDrawnValues() { + let before = ServerObjectInfo(kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384", detail: "1.3") + let after = ServerObjectInfo(kindId: "extension", name: "pgcrypto", schema: "public", identity: "16384", detail: "1.4") + #expect(before == after) + + let oldRow = ServerObjectRowView(object: before, kind: extensions) + let newRow = ServerObjectRowView(object: after, kind: extensions) + + #expect(oldRow.detail == "1.3") + #expect(newRow.detail == "1.4") + #expect(oldRow.tooltip != newRow.tooltip) + #expect(newRow.iconName == "puzzlepiece.extension") + } +} diff --git a/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift b/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift index 41044fa125..fde04c7c82 100644 --- a/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift +++ b/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift @@ -5,12 +5,14 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing @MainActor struct ClickHousePartsGridTests { private let part = ClickHousePartInfo( partition: "202601", + partitionId: "202601", name: "202601_1_4_1", rows: 1_234_567, bytesOnDisk: 5_242_880, @@ -35,7 +37,7 @@ struct ClickHousePartsGridTests { @Test("An inactive part reads NO") func inactivePart() { let inactive = ClickHousePartInfo( - partition: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false + partition: "", partitionId: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false ) #expect(ClickHousePartsView.gridRow(for: inactive).values.last == "NO") } @@ -43,7 +45,7 @@ struct ClickHousePartsGridTests { @Test("Active reads aloud as Active or Inactive rather than YES or NO") func activeIsSpokenByName() { let inactive = ClickHousePartInfo( - partition: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false + partition: "", partitionId: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false ) let active = ClickHousePartsView.activeColumn @@ -59,23 +61,25 @@ struct ClickHousePartsGridTests { #expect(zip(ClickHousePartsView.columnMinimumWidths, ClickHousePartsView.columnWidths).allSatisfy { $0 <= $1 }) } - @Test("Drop and Detach name the one selected part's partition, and none while several are selected") + @Test("Drop and Detach act on the one selected part, and on none while several are selected") func selectedPartition() { let other = ClickHousePartInfo( - partition: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, modificationTime: "", active: true + partition: "202602", partitionId: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, + modificationTime: "", active: true ) let parts = [part, other] - #expect(ClickHousePartsView.partition(of: [other.id], in: parts) == "202602") - #expect(ClickHousePartsView.partition(of: [part.id, other.id], in: parts) == nil) - #expect(ClickHousePartsView.partition(of: [], in: parts) == nil) - #expect(ClickHousePartsView.partition(of: [UUID()], in: parts) == nil) + #expect(ClickHousePartsView.selectedPart(of: [other.id], in: parts)?.id == other.id) + #expect(ClickHousePartsView.selectedPart(of: [part.id, other.id], in: parts)?.id == nil) + #expect(ClickHousePartsView.selectedPart(of: [], in: parts)?.id == nil) + #expect(ClickHousePartsView.selectedPart(of: [UUID()], in: parts)?.id == nil) } @Test("A selected grid row names its own part") func selectedRowResolvesToItsPart() throws { let other = ClickHousePartInfo( - partition: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, modificationTime: "", active: true + partition: "202602", partitionId: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, + modificationTime: "", active: true ) let parts = [part, other] let grid = StructureListGridModel() @@ -95,7 +99,84 @@ struct ClickHousePartsGridTests { #expect(grid.id(for: second) == other.id) #expect(selected == [other.id]) - #expect(ClickHousePartsView.partition(of: selected, in: parts) == "202602") + #expect(ClickHousePartsView.selectedPart(of: selected, in: parts)?.partition == "202602") #expect(grid.model.rowID(at: 2) == nil) } + + @Test("A parts row keeps the partition ID a statement names beside the value the grid shows") + func rowCarriesThePartitionId() throws { + let row: [PluginCellValue] = [ + "(202401,'eu')", + "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f", + "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f_1_1_0", + "12", + "4096", + "2026-01-05 10:00:00", + "1", + ] + + let parsed = try #require(ClickHousePartsView.part(from: row)) + + #expect(parsed.partition == "(202401,'eu')") + #expect(parsed.partitionId == "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f") + #expect(parsed.name == "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f_1_1_0") + #expect(parsed.rows == 12) + #expect(parsed.bytesOnDisk == 4096) + #expect(parsed.modificationTime == "2026-01-05 10:00:00") + #expect(parsed.active) + #expect(ClickHousePartsView.gridRow(for: parsed).values[0] == "(202401,'eu')") + } + + // The pane used to swap the list and its toolbar for the error text until the tab was refreshed. + @Test("A refused or failed action keeps the parts list and its toolbar on screen") + func actionFailureKeepsTheList() { + var pane = ClickHousePartsPaneState() + pane.beginLoad() + pane.finishLoad([part]) + + pane.actionEnded(.refused("Safe Mode is read-only"), title: "Couldn't Drop Partition") + #expect(pane.display == .parts) + #expect(pane.alert == ClickHousePartsAlert(title: "Couldn't Drop Partition", message: "Safe Mode is read-only")) + + pane.alert = nil + pane.actionEnded(.failed("Wrong partition value"), title: "Couldn't Detach Partition") + #expect(pane.display == .parts) + #expect(pane.alert == ClickHousePartsAlert(title: "Couldn't Detach Partition", message: "Wrong partition value")) + } + + @Test("A Cancel at the gate's confirmation shows nothing") + func cancelIsQuiet() { + var pane = ClickHousePartsPaneState() + pane.finishLoad([part]) + + pane.actionEnded( + GatedOperationFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")), + title: "Couldn't Drop Partition" + ) + + #expect(pane.alert == nil) + #expect(pane.display == .parts) + } + + @Test("A load that fails with nothing shown offers Try Again, and a reload that fails keeps the list") + func loadFailure() { + var pane = ClickHousePartsPaneState() + #expect(pane.display == .loading) + + pane.failLoad("Connection refused") + #expect(pane.display == .loadFailed("Connection refused")) + #expect(pane.alert == nil) + + pane.beginLoad() + #expect(pane.display == .loading) + pane.finishLoad([]) + #expect(pane.display == .empty) + + pane.finishLoad([part]) + pane.beginLoad() + #expect(pane.display == .parts) + pane.failLoad("Timeout exceeded") + #expect(pane.display == .parts) + #expect(pane.alert?.message == "Timeout exceeded") + } } diff --git a/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift b/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift index b4e8440113..7910b8ae10 100644 --- a/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift +++ b/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift @@ -109,7 +109,8 @@ struct StructureListGridAccessibilityTests { defer { DataGridAccessibility.isActive = false } let parts = [true, false].map { active in ClickHousePartInfo( - partition: "202601", name: "p", rows: 1, bytesOnDisk: 1, modificationTime: "", active: active + partition: "202601", partitionId: "202601", name: "p", rows: 1, bytesOnDisk: 1, modificationTime: "", + active: active ) } let rows = parts.map { part in diff --git a/apple/TableProTests/Views/Structure/StructureListGridModelTests.swift b/apple/TableProTests/Views/Structure/StructureListGridModelTests.swift index bceac298d7..731c1126a8 100644 --- a/apple/TableProTests/Views/Structure/StructureListGridModelTests.swift +++ b/apple/TableProTests/Views/Structure/StructureListGridModelTests.swift @@ -222,6 +222,7 @@ struct StructureListGridToneTests { placeholderText: .systemBrown, booleanTrueText: .labelColor, booleanFalseText: .labelColor, + valueColors: DataGridValueColors(all: .systemPurple), rowNumberText: .secondaryLabelColor, deletedRowText: .systemPink, modifiedColumnTint: .systemYellow, diff --git a/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift b/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift index 348093f635..fcb0310781 100644 --- a/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift +++ b/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift @@ -235,7 +235,7 @@ struct StructureSaveConfirmationTests { ).authorize(request) let error = try #require(decision.denialError) - #expect(StructureApplyFailure(error) == .cancelledByUser) + #expect(GatedOperationFailure(error) == .cancelledByUser) } @Test("A declined Touch ID is a refusal that says why, not a quiet Cancel") @@ -252,7 +252,7 @@ struct StructureSaveConfirmationTests { ).authorize(request) let error = try #require(decision.denialError) - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) #expect(failure.outcome == .refused) #expect(failure.message != nil) #expect(!failure.reportsFailure) @@ -404,7 +404,7 @@ struct StructureSaveConfirmationTests { @Test("A Cancel leaves the edits staged and shows nothing") func cancelIsQuiet() { - let failure = StructureApplyFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")) + let failure = GatedOperationFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")) #expect(failure == .cancelledByUser) #expect(failure.outcome == .refused) @@ -414,7 +414,7 @@ struct StructureSaveConfirmationTests { @Test("A Safe Mode refusal leaves the edits staged and says why, and is not a failed operation") func refusalExplainsItself() { - let failure = StructureApplyFailure(ExecutionGateError.denied("Safe Mode is read-only")) + let failure = GatedOperationFailure(ExecutionGateError.denied("Safe Mode is read-only")) #expect(failure.outcome == .refused) #expect(failure.message == "Safe Mode is read-only") @@ -423,7 +423,7 @@ struct StructureSaveConfirmationTests { @Test("A statement the server rejected is a failed save") func serverRejectionIsAFailure() { - let failure = StructureApplyFailure(ServerRejection()) + let failure = GatedOperationFailure(ServerRejection()) #expect(failure.outcome == .failed("Duplicate column name 'notes'")) #expect(failure.message == "Duplicate column name 'notes'") diff --git a/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift b/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift new file mode 100644 index 0000000000..3745115a46 --- /dev/null +++ b/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift @@ -0,0 +1,25 @@ +import AppKit +@testable import TablePro +import TableProEditorKit +import Testing + +@MainActor +struct TriggerEditorFontSizeTests { + @Test("New and Edit Trigger open at the size the DDL pane is zoomed to") + func opensAtTheStoredSize() throws { + let zoomed = try EphemeralDefaults() + zoomed.defaults.set(18.0, forKey: TriggerEditorView.fontSizeKey) + + #expect(TriggerEditorView.initialConfiguration(in: zoomed.defaults).appearance.font.pointSize == 18) + } + + @Test("Without a stored size the sheet opens at the default") + func opensAtTheDefaultSize() throws { + let fresh = try EphemeralDefaults() + + #expect( + TriggerEditorView.initialConfiguration(in: fresh.defaults).appearance.font.pointSize + == CGFloat(TriggerEditorView.defaultFontSize) + ) + } +} diff --git a/apple/TableProUITests/MCPConnectClientUITests.swift b/apple/TableProUITests/MCPConnectClientUITests.swift new file mode 100644 index 0000000000..e650ab8b3c --- /dev/null +++ b/apple/TableProUITests/MCPConnectClientUITests.swift @@ -0,0 +1,45 @@ +import XCTest + +final class MCPConnectClientUITests: UITestCase { + func testConnectAClientListsTheCatalogAndShowsASetup() throws { + let app = try launchOpeningURL("tablepro://settings/mcp") + let settings = window("settings", in: app) + XCTAssertTrue(settings.waitToExist(timeout: 10), "The settings link must open the settings window") + + let enable = settings.switches["mcp-enable-server-toggle"] + XCTAssertTrue(waitUntilHittable(enable, timeout: 10), "The MCP pane must offer its Enable switch") + if !isOn(enable) { + enable.click() + } + + let connect = settings.buttons["mcp-connect-client"] + XCTAssertTrue(scrollUntilHittable(connect, in: settings, timeout: 10), "An enabled server offers Connect a Client") + connect.click() + + let sheet = settings.sheets.firstMatch + XCTAssertTrue(sheet.waitToExist(timeout: 10), "Connect a Client must open its sheet") + // The list draws only the rows in view, so only the catalog's first clients are asked for. + for name in ["Claude Desktop", "Claude Code"] { + XCTAssertTrue(text(name, in: sheet).waitToExist(timeout: 10), "The sheet should list \(name) from the bundled catalog") + } + + let claudeDesktop = text("Claude Desktop", in: sheet) + XCTAssertTrue(waitUntilHittable(claudeDesktop, timeout: 10)) + claudeDesktop.click() + // Selectable text is not always a static text, so the path is matched by value alone. + let path = sheet.descendants(matching: .any) + .matching(NSPredicate(format: "value == %@", "~/Library/Application Support/Claude/claude_desktop_config.json")) + .firstMatch + XCTAssertTrue(path.waitToExist(timeout: 5), "Selecting a client shows its settings file, inside the sandbox's home") + + let done = sheet.buttons["mcp-client-setup-done"] + XCTAssertTrue(waitUntilHittable(done, timeout: 5)) + done.click() + XCTAssertTrue(waitForPredicate(timeout: 5) { !sheet.exists }, "Done should close the sheet") + } + + // SwiftUI text carries its string in `value` with no label, so a subscript by title finds nothing. + private func text(_ value: String, in container: XCUIElement) -> XCUIElement { + container.staticTexts.matching(NSPredicate(format: "value == %@", value)).firstMatch + } +} diff --git a/apple/TableProUITests/ManualCommitUITests.swift b/apple/TableProUITests/ManualCommitUITests.swift new file mode 100644 index 0000000000..2522edba99 --- /dev/null +++ b/apple/TableProUITests/ManualCommitUITests.swift @@ -0,0 +1,51 @@ +import XCTest + +final class ManualCommitUITests: UITestCase { + func testManualCommitHoldsAnUpdateUntilItIsRolledBack() throws { + let launch = try launchWithFixture() + let window = launch.window + let app = launch.app + XCTAssertTrue(app.menuBars.firstMatch.waitToExist(timeout: 20), "The app must publish its menu bar") + + menuItem("Manual Commit", under: "Database", in: app).click() + typeQuery("UPDATE note SET body = 'Held' WHERE id = 1;", in: window) + app.typeShortcut(.runQuery) + + let bar = window.descendants(matching: .any)["connection-transaction-bar"].firstMatch + XCTAssertTrue(bar.waitToExist(timeout: 20), "A write in Manual Commit must leave its transaction open") + + let rollBack = bar.buttons["Roll Back"].firstMatch + XCTAssertTrue(waitUntilHittable(rollBack, timeout: 10), "The bar must offer Roll Back") + rollBack.click() + let confirm = window.sheets.buttons["Roll Back"].firstMatch + XCTAssertTrue(waitUntilHittable(confirm, timeout: 10), "Roll Back must ask first") + confirm.click() + + XCTAssertTrue(bar.waitForNonExistence(timeout: 20), "Rolling back must end the transaction the bar shows") + let rows = FixtureFile.rows("SELECT body FROM note WHERE id = 1", in: launch.database) + XCTAssertEqual(rows, [["Buy milk"]], "The rolled-back UPDATE must leave the row as it was") + } + + func testDisconnectCommitsTheHeldUpdate() throws { + let launch = try launchWithFixture() + let window = launch.window + let app = launch.app + XCTAssertTrue(app.menuBars.firstMatch.waitToExist(timeout: 20), "The app must publish its menu bar") + + menuItem("Manual Commit", under: "Database", in: app).click() + typeQuery("UPDATE note SET body = 'Kept' WHERE id = 1;", in: window) + app.typeShortcut(.runQuery) + let bar = window.descendants(matching: .any)["connection-transaction-bar"].firstMatch + XCTAssertTrue(bar.waitToExist(timeout: 20), "A write in Manual Commit must leave its transaction open") + + menuItem("Disconnect", under: "Database", in: app).click() + let commit = window.sheets.buttons["Commit"].firstMatch + XCTAssertTrue(waitUntilHittable(commit, timeout: 10), "Disconnect must offer Commit for the open transaction") + XCTAssertTrue(window.sheets.buttons["Roll Back"].firstMatch.exists, "Disconnect must offer Roll Back beside Commit") + commit.click() + + XCTAssertTrue(bar.waitForNonExistence(timeout: 20), "Committing must end the transaction before the disconnect") + let rows = FixtureFile.rows("SELECT body FROM note WHERE id = 1", in: launch.database) + XCTAssertEqual(rows, [["Kept"]], "Commit on Disconnect must keep the held UPDATE") + } +} diff --git a/apple/TableProUITests/TableAutoRefreshUITests.swift b/apple/TableProUITests/TableAutoRefreshUITests.swift new file mode 100644 index 0000000000..fa6980e740 --- /dev/null +++ b/apple/TableProUITests/TableAutoRefreshUITests.swift @@ -0,0 +1,26 @@ +import XCTest + +final class TableAutoRefreshUITests: UITestCase { + func testAutoRefreshShowsAWriteMadeElsewhereWithoutARefresh() throws { + let (launch, grid) = try launchWithFixtureTable("document") + let app = launch.app + XCTAssertTrue(grid.waitForValue("First", row: 1, column: 2), "document must open on its first row") + + // The 1024x768 runner gives the status bar its narrowest tier, which leaves Auto Refresh to this menu. + let autoRefresh = menuItem("Auto Refresh", under: "Database", in: app) + autoRefresh.click() + let everyFiveSeconds = autoRefresh.menuItems["Every 5 Seconds"].firstMatch + XCTAssertTrue(everyFiveSeconds.waitToExist(timeout: 10), "Database > Auto Refresh must list Every 5 Seconds") + everyFiveSeconds.click() + + XCTAssertTrue( + FixtureFile.execute("UPDATE document SET label = 'Written Elsewhere' WHERE id = 1", in: launch.database), + "The fixture file must take the write" + ) + XCTAssertTrue( + grid.waitForValue("Written Elsewhere", row: 1, column: 2, timeout: 30), + "Auto Refresh must show the write with no Refresh, the grid shows " + + "'\(grid.value(row: 1, column: 2) ?? "nil")'" + ) + } +} diff --git a/apple/TestPlans/Unit.xctestplan b/apple/TestPlans/Unit.xctestplan index 2ac2d49434..7872b0c949 100644 --- a/apple/TestPlans/Unit.xctestplan +++ b/apple/TestPlans/Unit.xctestplan @@ -116,6 +116,13 @@ "name" : "TableProCommandLineTests" } }, + { + "target" : { + "containerPath" : "container:Packages\/TableProMacCore", + "identifier" : "TableProMCPClientsTests", + "name" : "TableProMCPClientsTests" + } + }, { "target" : { "containerPath" : "container:Packages\/TableProMacCore", diff --git a/apple/changelog.d/4181.fixed.1.md b/apple/changelog.d/4181.fixed.1.md new file mode 100644 index 0000000000..fe23d7f460 --- /dev/null +++ b/apple/changelog.d/4181.fixed.1.md @@ -0,0 +1 @@ +Column suggestions before FROM were slow to appear in schemas with many tables. diff --git a/apple/changelog.d/4181.fixed.10.md b/apple/changelog.d/4181.fixed.10.md new file mode 100644 index 0000000000..ec73a7630f --- /dev/null +++ b/apple/changelog.d/4181.fixed.10.md @@ -0,0 +1 @@ +Stopping an export from SAP HANA, Teradata, InfluxDB, Cloudflare R2 SQL or Memcached left its query running. diff --git a/apple/changelog.d/4181.fixed.11.md b/apple/changelog.d/4181.fixed.11.md new file mode 100644 index 0000000000..3fe26a5af6 --- /dev/null +++ b/apple/changelog.d/4181.fixed.11.md @@ -0,0 +1 @@ +AI chat could read foreign keys for every table in a large schema on Snowflake, Trino, ClickHouse and others. diff --git a/apple/changelog.d/4181.fixed.12.md b/apple/changelog.d/4181.fixed.12.md new file mode 100644 index 0000000000..d25eeefa74 --- /dev/null +++ b/apple/changelog.d/4181.fixed.12.md @@ -0,0 +1 @@ +Exporting more than 5,000,000 rows from SAP HANA wrote only the first 5,000,000 and reported success. diff --git a/apple/changelog.d/4181.fixed.13.md b/apple/changelog.d/4181.fixed.13.md new file mode 100644 index 0000000000..fc42943d59 --- /dev/null +++ b/apple/changelog.d/4181.fixed.13.md @@ -0,0 +1 @@ +Fetch All and large query results froze the window while their rows were loaded. diff --git a/apple/changelog.d/4181.fixed.14.md b/apple/changelog.d/4181.fixed.14.md new file mode 100644 index 0000000000..f6abe06d63 --- /dev/null +++ b/apple/changelog.d/4181.fixed.14.md @@ -0,0 +1 @@ +Run All on a script with thousands of statements froze the window and its result chooser. diff --git a/apple/changelog.d/4181.fixed.15.md b/apple/changelog.d/4181.fixed.15.md new file mode 100644 index 0000000000..14b9122b65 --- /dev/null +++ b/apple/changelog.d/4181.fixed.15.md @@ -0,0 +1 @@ +A capped PostgreSQL query or a cancelled export inside a transaction read the whole result. diff --git a/apple/changelog.d/4181.fixed.16.md b/apple/changelog.d/4181.fixed.16.md new file mode 100644 index 0000000000..a7045f946b --- /dev/null +++ b/apple/changelog.d/4181.fixed.16.md @@ -0,0 +1 @@ +Exporting a large Oracle table was far slower than the same export from other databases. diff --git a/apple/changelog.d/4181.fixed.17.md b/apple/changelog.d/4181.fixed.17.md new file mode 100644 index 0000000000..9444c35513 --- /dev/null +++ b/apple/changelog.d/4181.fixed.17.md @@ -0,0 +1 @@ +Browsing, filtering or exporting a large etcd prefix downloaded every key and value at once. diff --git a/apple/changelog.d/4181.fixed.2.md b/apple/changelog.d/4181.fixed.2.md new file mode 100644 index 0000000000..292e54419a --- /dev/null +++ b/apple/changelog.d/4181.fixed.2.md @@ -0,0 +1 @@ +The app froze during the first iCloud sync of a large connection library. diff --git a/apple/changelog.d/4181.fixed.3.md b/apple/changelog.d/4181.fixed.3.md new file mode 100644 index 0000000000..a38e0f31f3 --- /dev/null +++ b/apple/changelog.d/4181.fixed.3.md @@ -0,0 +1 @@ +XLSX export held the whole workbook in memory several times and hit the 4 GB limit only after reading every row. diff --git a/apple/changelog.d/4181.fixed.4.md b/apple/changelog.d/4181.fixed.4.md new file mode 100644 index 0000000000..9c83f750b0 --- /dev/null +++ b/apple/changelog.d/4181.fixed.4.md @@ -0,0 +1 @@ +JSON import used several times the file's size in memory and silently skipped array items that were not objects. diff --git a/apple/changelog.d/4181.fixed.5.md b/apple/changelog.d/4181.fixed.5.md new file mode 100644 index 0000000000..6fd33d00f9 --- /dev/null +++ b/apple/changelog.d/4181.fixed.5.md @@ -0,0 +1 @@ +Editing, adding or removing rows with Find open in a large result froze the app. diff --git a/apple/changelog.d/4181.fixed.6.md b/apple/changelog.d/4181.fixed.6.md new file mode 100644 index 0000000000..2651f531ab --- /dev/null +++ b/apple/changelog.d/4181.fixed.6.md @@ -0,0 +1 @@ +Filling a column, pasting over many rows or undoing it in a large result hung the app. diff --git a/apple/changelog.d/4181.fixed.7.md b/apple/changelog.d/4181.fixed.7.md new file mode 100644 index 0000000000..369cddca38 --- /dev/null +++ b/apple/changelog.d/4181.fixed.7.md @@ -0,0 +1 @@ +Typing in the row inspector after Fetch All on a large result lagged on every key. diff --git a/apple/changelog.d/4181.fixed.8.md b/apple/changelog.d/4181.fixed.8.md new file mode 100644 index 0000000000..46de700998 --- /dev/null +++ b/apple/changelog.d/4181.fixed.8.md @@ -0,0 +1 @@ +Opening Filter Values on a column with many distinct values froze the app while it sorted them. diff --git a/apple/changelog.d/4181.fixed.9.md b/apple/changelog.d/4181.fixed.9.md new file mode 100644 index 0000000000..3fe3b283dd --- /dev/null +++ b/apple/changelog.d/4181.fixed.9.md @@ -0,0 +1 @@ +Refreshing many databases or schemas at once could open more server connections than the server allows. diff --git a/apple/changelog.d/4181.fixed.md b/apple/changelog.d/4181.fixed.md new file mode 100644 index 0000000000..59c28971d3 --- /dev/null +++ b/apple/changelog.d/4181.fixed.md @@ -0,0 +1 @@ +Typing or moving the caret near the end of a long SQL script lagged while the current statement was highlighted. diff --git a/apple/changelog.d/4183.added.1.md b/apple/changelog.d/4183.added.1.md new file mode 100644 index 0000000000..57b421accc --- /dev/null +++ b/apple/changelog.d/4183.added.1.md @@ -0,0 +1 @@ +Redshift AWS IAM sign-in accepts a custom domain name as Host. diff --git a/apple/changelog.d/4183.added.md b/apple/changelog.d/4183.added.md new file mode 100644 index 0000000000..e7f62dfa45 --- /dev/null +++ b/apple/changelog.d/4183.added.md @@ -0,0 +1 @@ +AWS IAM connections to Amazon RDS accept a custom CNAME as Host and follow it to the RDS endpoint on every connect. diff --git a/apple/changelog.d/4183.fixed.1.md b/apple/changelog.d/4183.fixed.1.md new file mode 100644 index 0000000000..4096669b2a --- /dev/null +++ b/apple/changelog.d/4183.fixed.1.md @@ -0,0 +1 @@ +Reconnect used ${VAR} references in a connection as typed instead of resolving them. diff --git a/apple/changelog.d/4183.fixed.md b/apple/changelog.d/4183.fixed.md new file mode 100644 index 0000000000..c56309813e --- /dev/null +++ b/apple/changelog.d/4183.fixed.md @@ -0,0 +1 @@ +MySQL and MariaDB connections on AWS IAM failed to reconnect after a dropped connection once the first token expired. diff --git a/apple/changelog.d/4183.security.1.md b/apple/changelog.d/4183.security.1.md new file mode 100644 index 0000000000..ae49313814 --- /dev/null +++ b/apple/changelog.d/4183.security.1.md @@ -0,0 +1 @@ +A shared Elasticsearch, Kafka or MongoDB connection could mint an AWS IAM token on this Mac and send it to its servers. diff --git a/apple/changelog.d/4183.security.md b/apple/changelog.d/4183.security.md new file mode 100644 index 0000000000..ffb8a8437f --- /dev/null +++ b/apple/changelog.d/4183.security.md @@ -0,0 +1 @@ +A shared connection could send an AWS IAM token to its author's server through a host like 127.0.0.1.example.com. diff --git a/apple/changelog.d/4184.fixed.1.md b/apple/changelog.d/4184.fixed.1.md new file mode 100644 index 0000000000..ba9a7c9dea --- /dev/null +++ b/apple/changelog.d/4184.fixed.1.md @@ -0,0 +1 @@ +A synced change CloudKit failed to deliver during a pull was skipped for good instead of arriving on the next sync. diff --git a/apple/changelog.d/4184.fixed.10.md b/apple/changelog.d/4184.fixed.10.md new file mode 100644 index 0000000000..d01be0fc3f --- /dev/null +++ b/apple/changelog.d/4184.fixed.10.md @@ -0,0 +1 @@ +An AI chat tool call whose arguments did not parse ran with empty arguments. diff --git a/apple/changelog.d/4184.fixed.11.md b/apple/changelog.d/4184.fixed.11.md new file mode 100644 index 0000000000..b26586b441 --- /dev/null +++ b/apple/changelog.d/4184.fixed.11.md @@ -0,0 +1 @@ +ChatGPT sign-in failed if any other request reached its callback port, and waited 5 minutes after you denied access. diff --git a/apple/changelog.d/4184.fixed.12.md b/apple/changelog.d/4184.fixed.12.md new file mode 100644 index 0000000000..6d4796b5e1 --- /dev/null +++ b/apple/changelog.d/4184.fixed.12.md @@ -0,0 +1 @@ +Vault OIDC sign-in ended early when any other request reached its callback port. diff --git a/apple/changelog.d/4184.fixed.13.md b/apple/changelog.d/4184.fixed.13.md new file mode 100644 index 0000000000..fb5b3a0f4f --- /dev/null +++ b/apple/changelog.d/4184.fixed.13.md @@ -0,0 +1 @@ +Snowflake browser sign-in failed when any other request reached its local callback port. diff --git a/apple/changelog.d/4184.fixed.2.md b/apple/changelog.d/4184.fixed.2.md new file mode 100644 index 0000000000..5c09552a5c --- /dev/null +++ b/apple/changelog.d/4184.fixed.2.md @@ -0,0 +1 @@ +Upgrading from an old release could delete saved queries or query history when copying them failed. diff --git a/apple/changelog.d/4184.fixed.3.md b/apple/changelog.d/4184.fixed.3.md new file mode 100644 index 0000000000..edf719bad3 --- /dev/null +++ b/apple/changelog.d/4184.fixed.3.md @@ -0,0 +1 @@ +A damaged query history or saved queries file left both empty and silently stopped saving anything. diff --git a/apple/changelog.d/4184.fixed.4.md b/apple/changelog.d/4184.fixed.4.md new file mode 100644 index 0000000000..0dcea2f734 --- /dev/null +++ b/apple/changelog.d/4184.fixed.4.md @@ -0,0 +1 @@ +A taken local port made an SSH tunnel sign in twice, asking for the passphrase or TOTP code again. diff --git a/apple/changelog.d/4184.fixed.5.md b/apple/changelog.d/4184.fixed.5.md new file mode 100644 index 0000000000..291a877e12 --- /dev/null +++ b/apple/changelog.d/4184.fixed.5.md @@ -0,0 +1 @@ +A damaged query history file on iPhone or iPad was replaced by the next query run. diff --git a/apple/changelog.d/4184.fixed.6.md b/apple/changelog.d/4184.fixed.6.md new file mode 100644 index 0000000000..85a154693a --- /dev/null +++ b/apple/changelog.d/4184.fixed.6.md @@ -0,0 +1 @@ +A damaged linked folder index left linked SQL files and their keywords missing until the file was deleted. diff --git a/apple/changelog.d/4184.fixed.7.md b/apple/changelog.d/4184.fixed.7.md new file mode 100644 index 0000000000..de8a0f3864 --- /dev/null +++ b/apple/changelog.d/4184.fixed.7.md @@ -0,0 +1 @@ +A damaged MCP activity log stopped recording MCP activity until the file was deleted. diff --git a/apple/changelog.d/4184.fixed.8.md b/apple/changelog.d/4184.fixed.8.md new file mode 100644 index 0000000000..f7ec2f9d15 --- /dev/null +++ b/apple/changelog.d/4184.fixed.8.md @@ -0,0 +1 @@ +The first MCP activity after each launch could be missing from the activity log. diff --git a/apple/changelog.d/4184.fixed.9.md b/apple/changelog.d/4184.fixed.9.md new file mode 100644 index 0000000000..0e0c7eeee7 --- /dev/null +++ b/apple/changelog.d/4184.fixed.9.md @@ -0,0 +1 @@ +A request from an MCP client using the bridge could be cancelled when the bridge renewed its credential. diff --git a/apple/changelog.d/4184.fixed.md b/apple/changelog.d/4184.fixed.md new file mode 100644 index 0000000000..0cf0b208f0 --- /dev/null +++ b/apple/changelog.d/4184.fixed.md @@ -0,0 +1 @@ +Deleting a Local only connection removed its passwords from your other devices that still sync it. diff --git a/apple/changelog.d/4184.security.1.md b/apple/changelog.d/4184.security.1.md new file mode 100644 index 0000000000..9f8e493e79 --- /dev/null +++ b/apple/changelog.d/4184.security.1.md @@ -0,0 +1 @@ +A connection deleted on another device left its passwords and other secrets in this Mac's Keychain. diff --git a/apple/changelog.d/4184.security.2.md b/apple/changelog.d/4184.security.2.md new file mode 100644 index 0000000000..739d565123 --- /dev/null +++ b/apple/changelog.d/4184.security.2.md @@ -0,0 +1 @@ +Plugin secure fields could stay in plain text in connections.json after the upgrade that moves them to the Keychain. diff --git a/apple/changelog.d/4184.security.3.md b/apple/changelog.d/4184.security.3.md new file mode 100644 index 0000000000..d441f56093 --- /dev/null +++ b/apple/changelog.d/4184.security.3.md @@ -0,0 +1 @@ +Another app on the Mac could take the IPv6 side of a sign-in callback port and receive the browser's redirect. diff --git a/apple/changelog.d/4184.security.md b/apple/changelog.d/4184.security.md new file mode 100644 index 0000000000..5a36175cfd --- /dev/null +++ b/apple/changelog.d/4184.security.md @@ -0,0 +1 @@ +A plugin could load with any valid signature when TablePro's signing requirement failed to build. diff --git a/apple/changelog.d/4185.fixed.1.md b/apple/changelog.d/4185.fixed.1.md new file mode 100644 index 0000000000..625f6be5b1 --- /dev/null +++ b/apple/changelog.d/4185.fixed.1.md @@ -0,0 +1 @@ +CSV export with a decimal comma rewrote text values that looked like decimals, such as a version 1.10. diff --git a/apple/changelog.d/4185.fixed.10.md b/apple/changelog.d/4185.fixed.10.md new file mode 100644 index 0000000000..c4812cbb1b --- /dev/null +++ b/apple/changelog.d/4185.fixed.10.md @@ -0,0 +1 @@ +Rewinding a save could fail on a unique key when an earlier edit in it had been undone before saving. diff --git a/apple/changelog.d/4185.fixed.11.md b/apple/changelog.d/4185.fixed.11.md new file mode 100644 index 0000000000..03913855f0 --- /dev/null +++ b/apple/changelog.d/4185.fixed.11.md @@ -0,0 +1 @@ +Rewind reported a row as missing when the save had changed its primary key. diff --git a/apple/changelog.d/4185.fixed.12.md b/apple/changelog.d/4185.fixed.12.md new file mode 100644 index 0000000000..427ed680e8 --- /dev/null +++ b/apple/changelog.d/4185.fixed.12.md @@ -0,0 +1 @@ +A value of __DEFAULT__ was saved as the column default when duplicating a row or typing it into a cell. diff --git a/apple/changelog.d/4185.fixed.13.md b/apple/changelog.d/4185.fixed.13.md new file mode 100644 index 0000000000..f3f1e23763 --- /dev/null +++ b/apple/changelog.d/4185.fixed.13.md @@ -0,0 +1 @@ +A table opened while another query waited for Safe Mode approval stayed empty with its loading indicator. diff --git a/apple/changelog.d/4185.fixed.14.md b/apple/changelog.d/4185.fixed.14.md new file mode 100644 index 0000000000..bfaa1727f7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.14.md @@ -0,0 +1 @@ +Saving while closing a window could close it over unsaved edits in another connection or tab. diff --git a/apple/changelog.d/4185.fixed.15.md b/apple/changelog.d/4185.fixed.15.md new file mode 100644 index 0000000000..157fe467ec --- /dev/null +++ b/apple/changelog.d/4185.fixed.15.md @@ -0,0 +1 @@ +Saving a Cassandra row that another client deleted recreated it with every other column empty. diff --git a/apple/changelog.d/4185.fixed.16.md b/apple/changelog.d/4185.fixed.16.md new file mode 100644 index 0000000000..9b9cdee113 --- /dev/null +++ b/apple/changelog.d/4185.fixed.16.md @@ -0,0 +1 @@ +Running SET with NX or XX in the Redis console showed OK when the key was not written. diff --git a/apple/changelog.d/4185.fixed.17.md b/apple/changelog.d/4185.fixed.17.md new file mode 100644 index 0000000000..27e55c9db4 --- /dev/null +++ b/apple/changelog.d/4185.fixed.17.md @@ -0,0 +1 @@ +Pressing Stop as a MySQL tab closed could crash the app. diff --git a/apple/changelog.d/4185.fixed.18.md b/apple/changelog.d/4185.fixed.18.md new file mode 100644 index 0000000000..18a4d89177 --- /dev/null +++ b/apple/changelog.d/4185.fixed.18.md @@ -0,0 +1 @@ +A startup command that failed, such as SET ROLE, left the session running without it and nothing said so. diff --git a/apple/changelog.d/4185.fixed.19.md b/apple/changelog.d/4185.fixed.19.md new file mode 100644 index 0000000000..e052ba407c --- /dev/null +++ b/apple/changelog.d/4185.fixed.19.md @@ -0,0 +1 @@ +Exporting more than 5,000,000 MongoDB documents wrote the first 5,000,000 and reported success. diff --git a/apple/changelog.d/4185.fixed.2.md b/apple/changelog.d/4185.fixed.2.md new file mode 100644 index 0000000000..51fe78d8f7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.2.md @@ -0,0 +1 @@ +Changing export or import options in another window changed a running export or import, or left foreign keys off. diff --git a/apple/changelog.d/4185.fixed.20.md b/apple/changelog.d/4185.fixed.20.md new file mode 100644 index 0000000000..a4aa8a48a5 --- /dev/null +++ b/apple/changelog.d/4185.fixed.20.md @@ -0,0 +1 @@ +Stopping a Teradata query ended the whole session, so volatile tables were lost and the next query failed. diff --git a/apple/changelog.d/4185.fixed.21.md b/apple/changelog.d/4185.fixed.21.md new file mode 100644 index 0000000000..21106f7c75 --- /dev/null +++ b/apple/changelog.d/4185.fixed.21.md @@ -0,0 +1 @@ +A query parameter such as zip code 00123 was sent to Trino as the number 123. diff --git a/apple/changelog.d/4185.fixed.22.md b/apple/changelog.d/4185.fixed.22.md new file mode 100644 index 0000000000..05b1cf291d --- /dev/null +++ b/apple/changelog.d/4185.fixed.22.md @@ -0,0 +1 @@ +TablePro could quit when the Copilot language server exited while TablePro was sending it a message. diff --git a/apple/changelog.d/4185.fixed.23.md b/apple/changelog.d/4185.fixed.23.md new file mode 100644 index 0000000000..244113e6d1 --- /dev/null +++ b/apple/changelog.d/4185.fixed.23.md @@ -0,0 +1 @@ +Run Current and the gutter cut a SQL Server IF, WHILE or TRY...CATCH block at each semicolon inside it. diff --git a/apple/changelog.d/4185.fixed.24.md b/apple/changelog.d/4185.fixed.24.md new file mode 100644 index 0000000000..d34f707ee7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.24.md @@ -0,0 +1 @@ +Run Current split a PostgreSQL rule with several commands in parentheses at each semicolon. diff --git a/apple/changelog.d/4185.fixed.25.md b/apple/changelog.d/4185.fixed.25.md new file mode 100644 index 0000000000..4ff326bcbd --- /dev/null +++ b/apple/changelog.d/4185.fixed.25.md @@ -0,0 +1 @@ +New Trigger and Edit Trigger opened at 13 pt instead of the size the Structure tab's DDL pane was zoomed to. diff --git a/apple/changelog.d/4185.fixed.26.md b/apple/changelog.d/4185.fixed.26.md new file mode 100644 index 0000000000..65af010304 --- /dev/null +++ b/apple/changelog.d/4185.fixed.26.md @@ -0,0 +1 @@ +Moving the caret in a long MongoDB or Redis query re-read the whole document every time. diff --git a/apple/changelog.d/4185.fixed.27.md b/apple/changelog.d/4185.fixed.27.md new file mode 100644 index 0000000000..f0afd7c7e2 --- /dev/null +++ b/apple/changelog.d/4185.fixed.27.md @@ -0,0 +1 @@ +Column and table suggestions on iPhone and iPad were rebuilt on every keystroke. diff --git a/apple/changelog.d/4185.fixed.28.md b/apple/changelog.d/4185.fixed.28.md new file mode 100644 index 0000000000..27178d8fa0 --- /dev/null +++ b/apple/changelog.d/4185.fixed.28.md @@ -0,0 +1 @@ +The ClickHouse Parts tab hid its list and toolbar after a cancelled or failed partition action. diff --git a/apple/changelog.d/4185.fixed.29.md b/apple/changelog.d/4185.fixed.29.md new file mode 100644 index 0000000000..0ac47956a5 --- /dev/null +++ b/apple/changelog.d/4185.fixed.29.md @@ -0,0 +1 @@ +Dropping or detaching a ClickHouse partition failed on tables partitioned by several columns or not at all. diff --git a/apple/changelog.d/4185.fixed.3.md b/apple/changelog.d/4185.fixed.3.md new file mode 100644 index 0000000000..8fa5c94f07 --- /dev/null +++ b/apple/changelog.d/4185.fixed.3.md @@ -0,0 +1 @@ +Data sync and Copy as SQL wrote PostgreSQL binary values that stored other bytes with standard_conforming_strings off. diff --git a/apple/changelog.d/4185.fixed.30.md b/apple/changelog.d/4185.fixed.30.md new file mode 100644 index 0000000000..6be6044706 --- /dev/null +++ b/apple/changelog.d/4185.fixed.30.md @@ -0,0 +1 @@ +Cancelling the Safe Mode confirmation in an object editor or sheet showed an error. diff --git a/apple/changelog.d/4185.fixed.31.md b/apple/changelog.d/4185.fixed.31.md new file mode 100644 index 0000000000..c6d66b0771 --- /dev/null +++ b/apple/changelog.d/4185.fixed.31.md @@ -0,0 +1 @@ +Applying user changes ran ALTER and password statements for a user staged for drop, and overcounted pending changes. diff --git a/apple/changelog.d/4185.fixed.32.md b/apple/changelog.d/4185.fixed.32.md new file mode 100644 index 0000000000..98e47dea15 --- /dev/null +++ b/apple/changelog.d/4185.fixed.32.md @@ -0,0 +1 @@ +Moving the caret next to a bracket far below the visible text could crash the editor. diff --git a/apple/changelog.d/4185.fixed.4.md b/apple/changelog.d/4185.fixed.4.md new file mode 100644 index 0000000000..a1b1cd6aa7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.4.md @@ -0,0 +1 @@ +Copy as SQL wrote DuckDB and Oracle binary values that DuckDB stored as text and Oracle rejected. diff --git a/apple/changelog.d/4185.fixed.5.md b/apple/changelog.d/4185.fixed.5.md new file mode 100644 index 0000000000..2aea41ab0b --- /dev/null +++ b/apple/changelog.d/4185.fixed.5.md @@ -0,0 +1 @@ +Data sync wrote an empty Oracle BLOB as NULL, which a NOT NULL column refused. diff --git a/apple/changelog.d/4185.fixed.6.md b/apple/changelog.d/4185.fixed.6.md new file mode 100644 index 0000000000..77fcfcfd1f --- /dev/null +++ b/apple/changelog.d/4185.fixed.6.md @@ -0,0 +1 @@ +Data sync and Copy as SQL wrote Redshift binary values in a form Redshift refused. diff --git a/apple/changelog.d/4185.fixed.7.md b/apple/changelog.d/4185.fixed.7.md new file mode 100644 index 0000000000..2b5c350f28 --- /dev/null +++ b/apple/changelog.d/4185.fixed.7.md @@ -0,0 +1 @@ +Add Rows in Shortcuts refused CSV with bracketed headers such as [id], as SQL Server exports them. diff --git a/apple/changelog.d/4185.fixed.8.md b/apple/changelog.d/4185.fixed.8.md new file mode 100644 index 0000000000..88dc60ec1c --- /dev/null +++ b/apple/changelog.d/4185.fixed.8.md @@ -0,0 +1 @@ +Add Rows in Shortcuts refused a header that differed from its column's name only in case. diff --git a/apple/changelog.d/4185.fixed.9.md b/apple/changelog.d/4185.fixed.9.md new file mode 100644 index 0000000000..081e98b9ba --- /dev/null +++ b/apple/changelog.d/4185.fixed.9.md @@ -0,0 +1 @@ +Copy to another database and Data Compare missed rows past a database's row limit without saying so. diff --git a/apple/changelog.d/4185.fixed.md b/apple/changelog.d/4185.fixed.md new file mode 100644 index 0000000000..a90c14e4ae --- /dev/null +++ b/apple/changelog.d/4185.fixed.md @@ -0,0 +1 @@ +CSV export put a single quote before values starting with =, +, - or @ by default, which then stayed in imports. diff --git a/apple/changelog.d/4190.fixed.1.md b/apple/changelog.d/4190.fixed.1.md new file mode 100644 index 0000000000..ecc80d29b8 --- /dev/null +++ b/apple/changelog.d/4190.fixed.1.md @@ -0,0 +1 @@ +Saving the text __DEFAULT__ in a SQL Server, Oracle, MongoDB or other driver-written table was refused. diff --git a/apple/changelog.d/4190.fixed.2.md b/apple/changelog.d/4190.fixed.2.md new file mode 100644 index 0000000000..74b5117d28 --- /dev/null +++ b/apple/changelog.d/4190.fixed.2.md @@ -0,0 +1 @@ +Exporting a large Teradata, InfluxDB, SAP HANA or Memcached table loaded every row into memory first. diff --git a/apple/changelog.d/4190.fixed.3.md b/apple/changelog.d/4190.fixed.3.md new file mode 100644 index 0000000000..f49fe3cc89 --- /dev/null +++ b/apple/changelog.d/4190.fixed.3.md @@ -0,0 +1 @@ +Stopping a Teradata query in ANSI mode did not say that Teradata rolled back the open transaction. diff --git a/apple/changelog.d/4190.fixed.4.md b/apple/changelog.d/4190.fixed.4.md new file mode 100644 index 0000000000..ce44ae8112 --- /dev/null +++ b/apple/changelog.d/4190.fixed.4.md @@ -0,0 +1 @@ +Oracle TIMESTAMP WITH TIME ZONE values stored with a region name showed a placeholder instead of the time. diff --git a/apple/changelog.d/4190.fixed.5.md b/apple/changelog.d/4190.fixed.5.md new file mode 100644 index 0000000000..7a22306a09 --- /dev/null +++ b/apple/changelog.d/4190.fixed.5.md @@ -0,0 +1 @@ +A Teradata query with the row limit on read every row of the result. diff --git a/apple/changelog.d/4190.fixed.6.md b/apple/changelog.d/4190.fixed.6.md new file mode 100644 index 0000000000..76236db0a8 --- /dev/null +++ b/apple/changelog.d/4190.fixed.6.md @@ -0,0 +1 @@ +A long SAP HANA export failed when it ran past the query timeout. diff --git a/apple/changelog.d/4190.fixed.7.md b/apple/changelog.d/4190.fixed.7.md new file mode 100644 index 0000000000..b130b47acb --- /dev/null +++ b/apple/changelog.d/4190.fixed.7.md @@ -0,0 +1 @@ +Duplicating an Elasticsearch document saved the copy with the id __DEFAULT__. diff --git a/apple/changelog.d/4190.fixed.8.md b/apple/changelog.d/4190.fixed.8.md new file mode 100644 index 0000000000..29f5f55088 --- /dev/null +++ b/apple/changelog.d/4190.fixed.8.md @@ -0,0 +1 @@ +Stopping a Memcached export left the connection closed until a reconnect. diff --git a/apple/changelog.d/4190.fixed.md b/apple/changelog.d/4190.fixed.md new file mode 100644 index 0000000000..4d55e2759e --- /dev/null +++ b/apple/changelog.d/4190.fixed.md @@ -0,0 +1 @@ +Integer, Decimal and Boolean query parameters, and codes such as 1E5, were sent to Trino by how the value looked. diff --git a/apple/changelog.d/4190.security.md b/apple/changelog.d/4190.security.md new file mode 100644 index 0000000000..900541eb6a --- /dev/null +++ b/apple/changelog.d/4190.security.md @@ -0,0 +1 @@ +Browsing a Memcached cache with a key holding a line break could run a second command on the server. diff --git a/apple/changelog.d/4191.added.md b/apple/changelog.d/4191.added.md new file mode 100644 index 0000000000..537d2410ae --- /dev/null +++ b/apple/changelog.d/4191.added.md @@ -0,0 +1 @@ +Meta (Muse Spark, Muse Wonder, Muse Ember) is an AI provider in Add Provider, with its Base URL filled in. diff --git a/apple/changelog.d/4191.fixed.1.md b/apple/changelog.d/4191.fixed.1.md new file mode 100644 index 0000000000..a51ef53301 --- /dev/null +++ b/apple/changelog.d/4191.fixed.1.md @@ -0,0 +1 @@ +The chat panel kept a provider's old model list after its key or Base URL changed in Settings. diff --git a/apple/changelog.d/4191.fixed.2.md b/apple/changelog.d/4191.fixed.2.md new file mode 100644 index 0000000000..325fac2121 --- /dev/null +++ b/apple/changelog.d/4191.fixed.2.md @@ -0,0 +1 @@ +A chat panel model picked for a removed provider was sent to the active provider. diff --git a/apple/changelog.d/4191.fixed.3.md b/apple/changelog.d/4191.fixed.3.md new file mode 100644 index 0000000000..d1245f5d5d --- /dev/null +++ b/apple/changelog.d/4191.fixed.3.md @@ -0,0 +1 @@ +OpenAI and xAI model lists offered embedding, speech and image models that cannot chat. diff --git a/apple/changelog.d/4191.fixed.4.md b/apple/changelog.d/4191.fixed.4.md new file mode 100644 index 0000000000..620613a971 --- /dev/null +++ b/apple/changelog.d/4191.fixed.4.md @@ -0,0 +1 @@ +A new provider started with no model when its model list could not load yet. diff --git a/apple/changelog.d/4191.fixed.5.md b/apple/changelog.d/4191.fixed.5.md new file mode 100644 index 0000000000..c6cf9d90c8 --- /dev/null +++ b/apple/changelog.d/4191.fixed.5.md @@ -0,0 +1 @@ +Test Connection failed with a not found error when no model was picked yet. diff --git a/apple/changelog.d/4191.fixed.md b/apple/changelog.d/4191.fixed.md new file mode 100644 index 0000000000..458b4a4909 --- /dev/null +++ b/apple/changelog.d/4191.fixed.md @@ -0,0 +1 @@ +The chat sent a reasoning level that the model picked in the chat panel does not accept. diff --git a/apple/changelog.d/4200.added.md b/apple/changelog.d/4200.added.md new file mode 100644 index 0000000000..5ea239390d --- /dev/null +++ b/apple/changelog.d/4200.added.md @@ -0,0 +1 @@ +PostgreSQL connections set to Verify Identity check the server against this Mac's trusted roots without a CA file. diff --git a/apple/changelog.d/4200.changed.md b/apple/changelog.d/4200.changed.md new file mode 100644 index 0000000000..0cebd82ff0 --- /dev/null +++ b/apple/changelog.d/4200.changed.md @@ -0,0 +1 @@ +Run All shows statements that return no rows as one result, and keeps the last 100 results that return rows. diff --git a/apple/changelog.d/4200.fixed.1.md b/apple/changelog.d/4200.fixed.1.md new file mode 100644 index 0000000000..c57c00f578 --- /dev/null +++ b/apple/changelog.d/4200.fixed.1.md @@ -0,0 +1 @@ +Filter Values on a large column froze the window while it listed the values. diff --git a/apple/changelog.d/4200.fixed.2.md b/apple/changelog.d/4200.fixed.2.md new file mode 100644 index 0000000000..bd442f13c9 --- /dev/null +++ b/apple/changelog.d/4200.fixed.2.md @@ -0,0 +1 @@ +Grid edits to a row after another row's change saved out of order, failing saves such as a swap of unique values. diff --git a/apple/changelog.d/4200.fixed.3.md b/apple/changelog.d/4200.fixed.3.md new file mode 100644 index 0000000000..eb198d3b4d --- /dev/null +++ b/apple/changelog.d/4200.fixed.3.md @@ -0,0 +1 @@ +A sidebar Refresh held up autocomplete, and each schema of a database kept its own background connection. diff --git a/apple/changelog.d/4200.fixed.4.md b/apple/changelog.d/4200.fixed.4.md new file mode 100644 index 0000000000..9922a5daf1 --- /dev/null +++ b/apple/changelog.d/4200.fixed.4.md @@ -0,0 +1 @@ +After a dropped connection reconnected, statements ran without the temporary tables or settings the old session had. diff --git a/apple/changelog.d/4200.fixed.5.md b/apple/changelog.d/4200.fixed.5.md new file mode 100644 index 0000000000..ffb422e736 --- /dev/null +++ b/apple/changelog.d/4200.fixed.5.md @@ -0,0 +1 @@ +JSON import read UTF-16 and UTF-32 files, and files holding a single object, whole into memory. diff --git a/apple/changelog.d/4200.fixed.6.md b/apple/changelog.d/4200.fixed.6.md new file mode 100644 index 0000000000..cf51aee607 --- /dev/null +++ b/apple/changelog.d/4200.fixed.6.md @@ -0,0 +1 @@ +A chat reply cut off at the model's output limit gave no sign of it, and a tool call cut with it could run. diff --git a/apple/changelog.d/4200.fixed.md b/apple/changelog.d/4200.fixed.md new file mode 100644 index 0000000000..e2f2be9b9e --- /dev/null +++ b/apple/changelog.d/4200.fixed.md @@ -0,0 +1 @@ +Typing a Find term in a large grid froze the window while every cell was searched. diff --git a/apple/changelog.d/4200.security.md b/apple/changelog.d/4200.security.md new file mode 100644 index 0000000000..8497b232cf --- /dev/null +++ b/apple/changelog.d/4200.security.md @@ -0,0 +1 @@ +Marking a connection Local only left its passwords in iCloud Keychain on your other devices. diff --git a/apple/changelog.d/4201.changed.md b/apple/changelog.d/4201.changed.md new file mode 100644 index 0000000000..8367d4080e --- /dev/null +++ b/apple/changelog.d/4201.changed.md @@ -0,0 +1 @@ +On iPad, every window of a connection now shows Reconnecting or Lost at the same time. diff --git a/apple/changelog.d/4201.fixed.1.md b/apple/changelog.d/4201.fixed.1.md new file mode 100644 index 0000000000..bfdd221262 --- /dev/null +++ b/apple/changelog.d/4201.fixed.1.md @@ -0,0 +1 @@ +A DuckDB BLOB on iOS showed as base64 text, and saving an edit stored that text as the blob's bytes. diff --git a/apple/changelog.d/4201.fixed.2.md b/apple/changelog.d/4201.fixed.2.md new file mode 100644 index 0000000000..64e63ef093 --- /dev/null +++ b/apple/changelog.d/4201.fixed.2.md @@ -0,0 +1 @@ +Oracle BINARY_DOUBLE, BINARY_FLOAT and BINARY_INTEGER values on iOS showed as BLOBs and could not be copied or searched. diff --git a/apple/changelog.d/4201.fixed.3.md b/apple/changelog.d/4201.fixed.3.md new file mode 100644 index 0000000000..e01d17ded8 --- /dev/null +++ b/apple/changelog.d/4201.fixed.3.md @@ -0,0 +1 @@ +A Redis value over 4 KB on iOS could never be read in full. diff --git a/apple/changelog.d/4201.fixed.4.md b/apple/changelog.d/4201.fixed.4.md new file mode 100644 index 0000000000..19cd0c3680 --- /dev/null +++ b/apple/changelog.d/4201.fixed.4.md @@ -0,0 +1 @@ +Copy Row as SQL INSERT from an iOS query result quoted names the SQLite way on every engine. diff --git a/apple/changelog.d/4201.fixed.5.md b/apple/changelog.d/4201.fixed.5.md new file mode 100644 index 0000000000..e161edca9b --- /dev/null +++ b/apple/changelog.d/4201.fixed.5.md @@ -0,0 +1 @@ +An iOS connection could show Connected with no session after a failed switch back to its last database. diff --git a/apple/changelog.d/4201.fixed.6.md b/apple/changelog.d/4201.fixed.6.md new file mode 100644 index 0000000000..2ee86b2fa6 --- /dev/null +++ b/apple/changelog.d/4201.fixed.6.md @@ -0,0 +1 @@ +After a database switch on SQL Server on iOS, the schema list still showed the old database's schemas. diff --git a/apple/changelog.d/4201.fixed.7.md b/apple/changelog.d/4201.fixed.7.md new file mode 100644 index 0000000000..302f601988 --- /dev/null +++ b/apple/changelog.d/4201.fixed.7.md @@ -0,0 +1 @@ +On iPad, Reconnect on the lost-connection screen moved the connection's other windows to the saved database and back. diff --git a/apple/changelog.d/4201.fixed.8.md b/apple/changelog.d/4201.fixed.8.md new file mode 100644 index 0000000000..6458673e7f --- /dev/null +++ b/apple/changelog.d/4201.fixed.8.md @@ -0,0 +1 @@ +On iPad, a transaction rolled back in the background went unreported when another window reopened the connection first. diff --git a/apple/changelog.d/4201.fixed.md b/apple/changelog.d/4201.fixed.md new file mode 100644 index 0000000000..f2edc063f6 --- /dev/null +++ b/apple/changelog.d/4201.fixed.md @@ -0,0 +1 @@ +Text over 4 KB in an iOS query result could not be read in full or copied. diff --git a/apple/changelog.d/4205.added.1.md b/apple/changelog.d/4205.added.1.md new file mode 100644 index 0000000000..fe6fe25f96 --- /dev/null +++ b/apple/changelog.d/4205.added.1.md @@ -0,0 +1 @@ +A Values section in the Theme Editor sets the NULL, boolean and per-type value colors. diff --git a/apple/changelog.d/4205.added.md b/apple/changelog.d/4205.added.md new file mode 100644 index 0000000000..0def50c40b --- /dev/null +++ b/apple/changelog.d/4205.added.md @@ -0,0 +1 @@ +Color values by type in Settings > Data colors strings, numbers, dates, binary, JSON, UUID and enum values. diff --git a/apple/changelog.d/4205.changed.md b/apple/changelog.d/4205.changed.md new file mode 100644 index 0000000000..9dfff3253f --- /dev/null +++ b/apple/changelog.d/4205.changed.md @@ -0,0 +1 @@ +Bundled palette themes like Dracula and Nord color each value type, and booleans and values reach 4.5:1 on every row. diff --git a/apple/changelog.d/4205.fixed.md b/apple/changelog.d/4205.fixed.md new file mode 100644 index 0000000000..a1f611d878 --- /dev/null +++ b/apple/changelog.d/4205.fixed.md @@ -0,0 +1 @@ +MySQL and MariaDB BOOLEAN columns showed as numbers in query results, and BIT(8) columns as booleans. diff --git a/apple/changelog.d/4207.changed.1.md b/apple/changelog.d/4207.changed.1.md new file mode 100644 index 0000000000..54c382b929 --- /dev/null +++ b/apple/changelog.d/4207.changed.1.md @@ -0,0 +1 @@ +Themes that set Values colors, copies of built-in themes included, sync only to Macs on this version or later. diff --git a/apple/changelog.d/4207.changed.md b/apple/changelog.d/4207.changed.md new file mode 100644 index 0000000000..143021a1a8 --- /dev/null +++ b/apple/changelog.d/4207.changed.md @@ -0,0 +1 @@ +Edited cells in Dracula and Nord use a lighter tint, so values stay readable on them. diff --git a/apple/changelog.d/4207.fixed.md b/apple/changelog.d/4207.fixed.md new file mode 100644 index 0000000000..bea94351bf --- /dev/null +++ b/apple/changelog.d/4207.fixed.md @@ -0,0 +1 @@ +Sorting and highlight rules on boolean columns holding numbers compared them as text. diff --git a/apple/changelog.d/4209.changed.1.md b/apple/changelog.d/4209.changed.1.md new file mode 100644 index 0000000000..e2e6336f77 --- /dev/null +++ b/apple/changelog.d/4209.changed.1.md @@ -0,0 +1 @@ +Query tabs on Trino, Neo4j and similar drivers no longer read every table's columns up front. diff --git a/apple/changelog.d/4209.changed.md b/apple/changelog.d/4209.changed.md new file mode 100644 index 0000000000..0f0cdf1620 --- /dev/null +++ b/apple/changelog.d/4209.changed.md @@ -0,0 +1 @@ +PostgreSQL connections open with five fewer round trips, so the sidebar lists tables sooner after a connect. diff --git a/apple/changelog.d/4210.added.md b/apple/changelog.d/4210.added.md new file mode 100644 index 0000000000..242c829ee8 --- /dev/null +++ b/apple/changelog.d/4210.added.md @@ -0,0 +1 @@ +Autocomplete offers join conditions from foreign keys right after the ON of a join. diff --git a/apple/changelog.d/4210.fixed.1.md b/apple/changelog.d/4210.fixed.1.md new file mode 100644 index 0000000000..8585969013 --- /dev/null +++ b/apple/changelog.d/4210.fixed.1.md @@ -0,0 +1 @@ +The editor read a MySQL DELIMITER line as SQL, so a routine between DELIMITER lines was cut at each semicolon. diff --git a/apple/changelog.d/4210.fixed.2.md b/apple/changelog.d/4210.fixed.2.md new file mode 100644 index 0000000000..c7e6a59385 --- /dev/null +++ b/apple/changelog.d/4210.fixed.2.md @@ -0,0 +1 @@ +The rows after COPY FROM STDIN in a PostgreSQL script ran as SQL in the editor and in SQL import. diff --git a/apple/changelog.d/4210.fixed.3.md b/apple/changelog.d/4210.fixed.3.md new file mode 100644 index 0000000000..7583794837 --- /dev/null +++ b/apple/changelog.d/4210.fixed.3.md @@ -0,0 +1 @@ +SQL import of a plain pg_dump file failed at its start, on the `\restrict` line current pg_dump versions write. diff --git a/apple/changelog.d/4210.fixed.md b/apple/changelog.d/4210.fixed.md new file mode 100644 index 0000000000..08b9f34840 --- /dev/null +++ b/apple/changelog.d/4210.fixed.md @@ -0,0 +1 @@ +Run Current, the gutter, folding and SQL import split a SQL Server IF ... ELSE at a semicolon before the ELSE. diff --git a/apple/changelog.d/4211.added.1.md b/apple/changelog.d/4211.added.1.md new file mode 100644 index 0000000000..7921fdf593 --- /dev/null +++ b/apple/changelog.d/4211.added.1.md @@ -0,0 +1 @@ +The MCP Activity Log shows and filters by the client that made each call. diff --git a/apple/changelog.d/4211.added.md b/apple/changelog.d/4211.added.md new file mode 100644 index 0000000000..b040e36e0e --- /dev/null +++ b/apple/changelog.d/4211.added.md @@ -0,0 +1 @@ +Connect a Client lists 21 AI clients, including Codex and VS Code, sets most up in one click and says when one connects. diff --git a/apple/changelog.d/4211.changed.md b/apple/changelog.d/4211.changed.md new file mode 100644 index 0000000000..764926429e --- /dev/null +++ b/apple/changelog.d/4211.changed.md @@ -0,0 +1 @@ +AI provider and MCP client lists show each brand's logo instead of a generic symbol. diff --git a/apple/changelog.d/4211.fixed.1.md b/apple/changelog.d/4211.fixed.1.md new file mode 100644 index 0000000000..66b365425f --- /dev/null +++ b/apple/changelog.d/4211.fixed.1.md @@ -0,0 +1 @@ +A Claude Desktop, Cursor or Zed client showed twice in Connected Clients, and Disconnect cut off every other one. diff --git a/apple/changelog.d/4211.fixed.2.md b/apple/changelog.d/4211.fixed.2.md new file mode 100644 index 0000000000..6d108fd695 --- /dev/null +++ b/apple/changelog.d/4211.fixed.2.md @@ -0,0 +1 @@ +The new token sheet showed the hidden token in its setup snippet. diff --git a/apple/changelog.d/4211.fixed.3.md b/apple/changelog.d/4211.fixed.3.md new file mode 100644 index 0000000000..485891f5fe --- /dev/null +++ b/apple/changelog.d/4211.fixed.3.md @@ -0,0 +1 @@ +Generate Token did nothing when the token could not be saved, and the MCP port took any number. diff --git a/apple/changelog.d/4211.fixed.4.md b/apple/changelog.d/4211.fixed.4.md new file mode 100644 index 0000000000..5e0a7d782b --- /dev/null +++ b/apple/changelog.d/4211.fixed.4.md @@ -0,0 +1 @@ +Claude Agent lost its database tools silently, and pairing failed with a vague message, when a token was not saved. diff --git a/apple/changelog.d/4211.fixed.md b/apple/changelog.d/4211.fixed.md new file mode 100644 index 0000000000..e67edd5a74 --- /dev/null +++ b/apple/changelog.d/4211.fixed.md @@ -0,0 +1 @@ +The Claude Code setup command added TablePro to one project only, and broke on a path with a space. diff --git a/apple/changelog.d/4212.changed.md b/apple/changelog.d/4212.changed.md new file mode 100644 index 0000000000..446e2b3b8f --- /dev/null +++ b/apple/changelog.d/4212.changed.md @@ -0,0 +1 @@ +Queries start sooner: running one no longer rewrites the whole execution log first. diff --git a/apple/changelog.d/4212.fixed.1.md b/apple/changelog.d/4212.fixed.1.md new file mode 100644 index 0000000000..94ce9e289f --- /dev/null +++ b/apple/changelog.d/4212.fixed.1.md @@ -0,0 +1 @@ +Open tabs were written to disk every 30 seconds and on every tab switch even when nothing had changed. diff --git a/apple/changelog.d/4212.fixed.2.md b/apple/changelog.d/4212.fixed.2.md new file mode 100644 index 0000000000..7aad7d83bf --- /dev/null +++ b/apple/changelog.d/4212.fixed.2.md @@ -0,0 +1 @@ +Closed agent sessions kept their transcripts and schema in memory until TablePro quit. diff --git a/apple/changelog.d/4212.fixed.3.md b/apple/changelog.d/4212.fixed.3.md new file mode 100644 index 0000000000..81e16ab59c --- /dev/null +++ b/apple/changelog.d/4212.fixed.3.md @@ -0,0 +1 @@ +An agent session restored at launch could lose its conversation the next time another session was saved. diff --git a/apple/changelog.d/4212.fixed.4.md b/apple/changelog.d/4212.fixed.4.md new file mode 100644 index 0000000000..30c58520b5 --- /dev/null +++ b/apple/changelog.d/4212.fixed.4.md @@ -0,0 +1 @@ +Opening a .gz file, or checking the Copilot and Cloud SQL Proxy helpers, held the whole file in memory. diff --git a/apple/changelog.d/4212.fixed.5.md b/apple/changelog.d/4212.fixed.5.md new file mode 100644 index 0000000000..a324f1b9ff --- /dev/null +++ b/apple/changelog.d/4212.fixed.5.md @@ -0,0 +1 @@ +An execution log TablePro could not read was overwritten by a new one instead of being kept. diff --git a/apple/changelog.d/4212.fixed.6.md b/apple/changelog.d/4212.fixed.6.md new file mode 100644 index 0000000000..9fdbb1a84a --- /dev/null +++ b/apple/changelog.d/4212.fixed.6.md @@ -0,0 +1 @@ +Picking another file in the Import dialog while a .gz file was still opening could import the first file. diff --git a/apple/changelog.d/4212.fixed.md b/apple/changelog.d/4212.fixed.md new file mode 100644 index 0000000000..91e2ecdee3 --- /dev/null +++ b/apple/changelog.d/4212.fixed.md @@ -0,0 +1 @@ +Claude Agent replies appeared only once finished, and a long reply never finished. diff --git a/apple/changelog.d/4214.added.1.md b/apple/changelog.d/4214.added.1.md new file mode 100644 index 0000000000..626ed8ec13 --- /dev/null +++ b/apple/changelog.d/4214.added.1.md @@ -0,0 +1 @@ +A table tab can refresh itself on an interval, set from its status bar or Database > Auto Refresh. diff --git a/apple/changelog.d/4214.added.2.md b/apple/changelog.d/4214.added.2.md new file mode 100644 index 0000000000..83152ab3c9 --- /dev/null +++ b/apple/changelog.d/4214.added.2.md @@ -0,0 +1 @@ +Query tabs keep earlier versions of their text, listed under File > Revert To. diff --git a/apple/changelog.d/4214.added.3.md b/apple/changelog.d/4214.added.3.md new file mode 100644 index 0000000000..e9ea1e5804 --- /dev/null +++ b/apple/changelog.d/4214.added.3.md @@ -0,0 +1 @@ +The sidebar lists MySQL and MariaDB events and PostgreSQL extensions, and opens their DDL. diff --git a/apple/changelog.d/4214.added.4.md b/apple/changelog.d/4214.added.4.md new file mode 100644 index 0000000000..9d91cb3400 --- /dev/null +++ b/apple/changelog.d/4214.added.4.md @@ -0,0 +1 @@ +The ER diagram's Add Foreign Key sheet builds keys over several columns. diff --git a/apple/changelog.d/4214.added.md b/apple/changelog.d/4214.added.md new file mode 100644 index 0000000000..3b8c703b40 --- /dev/null +++ b/apple/changelog.d/4214.added.md @@ -0,0 +1 @@ +Manual Commit keeps a connection's changes in a transaction until you commit or roll back. diff --git a/apple/changelog.d/4214.fixed.1.md b/apple/changelog.d/4214.fixed.1.md new file mode 100644 index 0000000000..3e96339e89 --- /dev/null +++ b/apple/changelog.d/4214.fixed.1.md @@ -0,0 +1 @@ +Closing, disconnecting or quitting did not mention an open transaction it was about to roll back. diff --git a/apple/changelog.d/4214.fixed.10.md b/apple/changelog.d/4214.fixed.10.md new file mode 100644 index 0000000000..67cc9c96bb --- /dev/null +++ b/apple/changelog.d/4214.fixed.10.md @@ -0,0 +1 @@ +An MCP insert_rows call that failed partway inside an open transaction left its earlier rows pending. diff --git a/apple/changelog.d/4214.fixed.2.md b/apple/changelog.d/4214.fixed.2.md new file mode 100644 index 0000000000..2015bc0a7e --- /dev/null +++ b/apple/changelog.d/4214.fixed.2.md @@ -0,0 +1 @@ +Commit and Roll Back in the transaction bar refused after any background read on the connection. diff --git a/apple/changelog.d/4214.fixed.3.md b/apple/changelog.d/4214.fixed.3.md new file mode 100644 index 0000000000..443c047748 --- /dev/null +++ b/apple/changelog.d/4214.fixed.3.md @@ -0,0 +1 @@ +The transaction bar's Commit button did not follow a Safe Mode change until the next statement. diff --git a/apple/changelog.d/4214.fixed.4.md b/apple/changelog.d/4214.fixed.4.md new file mode 100644 index 0000000000..349037c127 --- /dev/null +++ b/apple/changelog.d/4214.fixed.4.md @@ -0,0 +1 @@ +An MCP client could start a transaction over an open one, which on MySQL commits the open one. diff --git a/apple/changelog.d/4214.fixed.5.md b/apple/changelog.d/4214.fixed.5.md new file mode 100644 index 0000000000..873610b77c --- /dev/null +++ b/apple/changelog.d/4214.fixed.5.md @@ -0,0 +1 @@ +A refresh turned a selected cell or cell range into whole selected rows. diff --git a/apple/changelog.d/4214.fixed.6.md b/apple/changelog.d/4214.fixed.6.md new file mode 100644 index 0000000000..520d92aff3 --- /dev/null +++ b/apple/changelog.d/4214.fixed.6.md @@ -0,0 +1 @@ +Clear Query, loading a query, AI apply and inserting a saved query could not be undone. diff --git a/apple/changelog.d/4214.fixed.7.md b/apple/changelog.d/4214.fixed.7.md new file mode 100644 index 0000000000..b49cce4f28 --- /dev/null +++ b/apple/changelog.d/4214.fixed.7.md @@ -0,0 +1 @@ +Typing in a second window of a connection waited up to 30 seconds to be saved. diff --git a/apple/changelog.d/4214.fixed.8.md b/apple/changelog.d/4214.fixed.8.md new file mode 100644 index 0000000000..6b69890e33 --- /dev/null +++ b/apple/changelog.d/4214.fixed.8.md @@ -0,0 +1 @@ +The Structure tab accepted a foreign key whose column and referenced column counts differ. diff --git a/apple/changelog.d/4214.fixed.9.md b/apple/changelog.d/4214.fixed.9.md new file mode 100644 index 0000000000..32f12e10e1 --- /dev/null +++ b/apple/changelog.d/4214.fixed.9.md @@ -0,0 +1 @@ +The Server Dashboard's refresh interval was not shown in your language. diff --git a/apple/changelog.d/4214.fixed.md b/apple/changelog.d/4214.fixed.md new file mode 100644 index 0000000000..0019928d55 --- /dev/null +++ b/apple/changelog.d/4214.fixed.md @@ -0,0 +1 @@ +On DuckDB and PGlite, saving a structure, a new table or a trigger committed a transaction left open. diff --git a/apple/changelog.d/4215.fixed.1.md b/apple/changelog.d/4215.fixed.1.md new file mode 100644 index 0000000000..853119454b --- /dev/null +++ b/apple/changelog.d/4215.fixed.1.md @@ -0,0 +1 @@ +On iOS, a PostgreSQL Roll Back or a typed SET search_path left the schema menu on a schema the session had left. diff --git a/apple/changelog.d/4215.fixed.2.md b/apple/changelog.d/4215.fixed.2.md new file mode 100644 index 0000000000..118102133a --- /dev/null +++ b/apple/changelog.d/4215.fixed.2.md @@ -0,0 +1 @@ +Paging an iOS PostgreSQL or ClickHouse table without a sort repeated and skipped rows. diff --git a/apple/changelog.d/4215.fixed.3.md b/apple/changelog.d/4215.fixed.3.md new file mode 100644 index 0000000000..7874f9d633 --- /dev/null +++ b/apple/changelog.d/4215.fixed.3.md @@ -0,0 +1 @@ +Copying rows as JSON on iOS turned text that looked like a number or true and false into numbers and booleans. diff --git a/apple/changelog.d/4215.fixed.4.md b/apple/changelog.d/4215.fixed.4.md new file mode 100644 index 0000000000..48c210f1ff --- /dev/null +++ b/apple/changelog.d/4215.fixed.4.md @@ -0,0 +1 @@ +Searching an iOS table for a name with an apostrophe found nothing with Smart Punctuation on. diff --git a/apple/changelog.d/4215.fixed.5.md b/apple/changelog.d/4215.fixed.5.md new file mode 100644 index 0000000000..6299555e57 --- /dev/null +++ b/apple/changelog.d/4215.fixed.5.md @@ -0,0 +1 @@ +Each query run on iOS paused the app while it reread and rewrote the whole query history. diff --git a/apple/changelog.d/4215.fixed.6.md b/apple/changelog.d/4215.fixed.6.md new file mode 100644 index 0000000000..2550548b61 --- /dev/null +++ b/apple/changelog.d/4215.fixed.6.md @@ -0,0 +1 @@ +On iPad, a row delete confirmed after another window switched schema deleted the same key in that schema. diff --git a/apple/changelog.d/4215.fixed.md b/apple/changelog.d/4215.fixed.md new file mode 100644 index 0000000000..12697fa658 --- /dev/null +++ b/apple/changelog.d/4215.fixed.md @@ -0,0 +1 @@ +A USE run in the iOS Query tab moved where Tables tab edits, deletes, truncates and drops landed. diff --git a/apple/changelog.d/4215.security.md b/apple/changelog.d/4215.security.md new file mode 100644 index 0000000000..1e93f7c022 --- /dev/null +++ b/apple/changelog.d/4215.security.md @@ -0,0 +1 @@ +Deleting a connection on iOS left its decrypted client private key in a file on the device. diff --git a/apple/changelog.d/4218.changed.md b/apple/changelog.d/4218.changed.md new file mode 100644 index 0000000000..ecf1106a66 --- /dev/null +++ b/apple/changelog.d/4218.changed.md @@ -0,0 +1 @@ +Copilot's schema context covers up to Max schema tables tables, the ones the current statement names first. diff --git a/apple/changelog.d/4218.fixed.1.md b/apple/changelog.d/4218.fixed.1.md new file mode 100644 index 0000000000..32c2357d0b --- /dev/null +++ b/apple/changelog.d/4218.fixed.1.md @@ -0,0 +1 @@ +Vim section and paragraph motions froze on long scripts, and j and k slowed down as a script grew. diff --git a/apple/changelog.d/4218.fixed.2.md b/apple/changelog.d/4218.fixed.2.md new file mode 100644 index 0000000000..3b3fce3ea6 --- /dev/null +++ b/apple/changelog.d/4218.fixed.2.md @@ -0,0 +1 @@ +Copilot resent the whole script and schema on every keystroke, and stopped suggesting after a tab switch. diff --git a/apple/changelog.d/4218.fixed.3.md b/apple/changelog.d/4218.fixed.3.md new file mode 100644 index 0000000000..e653509202 --- /dev/null +++ b/apple/changelog.d/4218.fixed.3.md @@ -0,0 +1 @@ +Copilot ran one metadata query per table when an editor opened, thousands on a large database. diff --git a/apple/changelog.d/4218.fixed.4.md b/apple/changelog.d/4218.fixed.4.md new file mode 100644 index 0000000000..d71d686cde --- /dev/null +++ b/apple/changelog.d/4218.fixed.4.md @@ -0,0 +1 @@ +Moving the caret in the query editor redrew the whole window. diff --git a/apple/changelog.d/4218.fixed.5.md b/apple/changelog.d/4218.fixed.5.md new file mode 100644 index 0000000000..7a1b936cf4 --- /dev/null +++ b/apple/changelog.d/4218.fixed.5.md @@ -0,0 +1 @@ +The query history panel lagged on very long saved queries. diff --git a/apple/changelog.d/4218.fixed.md b/apple/changelog.d/4218.fixed.md new file mode 100644 index 0000000000..65870fa9d7 --- /dev/null +++ b/apple/changelog.d/4218.fixed.md @@ -0,0 +1 @@ +Typing lagged in large SQL scripts while code folding was on. diff --git a/apple/changelog.d/4219.added.md b/apple/changelog.d/4219.added.md new file mode 100644 index 0000000000..ecbd4e8753 --- /dev/null +++ b/apple/changelog.d/4219.added.md @@ -0,0 +1 @@ +Always Allow grants for assistant tools can be removed with Forget under AI Policy in the connection form. diff --git a/apple/changelog.d/4219.fixed.1.md b/apple/changelog.d/4219.fixed.1.md new file mode 100644 index 0000000000..df1d9ca804 --- /dev/null +++ b/apple/changelog.d/4219.fixed.1.md @@ -0,0 +1 @@ +A large MCP or assistant query result on PostgreSQL, MySQL, SQLite, DuckDB or ClickHouse could use up memory. diff --git a/apple/changelog.d/4219.fixed.2.md b/apple/changelog.d/4219.fixed.2.md new file mode 100644 index 0000000000..76a8511fa0 --- /dev/null +++ b/apple/changelog.d/4219.fixed.2.md @@ -0,0 +1 @@ +Stopping the assistant during a write reported a failure while the write went on and committed. diff --git a/apple/changelog.d/4219.fixed.3.md b/apple/changelog.d/4219.fixed.3.md new file mode 100644 index 0000000000..0a54c93703 --- /dev/null +++ b/apple/changelog.d/4219.fixed.3.md @@ -0,0 +1 @@ +A conversation left while a tool card waited reopened with Run buttons that did nothing. diff --git a/apple/changelog.d/4219.fixed.4.md b/apple/changelog.d/4219.fixed.4.md new file mode 100644 index 0000000000..405b8c509d --- /dev/null +++ b/apple/changelog.d/4219.fixed.4.md @@ -0,0 +1 @@ +The assistant saw a generic error instead of the database's message when a query failed. diff --git a/apple/changelog.d/4219.fixed.md b/apple/changelog.d/4219.fixed.md new file mode 100644 index 0000000000..e383eefdde --- /dev/null +++ b/apple/changelog.d/4219.fixed.md @@ -0,0 +1 @@ +A tunnel or proxy helper's error could lose its lines when they ended in CRLF or held non-ASCII text. diff --git a/apple/changelog.d/4219.security.1.md b/apple/changelog.d/4219.security.1.md new file mode 100644 index 0000000000..d21513e43d --- /dev/null +++ b/apple/changelog.d/4219.security.1.md @@ -0,0 +1 @@ +MCP clients could read the query history and tabs of connections the user had denied or never approved. diff --git a/apple/changelog.d/4219.security.2.md b/apple/changelog.d/4219.security.2.md new file mode 100644 index 0000000000..e9f626948d --- /dev/null +++ b/apple/changelog.d/4219.security.2.md @@ -0,0 +1 @@ +Claude Agent could run queries and read rows in Ask mode. diff --git a/apple/changelog.d/4219.security.3.md b/apple/changelog.d/4219.security.3.md new file mode 100644 index 0000000000..f0bfd79504 --- /dev/null +++ b/apple/changelog.d/4219.security.3.md @@ -0,0 +1 @@ +The assistant's list_connections sent every saved connection's host and database to the AI provider. diff --git a/apple/changelog.d/4219.security.4.md b/apple/changelog.d/4219.security.4.md new file mode 100644 index 0000000000..de0e79612d --- /dev/null +++ b/apple/changelog.d/4219.security.4.md @@ -0,0 +1 @@ +An MCP client could read files through a DuckDB table function or a file name used as a table. diff --git a/apple/changelog.d/4219.security.md b/apple/changelog.d/4219.security.md new file mode 100644 index 0000000000..9dcf1b7a1e --- /dev/null +++ b/apple/changelog.d/4219.security.md @@ -0,0 +1 @@ +Read-Only Safe Mode ran a query that called a function which writes data. diff --git a/apple/changelog.d/4220.fixed.1.md b/apple/changelog.d/4220.fixed.1.md new file mode 100644 index 0000000000..582a62eb1e --- /dev/null +++ b/apple/changelog.d/4220.fixed.1.md @@ -0,0 +1 @@ +Cassandra and ScyllaDB edits and undone deletes no longer remove the time to live of the cells they write. diff --git a/apple/changelog.d/4220.fixed.2.md b/apple/changelog.d/4220.fixed.2.md new file mode 100644 index 0000000000..0a4102b63c --- /dev/null +++ b/apple/changelog.d/4220.fixed.2.md @@ -0,0 +1 @@ +Kafka pages from the newest message no longer skip messages in topics with transactions or compaction. diff --git a/apple/changelog.d/4220.fixed.3.md b/apple/changelog.d/4220.fixed.3.md new file mode 100644 index 0000000000..80c1da3c1c --- /dev/null +++ b/apple/changelog.d/4220.fixed.3.md @@ -0,0 +1 @@ +MongoDB pages sorted by a column with repeated values no longer repeat or skip documents. diff --git a/apple/changelog.d/4220.fixed.4.md b/apple/changelog.d/4220.fixed.4.md new file mode 100644 index 0000000000..ac0200526b --- /dev/null +++ b/apple/changelog.d/4220.fixed.4.md @@ -0,0 +1 @@ +Redis, Cassandra and Bigtable keys that start with a byte order mark are no longer edited or deleted as another key. diff --git a/apple/changelog.d/4220.fixed.5.md b/apple/changelog.d/4220.fixed.5.md new file mode 100644 index 0000000000..74dd454109 --- /dev/null +++ b/apple/changelog.d/4220.fixed.5.md @@ -0,0 +1 @@ +Browsing a Redis database no longer reads every string value whole. diff --git a/apple/changelog.d/4220.fixed.6.md b/apple/changelog.d/4220.fixed.6.md new file mode 100644 index 0000000000..0e7ad833ef --- /dev/null +++ b/apple/changelog.d/4220.fixed.6.md @@ -0,0 +1 @@ +Elasticsearch edits and deletes of documents indexed with custom routing no longer fail or change another document. diff --git a/apple/changelog.d/4220.fixed.7.md b/apple/changelog.d/4220.fixed.7.md new file mode 100644 index 0000000000..373d89711e --- /dev/null +++ b/apple/changelog.d/4220.fixed.7.md @@ -0,0 +1 @@ +Editing an Elasticsearch text or keyword value that reads as a JSON array no longer stores an array. diff --git a/apple/changelog.d/4220.fixed.8.md b/apple/changelog.d/4220.fixed.8.md new file mode 100644 index 0000000000..457a1f2032 --- /dev/null +++ b/apple/changelog.d/4220.fixed.8.md @@ -0,0 +1 @@ +Editing a MongoDB number no longer changes its type to the one most of the column holds. diff --git a/apple/changelog.d/4220.fixed.9.md b/apple/changelog.d/4220.fixed.9.md new file mode 100644 index 0000000000..2bdcfdc29c --- /dev/null +++ b/apple/changelog.d/4220.fixed.9.md @@ -0,0 +1 @@ +Changing a Memcached item's flags no longer writes back the value the grid loaded. diff --git a/apple/changelog.d/4220.fixed.md b/apple/changelog.d/4220.fixed.md new file mode 100644 index 0000000000..bb2ef3c042 --- /dev/null +++ b/apple/changelog.d/4220.fixed.md @@ -0,0 +1 @@ +SurrealDB edits and deletes of a record whose id is a UUID, array or object now change that record. diff --git a/apple/changelog.d/4221.fixed.1.md b/apple/changelog.d/4221.fixed.1.md new file mode 100644 index 0000000000..9257f6ba9f --- /dev/null +++ b/apple/changelog.d/4221.fixed.1.md @@ -0,0 +1 @@ +Binary values exported to CSV, JSON or XLSX imported back as hex or Base64 text instead of the bytes. diff --git a/apple/changelog.d/4221.fixed.2.md b/apple/changelog.d/4221.fixed.2.md new file mode 100644 index 0000000000..551b2772f6 --- /dev/null +++ b/apple/changelog.d/4221.fixed.2.md @@ -0,0 +1 @@ +CSV and JSON import dropped rows of all NULL values, the way their own export writes them, and reported success. diff --git a/apple/changelog.d/4221.fixed.3.md b/apple/changelog.d/4221.fixed.3.md new file mode 100644 index 0000000000..9aa8798be9 --- /dev/null +++ b/apple/changelog.d/4221.fixed.3.md @@ -0,0 +1 @@ +Parquet export dropped the UTC offset of timestamps and times with time zone. diff --git a/apple/changelog.d/4221.fixed.4.md b/apple/changelog.d/4221.fixed.4.md new file mode 100644 index 0000000000..8217c58508 --- /dev/null +++ b/apple/changelog.d/4221.fixed.4.md @@ -0,0 +1 @@ +Importing true, false, yes or no into a boolean column failed on MySQL and SQL Server and stored text on SQLite. diff --git a/apple/changelog.d/4221.fixed.5.md b/apple/changelog.d/4221.fixed.5.md new file mode 100644 index 0000000000..ae04654c8c --- /dev/null +++ b/apple/changelog.d/4221.fixed.5.md @@ -0,0 +1 @@ +XLSX export of more than about 2.1 million rows wrote sheets past Excel's row limit. diff --git a/apple/changelog.d/4221.fixed.6.md b/apple/changelog.d/4221.fixed.6.md new file mode 100644 index 0000000000..f8af8f84ad --- /dev/null +++ b/apple/changelog.d/4221.fixed.6.md @@ -0,0 +1 @@ +Copy as IN Clause wrote TRUE and FALSE for SQL Server bit columns and left out binary values. diff --git a/apple/changelog.d/4221.fixed.md b/apple/changelog.d/4221.fixed.md new file mode 100644 index 0000000000..568e5f6590 --- /dev/null +++ b/apple/changelog.d/4221.fixed.md @@ -0,0 +1 @@ +Copying a binary cell copied only its first 64 bytes, and pasting it back wrote the hex text instead of the bytes. diff --git a/apple/changelog.d/4222.changed.1.md b/apple/changelog.d/4222.changed.1.md new file mode 100644 index 0000000000..6e9b0ae25a --- /dev/null +++ b/apple/changelog.d/4222.changed.1.md @@ -0,0 +1 @@ +A connection on an SSH profile asks once more before it signs in with this Mac's credentials or environment variables. diff --git a/apple/changelog.d/4222.changed.md b/apple/changelog.d/4222.changed.md new file mode 100644 index 0000000000..cd892381fb --- /dev/null +++ b/apple/changelog.d/4222.changed.md @@ -0,0 +1 @@ +Reset All Settings keeps AI providers, their API keys and the Enable AI Features switch. diff --git a/apple/changelog.d/4222.fixed.1.md b/apple/changelog.d/4222.fixed.1.md new file mode 100644 index 0000000000..ed4a5fe886 --- /dev/null +++ b/apple/changelog.d/4222.fixed.1.md @@ -0,0 +1 @@ +Copying or exporting a large ER diagram as an image could freeze TablePro and use gigabytes of memory. diff --git a/apple/changelog.d/4222.fixed.2.md b/apple/changelog.d/4222.fixed.2.md new file mode 100644 index 0000000000..db0e508cfa --- /dev/null +++ b/apple/changelog.d/4222.fixed.2.md @@ -0,0 +1 @@ +Removing an AI provider from its sheet asked twice. diff --git a/apple/changelog.d/4222.fixed.3.md b/apple/changelog.d/4222.fixed.3.md new file mode 100644 index 0000000000..c492baacf6 --- /dev/null +++ b/apple/changelog.d/4222.fixed.3.md @@ -0,0 +1 @@ +Deleting a custom slash command or resetting all keyboard shortcuts happened on one click. diff --git a/apple/changelog.d/4222.fixed.4.md b/apple/changelog.d/4222.fixed.4.md new file mode 100644 index 0000000000..86fe3cec32 --- /dev/null +++ b/apple/changelog.d/4222.fixed.4.md @@ -0,0 +1 @@ +Remove on a plugin that failed to load deleted it at once, with no confirmation, bundled plugins included. diff --git a/apple/changelog.d/4222.fixed.md b/apple/changelog.d/4222.fixed.md new file mode 100644 index 0000000000..281c72ce45 --- /dev/null +++ b/apple/changelog.d/4222.fixed.md @@ -0,0 +1 @@ +A connections file that could not be read loaded as an empty library, and the next save or sync wrote over it. diff --git a/apple/changelog.d/4222.security.1.md b/apple/changelog.d/4222.security.1.md new file mode 100644 index 0000000000..a1617698c3 --- /dev/null +++ b/apple/changelog.d/4222.security.1.md @@ -0,0 +1 @@ +Sign-in approvals through an SSH profile still held after the profile moved to another server. diff --git a/apple/changelog.d/4222.security.2.md b/apple/changelog.d/4222.security.2.md new file mode 100644 index 0000000000..9130adb4f5 --- /dev/null +++ b/apple/changelog.d/4222.security.2.md @@ -0,0 +1 @@ +Copied connection strings and MCP tokens were not marked concealed, so clipboard history apps could keep them. diff --git a/apple/changelog.d/4222.security.3.md b/apple/changelog.d/4222.security.3.md new file mode 100644 index 0000000000..858a25ccf1 --- /dev/null +++ b/apple/changelog.d/4222.security.3.md @@ -0,0 +1 @@ +Publishing a connection to the team library uploaded its tunnel command and startup commands. diff --git a/apple/changelog.d/4222.security.4.md b/apple/changelog.d/4222.security.4.md new file mode 100644 index 0000000000..bc91f22f62 --- /dev/null +++ b/apple/changelog.d/4222.security.4.md @@ -0,0 +1 @@ +Always Allow was offered for a local link to a connection that goes through a proxy, tunnel command or SSH profile. diff --git a/apple/changelog.d/4222.security.5.md b/apple/changelog.d/4222.security.5.md new file mode 100644 index 0000000000..448b24f3b1 --- /dev/null +++ b/apple/changelog.d/4222.security.5.md @@ -0,0 +1 @@ +Reset All Settings left iCloud password sync on until relaunch, though the Sync pane showed it off. diff --git a/apple/changelog.d/4222.security.md b/apple/changelog.d/4222.security.md new file mode 100644 index 0000000000..88b269c07d --- /dev/null +++ b/apple/changelog.d/4222.security.md @@ -0,0 +1 @@ +A reconnect TablePro started on its own could follow an edited SSH profile to another server under an open window. diff --git a/apple/changelog.d/4223.fixed.1.md b/apple/changelog.d/4223.fixed.1.md new file mode 100644 index 0000000000..e232389f4f --- /dev/null +++ b/apple/changelog.d/4223.fixed.1.md @@ -0,0 +1 @@ +Saving edits on a Teradata connection in ANSI mode failed and discarded the editor's uncommitted changes. diff --git a/apple/changelog.d/4223.fixed.10.md b/apple/changelog.d/4223.fixed.10.md new file mode 100644 index 0000000000..a7e9ae7a4c --- /dev/null +++ b/apple/changelog.d/4223.fixed.10.md @@ -0,0 +1 @@ +Run All of a Teradata script that creates a table and then uses it failed with error 3932. diff --git a/apple/changelog.d/4223.fixed.2.md b/apple/changelog.d/4223.fixed.2.md new file mode 100644 index 0000000000..9969846c4f --- /dev/null +++ b/apple/changelog.d/4223.fixed.2.md @@ -0,0 +1 @@ +SQL Server table DDL and SQL exports dropped computed columns, identity seeds, checks and composite foreign keys. diff --git a/apple/changelog.d/4223.fixed.3.md b/apple/changelog.d/4223.fixed.3.md new file mode 100644 index 0000000000..d0e8b00414 --- /dev/null +++ b/apple/changelog.d/4223.fixed.3.md @@ -0,0 +1 @@ +Changing a SQL Server column's type in the structure editor reset its collation to the database default. diff --git a/apple/changelog.d/4223.fixed.4.md b/apple/changelog.d/4223.fixed.4.md new file mode 100644 index 0000000000..2e74fe2f31 --- /dev/null +++ b/apple/changelog.d/4223.fixed.4.md @@ -0,0 +1 @@ +Oracle SQL exports wrote no primary keys, so every foreign key in the dump failed on restore. diff --git a/apple/changelog.d/4223.fixed.5.md b/apple/changelog.d/4223.fixed.5.md new file mode 100644 index 0000000000..54783e88e2 --- /dev/null +++ b/apple/changelog.d/4223.fixed.5.md @@ -0,0 +1 @@ +Copying, transferring or comparing a large PostgreSQL, MySQL or SQL Server table held all of it in memory. diff --git a/apple/changelog.d/4223.fixed.6.md b/apple/changelog.d/4223.fixed.6.md new file mode 100644 index 0000000000..6723e860f5 --- /dev/null +++ b/apple/changelog.d/4223.fixed.6.md @@ -0,0 +1 @@ +PostgreSQL DDL of a partitioned table dropped each partition's own indexes, checks and defaults. diff --git a/apple/changelog.d/4223.fixed.7.md b/apple/changelog.d/4223.fixed.7.md new file mode 100644 index 0000000000..cb84f549f0 --- /dev/null +++ b/apple/changelog.d/4223.fixed.7.md @@ -0,0 +1 @@ +Cancelling a DuckDB export inside an open transaction could abort that transaction. diff --git a/apple/changelog.d/4223.fixed.8.md b/apple/changelog.d/4223.fixed.8.md new file mode 100644 index 0000000000..fe3dc4fe16 --- /dev/null +++ b/apple/changelog.d/4223.fixed.8.md @@ -0,0 +1 @@ +SAP HANA exports wrote large objects over 64 MiB cut short without reporting it. diff --git a/apple/changelog.d/4223.fixed.9.md b/apple/changelog.d/4223.fixed.9.md new file mode 100644 index 0000000000..acf6a54d6c --- /dev/null +++ b/apple/changelog.d/4223.fixed.9.md @@ -0,0 +1 @@ +The MySQL structure view could show another table's check constraint. diff --git a/apple/changelog.d/4223.fixed.md b/apple/changelog.d/4223.fixed.md new file mode 100644 index 0000000000..28c3c6b397 --- /dev/null +++ b/apple/changelog.d/4223.fixed.md @@ -0,0 +1 @@ +Importing or copying into Oracle stored text such as 00123 as 123, and national text as question marks. diff --git a/apple/changelog.d/4225.fixed.1.md b/apple/changelog.d/4225.fixed.1.md new file mode 100644 index 0000000000..45dca2cf50 --- /dev/null +++ b/apple/changelog.d/4225.fixed.1.md @@ -0,0 +1 @@ +Stop on Run All dropped the results and history of the statements that had already run. diff --git a/apple/changelog.d/4225.fixed.2.md b/apple/changelog.d/4225.fixed.2.md new file mode 100644 index 0000000000..b2d9f77b0e --- /dev/null +++ b/apple/changelog.d/4225.fixed.2.md @@ -0,0 +1 @@ +A connection first opened on a table or query from a link never saved its tabs, so they were lost at quit. diff --git a/apple/changelog.d/4225.fixed.3.md b/apple/changelog.d/4225.fixed.3.md new file mode 100644 index 0000000000..dc305be738 --- /dev/null +++ b/apple/changelog.d/4225.fixed.3.md @@ -0,0 +1 @@ +Truncate and Copy in one window acted on the table selected in another window of the same connection. diff --git a/apple/changelog.d/4225.fixed.4.md b/apple/changelog.d/4225.fixed.4.md new file mode 100644 index 0000000000..844e7bbc9d --- /dev/null +++ b/apple/changelog.d/4225.fixed.4.md @@ -0,0 +1 @@ +Switching tabs, Clear Query, loading from history or applying an AI rewrite wiped the SQL editor's undo history. diff --git a/apple/changelog.d/4225.fixed.5.md b/apple/changelog.d/4225.fixed.5.md new file mode 100644 index 0000000000..c56a3c1fc9 --- /dev/null +++ b/apple/changelog.d/4225.fixed.5.md @@ -0,0 +1 @@ +Showing an earlier result read in another schema moved the query tab back to that schema. diff --git a/apple/changelog.d/4225.fixed.md b/apple/changelog.d/4225.fixed.md new file mode 100644 index 0000000000..ce9fe0a25d --- /dev/null +++ b/apple/changelog.d/4225.fixed.md @@ -0,0 +1 @@ +Changing a query tab's database while its query ran put the old database's rows under the new one, and Save wrote there. diff --git a/apple/changelog.d/4226.fixed.1.md b/apple/changelog.d/4226.fixed.1.md new file mode 100644 index 0000000000..cd7116c646 --- /dev/null +++ b/apple/changelog.d/4226.fixed.1.md @@ -0,0 +1 @@ +A save reported success for an edited row that was deleted elsewhere or hidden by a row-level security policy. diff --git a/apple/changelog.d/4226.fixed.10.md b/apple/changelog.d/4226.fixed.10.md new file mode 100644 index 0000000000..8b2a46888c --- /dev/null +++ b/apple/changelog.d/4226.fixed.10.md @@ -0,0 +1 @@ +A structure save that stopped part way lost a staged drop and re-add of a column with the same name. diff --git a/apple/changelog.d/4226.fixed.2.md b/apple/changelog.d/4226.fixed.2.md new file mode 100644 index 0000000000..7d54755eac --- /dev/null +++ b/apple/changelog.d/4226.fixed.2.md @@ -0,0 +1 @@ +Undoing a cell typed back to its value kept Restore Previous Values from bringing the old value back. diff --git a/apple/changelog.d/4226.fixed.3.md b/apple/changelog.d/4226.fixed.3.md new file mode 100644 index 0000000000..b5c1c10e16 --- /dev/null +++ b/apple/changelog.d/4226.fixed.3.md @@ -0,0 +1 @@ +Cmd+Z after Undo Delete on the row menu deleted the row again. diff --git a/apple/changelog.d/4226.fixed.4.md b/apple/changelog.d/4226.fixed.4.md new file mode 100644 index 0000000000..5f1a83ef1f --- /dev/null +++ b/apple/changelog.d/4226.fixed.4.md @@ -0,0 +1 @@ +Restore Previous Values overwrote changes made to a row after the restore was reviewed. diff --git a/apple/changelog.d/4226.fixed.5.md b/apple/changelog.d/4226.fixed.5.md new file mode 100644 index 0000000000..fe84392d51 --- /dev/null +++ b/apple/changelog.d/4226.fixed.5.md @@ -0,0 +1 @@ +Deleting more than 100 rows in one save failed part way on Cloudflare D1. diff --git a/apple/changelog.d/4226.fixed.6.md b/apple/changelog.d/4226.fixed.6.md new file mode 100644 index 0000000000..3afec1a872 --- /dev/null +++ b/apple/changelog.d/4226.fixed.6.md @@ -0,0 +1 @@ +Pasting rows replaced every pasted primary key with DEFAULT, even a key the server cannot fill. diff --git a/apple/changelog.d/4226.fixed.7.md b/apple/changelog.d/4226.fixed.7.md new file mode 100644 index 0000000000..2676f21e4a --- /dev/null +++ b/apple/changelog.d/4226.fixed.7.md @@ -0,0 +1 @@ +Editing a formatted JSON value in the row inspector or the JSON editor saved it on one line. diff --git a/apple/changelog.d/4226.fixed.8.md b/apple/changelog.d/4226.fixed.8.md new file mode 100644 index 0000000000..8768e3bcd4 --- /dev/null +++ b/apple/changelog.d/4226.fixed.8.md @@ -0,0 +1 @@ +Duplicate on the row menu copied only the first of the selected rows. diff --git a/apple/changelog.d/4226.fixed.9.md b/apple/changelog.d/4226.fixed.9.md new file mode 100644 index 0000000000..0806d5f7df --- /dev/null +++ b/apple/changelog.d/4226.fixed.9.md @@ -0,0 +1 @@ +Pasting CSV removed the spaces inside quoted fields. diff --git a/apple/changelog.d/4226.fixed.md b/apple/changelog.d/4226.fixed.md new file mode 100644 index 0000000000..c025502181 --- /dev/null +++ b/apple/changelog.d/4226.fixed.md @@ -0,0 +1 @@ +Edits typed while a save was running were wiped when the save finished. diff --git a/apple/changelog.d/4227.fixed.md b/apple/changelog.d/4227.fixed.md new file mode 100644 index 0000000000..d6509a0a02 --- /dev/null +++ b/apple/changelog.d/4227.fixed.md @@ -0,0 +1 @@ +Copying, comparing or exporting a large SQLite, Oracle, Teradata, Cassandra or cloud table held all of it in memory. diff --git a/apple/changelog.d/4228.changed.1.md b/apple/changelog.d/4228.changed.1.md new file mode 100644 index 0000000000..049458cd69 --- /dev/null +++ b/apple/changelog.d/4228.changed.1.md @@ -0,0 +1 @@ +On iOS, a PostgreSQL connection opens straight into the database and schema you last used. diff --git a/apple/changelog.d/4228.changed.2.md b/apple/changelog.d/4228.changed.2.md new file mode 100644 index 0000000000..22de99ce4d --- /dev/null +++ b/apple/changelog.d/4228.changed.2.md @@ -0,0 +1 @@ +Large iOS query results use less memory, so more rows fit before the app stops at its memory limit. diff --git a/apple/changelog.d/4228.changed.3.md b/apple/changelog.d/4228.changed.3.md new file mode 100644 index 0000000000..c4c25a1a3e --- /dev/null +++ b/apple/changelog.d/4228.changed.3.md @@ -0,0 +1 @@ +Returning to the iOS app syncs with iCloud at most once a minute. diff --git a/apple/changelog.d/4228.changed.4.md b/apple/changelog.d/4228.changed.4.md new file mode 100644 index 0000000000..4234868ae2 --- /dev/null +++ b/apple/changelog.d/4228.changed.4.md @@ -0,0 +1 @@ +The iOS app starts faster: it no longer rebuilds Spotlight entries, widgets and Shortcuts at every launch. diff --git a/apple/changelog.d/4228.changed.md b/apple/changelog.d/4228.changed.md new file mode 100644 index 0000000000..07f1cb4497 --- /dev/null +++ b/apple/changelog.d/4228.changed.md @@ -0,0 +1 @@ +Opening a table on iOS shows its rows before the row count finishes, and sorting no longer counts the table again. diff --git a/apple/changelog.d/4228.fixed.1.md b/apple/changelog.d/4228.fixed.1.md new file mode 100644 index 0000000000..16d899ae32 --- /dev/null +++ b/apple/changelog.d/4228.fixed.1.md @@ -0,0 +1 @@ +Going past the last page of an iOS table before its row count arrived showed it as empty with a wrong total. diff --git a/apple/changelog.d/4228.fixed.2.md b/apple/changelog.d/4228.fixed.2.md new file mode 100644 index 0000000000..ff7f77f776 --- /dev/null +++ b/apple/changelog.d/4228.fixed.2.md @@ -0,0 +1 @@ +Switching schema on iOS kept the previous schema's row count in the table pager. diff --git a/apple/changelog.d/4228.fixed.md b/apple/changelog.d/4228.fixed.md new file mode 100644 index 0000000000..2e6e2bde06 --- /dev/null +++ b/apple/changelog.d/4228.fixed.md @@ -0,0 +1 @@ +Share Results and Copy Results on iOS froze the screen while they formatted a large result. diff --git a/apple/changelog.d/4229.changed.1.md b/apple/changelog.d/4229.changed.1.md new file mode 100644 index 0000000000..13afcbf59e --- /dev/null +++ b/apple/changelog.d/4229.changed.1.md @@ -0,0 +1 @@ +Switching back to TablePro syncs with iCloud at most once a minute. diff --git a/apple/changelog.d/4229.changed.2.md b/apple/changelog.d/4229.changed.2.md new file mode 100644 index 0000000000..dd119f8563 --- /dev/null +++ b/apple/changelog.d/4229.changed.2.md @@ -0,0 +1 @@ +A theme edited in Finder or another app loads as soon as it is saved. diff --git a/apple/changelog.d/4229.changed.md b/apple/changelog.d/4229.changed.md new file mode 100644 index 0000000000..2465f5018b --- /dev/null +++ b/apple/changelog.d/4229.changed.md @@ -0,0 +1 @@ +Opening a remote database file over SFTP downloads it several times faster on high-latency links. diff --git a/apple/changelog.d/4229.fixed.1.md b/apple/changelog.d/4229.fixed.1.md new file mode 100644 index 0000000000..1c91c84e53 --- /dev/null +++ b/apple/changelog.d/4229.fixed.1.md @@ -0,0 +1 @@ +Every write another app made to a SQLite file reloaded the sidebar, even when the schema stayed the same. diff --git a/apple/changelog.d/4229.fixed.2.md b/apple/changelog.d/4229.fixed.2.md new file mode 100644 index 0000000000..30c8bf9d2e --- /dev/null +++ b/apple/changelog.d/4229.fixed.2.md @@ -0,0 +1 @@ +The Server Dashboard kept querying the server while its window was minimized, hidden or covered. diff --git a/apple/changelog.d/4229.fixed.3.md b/apple/changelog.d/4229.fixed.3.md new file mode 100644 index 0000000000..04050f3906 --- /dev/null +++ b/apple/changelog.d/4229.fixed.3.md @@ -0,0 +1 @@ +Each switch back to TablePro re-read every theme file and ran Git in each linked SQL folder. diff --git a/apple/changelog.d/4229.fixed.4.md b/apple/changelog.d/4229.fixed.4.md new file mode 100644 index 0000000000..c9a3d7f483 --- /dev/null +++ b/apple/changelog.d/4229.fixed.4.md @@ -0,0 +1 @@ +Saving any file in a linked SQL folder re-read every SQL file in it. diff --git a/apple/changelog.d/4229.fixed.5.md b/apple/changelog.d/4229.fixed.5.md new file mode 100644 index 0000000000..29824bd092 --- /dev/null +++ b/apple/changelog.d/4229.fixed.5.md @@ -0,0 +1 @@ +Snowflake leaked two network sessions each time a pooled connection opened. diff --git a/apple/changelog.d/4229.fixed.md b/apple/changelog.d/4229.fixed.md new file mode 100644 index 0000000000..6d24895ce0 --- /dev/null +++ b/apple/changelog.d/4229.fixed.md @@ -0,0 +1 @@ +An open SSH tunnel woke the Mac five times a second while TablePro sat idle. diff --git a/apple/changelog.d/4230.fixed.1.md b/apple/changelog.d/4230.fixed.1.md new file mode 100644 index 0000000000..5a08d62022 --- /dev/null +++ b/apple/changelog.d/4230.fixed.1.md @@ -0,0 +1 @@ +A connection with a credential profile signed in without the profile's credentials when it used a tunnel. diff --git a/apple/changelog.d/4230.fixed.2.md b/apple/changelog.d/4230.fixed.2.md new file mode 100644 index 0000000000..00a761099b --- /dev/null +++ b/apple/changelog.d/4230.fixed.2.md @@ -0,0 +1 @@ +A refused SQL Server login showed a generic message instead of the server's reason. diff --git a/apple/changelog.d/4230.fixed.3.md b/apple/changelog.d/4230.fixed.3.md new file mode 100644 index 0000000000..2612d2ff99 --- /dev/null +++ b/apple/changelog.d/4230.fixed.3.md @@ -0,0 +1 @@ +Quitting TablePro left every Vault database lease alive until it expired. diff --git a/apple/changelog.d/4230.fixed.md b/apple/changelog.d/4230.fixed.md new file mode 100644 index 0000000000..57611a3efa --- /dev/null +++ b/apple/changelog.d/4230.fixed.md @@ -0,0 +1 @@ +A Vault database role leased a new user on every sign-in, so a database slow to load users never accepted one. diff --git a/apple/changelog.d/4231.fixed.1.md b/apple/changelog.d/4231.fixed.1.md new file mode 100644 index 0000000000..db9f1b88db --- /dev/null +++ b/apple/changelog.d/4231.fixed.1.md @@ -0,0 +1 @@ +Scrolling a result with large binary values stuttered. diff --git a/apple/changelog.d/4231.fixed.2.md b/apple/changelog.d/4231.fixed.2.md new file mode 100644 index 0000000000..27614bb7f9 --- /dev/null +++ b/apple/changelog.d/4231.fixed.2.md @@ -0,0 +1 @@ +Moving from row to row with the inspector open was slow. diff --git a/apple/changelog.d/4231.fixed.md b/apple/changelog.d/4231.fixed.md new file mode 100644 index 0000000000..40dc3bec9f --- /dev/null +++ b/apple/changelog.d/4231.fixed.md @@ -0,0 +1 @@ +An ER diagram with hundreds of related tables took seconds to open, reset its layout or switch to Compact Mode. diff --git a/apple/changelog.d/4233.changed.md b/apple/changelog.d/4233.changed.md new file mode 100644 index 0000000000..63fb3e9c5d --- /dev/null +++ b/apple/changelog.d/4233.changed.md @@ -0,0 +1 @@ +TablePro and its plugins are smaller to download and install. diff --git a/apple/changelog.d/4236.fixed.1.md b/apple/changelog.d/4236.fixed.1.md new file mode 100644 index 0000000000..f80b087afc --- /dev/null +++ b/apple/changelog.d/4236.fixed.1.md @@ -0,0 +1 @@ +Vim mode commands such as `x`, `r`, `h` and `l` split emoji and accented letters, and `f` with an emoji closed the app. diff --git a/apple/changelog.d/4236.fixed.2.md b/apple/changelog.d/4236.fixed.2.md new file mode 100644 index 0000000000..30a31e7117 --- /dev/null +++ b/apple/changelog.d/4236.fixed.2.md @@ -0,0 +1 @@ +Vim mode `dl`, `dh` and `cl` did nothing and left the operator waiting for the next key. diff --git a/apple/changelog.d/4236.fixed.md b/apple/changelog.d/4236.fixed.md new file mode 100644 index 0000000000..cc7b33b07c --- /dev/null +++ b/apple/changelog.d/4236.fixed.md @@ -0,0 +1 @@ +An unquoted `begin` column in a one-statement MySQL procedure or event swallowed the rest of the script. diff --git a/apple/changelog.d/4236.security.md b/apple/changelog.d/4236.security.md new file mode 100644 index 0000000000..c1385bbfbe --- /dev/null +++ b/apple/changelog.d/4236.security.md @@ -0,0 +1 @@ +Safe Mode missed a statement after a string glued to a name, such as `x$e'\\'` on PostgreSQL. diff --git a/apple/changelog.d/4240.fixed.md b/apple/changelog.d/4240.fixed.md new file mode 100644 index 0000000000..d3d6899722 --- /dev/null +++ b/apple/changelog.d/4240.fixed.md @@ -0,0 +1 @@ +Remove Invisible Characters changed the text in a string that followed a non-breaking or zero-width space. diff --git a/apple/changelog.d/4242.changed.md b/apple/changelog.d/4242.changed.md new file mode 100644 index 0000000000..256009722f --- /dev/null +++ b/apple/changelog.d/4242.changed.md @@ -0,0 +1 @@ +MySQL, MariaDB, TiDB and OceanBase count an UPDATE's matched rows as affected, unchanged rows included. diff --git a/apple/changelog.d/4242.fixed.md b/apple/changelog.d/4242.fixed.md new file mode 100644 index 0000000000..52614570f2 --- /dev/null +++ b/apple/changelog.d/4242.fixed.md @@ -0,0 +1 @@ +Saving an edit that matched no row reported success on MySQL, MariaDB, SQL Server, Oracle, Snowflake and Dameng. diff --git a/apple/changelog.d/4244.changed.md b/apple/changelog.d/4244.changed.md new file mode 100644 index 0000000000..4e2b7024e5 --- /dev/null +++ b/apple/changelog.d/4244.changed.md @@ -0,0 +1 @@ +Read-Only blocks a query that calls a function which may change data, and names the level that runs it. diff --git a/apple/changelog.d/4244.fixed.1.md b/apple/changelog.d/4244.fixed.1.md new file mode 100644 index 0000000000..4b1d4f0e42 --- /dev/null +++ b/apple/changelog.d/4244.fixed.1.md @@ -0,0 +1 @@ +An Always Allow grant let the assistant's query tool run on a Read-Only connection. diff --git a/apple/changelog.d/4244.fixed.2.md b/apple/changelog.d/4244.fixed.2.md new file mode 100644 index 0000000000..1c35888d89 --- /dev/null +++ b/apple/changelog.d/4244.fixed.2.md @@ -0,0 +1 @@ +INSERT OVERWRITE, which empties the table first, was not treated as a destructive statement. diff --git a/apple/changelog.d/4244.fixed.md b/apple/changelog.d/4244.fixed.md new file mode 100644 index 0000000000..c7c8746842 --- /dev/null +++ b/apple/changelog.d/4244.fixed.md @@ -0,0 +1 @@ +Fetch All ran its query again without Safe Mode's check, so a level raised since the first run did not apply. diff --git a/apple/changelog.d/4244.security.1.md b/apple/changelog.d/4244.security.1.md new file mode 100644 index 0000000000..45675771ba --- /dev/null +++ b/apple/changelog.d/4244.security.1.md @@ -0,0 +1 @@ +Read-Only on iPhone and iPad ran a query that called a function which writes data. diff --git a/apple/changelog.d/4244.security.md b/apple/changelog.d/4244.security.md new file mode 100644 index 0000000000..92bb7c9495 --- /dev/null +++ b/apple/changelog.d/4244.security.md @@ -0,0 +1 @@ +On SQL databases the assistant's Edit mode ran schema, privilege, session and transaction statements, such as GRANT. diff --git a/apple/changelog.d/4245.fixed.1.md b/apple/changelog.d/4245.fixed.1.md new file mode 100644 index 0000000000..b37c874335 --- /dev/null +++ b/apple/changelog.d/4245.fixed.1.md @@ -0,0 +1 @@ +An Elasticsearch export could leave out the documents of a shard that stopped answering, without an error. diff --git a/apple/changelog.d/4245.fixed.md b/apple/changelog.d/4245.fixed.md new file mode 100644 index 0000000000..4d6cc8eb24 --- /dev/null +++ b/apple/changelog.d/4245.fixed.md @@ -0,0 +1 @@ +Copying, comparing or exporting a large Elasticsearch, Trino or MongoDB table held the whole table in memory. diff --git a/apple/changelog.d/4246.added.md b/apple/changelog.d/4246.added.md new file mode 100644 index 0000000000..26e0f0821f --- /dev/null +++ b/apple/changelog.d/4246.added.md @@ -0,0 +1 @@ +Settings > Data > Results kept per run limits how many results with rows a run keeps. diff --git a/apple/changelog.d/4246.changed.1.md b/apple/changelog.d/4246.changed.1.md new file mode 100644 index 0000000000..4592f97d1b --- /dev/null +++ b/apple/changelog.d/4246.changed.1.md @@ -0,0 +1 @@ +A Teradata script that starts with BT runs as its own transaction instead of inside one TablePro opens. diff --git a/apple/changelog.d/4246.changed.md b/apple/changelog.d/4246.changed.md new file mode 100644 index 0000000000..604becf744 --- /dev/null +++ b/apple/changelog.d/4246.changed.md @@ -0,0 +1 @@ +An AI agent's statement over an open Teradata transaction is refused, since a failure would roll it all back. diff --git a/apple/changelog.d/4246.fixed.1.md b/apple/changelog.d/4246.fixed.1.md new file mode 100644 index 0000000000..b6eaa8ac14 --- /dev/null +++ b/apple/changelog.d/4246.fixed.1.md @@ -0,0 +1 @@ +Tabbing through a binary field in the inspector without changing it marked the row as edited. diff --git a/apple/changelog.d/4246.fixed.2.md b/apple/changelog.d/4246.fixed.2.md new file mode 100644 index 0000000000..8714393345 --- /dev/null +++ b/apple/changelog.d/4246.fixed.2.md @@ -0,0 +1 @@ +An MCP client's insert_rows reply never said that Manual Commit held the inserted rows. diff --git a/apple/changelog.d/4246.fixed.md b/apple/changelog.d/4246.fixed.md new file mode 100644 index 0000000000..dff1c4e2e9 --- /dev/null +++ b/apple/changelog.d/4246.fixed.md @@ -0,0 +1 @@ +Moving from row to row with the inspector open was slow with text, JSON, binary, image or geometry fields. diff --git a/apple/changelog.d/4247.changed.md b/apple/changelog.d/4247.changed.md new file mode 100644 index 0000000000..abcaf1023f --- /dev/null +++ b/apple/changelog.d/4247.changed.md @@ -0,0 +1 @@ +Closing, disconnecting, quitting or relaunching with an open transaction now offers Commit as well as Roll Back. diff --git a/apple/changelog.d/4247.fixed.1.md b/apple/changelog.d/4247.fixed.1.md new file mode 100644 index 0000000000..b4409789be --- /dev/null +++ b/apple/changelog.d/4247.fixed.1.md @@ -0,0 +1 @@ +On Teradata, Run All failed on the statement after a DDL statement instead of stopping and asking you to commit. diff --git a/apple/changelog.d/4247.fixed.2.md b/apple/changelog.d/4247.fixed.2.md new file mode 100644 index 0000000000..8066cf29cd --- /dev/null +++ b/apple/changelog.d/4247.fixed.2.md @@ -0,0 +1 @@ +On Teradata, every statement after an uncommitted change was refused, a typed COMMIT included. diff --git a/apple/changelog.d/4247.fixed.3.md b/apple/changelog.d/4247.fixed.3.md new file mode 100644 index 0000000000..3abb4c2ee0 --- /dev/null +++ b/apple/changelog.d/4247.fixed.3.md @@ -0,0 +1 @@ +On Teradata, saving two staged table drops failed, and Truncate sent a TRUNCATE statement Teradata does not have. diff --git a/apple/changelog.d/4247.fixed.4.md b/apple/changelog.d/4247.fixed.4.md new file mode 100644 index 0000000000..164b9cbe2d --- /dev/null +++ b/apple/changelog.d/4247.fixed.4.md @@ -0,0 +1 @@ +Closing a connection under Manual Commit rolled back the edits you had just chosen to save. diff --git a/apple/changelog.d/4247.fixed.5.md b/apple/changelog.d/4247.fixed.5.md new file mode 100644 index 0000000000..3d1661f393 --- /dev/null +++ b/apple/changelog.d/4247.fixed.5.md @@ -0,0 +1 @@ +A TablePro opened only to serve MCP kept running after MCP refused it. diff --git a/apple/changelog.d/4247.fixed.md b/apple/changelog.d/4247.fixed.md new file mode 100644 index 0000000000..6aff4835cf --- /dev/null +++ b/apple/changelog.d/4247.fixed.md @@ -0,0 +1 @@ +On Teradata ANSI sessions, structure edits, new tables, renames and drops were left uncommitted and could be lost. diff --git a/apple/changelog.d/4248.fixed.1.md b/apple/changelog.d/4248.fixed.1.md new file mode 100644 index 0000000000..ce7f934492 --- /dev/null +++ b/apple/changelog.d/4248.fixed.1.md @@ -0,0 +1 @@ +A connection through an SSH tunnel could wait half a second for a reply while others were open. diff --git a/apple/changelog.d/4248.fixed.2.md b/apple/changelog.d/4248.fixed.2.md new file mode 100644 index 0000000000..96ced0ec41 --- /dev/null +++ b/apple/changelog.d/4248.fixed.2.md @@ -0,0 +1 @@ +An SSH tunnel could stall every connection through it after a keep-alive met a full network buffer. diff --git a/apple/changelog.d/4248.fixed.md b/apple/changelog.d/4248.fixed.md new file mode 100644 index 0000000000..37d521498d --- /dev/null +++ b/apple/changelog.d/4248.fixed.md @@ -0,0 +1 @@ +An open SSH tunnel woke TablePro twice a second for each connection and kept macOS from napping it. diff --git a/apple/changelog.d/4249.fixed.md b/apple/changelog.d/4249.fixed.md new file mode 100644 index 0000000000..d5c62eb041 --- /dev/null +++ b/apple/changelog.d/4249.fixed.md @@ -0,0 +1 @@ +A SQLite over SSH connection set to open the database on the server never connected. diff --git a/apple/changelog.d/4252.changed.md b/apple/changelog.d/4252.changed.md new file mode 100644 index 0000000000..9c8c26082e --- /dev/null +++ b/apple/changelog.d/4252.changed.md @@ -0,0 +1 @@ +On iOS, reopening a MySQL, MariaDB or TiDB connection on the database you last used takes fewer round trips. diff --git a/apple/changelog.d/4254.fixed.1.md b/apple/changelog.d/4254.fixed.1.md new file mode 100644 index 0000000000..480bf46a22 --- /dev/null +++ b/apple/changelog.d/4254.fixed.1.md @@ -0,0 +1 @@ +Remove Invisible Characters left a non-ASCII space between the two parts of a PostgreSQL string, so the query failed. diff --git a/apple/changelog.d/4254.fixed.md b/apple/changelog.d/4254.fixed.md new file mode 100644 index 0000000000..e9eb48b304 --- /dev/null +++ b/apple/changelog.d/4254.fixed.md @@ -0,0 +1 @@ +A MySQL routine with a `SET` or `IF` body naming a `begin` column swallowed the rest of the script on Run All. diff --git a/apple/changelog.d/4254.security.md b/apple/changelog.d/4254.security.md new file mode 100644 index 0000000000..56d30d1b2d --- /dev/null +++ b/apple/changelog.d/4254.security.md @@ -0,0 +1 @@ +Safe Mode and MCP missed a PostgreSQL statement hidden after a `BEGIN ATOMIC` body that selects `begin atomic`. diff --git a/apple/changelog.d/4255.added.md b/apple/changelog.d/4255.added.md new file mode 100644 index 0000000000..9cbf52dd22 --- /dev/null +++ b/apple/changelog.d/4255.added.md @@ -0,0 +1 @@ +SQL import and the query editor load the rows of a PostgreSQL COPY ... FROM STDIN, as psql sends them. diff --git a/apple/changelog.d/4257.changed.md b/apple/changelog.d/4257.changed.md new file mode 100644 index 0000000000..4fb4eef962 --- /dev/null +++ b/apple/changelog.d/4257.changed.md @@ -0,0 +1 @@ +iOS Read-Only runs PostgreSQL, MySQL and MariaDB queries in a read-only transaction, so the server refuses their writes. diff --git a/apple/changelog.d/4260.changed.md b/apple/changelog.d/4260.changed.md new file mode 100644 index 0000000000..8872a8fe57 --- /dev/null +++ b/apple/changelog.d/4260.changed.md @@ -0,0 +1 @@ +Read-Only and Edit mode treat Redis module and new data-type reads, and Elasticsearch POST reads, as reads. diff --git a/apple/changelog.d/4260.security.1.md b/apple/changelog.d/4260.security.1.md new file mode 100644 index 0000000000..438ef2350d --- /dev/null +++ b/apple/changelog.d/4260.security.1.md @@ -0,0 +1 @@ +Read-Only blocks an Oracle or Dameng function called without parentheses, and a SurrealDB fn:: call. diff --git a/apple/changelog.d/4260.security.md b/apple/changelog.d/4260.security.md new file mode 100644 index 0000000000..e133908a67 --- /dev/null +++ b/apple/changelog.d/4260.security.md @@ -0,0 +1 @@ +AI Edit mode refuses schema, privilege and session commands on MongoDB, Redis, Neo4j and every other engine. diff --git a/apple/changelog.d/4262.changed.1.md b/apple/changelog.d/4262.changed.1.md new file mode 100644 index 0000000000..86f415c409 --- /dev/null +++ b/apple/changelog.d/4262.changed.1.md @@ -0,0 +1 @@ +Read-Only runs Oracle MATCH_RECOGNIZE queries that use PREV, NEXT, FIRST, LAST, CLASSIFIER or MATCH_NUMBER. diff --git a/apple/changelog.d/4262.changed.md b/apple/changelog.d/4262.changed.md new file mode 100644 index 0000000000..e2e666f0a6 --- /dev/null +++ b/apple/changelog.d/4262.changed.md @@ -0,0 +1 @@ +Deleting one document, row or key by its ID is a write, so AI Edit mode and MCP's execute_query can run it. diff --git a/apple/changelog.d/4262.fixed.1.md b/apple/changelog.d/4262.fixed.1.md new file mode 100644 index 0000000000..33e66e9af5 --- /dev/null +++ b/apple/changelog.d/4262.fixed.1.md @@ -0,0 +1 @@ +An etcd del with a range end or --from-key deleted only its first key. diff --git a/apple/changelog.d/4262.fixed.md b/apple/changelog.d/4262.fixed.md new file mode 100644 index 0000000000..23dc692d20 --- /dev/null +++ b/apple/changelog.d/4262.fixed.md @@ -0,0 +1 @@ +Deleting a Weaviate row in the grid also deleted objects with the same ID in other collections. diff --git a/apple/changelog.d/4262.security.1.md b/apple/changelog.d/4262.security.1.md new file mode 100644 index 0000000000..fb17948cce --- /dev/null +++ b/apple/changelog.d/4262.security.1.md @@ -0,0 +1 @@ +A Server Dashboard tile runs in the schema Read-Only checked, after any statement a query tab is running. diff --git a/apple/changelog.d/4262.security.2.md b/apple/changelog.d/4262.security.2.md new file mode 100644 index 0000000000..2f3e142407 --- /dev/null +++ b/apple/changelog.d/4262.security.2.md @@ -0,0 +1 @@ +Replacing a Realtime Database top-level node, or an escaped Memcached flush_all, now counts as destructive. diff --git a/apple/changelog.d/4262.security.md b/apple/changelog.d/4262.security.md new file mode 100644 index 0000000000..75e50bb03f --- /dev/null +++ b/apple/changelog.d/4262.security.md @@ -0,0 +1 @@ +AI Edit mode refuses writes into ScyllaDB's audit log and Kafka topics of Schema Registry, Connect or Streams. diff --git a/apple/changelog.d/4263.changed.md b/apple/changelog.d/4263.changed.md new file mode 100644 index 0000000000..783b896f14 --- /dev/null +++ b/apple/changelog.d/4263.changed.md @@ -0,0 +1 @@ +The SQL import progress bar follows how much of the file has been read, so it moves while a large COPY block loads. diff --git a/apple/changelog.d/4264.fixed.1.md b/apple/changelog.d/4264.fixed.1.md new file mode 100644 index 0000000000..1f4b615872 --- /dev/null +++ b/apple/changelog.d/4264.fixed.1.md @@ -0,0 +1 @@ +Read-Only refused every query while a transaction was open, as on MySQL and MariaDB with autocommit off. diff --git a/apple/changelog.d/4264.fixed.md b/apple/changelog.d/4264.fixed.md new file mode 100644 index 0000000000..6908cd22c4 --- /dev/null +++ b/apple/changelog.d/4264.fixed.md @@ -0,0 +1 @@ +Read-Only failed every query on MySQL before 5.6.5 and MariaDB before 10.0. diff --git a/apple/changelog.d/4265.fixed.1.md b/apple/changelog.d/4265.fixed.1.md new file mode 100644 index 0000000000..b70cf7fbee --- /dev/null +++ b/apple/changelog.d/4265.fixed.1.md @@ -0,0 +1 @@ +TablePro could quit when the SQLite session on an SSH server ended while its connection stayed open. diff --git a/apple/changelog.d/4265.fixed.2.md b/apple/changelog.d/4265.fixed.2.md new file mode 100644 index 0000000000..fde3bf2565 --- /dev/null +++ b/apple/changelog.d/4265.fixed.2.md @@ -0,0 +1 @@ +A query on a SQLite file on an SSH server could wait forever after the session on the server ended. diff --git a/apple/changelog.d/4265.fixed.md b/apple/changelog.d/4265.fixed.md new file mode 100644 index 0000000000..538172a465 --- /dev/null +++ b/apple/changelog.d/4265.fixed.md @@ -0,0 +1 @@ +Copying, comparing or exporting a large table from a SQLite file on an SSH server held the whole table in memory. diff --git a/apple/changelog.d/4266.changed.1.md b/apple/changelog.d/4266.changed.1.md new file mode 100644 index 0000000000..6c71167c7a --- /dev/null +++ b/apple/changelog.d/4266.changed.1.md @@ -0,0 +1 @@ +On iOS, changes from your other devices arrive within seconds while the app is open or iOS wakes it. diff --git a/apple/changelog.d/4266.changed.md b/apple/changelog.d/4266.changed.md new file mode 100644 index 0000000000..12b7a29586 --- /dev/null +++ b/apple/changelog.d/4266.changed.md @@ -0,0 +1 @@ +With iCloud sync on, changes from your other devices arrive within seconds while TablePro runs, in the background too. diff --git a/apple/changelog.d/4278.fixed.md b/apple/changelog.d/4278.fixed.md new file mode 100644 index 0000000000..d0403e47fe --- /dev/null +++ b/apple/changelog.d/4278.fixed.md @@ -0,0 +1 @@ +Copying, comparing or exporting a SQLite or libSQL table kept other programs from writing the file until it ended. diff --git a/apple/changelog.d/4279.fixed.1.md b/apple/changelog.d/4279.fixed.1.md new file mode 100644 index 0000000000..b52a9b633b --- /dev/null +++ b/apple/changelog.d/4279.fixed.1.md @@ -0,0 +1 @@ +A read error while previewing or importing a SQL file quit TablePro instead of failing the import. diff --git a/apple/changelog.d/4279.fixed.md b/apple/changelog.d/4279.fixed.md new file mode 100644 index 0000000000..8c95d60df1 --- /dev/null +++ b/apple/changelog.d/4279.fixed.md @@ -0,0 +1 @@ +Importing a large SQL file held memory the size of the file while counting its statements or loading COPY rows. diff --git a/apple/project.yml b/apple/project.yml index 41c0ec9abe..482c603ffd 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -103,6 +103,7 @@ targetTemplates: COMBINE_HIDPI_IMAGES: YES DEVELOPMENT_TEAM: $(TABLEPRO_DEVELOPMENT_TEAM) ENABLE_HARDENED_RUNTIME: YES + EXPORTED_SYMBOLS_FILE: $(SRCROOT)/Configs/NoExportedSymbols.exp GENERATE_INFOPLIST_FILE: YES INFOPLIST_FILE: Plugins/${folder}/Info.plist INFOPLIST_KEY_NSPrincipalClass: $(PRODUCT_MODULE_NAME).${principalClass} @@ -188,7 +189,7 @@ targetTemplates: PostgreSQLLinkage: dependencies: - package: TableProCore - products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary] + products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary, TableProTLSClientIdentity] settings: base: HEADER_SEARCH_PATHS: @@ -245,9 +246,11 @@ targetTemplates: # Links its own SQLite (scripts/build-sqlite.sh), never -lsqlite3: macOS's library is built without # extension loading. SQLiteLinkage: + sources: + - Plugins/SharedSpill dependencies: - package: TableProCore - products: [CSQLite, TableProSQLGrammar, TableProSQLiteCore, TableProSQLVocabulary] + products: [CSQLite, TableProSQLGrammar, TableProSQLiteCore, TableProSQLVocabulary, TableProWireIO] settings: base: OTHER_LDFLAGS: @@ -310,6 +313,9 @@ targets: - path: TablePro/Resources/ThirdPartyLicenses type: folder buildPhase: resources + # The MCP client catalog, one file the Linux app reads too. + - path: ../core/crates/mcp/clients.json + buildPhase: resources - path: TablePro/CLI/tablepro buildPhase: copyFiles: @@ -318,6 +324,9 @@ targets: # The etcd plugin's own command parser, so the app tiers an etcd statement by the operation # the driver parses it into. - Plugins/EtcdDriverPlugin/EtcdCommandParser.swift + # The Kafka plugin's KafkaQL parser, so the app reads a KafkaQL statement as the driver does. + - Packages/TableProCore/Sources/TableProKafkaCore/KafkaQL.swift + - Packages/TableProCore/Sources/TableProKafkaCore/KafkaStatement.swift # The MySQL plugin's socket key and path rules, so the app writes and checks the socket path # exactly as the driver reads it. - Plugins/MySQLDriverPlugin/MySQLLocalSocket.swift @@ -412,9 +421,9 @@ targets: - package: Yams - package: PostHog - package: TableProCore - products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] + products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProMemcachedCore, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTLSClientIdentity, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] - package: TableProMacCore - products: [TableProCloudAPI, TableProMCP, TableProSSHConfig, TableProUtilities] + products: [TableProCloudAPI, TableProMCP, TableProMCPClients, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. postCompileScripts: - name: Write WhatsNew.md @@ -474,6 +483,8 @@ targets: Release: CODE_SIGN_ENTITLEMENTS: TablePro/TablePro.entitlements COPY_PHASE_STRIP: YES + # Debug keeps its exports: the test bundle links against the app. + EXPORTED_SYMBOLS_FILE: $(SRCROOT)/Configs/NoExportedSymbols.exp # ─── Plugin ABI ─────────────────────────────────────────────────────────────── @@ -518,11 +529,14 @@ targets: - TablePro/CLI/BridgeMain.swift dependencies: - package: TableProMacCore - products: [TableProMCP, TableProUtilities] + product: TableProMCP + - package: TableProCore + product: TableProProcess scheme: {} settings: base: ENABLE_HARDENED_RUNTIME: YES + EXPORTED_SYMBOLS_FILE: $(SRCROOT)/Configs/NoExportedSymbols.exp PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.tablepro-mcp # The app embeds it; installed on its own it turns the app archive into a generic one. SKIP_INSTALL: YES @@ -544,6 +558,7 @@ targets: settings: base: ENABLE_HARDENED_RUNTIME: YES + EXPORTED_SYMBOLS_FILE: $(SRCROOT)/Configs/NoExportedSymbols.exp PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.tablepro-cli # The app embeds it; installed on its own it turns the app archive into a generic one. SKIP_INSTALL: YES @@ -826,9 +841,12 @@ targets: - path: Plugins/SQLExportPlugin excludes: - Info.plist + # The agent the app launches over SSH, so the agent backend's tests run the shipped program. + - TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift + - TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift dependencies: - package: TableProCore - products: [TableProProcess, TableProSQLGrammar] + products: [TableProProcess, TableProSQLGrammar, TableProTestSupport] ClickHouseDriverTests: templates: [PluginTests, ClickHouseLinkage] @@ -1012,7 +1030,7 @@ targets: principalClass: DynamoDBPlugin dependencies: - package: TableProCore - product: TableProNumberFormatting + products: [TableProNumberFormatting, TableProWireIO] DamengDriver: templates: [DriverPlugin] @@ -1122,6 +1140,8 @@ targets: templateAttributes: folder: LibSQLDriverPlugin principalClass: LibSQLPlugin + sources: + - Plugins/SharedSpill dependencies: - package: TableProCore product: CSQLite @@ -1129,6 +1149,8 @@ targets: product: TableProSQLGrammar - package: TableProCore product: TableProSQLiteCore + - package: TableProCore + product: TableProWireIO settings: base: OTHER_LDFLAGS: @@ -1319,6 +1341,8 @@ targets: product: TableProGoogleCloud - package: TableProCore product: TableProFirebaseCore + - package: TableProCore + product: TableProFirebaseClient - package: TableProCore product: TableProLogRedaction @@ -1330,6 +1354,8 @@ targets: dependencies: - package: TableProCore product: TableProQdrantCore + - package: TableProCore + product: TableProQdrantClient - package: TableProCore product: TableProTLSClientIdentity @@ -1341,6 +1367,8 @@ targets: dependencies: - package: TableProCore product: TableProWeaviateCore + - package: TableProCore + product: TableProWeaviateClient InfluxDBDriverPlugin: templates: [DriverPlugin] diff --git a/apple/scripts/ci/check-entitlements.py b/apple/scripts/ci/check-entitlements.py index b4064bb42d..c055c8a1c3 100755 --- a/apple/scripts/ci/check-entitlements.py +++ b/apple/scripts/ci/check-entitlements.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -"""Check both apps pin the Production CloudKit environment. +"""Check both apps pin the Production CloudKit environment and can receive CloudKit's change notices. -A signed build without the key talks to Development, which the shipped schema never reaches. +A signed build without the environment key talks to Development, which the shipped schema never reaches. +A build without the push key never hears of another device's change. The iOS file holds the development +value Xcode writes, and the App Store export signs it as production. """ from __future__ import annotations @@ -10,20 +12,22 @@ import sys from pathlib import Path -KEY = "com.apple.developer.icloud-container-environment" -ENTITLEMENTS = ( - "TablePro/TablePro.entitlements", - "TableProMobile/TableProMobile/TableProMobileRelease.entitlements", +ENVIRONMENT_KEY = "com.apple.developer.icloud-container-environment" +EXPECTED = ( + ("TablePro/TablePro.entitlements", ENVIRONMENT_KEY, "Production"), + ("TablePro/TablePro.entitlements", "com.apple.developer.aps-environment", "production"), + ("TableProMobile/TableProMobile/TableProMobileRelease.entitlements", ENVIRONMENT_KEY, "Production"), + ("TableProMobile/TableProMobile/TableProMobileRelease.entitlements", "aps-environment", "development"), ) def main() -> int: root = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path(__file__).resolve().parents[2] status = 0 - for relative in ENTITLEMENTS: - value = plistlib.loads((root / relative).read_bytes()).get(KEY) - if value != "Production": - print(f"{relative}: {KEY} is {value!r}, expected 'Production'") + for relative, key, expected in EXPECTED: + value = plistlib.loads((root / relative).read_bytes()).get(key) + if value != expected: + print(f"{relative}: {key} is {value!r}, expected {expected!r}") status = 1 return status diff --git a/apple/scripts/ci/check-string-catalogs.py b/apple/scripts/ci/check-string-catalogs.py index 38dc17624d..9cb91e5e68 100755 --- a/apple/scripts/ci/check-string-catalogs.py +++ b/apple/scripts/ci/check-string-catalogs.py @@ -16,6 +16,7 @@ CATALOGS = { "TablePro": "TablePro/Resources/Localizable.xcstrings", "TableProMobile": "TableProMobile/TableProMobile/Localizable.xcstrings", + "TableProWidget": "TableProMobile/TableProWidget/Localizable.xcstrings", } TERMINAL_MARKS = {":": (":", ":"), "?": ("?", "?"), ".": (".", "。")} # SQL notation, written the way SQL documentation writes it, not a UI ellipsis. diff --git a/apple/scripts/localization.py b/apple/scripts/localization.py index 2c3b5ee71e..97515daf9f 100755 --- a/apple/scripts/localization.py +++ b/apple/scripts/localization.py @@ -10,6 +10,9 @@ scripts/localization.py plugins --target ios -> the same for the package targets the iOS app links scripts/localization.py extracted [--add] -> strings the app's last build extracted, missing +`--target widget` names the iOS widget extension's own catalog, for `status` and `extracted`. It +links no package, so `plugins` has nothing to check there. + Xcode extracts strings per target. Code in a plugin bundle or in a package under `Packages/` resolves `String(localized:)` against `Bundle.main`, which is the host app, so its strings must be in the app's catalog, yet no sync of the app target ever finds them in its sources. Xcode deletes @@ -35,13 +38,16 @@ CATALOGS = { "mac": Path("TablePro/Resources/Localizable.xcstrings"), "ios": Path("TableProMobile/TableProMobile/Localizable.xcstrings"), + "widget": Path("TableProMobile/TableProWidget/Localizable.xcstrings"), } +PLUGIN_TARGETS = ("mac", "ios") HIDDEN_SOURCE_ROOTS = (Path("Plugins"), Path("Packages")) IOS_PROJECT = Path("TableProMobile/project.yml") # The project, its scheme, and the targets whose sources hold the catalog. APP_BUILDS = { "mac": ("TablePro.xcodeproj", "TablePro", ("TablePro",)), - "ios": ("TableProMobile/TableProMobile.xcodeproj", "TableProMobile", ("TableProMobile", "TableProWidgetExtension")), + "ios": ("TableProMobile/TableProMobile.xcodeproj", "TableProMobile", ("TableProMobile",)), + "widget": ("TableProMobile/TableProMobile.xcodeproj", "TableProMobile", ("TableProWidgetExtension",)), } # Xcode writes a catalog through JSONSerialization with exactly these options. Writing through the @@ -332,6 +338,8 @@ def main() -> int: parser.add_argument("--derived-data", type=Path, help="for extracted: the DerivedData folder of the app's build") args = parser.parse_args() + if args.command == "plugins" and args.target not in PLUGIN_TARGETS: + parser.error(f"plugins checks {' and '.join(PLUGIN_TARGETS)}: the {args.target} target links no package") if args.command == "plugins": return plugins(args.add, args.target) if args.command == "extracted": diff --git a/apple/scripts/probes/check-bare-function-names.sh b/apple/scripts/probes/check-bare-function-names.sh new file mode 100755 index 0000000000..0f4cf05036 --- /dev/null +++ b/apple/scripts/probes/check-bare-function-names.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Check the name resolution SQLBareNameCalls hard-codes against a real Oracle and DM8 server. +# +# Oracle and Dameng run a function named without parentheses once the name is not a column of a table in scope. +# Packages/TableProCore/Sources/TableProSQLGrammar/SQLBareNameCalls+Parser.swift reads which tables a name can see +# (correlated subqueries see the outer query, inline views do not, ORDER BY sees select-list aliases) and treats a +# dotted name no table starts as a call. This runs each shape and reports whether the function ran. +# +# Usage: +# scripts/probes/check-bare-function-names.sh oracle # gvenzl/oracle-free:23-slim-faststart, APP_USER probe +# scripts/probes/check-bare-function-names.sh dameng # a DM8 image, SYSDBA password in DM8_PASSWORD +# +# Creates PURGE, WIPE, package PK, tables ORDERS and AUDIT_LOG in the probe schema. Each line prints the query and +# `calls` or `runs` (the function did not run); compare with SQLBareNameCallsTests. + +set -uo pipefail + +ENGINE="${1:?oracle or dameng}" +CONTAINER="${2:?container name}" + +case "$ENGINE" in + oracle) + run_sql() { docker exec -i "$CONTAINER" sqlplus -s probe/probe_pw@FREEPDB1; } + ;; + dameng) + PASSWORD="${DM8_PASSWORD:?set DM8_PASSWORD}" + run_sql() { + docker exec -i "$CONTAINER" bash -c "cat > /tmp/tp-probe.sql && cd /home/dmdba/dmdbms/bin \ + && ./disql -S SYSDBA/$PASSWORD@localhost:5236 \\\`/tmp/tp-probe.sql" + } + ;; + *) + echo "unknown engine: $ENGINE" >&2 + exit 3 + ;; +esac + +run_sql > /dev/null <<'SQL' +CREATE TABLE orders (id NUMBER, purge NUMBER); +INSERT INTO orders VALUES (1, 100); +CREATE TABLE audit_log (n NUMBER); +COMMIT; +CREATE OR REPLACE FUNCTION purge RETURN NUMBER IS PRAGMA AUTONOMOUS_TRANSACTION; BEGIN INSERT INTO audit_log VALUES (1); COMMIT; RETURN 42; END; +/ +CREATE OR REPLACE FUNCTION wipe RETURN NUMBER IS PRAGMA AUTONOMOUS_TRANSACTION; BEGIN INSERT INTO audit_log VALUES (2); COMMIT; RETURN 7; END; +/ +CREATE OR REPLACE PACKAGE pk AS FUNCTION f RETURN NUMBER; END; +/ +CREATE OR REPLACE PACKAGE BODY pk AS FUNCTION f RETURN NUMBER IS PRAGMA AUTONOMOUS_TRANSACTION; BEGIN INSERT INTO audit_log VALUES (3); COMMIT; RETURN 3; END; END; +/ +EXIT +SQL + +while IFS= read -r query; do + out="$(printf 'DELETE FROM audit_log;\nCOMMIT;\n%s\nSELECT '"'"'audit='"'"' || COUNT(*) FROM audit_log;\nEXIT\n' "$query" | run_sql 2>&1)" + if printf '%s' "$out" | grep -q 'audit=0'; then verdict=runs; else verdict=calls; fi + printf '%-6s %s\n' "$verdict" "$query" +done <<'QUERIES' +SELECT purge FROM orders; +SELECT purge FROM dual; +SELECT id FROM orders WHERE EXISTS (SELECT 1 FROM dual WHERE purge = 100); +SELECT * FROM orders, (SELECT purge FROM dual); +SELECT 1 AS wipe FROM dual ORDER BY wipe; +SELECT 1 AS wipe FROM dual WHERE wipe = 7; +SELECT 1 wipe FROM dual GROUP BY wipe; +SELECT pk.f FROM dual; +SELECT pk.f FROM orders pk; +WITH c AS (SELECT 1 wipe FROM dual) SELECT wipe FROM c; +SELECT purge FROM (SELECT * FROM orders); +SELECT "WIPE" FROM dual; +SELECT id, purge FROM orders UNION ALL SELECT 1, purge FROM dual; +SELECT * FROM (VALUES (wipe)) t (a); +SELECT * FROM orders, (VALUES (purge)) v (a); +SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id ALL ROWS PER MATCH PATTERN (x) DEFINE x AS wipe = 7); +SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id ALL ROWS PER MATCH PATTERN (x) DEFINE x AS purge = 100); +QUERIES diff --git a/apple/scripts/probes/check-column-type-corpus.sh b/apple/scripts/probes/check-column-type-corpus.sh index 88797c1898..fd2bad9ed0 100755 --- a/apple/scripts/probes/check-column-type-corpus.sh +++ b/apple/scripts/probes/check-column-type-corpus.sh @@ -116,7 +116,7 @@ case "$ENGINE" in FIELDS="oracleServiceName=FREEPDB1" ;; sqlite) - FOLDER=SQLiteDriverPlugin CLASS=SQLitePlugin PRODUCTS="CSQLite TableProSQLiteCore" + FOLDER=SQLiteDriverPlugin CLASS=SQLitePlugin PRODUCTS="CSQLite TableProSQLiteCore TableProWireIO" LINK="$ROOT/Libs/libsqlite3_vendored.a" ;; *) @@ -186,6 +186,7 @@ link_sources corpus "$PLUGINS/ColumnTypeCorpus" link_sources csv "$PLUGINS/CSVExportPlugin" link_sources json "$PLUGINS/JSONExportPlugin" link_sources sql "$PLUGINS/SQLExportPlugin" +[ "$FOLDER" = SQLiteDriverPlugin ] && link_sources spill "$PLUGINS/SharedSpill" for shared in PluginConnectTimeout PluginQueryTimeout PluginURLSessionTaskBox; do ln -s "$PLUGINS/Shared/$shared.swift" "$SOURCES/shared_$shared.swift" done diff --git a/apple/scripts/probes/check-freetds-login-messages.sh b/apple/scripts/probes/check-freetds-login-messages.sh new file mode 100755 index 0000000000..22dd78d255 --- /dev/null +++ b/apple/scripts/probes/check-freetds-login-messages.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# Check how FreeTDS db-lib reports a refused login to the SQL Server driver, through the shipped Libs/libsybdb.a +# and the stub headers in Plugins/MSSQLDriverPlugin/CFreeTDS that the driver compiles against. +# +# db-lib hands the message and error handlers the DBPROCESS it is still building, not NULL, and frees it when the +# login fails. FreeTDSConnection filed those messages under that handle, so a failed open read an empty reason and +# reported only "Check host, port, credentials, and TLS settings", and nothing could tell a refused login apart. +# freetdsRecordMessage now sends an error for a handle with no gate yet to the open's own reason, first one first. +# A 4060 (the database cannot be opened) arrives before its 18456, so the reason stays the 4060. Each server message +# is followed by db-lib's 20018 notice, which the driver skips (MSSQLLibraryError.serverMessageNotice). +# +# No SQL Server or Docker: freetds-login-messages/fake_login.py answers PRELOGIN and refuses LOGIN7 with the given +# ERROR tokens, and freetds-login-messages/client.c logs in the way the driver does. Run it after a FreeTDS bump. +# +# Usage: +# scripts/probes/check-freetds-login-messages.sh +# +# Exits 0 when db-lib answers as expected, 1 when it differs, 3 when it cannot run. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HELPERS="$ROOT/scripts/probes/freetds-login-messages" +HEADERS="$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS/include" +DYLIBS="$ROOT/Libs/dylibs" +WORK="$(mktemp -d)" +SERVER_PID="" +trap '[ -z "$SERVER_PID" ] || { kill "$SERVER_PID"; wait "$SERVER_PID"; } 2> /dev/null; rm -rf "$WORK"' EXIT + +cannot_run() { + echo "$1" >&2 + exit 3 +} + +command -v python3 > /dev/null 2>&1 || cannot_run "no python3" +command -v cc > /dev/null 2>&1 || cannot_run "no cc" + +LIB="" +for candidate in "$ROOT/Libs/libsybdb.a" "$ROOT/Libs/libsybdb_$(uname -m).a"; do + if [ -f "$candidate" ]; then + LIB="$candidate" + break + fi +done +[ -n "$LIB" ] || cannot_run "not found: Libs/libsybdb.a (run scripts/download-libs.sh)" +for dylib in libssl.3.dylib libcrypto.3.dylib; do + [ -f "$DYLIBS/$dylib" ] || cannot_run "not found: Libs/dylibs/$dylib (run scripts/download-libs.sh)" +done + +# The MSSQLDriver target's link line in project.yml. +if ! cc -I "$HEADERS" "$HELPERS/client.c" "$LIB" -L "$DYLIBS" -lssl.3 -lcrypto.3 -liconv -framework GSS -lcom_err \ + -Wl,-rpath,"$DYLIBS" -o "$WORK/client" > "$WORK/cc.log" 2>&1; then + cat "$WORK/cc.log" >&2 + cannot_run "client.c failed to build" +fi + +python3 -I "$HELPERS/fake_login.py" \ + "4060:11:Cannot open database \"inventory\" requested by the login. The login failed." \ + "18456:14:Login failed for user 'v-token-ro-1'." \ + > "$WORK/port" 2> "$WORK/server.log" & +SERVER_PID=$! +PORT="" +for _ in $(seq 1 50); do + PORT="$(head -n 1 "$WORK/port")" + [ -n "$PORT" ] && break + sleep 0.1 +done +if [ -z "$PORT" ]; then + cat "$WORK/server.log" >&2 + cannot_run "fake_login.py never reported a port" +fi + +printf '[fake]\n\thost = 127.0.0.1\n\tport = %s\n\ttds version = 7.4\n\tconnect timeout = 5\n\tencryption = off\n' \ + "$PORT" > "$WORK/freetds.conf" +"$WORK/client" "$WORK/freetds.conf" fake > "$WORK/actual" 2> "$WORK/client.log" + +cat > "$WORK/expected" << 'EXPECTED' +msg proc=SET 4060 sev 11: Cannot open database "inventory" requested by the login. The login failed. +err proc=SET dberr=20018: General SQL Server error: Check messages from the SQL Server +msg proc=SET 18456 sev 14: Login failed for user 'v-token-ro-1'. +err proc=SET dberr=20018: General SQL Server error: Check messages from the SQL Server +err proc=SET dberr=20002: Adaptive Server connection failed (fake) +dbopen=NULL +EXPECTED + +cat "$WORK/actual" +if diff -u "$WORK/expected" "$WORK/actual" > "$WORK/diff"; then + echo "OK: db-lib reports the login refusal as expected" + exit 0 +fi + +echo "" >&2 +echo "FAIL: db-lib's answers differ (- expected, + read):" >&2 +tail -n +3 "$WORK/diff" >&2 +cat "$WORK/client.log" "$WORK/server.log" >&2 +exit 1 diff --git a/apple/scripts/probes/check-libssh2-keepalive-pending.sh b/apple/scripts/probes/check-libssh2-keepalive-pending.sh new file mode 100755 index 0000000000..3b0466c0c5 --- /dev/null +++ b/apple/scripts/probes/check-libssh2-keepalive-pending.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# Shows why LibSSH2KeepAlive sends only into a socket with room, against the vendored libssh2 and a +# real OpenSSH server. +# +# libssh2_keepalive_send builds its packet on its own stack (keepalive.c) and counts a send that would +# block as sent. libssh2 keeps a packet it could not send whole and refuses every other send until a call +# with the same buffer address finishes it (transport_send_existing in transport.c). So a keep-alive that +# blocks stops every channel write on the session, the next keep-alive comes only an interval later, and +# from another stack frame it finishes nothing. The probe makes one send would-block through libssh2's +# send callback, then counts the channel writes refused while the socket is writable. Rerun after a +# libssh2 bump: exit 1 means the keep-alive no longer blocks the session and the guard can go. +# +# Usage: +# scripts/probes/check-libssh2-keepalive-pending.sh +# +# Needs Docker. Starts a throwaway OpenSSH container and removes it on exit. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../lib/openssh-fixture.sh +source "$ROOT/scripts/lib/openssh-fixture.sh" + +CONTAINER="tp-keepalive-pending" +PORT="${TABLEPRO_KEEPALIVE_PENDING_PORT:-22024}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"; docker rm -f -v "$CONTAINER" > /dev/null 2>&1 || true' EXIT + +docker rm -f -v "$CONTAINER" > /dev/null 2>&1 || true +docker run -d --name "$CONTAINER" -p "127.0.0.1:$PORT:2222" \ + -e USER_NAME=tp -e USER_PASSWORD=tppass -e PASSWORD_ACCESS=true -e SUDO_ACCESS=false \ + "$OPENSSH_FIXTURE_IMAGE_10" > /dev/null +openssh_wait_for_banner 127.0.0.1 "$PORT" 60 || { echo "no SSH banner within 60s" >&2; exit 2; } + +cat > "$WORK/probe.c" << 'PROBE' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "libssh2.h" + +static int refuse_next_send = 0; + +static LIBSSH2_SEND_FUNC(gated_send) { + (void)abstract; + if (refuse_next_send) { + refuse_next_send = 0; + return -EAGAIN; + } + ssize_t rc = send(socket, buffer, length, flags); + return rc < 0 ? -errno : rc; +} + +static double now(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return t.tv_sec + t.tv_nsec / 1e9; +} + +__attribute__((noinline)) static int keepalive_here(LIBSSH2_SESSION *s) { + int next = 0; + return libssh2_keepalive_send(s, &next); +} + +__attribute__((noinline)) static int keepalive_deeper(LIBSSH2_SESSION *s) { + volatile char pad[512]; + pad[0] = 1; + return keepalive_here(s) + pad[0] - 1; +} + +static int refused_writes(LIBSSH2_CHANNEL *c, int sock, double seconds, int *accepted) { + const char *line = "ping\n"; + int refused = 0; + double until = now() + seconds; + *accepted = 0; + while (now() < until) { + ssize_t rc = libssh2_channel_write(c, line, strlen(line)); + if (rc > 0) { *accepted = 1; return refused; } + if (rc != LIBSSH2_ERROR_EAGAIN) { printf("write failed %zd\n", rc); exit(2); } + refused++; + struct pollfd p = { sock, POLLOUT, 0 }; + if (poll(&p, 1, 100) <= 0) { puts("socket not writable"); exit(2); } + } + return refused; +} + +int main(int argc, char **argv) { + int port = atoi(argv[1]); + libssh2_init(0); + int sock = socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_in a; + memset(&a, 0, sizeof(a)); + a.sin_family = AF_INET; + a.sin_port = htons((unsigned short)port); + a.sin_addr.s_addr = inet_addr("127.0.0.1"); + if (connect(sock, (struct sockaddr *)&a, sizeof(a))) { puts("tcp connect failed"); exit(2); } + LIBSSH2_SESSION *s = libssh2_session_init(); + if (libssh2_session_handshake(s, sock)) { puts("handshake failed"); exit(2); } + if (libssh2_userauth_password(s, "tp", "tppass")) { puts("auth failed"); exit(2); } + LIBSSH2_CHANNEL *c = libssh2_channel_open_session(s); + if (!c || libssh2_channel_exec(c, "cat")) { puts("could not start cat"); exit(2); } + libssh2_session_callback_set2(s, LIBSSH2_CALLBACK_SEND, (libssh2_cb_generic *)gated_send); + libssh2_session_set_blocking(s, 0); + libssh2_keepalive_config(s, 1, 2); + sleep(3); + + refuse_next_send = 1; + int rc = keepalive_here(s); + int accepted; + int blocked = refused_writes(c, sock, 3.0, &accepted); + printf("keep-alive whose send would block: rc %d; channel writes refused over 3 s with the socket writable: %d\n", rc, blocked); + int pending = blocked > 0 && !accepted; + + sleep(2); + rc = keepalive_deeper(s); + int refusedAfterOther = refused_writes(c, sock, 1.0, &accepted); + printf("after the next keep-alive from another stack frame (rc %d): refused %d, accepted %d\n", rc, refusedAfterOther, accepted); + int stillPending = refusedAfterOther > 0 && !accepted; + + sleep(2); + rc = keepalive_here(s); + int refusedAfterSame = refused_writes(c, sock, 1.0, &accepted); + printf("after a keep-alive from the first frame (rc %d): refused %d, accepted %d\n", rc, refusedAfterSame, accepted); + libssh2_exit(); + return pending && stillPending && accepted ? 0 : 1; +} +PROBE + +clang -w -arch arm64 -o "$WORK/probe" "$WORK/probe.c" \ + -I "$ROOT/TablePro/Core/SSH/CLibSSH2/include" \ + "$ROOT/Libs/libssh2_arm64.a" "$ROOT/Libs/libssl_arm64.a" "$ROOT/Libs/libcrypto_arm64.a" \ + -lz -framework Security -framework CoreFoundation + +"$WORK/probe" "$PORT" diff --git a/apple/scripts/probes/check-match-recognize-calls.sh b/apple/scripts/probes/check-match-recognize-calls.sh new file mode 100755 index 0000000000..5aa5b6e57e --- /dev/null +++ b/apple/scripts/probes/check-match-recognize-calls.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# Check that MATCH_RECOGNIZE's navigation calls never reach a user function of the same name, which +# Packages/TableProCore/Sources/TableProSQLGrammar/SQLReadOnlyFunctionCalls.swift (RowPatternClauses) and +# SQLBareNameCalls+Parser.swift (the row pattern clause) rely on, and that the same names outside the clause do. +# +# Usage: +# scripts/probes/check-match-recognize-calls.sh oracle # gvenzl/oracle-free:23-slim-faststart, APP_USER probe +# scripts/probes/check-match-recognize-calls.sh dameng # a DM8 image, SYSDBA password in DM8_PASSWORD +# +# Creates functions PREV, NEXT, FIRST, LAST, CLASSIFIER, MATCH_NUMBER, RUNNING, FINAL and WIPE, each writing a row, +# and tables ORDERS and AUDIT_LOG. Each line prints the query and `calls` (a user function ran), `runs` (none ran) +# or `error` (the server refused the query); compare with SQLReadOnlyFunctionCallsTests and SQLBareNameCallsTests. + +set -uo pipefail + +ENGINE="${1:?oracle or dameng}" +CONTAINER="${2:?container name}" + +case "$ENGINE" in + oracle) + run_sql() { docker exec -i "$CONTAINER" sqlplus -s probe/probe_pw@FREEPDB1; } + ;; + dameng) + PASSWORD="${DM8_PASSWORD:?set DM8_PASSWORD}" + run_sql() { + docker exec -i "$CONTAINER" bash -c "cat > /tmp/tp-probe.sql && cd /home/dmdba/dmdbms/bin \ + && ./disql -S SYSDBA/$PASSWORD@localhost:5236 \\\`/tmp/tp-probe.sql" + } + ;; + *) + echo "unknown engine: $ENGINE" >&2 + exit 3 + ;; +esac + +{ + echo "CREATE TABLE orders (id NUMBER, purge NUMBER);" + echo "INSERT INTO orders VALUES (1, 100);" + echo "INSERT INTO orders VALUES (2, 50);" + echo "INSERT INTO orders VALUES (3, 80);" + echo "CREATE TABLE audit_log (n NUMBER);" + echo "COMMIT;" + for name in prev next first last; do + printf 'CREATE OR REPLACE FUNCTION %s(v NUMBER) RETURN NUMBER IS PRAGMA AUTONOMOUS_TRANSACTION; BEGIN INSERT INTO audit_log VALUES (1); COMMIT; RETURN v; END;\n/\n' "$name" + done + for name in classifier match_number running final wipe; do + printf 'CREATE OR REPLACE FUNCTION %s RETURN NUMBER IS PRAGMA AUTONOMOUS_TRANSACTION; BEGIN INSERT INTO audit_log VALUES (2); COMMIT; RETURN 7; END;\n/\n' "$name" + done + echo "EXIT" +} | run_sql > /dev/null + +MR="SELECT * FROM orders MATCH_RECOGNIZE (ORDER BY id" + +while IFS= read -r query; do + out="$(printf 'DELETE FROM audit_log;\nCOMMIT;\n%s\nSELECT '"'"'audit='"'"' || COUNT(*) FROM audit_log;\nEXIT\n' "$query" | run_sql 2>&1)" + if printf '%s' "$out" | grep -qE 'ORA-|\[-[0-9]+\]'; then + verdict=error + elif printf '%s' "$out" | grep -q 'audit=0'; then + verdict=runs + else + verdict=calls + fi + printf '%-6s %s\n' "$verdict" "$query" +done < 0); +$MR MEASURES CLASSIFIER() AS c, MATCH_NUMBER() AS m ALL ROWS PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES FINAL LAST(x.purge) AS l, RUNNING COUNT(*) AS c ALL ROWS PER MATCH PATTERN (x+) DEFINE x AS 1 = 1); +$MR MEASURES RUNNING LAST(x.purge) AS l ALL ROWS PER MATCH PATTERN (x+) DEFINE x AS RUNNING COUNT(*) < 9); +$MR MEASURES x.id AS wipe ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES wipe AS w ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES wipe() AS w ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES x.id AS i ONE ROW PER MATCH PATTERN (x) DEFINE x AS wipe = 7); +$MR MEASURES x.id AS i ONE ROW PER MATCH PATTERN (PERMUTE(x, y)) DEFINE x AS x.id = 1, y AS y.id = 2); +$MR MEASURES x.id AS i ONE ROW PER MATCH AFTER MATCH SKIP TO LAST y PATTERN (x y) DEFINE x AS 1 = 1, y AS 1 = 1); +$MR MEASURES x.id AS i ONE ROW PER MATCH PATTERN (x) DEFINE x AS x.purge > (SELECT prev(1) FROM dual)); +$MR MEASURES x.id AS i ALL ROWS PER MATCH PATTERN (^ x {- y -} z* $) DEFINE x AS PREV(x.purge, 1) IS NULL, y AS ABS(PREV(y.purge)) > 0, z AS 1 = 1); +SELECT * FROM orders MATCH_RECOGNIZE (PARTITION BY prev(purge) ORDER BY id MEASURES x.id AS i ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES x.id AS subset, wipe AS w ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES x.id AS define, wipe AS w ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +$MR MEASURES x.id AS i ONE ROW PER MATCH PATTERN (x) DEFINE x AS x.purge > ((SELECT 1 FROM dual WHERE 1 = 0) UNION ALL SELECT prev(1) FROM dual)); +$MR MEASURES x.id AS i ONE ROW PER MATCH PATTERN (x) DEFINE x AS EXISTS ((SELECT 1 FROM dual WHERE 1 = 0) UNION ALL SELECT prev(1) FROM dual)); +SELECT * FROM (SELECT id, purge, 1 AS subset FROM orders) MATCH_RECOGNIZE (ORDER BY id MEASURES x.subset + wipe AS m ONE ROW PER MATCH PATTERN (x) DEFINE x AS 1 = 1); +QUERIES diff --git a/apple/scripts/probes/check-mssql-keyed-row-counts.sh b/apple/scripts/probes/check-mssql-keyed-row-counts.sh new file mode 100755 index 0000000000..09e1a9fa3b --- /dev/null +++ b/apple/scripts/probes/check-mssql-keyed-row-counts.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# +# Check that the count SQL Server reports for a grid save's UPDATE and DELETE is the rows the statement matched. +# +# MSSQLStatementGenerator tells the host each update and delete matches one row (PluginRowWrite.expectedRowCount), and +# the host then fails a save whose count is higher, or lower for an UPDATE. That is only right while these hold, and +# no unit test can see them because each depends on what the server sends and how db-lib reads it: +# +# - An UPDATE that writes the value a row already holds still counts the row. +# - An UPDATE or DELETE that matches nothing counts 0. +# - A session left at SET NOCOUNT ON still reports the count, through the SET NOCOUNT OFF the statement carries. +# - Rows an AFTER trigger writes are not added, with or without SET NOCOUNT ON inside the trigger, and an +# INSTEAD OF trigger leaves the statement's own count. +# +# So this builds a harness from the real Plugins/MSSQLDriverPlugin sources, the TableProCore package and the shipped +# Libs/libsybdb.a, writes each statement with MSSQLStatementGenerator, runs it through MSSQLPluginDriver, and fails +# when a count differs. Verified against SQL Server 2022 CU14 with FreeTDS 1.4.22. +# +# Usage: +# MSSQL_SA_PASSWORD=secret scripts/probes/check-mssql-keyed-row-counts.sh [host] [port] [user] [database] +# +# The database defaults to tablepro_row_count_check and is created when missing. Exits 1 when a check fails, 3 when it +# cannot run. + +set -uo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-1433}" +USER_NAME="${3:-sa}" +DATABASE="${4:-tablepro_row_count_check}" +PASSWORD="${MSSQL_SA_PASSWORD:-}" + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +[ -f "$ROOT/Libs/libsybdb.a" ] || { + echo "not found: Libs/libsybdb.a (run scripts/download-libs.sh)" >&2 + exit 3 +} +[ -n "$PASSWORD" ] || { + echo "no password: set MSSQL_SA_PASSWORD for the server at $HOST:$PORT" >&2 + exit 3 +} +nc -z "$HOST" "$PORT" > /dev/null 2>&1 || { + echo "no SQL Server listening at $HOST:$PORT" >&2 + exit 3 +} + +mkdir -p "$WORK/Sources/Check" +ln -s "$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS" "$WORK/CFreeTDS" +# The plugin target's sources: its folder plus the shared files the DriverPlugin template in project.yml adds. +for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift "$ROOT"/Plugins/Shared/PluginConnectTimeout.swift \ + "$ROOT"/Plugins/Shared/PluginQueryTimeout.swift "$ROOT"/Plugins/Shared/PluginURLSessionTaskBox.swift \ + "$ROOT"/Plugins/Shared/PluginRedirectRefusingDelegate.swift; do + ln -s "$source" "$WORK/Sources/Check/$(basename "$source")" +done + +cat > "$WORK/Package.swift" << MANIFEST +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "MSSQLRowCountCheck", + platforms: [.macOS(.v14)], + dependencies: [.package(path: "$ROOT/Packages/TableProCore")], + targets: [ + .systemLibrary(name: "CFreeTDS", path: "CFreeTDS"), + .executableTarget( + name: "Check", + dependencies: [ + "CFreeTDS", + .product(name: "TableProPluginKit", package: "TableProCore"), + .product(name: "TableProCoreTypes", package: "TableProCore"), + .product(name: "TableProMSSQLCore", package: "TableProCore"), + .product(name: "TableProLogRedaction", package: "TableProCore"), + .product(name: "TableProKerberos", package: "TableProCore"), + .product(name: "TableProSQLQuoting", package: "TableProCore"), + .product(name: "TableProSQLVocabulary", package: "TableProCore"), + ], + path: "Sources/Check", + swiftSettings: [.swiftLanguageMode(.v6)], + linkerSettings: [.unsafeFlags([ + "-L$ROOT/Libs", "-L$ROOT/Libs/dylibs", "-lsybdb", "-lssl.3", "-lcrypto.3", "-liconv", + "-framework", "GSS", "-lcom_err", "-Xlinker", "-rpath", "-Xlinker", "$ROOT/Libs/dylibs", + ])] + ), + ] +) +MANIFEST + +cat > "$WORK/Sources/Check/Check.swift" << 'SWIFT' +import Foundation +import TableProPluginKit + +@main +enum Check { + nonisolated(unsafe) static var failures = 0 + + static let database = ProcessInfo.processInfo.environment["TP_CHECK_DATABASE"] ?? "tablepro_row_count_check" + static let columns = ["id", "a"] + + static func expect(_ condition: Bool, _ label: String, _ detail: @autoclosure () -> String = "") { + if condition { + print("PASS: \(label)") + } else { + failures += 1 + print("FAIL: \(label) \(detail())") + } + } + + static func connect(_ database: String) async throws -> MSSQLPluginDriver { + let environment = ProcessInfo.processInfo.environment + let config = DriverConnectionConfig( + host: environment["TP_CHECK_HOST"] ?? "127.0.0.1", + port: Int(environment["TP_CHECK_PORT"] ?? "") ?? 1433, + username: environment["TP_CHECK_USER"] ?? "sa", + password: environment["TP_CHECK_PASSWORD"] ?? "", + database: database + ) + let driver = MSSQLPluginDriver(config: config) + try await driver.connect() + return driver + } + + static func main() async { + setvbuf(stdout, nil, _IOLBF, 0) + let driver: MSSQLPluginDriver + do { + let admin = try await connect("master") + _ = try await admin.execute(query: "IF DB_ID(N'\(database)') IS NULL CREATE DATABASE [\(database)]") + admin.disconnect() + driver = try await connect(database) + } catch { + print("cannot connect: \(error)") + exit(3) + } + do { + try await run(driver) + } catch { + failures += 1 + print("FAIL: unexpected error \(error)") + } + print(failures == 0 ? "OK: every check passed" : "\(failures) check(s) failed") + exit(failures == 0 ? 0 : 1) + } + + static func update(id: String, to value: String) -> PluginRowChange { + PluginRowChange( + rowIndex: 0, type: .update, + cellChanges: [(columnIndex: 1, columnName: "a", oldValue: .text(value), newValue: .text(value))], + originalRow: [.text(id), .text(value)] + ) + } + + static func delete(id: String) -> PluginRowChange { + PluginRowChange(rowIndex: 0, type: .delete, cellChanges: [], originalRow: [.text(id), .text("1")]) + } + + static func write(_ change: PluginRowChange, keyed: Bool) throws -> PluginRowWrite { + let generator = MSSQLStatementGenerator( + qualifiedTable: "[dbo].[rc_t]", columns: columns, primaryKeyColumns: keyed ? ["id"] : [] + ) + let writes = try generator.rowWrites( + for: [change], insertedRowData: [:], + deletedRowIndices: change.type == .delete ? [0] : [], insertedRowIndices: [] + ) + guard let write = writes.first else { throw CocoaError(.featureUnsupported) } + return write + } + + static func count(_ write: PluginRowWrite, on driver: MSSQLPluginDriver) async throws -> Int { + try await driver.executeParameterized(query: write.statement, parameters: write.parameters).rowsAffected + } + + static func reset(_ driver: MSSQLPluginDriver, trigger: String?) async throws { + _ = try await driver.execute(query: "IF OBJECT_ID(N'dbo.rc_t') IS NOT NULL DROP TABLE dbo.rc_t") + _ = try await driver.execute(query: "IF OBJECT_ID(N'dbo.rc_audit') IS NOT NULL DROP TABLE dbo.rc_audit") + _ = try await driver.execute(query: "CREATE TABLE dbo.rc_t (id INT PRIMARY KEY, a INT)") + _ = try await driver.execute(query: "CREATE TABLE dbo.rc_audit (n INT)") + _ = try await driver.execute(query: "INSERT INTO dbo.rc_t VALUES (1, 1), (2, 1), (3, 1)") + if let trigger { _ = try await driver.execute(query: trigger) } + } + + static func checkCounts(_ driver: MSSQLPluginDriver, _ label: String) async throws { + let same = try write(update(id: "1", to: "1"), keyed: true) + expect(same.expectedRowCount == 1, "\(label): a keyed update says it matches one row") + let sameCount = try await count(same, on: driver) + expect(sameCount == 1, "\(label): a keyed update writing the value the row holds counts 1", "got \(sameCount)") + let missCount = try await count(try write(update(id: "99", to: "1"), keyed: true), on: driver) + expect(missCount == 0, "\(label): a keyed update that matches nothing counts 0", "got \(missCount)") + let deleteCount = try await count(try write(delete(id: "2"), keyed: true), on: driver) + expect(deleteCount == 1, "\(label): a keyed delete counts 1", "got \(deleteCount)") + let goneCount = try await count(try write(delete(id: "2"), keyed: true), on: driver) + expect(goneCount == 0, "\(label): a keyed delete of a row already gone counts 0", "got \(goneCount)") + let keylessCount = try await count(try write(update(id: "3", to: "1"), keyed: false), on: driver) + expect(keylessCount == 1, "\(label): a keyless update writing the value the row holds counts 1", "got \(keylessCount)") + } + + static func run(_ driver: MSSQLPluginDriver) async throws { + try await reset(driver, trigger: nil) + try await checkCounts(driver, "no trigger") + + try await reset(driver, trigger: nil) + _ = try await driver.execute(query: "SET NOCOUNT ON") + try await checkCounts(driver, "session at SET NOCOUNT ON") + let options = try await driver.execute(query: "SELECT @@OPTIONS & 512") + expect(options.rows.first?.first?.asText == "512", "the session is still at SET NOCOUNT ON afterwards") + _ = try await driver.execute(query: "SET NOCOUNT OFF") + + // A keyless row of NULLs set to DEFAULT binds nothing, so the statement runs in its own sp_executesql. + _ = try await driver.execute(query: "IF OBJECT_ID(N'dbo.rc_k') IS NOT NULL DROP TABLE dbo.rc_k") + _ = try await driver.execute(query: "CREATE TABLE dbo.rc_k (id INT NULL, a INT NULL)") + _ = try await driver.execute(query: "INSERT INTO dbo.rc_k VALUES (NULL, NULL)") + var toDefault = PluginRowChange( + rowIndex: 0, type: .update, + cellChanges: [(columnIndex: 1, columnName: "a", oldValue: .null, newValue: .null)], + originalRow: [.null, .null] + ) + toDefault.defaultColumns = [1] + let unbound = try MSSQLStatementGenerator(qualifiedTable: "[dbo].[rc_k]", columns: columns, primaryKeyColumns: []) + .rowWrites(for: [toDefault], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: []) + guard let unboundWrite = unbound.first else { throw CocoaError(.featureUnsupported) } + expect(unboundWrite.parameters.isEmpty && unboundWrite.statement.hasPrefix("EXEC sp_executesql"), + "a statement binding nothing runs in its own sp_executesql", unboundWrite.statement) + _ = try await driver.execute(query: "SET NOCOUNT ON") + let unboundCount = try await count(unboundWrite, on: driver) + expect(unboundCount == 1, "a statement binding nothing counts 1 under SET NOCOUNT ON", "got \(unboundCount)") + _ = try await driver.execute(query: "SET NOCOUNT OFF") + _ = try await driver.execute(query: "DROP TABLE dbo.rc_k") + + let audit = "INSERT INTO dbo.rc_audit SELECT id FROM inserted UNION ALL SELECT id FROM deleted" + try await reset(driver, trigger: "CREATE TRIGGER dbo.rc_trg ON dbo.rc_t AFTER UPDATE, DELETE AS \(audit)") + try await checkCounts(driver, "AFTER trigger") + let audited = try await driver.execute(query: "SELECT COUNT(*) FROM dbo.rc_audit") + expect(audited.rows.first?.first?.asText != "0", "AFTER trigger: the trigger wrote its rows") + + try await reset( + driver, + trigger: "CREATE TRIGGER dbo.rc_trg ON dbo.rc_t AFTER UPDATE, DELETE AS BEGIN SET NOCOUNT ON; \(audit) END" + ) + try await checkCounts(driver, "AFTER trigger at SET NOCOUNT ON") + + try await reset(driver, trigger: """ + CREATE TRIGGER dbo.rc_trg ON dbo.rc_t INSTEAD OF UPDATE AS \ + UPDATE t SET a = i.a FROM dbo.rc_t t JOIN inserted i ON i.id = t.id + """) + let insteadCount = try await count(try write(update(id: "1", to: "1"), keyed: true), on: driver) + expect(insteadCount == 1, "INSTEAD OF trigger: a keyed update counts 1", "got \(insteadCount)") + + _ = try await driver.execute(query: "DROP TABLE dbo.rc_t") + _ = try await driver.execute(query: "DROP TABLE dbo.rc_audit") + driver.disconnect() + } +} +SWIFT + +export DEVELOPER_DIR="${DEVELOPER_DIR:-$(xcode-select -p)}" +swift build --package-path "$WORK" --scratch-path "$WORK/.build" > "$WORK/build.log" 2>&1 || { + echo "the check failed to build" >&2 + grep -E "error:" "$WORK/build.log" >&2 + exit 3 +} + +TP_CHECK_HOST="$HOST" TP_CHECK_PORT="$PORT" TP_CHECK_USER="$USER_NAME" TP_CHECK_PASSWORD="$PASSWORD" \ + TP_CHECK_DATABASE="$DATABASE" "$WORK/.build/debug/Check" diff --git a/apple/scripts/probes/check-mysql-dial-database-refusal.sh b/apple/scripts/probes/check-mysql-dial-database-refusal.sh new file mode 100755 index 0000000000..6d29d47761 --- /dev/null +++ b/apple/scripts/probes/check-mysql-dial-database-refusal.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# +# Check that a MySQL-family server refuses the database a handshake names exactly as it refuses a USE there, and keeps +# a failed sign-in apart from both. +# +# The iOS open dials a MySQL, MariaDB or TiDB connection straight into the database it was last on, and falls back to +# the saved one in the same open only when the dial is refused with 1044 or 1049 (MySQLDatabaseRefusal). That is right +# only while: +# +# - a database the user may not use, or that is missing, is refused with 1044 (ER_DBACCESS_DENIED_ERROR) or 1049 +# (ER_BAD_DB_ERROR), with the same code and text a USE there gets after the sign-in; +# - a wrong password or an unknown user is refused with 1045 (ER_ACCESS_DENIED_ERROR) whatever the database names, +# so the sign-in is checked first and never depends on the database. +# +# It builds a client from the shipped Libs/libmariadb.a, creates the users dr_plain (one database) and dr_wide (global +# SELECT) and the databases dr_kept and dr_locked, and drops them again. +# +# Verified against MySQL 8.4.13, MariaDB 11.4.13 and TiDB 8.5.1 with libmariadb 3.4.4. +# +# Usage: +# MYSQL_PWD=secret scripts/probes/check-mysql-dial-database-refusal.sh [host] [port] [admin user] +# +# The admin password is read from MYSQL_PWD only. TLS is required unless the host is a loopback address. Exits 1 when +# an answer differs, 3 when it cannot run. + +set -uo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-3306}" +ADMIN="${3:-root}" + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +[ -f "$ROOT/Libs/libmariadb.a" ] || { + echo "not found: Libs/libmariadb.a (run scripts/download-libs.sh)" >&2 + exit 3 +} + +# Connector/C dials localhost through the Unix socket, which a server in a container does not share. +[ "$HOST" = localhost ] && HOST=127.0.0.1 +LOOPBACK=0 +case "$HOST" in + 127.*|::1) LOOPBACK=1 ;; +esac + +cat > "$WORK/check.c" << 'C' +#include +#include +#include +#include + +static const char *host; +static unsigned port; +static int loopback; +static int failures = 0; + +struct answer { + unsigned code; + char text[512]; +}; + +static MYSQL *dial(const char *user, const char *password, const char *database, struct answer *answer) { + MYSQL *m = mysql_init(NULL); + if (loopback) { + my_bool off = 0; + mysql_options(m, MYSQL_OPT_SSL_ENFORCE, &off); + mysql_options(m, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, &off); + } + mysql_options(m, MYSQL_SET_CHARSET_NAME, "utf8mb4"); + if (mysql_real_connect(m, host, user, password, database, port, NULL, 0)) { + answer->code = 0; + answer->text[0] = 0; + return m; + } + answer->code = mysql_errno(m); + snprintf(answer->text, sizeof answer->text, "%s", mysql_error(m)); + mysql_close(m); + return NULL; +} + +static void use(MYSQL *m, const char *database, struct answer *answer) { + char statement[256]; + snprintf(statement, sizeof statement, "USE `%s`", database); + if (mysql_query(m, statement) == 0) { + answer->code = 0; + answer->text[0] = 0; + return; + } + answer->code = mysql_errno(m); + snprintf(answer->text, sizeof answer->text, "%s", mysql_error(m)); +} + +static void admin(MYSQL *m, const char *sql) { + if (mysql_query(m, sql) != 0) { + printf("FAIL: %s: %s\n", sql, mysql_error(m)); + failures++; + } +} + +static void expect_code(const struct answer *answer, unsigned expected, const char *label) { + if (answer->code == expected) { + printf("PASS: %s (%u %s)\n", label, answer->code, answer->text); + } else { + failures++; + printf("FAIL: %s (expected %u, got %u %s)\n", label, expected, answer->code, answer->text); + } +} + +// The dial's refusal and the USE's are one answer: same code, same text. +static void expect_same_as_use(const char *user, const char *password, const char *database, unsigned expected, + const char *label) { + struct answer dialed, used, signed_in; + MYSQL *unexpected = dial(user, password, database, &dialed); + if (unexpected) mysql_close(unexpected); + expect_code(&dialed, expected, label); + MYSQL *m = dial(user, password, NULL, &signed_in); + if (!m) { + failures++; + printf("FAIL: %s cannot sign in without a database: %u %s\n", user, signed_in.code, signed_in.text); + return; + } + use(m, database, &used); + mysql_close(m); + if (used.code == dialed.code && strcmp(used.text, dialed.text) == 0) { + printf("PASS: a USE there answers the same\n"); + } else { + failures++; + printf("FAIL: a USE there answers %u %s\n", used.code, used.text); + } +} + +int main(int argc, char **argv) { + (void)argc; + host = argv[1]; + port = (unsigned)atoi(argv[2]); + loopback = atoi(argv[4]); + + struct answer answer; + MYSQL *a = dial(argv[3], getenv("MYSQL_PWD"), NULL, &answer); + if (!a) { + fprintf(stderr, "cannot connect as %s: %u %s\n", argv[3], answer.code, answer.text); + return 3; + } + printf("server %s, libmariadb %s\n", mysql_get_server_info(a), mysql_get_client_info()); + const char *cleanup[] = { + "DROP USER IF EXISTS 'dr_plain'@'%'", "DROP USER IF EXISTS 'dr_wide'@'%'", + "DROP DATABASE IF EXISTS dr_kept", "DROP DATABASE IF EXISTS dr_locked", "DROP DATABASE IF EXISTS dr_missing", + }; + for (size_t i = 0; i < sizeof cleanup / sizeof *cleanup; i++) admin(a, cleanup[i]); + admin(a, "CREATE DATABASE dr_kept"); + admin(a, "CREATE DATABASE dr_locked"); + admin(a, "CREATE USER 'dr_plain'@'%' IDENTIFIED BY 'dr-probe-Pass1'"); + admin(a, "GRANT ALL ON dr_kept.* TO 'dr_plain'@'%'"); + admin(a, "CREATE USER 'dr_wide'@'%' IDENTIFIED BY 'dr-probe-Pass1'"); + admin(a, "GRANT SELECT ON *.* TO 'dr_wide'@'%'"); + + MYSQL *kept = dial("dr_plain", "dr-probe-Pass1", "dr_kept", &answer); + expect_code(&answer, 0, "a granted database opens in the handshake"); + if (kept) mysql_close(kept); + + expect_same_as_use("dr_plain", "dr-probe-Pass1", "dr_locked", 1044, "a database without a grant is refused with 1044"); + expect_same_as_use("dr_plain", "dr-probe-Pass1", "dr_missing", 1044, + "a missing database is refused with 1044 to a user without global privileges"); + expect_same_as_use("dr_wide", "dr-probe-Pass1", "dr_missing", 1049, + "a missing database is refused with 1049 to a user with global privileges"); + + const char *databases[] = { "dr_kept", "dr_locked", "dr_missing" }; + for (size_t i = 0; i < 3; i++) { + char label[128]; + MYSQL *unexpected = dial("dr_plain", "wrong-password", databases[i], &answer); + if (unexpected) mysql_close(unexpected); + snprintf(label, sizeof label, "a wrong password is refused with 1045 when the dial names %s", databases[i]); + expect_code(&answer, 1045, label); + unexpected = dial("dr_wide", "wrong-password", databases[i], &answer); + if (unexpected) mysql_close(unexpected); + snprintf(label, sizeof label, "and for a user with global privileges, naming %s", databases[i]); + expect_code(&answer, 1045, label); + } + MYSQL *nobody = dial("dr_nobody", "wrong-password", "dr_missing", &answer); + if (nobody) mysql_close(nobody); + expect_code(&answer, 1045, "an unknown user is refused with 1045 when the dial names a missing database"); + + for (size_t i = 0; i < sizeof cleanup / sizeof *cleanup; i++) admin(a, cleanup[i]); + mysql_close(a); + + printf(failures == 0 ? "OK: every check passed\n" : "%d check(s) failed\n", failures); + return failures == 0 ? 0 : 1; +} +C + +clang -O0 -o "$WORK/check" "$WORK/check.c" -I"$ROOT/Plugins/MySQLDriverPlugin/CMariaDB/include" "$ROOT/Libs/libmariadb.a" \ + -L"$ROOT/Libs/dylibs" -lssl.3 -lcrypto.3 -liconv -lz -framework GSS -Wl,-rpath,"$ROOT/Libs/dylibs" \ + > "$WORK/build.log" 2>&1 || { + echo "the check failed to build" >&2 + cat "$WORK/build.log" >&2 + exit 3 +} + +"$WORK/check" "$HOST" "$PORT" "$ADMIN" "$LOOPBACK" diff --git a/apple/scripts/probes/check-mysql-found-rows.sh b/apple/scripts/probes/check-mysql-found-rows.sh new file mode 100755 index 0000000000..30f477d1bb --- /dev/null +++ b/apple/scripts/probes/check-mysql-found-rows.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# +# Check that a MySQL-family session opened with CLIENT_FOUND_ROWS counts the rows an UPDATE matched. +# +# The MySQL driver connects with CLIENT_FOUND_ROWS (MySQLTransport.clientFlags), and DataWriteRowCounts then holds a +# keyed UPDATE to the one row it was written for: a count of 0 fails the save. That is only right while the server +# counts a row that already held the value, which a session without the flag does not. No unit test can see it, so +# this builds a client from the shipped Libs/libmariadb.a and measures, with and without the flag: +# +# - an UPDATE writing the value the row holds counts 1 (0 without the flag), as text and as a prepared statement, +# and ROW_COUNT() agrees; +# - an UPDATE that matches nothing counts 0; +# - INSERT ... ON DUPLICATE KEY UPDATE onto a row holding those values counts 1; +# - rows an AFTER UPDATE trigger writes are not added (skipped on a server with no triggers, as TiDB). +# +# Verified against MySQL 8.4.13, MariaDB 11.4.13, TiDB 8.5.1 and OceanBase 4.4.2.1 with libmariadb 3.4.4. +# +# Usage: +# MYSQL_PWD=secret scripts/probes/check-mysql-found-rows.sh [host] [port] [user] [database] +# +# The password is read from MYSQL_PWD only, never from an argument. TLS is required unless the host is a loopback +# address. It creates and drops the tables fr_check and fr_check_audit in the database, test by default. Exits 1 when +# a count differs, 3 when it cannot run. + +set -uo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-3306}" +USER_NAME="${3:-root}" +DATABASE="${4:-test}" + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +[ -f "$ROOT/Libs/libmariadb.a" ] || { + echo "not found: Libs/libmariadb.a (run scripts/download-libs.sh)" >&2 + exit 3 +} + +LOOPBACK=0 +case "$HOST" in + 127.*|localhost|::1) LOOPBACK=1 ;; +esac + +cat > "$WORK/check.c" << 'C' +#include +#include +#include +#include + +static int failures = 0; + +static void expect(long long actual, long long expected, const char *label) { + if (actual == expected) { + printf("PASS: %s\n", label); + } else { + failures++; + printf("FAIL: %s (expected %lld, got %lld)\n", label, expected, actual); + } +} + +static long long affected(MYSQL *m, const char *sql) { + if (mysql_query(m, sql) != 0) { + printf("FAIL: %s: %s\n", sql, mysql_error(m)); + failures++; + return -1; + } + return (long long)mysql_affected_rows(m); +} + +static long long scalar(MYSQL *m, const char *sql) { + long long value = -1; + if (mysql_query(m, sql) != 0) { + printf("FAIL: %s: %s\n", sql, mysql_error(m)); + failures++; + return value; + } + MYSQL_RES *result = mysql_store_result(m); + MYSQL_ROW row = result ? mysql_fetch_row(result) : NULL; + if (row && row[0]) value = atoll(row[0]); + if (result) mysql_free_result(result); + return value; +} + +static long long prepared(MYSQL *m, int value, int id) { + const char *sql = "UPDATE fr_check SET a = ? WHERE id = ?"; + MYSQL_STMT *s = mysql_stmt_init(m); + MYSQL_BIND b[2]; + memset(b, 0, sizeof b); + b[0].buffer_type = MYSQL_TYPE_LONG; + b[0].buffer = &value; + b[1].buffer_type = MYSQL_TYPE_LONG; + b[1].buffer = &id; + long long count = -1; + if (mysql_stmt_prepare(s, sql, strlen(sql)) == 0 && mysql_stmt_bind_param(s, b) == 0 && mysql_stmt_execute(s) == 0) { + count = (long long)mysql_stmt_affected_rows(s); + } else { + printf("FAIL: prepared UPDATE: %s\n", mysql_stmt_error(s)); + failures++; + } + mysql_stmt_close(s); + return count; +} + +static MYSQL *open_session(char **argv, int loopback, unsigned long flags) { + MYSQL *m = mysql_init(NULL); + if (loopback) { + my_bool off = 0; + mysql_options(m, MYSQL_OPT_SSL_ENFORCE, &off); + mysql_options(m, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, &off); + } + if (!mysql_real_connect(m, argv[1], argv[3], getenv("MYSQL_PWD"), argv[4], (unsigned)atoi(argv[2]), NULL, flags)) { + fprintf(stderr, "cannot connect: %s\n", mysql_error(m)); + exit(3); + } + return m; +} + +static void reset(MYSQL *m) { + affected(m, "DROP TABLE IF EXISTS fr_check_audit"); + affected(m, "DROP TABLE IF EXISTS fr_check"); + affected(m, "CREATE TABLE fr_check (id INT PRIMARY KEY, a INT)"); + affected(m, "INSERT INTO fr_check VALUES (1, 1), (2, 1)"); +} + +int main(int argc, char **argv) { + (void)argc; + int loopback = atoi(argv[5]); + + MYSQL *plain = open_session(argv, loopback, 0); + printf("server %s, libmariadb %s\n", mysql_get_server_info(plain), mysql_get_client_info()); + reset(plain); + expect(affected(plain, "UPDATE fr_check SET a = 1 WHERE id = 1"), 0, + "without the flag an UPDATE writing the value the row holds counts 0"); + mysql_close(plain); + + MYSQL *m = open_session(argv, loopback, CLIENT_FOUND_ROWS); + unsigned long server = 0; + mariadb_get_infov(m, MARIADB_CONNECTION_SERVER_CAPABILITIES, &server); + expect((server & CLIENT_FOUND_ROWS) != 0, 1, "the server offers CLIENT_FOUND_ROWS"); + reset(m); + expect(affected(m, "UPDATE fr_check SET a = 1 WHERE id = 1"), 1, + "an UPDATE writing the value the row holds counts 1"); + expect(scalar(m, "SELECT ROW_COUNT()"), 1, "ROW_COUNT() after it is 1"); + expect(prepared(m, 1, 1), 1, "a prepared UPDATE writing the value the row holds counts 1"); + expect(affected(m, "UPDATE fr_check SET a = 1 WHERE id = 99"), 0, "an UPDATE that matches nothing counts 0"); + expect(prepared(m, 1, 99), 0, "a prepared UPDATE that matches nothing counts 0"); + expect(affected(m, "UPDATE fr_check SET a = 1"), 2, "an UPDATE over two rows holding the value counts 2"); + expect(affected(m, "INSERT INTO fr_check VALUES (2, 1) ON DUPLICATE KEY UPDATE a = VALUES(a)"), 1, + "ON DUPLICATE KEY UPDATE onto a row holding the values counts 1"); + + affected(m, "CREATE TABLE fr_check_audit (n INT)"); + if (mysql_query(m, "CREATE TRIGGER fr_check_trg AFTER UPDATE ON fr_check FOR EACH ROW " + "INSERT INTO fr_check_audit VALUES (NEW.id), (NEW.id)") == 0) { + expect(affected(m, "UPDATE fr_check SET a = 1 WHERE id = 1"), 1, + "rows an AFTER UPDATE trigger writes are not counted"); + expect(scalar(m, "SELECT COUNT(*) FROM fr_check_audit"), 2, "the trigger wrote its two rows"); + } else { + printf("SKIP: no triggers here: %s\n", mysql_error(m)); + } + affected(m, "DROP TABLE fr_check_audit"); + affected(m, "DROP TABLE fr_check"); + mysql_close(m); + + printf(failures == 0 ? "OK: every check passed\n" : "%d check(s) failed\n", failures); + return failures == 0 ? 0 : 1; +} +C + +clang -O0 -o "$WORK/check" "$WORK/check.c" -I"$ROOT/Plugins/MySQLDriverPlugin/CMariaDB/include" "$ROOT/Libs/libmariadb.a" \ + -L"$ROOT/Libs/dylibs" -lssl.3 -lcrypto.3 -liconv -lz -framework GSS -Wl,-rpath,"$ROOT/Libs/dylibs" \ + > "$WORK/build.log" 2>&1 || { + echo "the check failed to build" >&2 + cat "$WORK/build.log" >&2 + exit 3 +} + +"$WORK/check" "$HOST" "$PORT" "$USER_NAME" "$DATABASE" "$LOOPBACK" diff --git a/apple/scripts/probes/check-nul-text-drivers.sh b/apple/scripts/probes/check-nul-text-drivers.sh index a916a1c271..83e25fe6ef 100755 --- a/apple/scripts/probes/check-nul-text-drivers.sh +++ b/apple/scripts/probes/check-nul-text-drivers.sh @@ -10,7 +10,7 @@ LINK="" case "$ENGINE" in libsql) FOLDER=LibSQLDriverPlugin - PRODUCTS="TableProSQLiteCore CSQLite" + PRODUCTS="TableProSQLiteCore CSQLite TableProWireIO" LINK="\"$ROOT/Libs/libsqlite3_vendored.a\"" ;; beancount) @@ -32,6 +32,9 @@ mkdir -p "$SOURCES" for source in "$ROOT/Plugins/$FOLDER/"*.swift; do ln -sf "$source" "$SOURCES/$(basename "$source")" done +if [ "$ENGINE" = libsql ]; then + ln -sf "$ROOT/Plugins/SharedSpill/PluginRowSpill.swift" "$SOURCES/PluginRowSpill.swift" +fi for shared in PluginConnectTimeout PluginQueryTimeout PluginURLSessionTaskBox PluginRedirectRefusingDelegate; do ln -sf "$ROOT/Plugins/Shared/$shared.swift" "$SOURCES/$shared.swift" done diff --git a/apple/scripts/probes/check-oracle-time-zone-regions.sh b/apple/scripts/probes/check-oracle-time-zone-regions.sh new file mode 100755 index 0000000000..bd6a1beab8 --- /dev/null +++ b/apple/scripts/probes/check-oracle-time-zone-regions.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +# +# Check that a TIMESTAMP WITH TIME ZONE stored with a region name reads as its wall clock and Oracle's name, and that +# the grid's write-back masks parse that text again. +# +# Such a value carries an id into the server's time zone file. OracleTimeZoneRegions reads the ids once per connection +# from SYS.V_$TIMEZONE_NAMES and assumes a layout no Oracle document states: 0x80 in byte 11 marks a region, the id is +# 13 bits over bytes 11 and 12 with the low two bits of byte 12 unused, and bytes 1 to 7 are UTC. This reads the wire +# bytes through oracle-nio and the cell text through the same OracleCoreConnection the app uses, and checks: +# 1. whether the account can read the view; ORA-00942 is a normal answer, and the cells then read as UTC +# 2. the map query's rows, for AT TIME ZONE and FROM_TZ: every row a region, no id repeated, the low bits clear +# 3. which names Foundation's TimeZone does not know +# 4. stored region values: wire bytes against SYS_EXTRACT_UTC and DUMP, cell text against TO_CHAR and EXTRACT +# 5. a session zone set to a region: CURRENT_TIMESTAMP and friends read with it, and the map outlives the ALTER +# 6. the masks in OracleRowWriter.swift parse the region, offset and zone-less spellings, into TSTZ and LTZ +# +# Usage: +# scripts/probes/check-oracle-time-zone-regions.sh [host] [port] [service] [user] [password] +# +# Defaults suit a throwaway Oracle Free container: +# docker run -d --name oracle-probe -p 1521:1521 -e ORACLE_PASSWORD=probe_pw \ +# -e APP_USER=probe -e APP_USER_PASSWORD=probe_pw gvenzl/oracle-free:23-slim-faststart +# +# Hand-run only, never in CI. The user needs CREATE TABLE. Every object it creates is prefixed TP_TZR_ and dropped +# again. Exits non-zero on a disagreement, 3 when a prerequisite is missing. + +set -euo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-1521}" +SERVICE="${3:-FREEPDB1}" +USER_NAME="${4:-probe}" +PASSWORD="${5:-probe_pw}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +if [ -z "${DEVELOPER_DIR:-}" ]; then + for candidate in /Applications/Xcode-beta.app /Applications/Xcode.app; do + if [ -d "$candidate/Contents/Developer" ]; then + export DEVELOPER_DIR="$candidate/Contents/Developer" + break + fi + done +fi +command -v swift > /dev/null || { + echo "swift not found" >&2 + exit 3 +} + +MASK="$(sed -n 's/.*static let timestampWithTimeZoneMask = "\(.*\)"$/\1/p' "$ROOT/Plugins/OracleDriverPlugin/OracleRowWriter.swift")" +OFFSET_MASK="$(sed -n 's/.*static let timestampWithOffsetMask = "\(.*\)"$/\1/p' "$ROOT/Plugins/OracleDriverPlugin/OracleRowWriter.swift")" +NIO_REVISION="$(sed -n 's/.*revision: "\([0-9a-f]*\)".*/\1/p' "$ROOT/Packages/TableProOracle/Package.swift")" +if [ -z "$MASK" ] || [ -z "$OFFSET_MASK" ] || [ -z "$NIO_REVISION" ]; then + echo "cannot read the writer's mask or the oracle-nio pin" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/Sources/Probe" + +cat > "$WORK/Package.swift" < "$WORK/Sources/Probe/main.swift" <<'EOF' +import Foundation +import NIOCore +import OracleNIO +import TableProOracleCore + +let arguments = CommandLine.arguments +let host = arguments[1] +let port = Int(arguments[2]) ?? 1_521 +let service = arguments[3] +let user = arguments[4] +let password = arguments[5] +let regionMask = arguments[6] +let offsetMask = arguments[7] +let view = "SYS.V_$TIMEZONE_NAMES" + +let connection = OracleCoreConnection(options: OracleConnectionOptions( + host: host, port: port, user: user, password: password, identifierMode: .service, serviceName: service +)) +do { + try await connection.connect() +} catch { + print("cannot connect: \(error)") + exit(3) +} + +func dialRaw() async -> OracleNIO.OracleConnection { + do { + return try await OracleNIO.OracleConnection.connect( + configuration: OracleNIO.OracleConnection.Configuration( + host: host, port: port, service: .serviceName(service), username: user, password: password + ), + id: 1 + ) + } catch { + print("cannot connect through oracle-nio: \(error)") + exit(3) + } +} + +let raw = await dialRaw() + +var disagreements = 0 + +@MainActor +func check(_ holds: Bool, _ claim: String) { + print("\(holds ? "ok " : "FAIL") \(claim)") + if !holds { disagreements += 1 } +} + +func info(_ line: String) { + print("info \(line)") +} + +func attempt(_ sql: String) async -> (result: OracleRawResult?, failure: String?) { + do { + return (try await connection.executeQuery(sql), nil) + } catch { + return (nil, String(describing: error)) + } +} + +@MainActor +func result(_ sql: String) async -> OracleRawResult? { + let answer = await attempt(sql) + if let failure = answer.failure { + print("FAIL \(sql.prefix(80)): \(failure)") + disagreements += 1 + } + return answer.result +} + +func scalar(_ sql: String) async -> String? { + await attempt(sql).result?.rows.first?.first?.stringValue +} + +func run(_ sql: String) async { + _ = await attempt(sql) +} + +// The cells as oracle-nio hands them over, before the core decodes anything. +func rawRows(_ sql: String) async throws -> [[OracleCell]] { + let stream = try await raw.execute( + OracleStatement(unsafeSQL: sql), options: StatementOptions(prefetchRows: 1_000, arraySize: 1_000) + ) + var rows: [[OracleCell]] = [] + for try await row in stream { + rows.append(Array(row)) + } + return rows +} + +func bytes(of cell: OracleCell) -> [UInt8] { + cell.bytes.map { Array($0.readableBytesView) } ?? [] +} + +func hex(_ bytes: [UInt8]) -> String { + bytes.map { String(format: "%02x", $0) }.joined() +} + +func listed(_ items: [String]) -> String { + items.isEmpty ? "" : " (\(items.count): \(items.prefix(5).joined(separator: "; ")))" +} + +// The writer's own choice: a region name after a blank takes the region mask, anything else the offset mask. +func mask(for text: String) -> String { + text.split(separator: " ").last?.first?.isLetter == true ? regionMask : offsetMask +} + +func parsed(_ text: String) -> String { + "TO_TIMESTAMP_TZ('\(text)', '\(mask(for: text))')" +} + +guard let release = connection.serverRelease else { + print("the login reply carried no server release") + exit(3) +} +print("Oracle \(release.major).\(release.update), writer masks '\(regionMask)' and '\(offsetMask)'") + +print("\n1. Who can read \(view)") +let roles = await result("SELECT ROLE FROM SESSION_ROLES ORDER BY ROLE")?.rows.compactMap { $0.first?.stringValue } ?? [] +info("session roles: \(roles.isEmpty ? "none" : roles.joined(separator: ", "))") +let publicGrants = await scalar(""" + SELECT COUNT(*) FROM ALL_TAB_PRIVS + WHERE TABLE_SCHEMA = 'SYS' AND TABLE_NAME = 'V_$TIMEZONE_NAMES' AND GRANTEE = 'PUBLIC' + """) +info("PUBLIC grants on \(view) in ALL_TAB_PRIVS: \(publicGrants ?? "unreadable")") +let access = await attempt("SELECT COUNT(*) FROM \(view)") +let mapReadable = access.result != nil +if let count = access.result?.rows.first?.first?.stringValue { + check(true, "\(user) reads \(view): \(count) rows") +} else { + let failure = access.failure ?? "" + check(failure.contains("ORA-00942"), "\(user) is refused \(view) with ORA-00942: \(failure.prefix(120))") +} +if let file = await scalar("SELECT FILENAME || ' version ' || VERSION FROM SYS.V_$TIMEZONE_FILE") { + info("time zone file \(file)") +} + +print("\n2. The map query, read raw") +var mapRows: [(name: String, wire: [UInt8])] = [] +var idsByForm: [String: [String: Int]] = [:] +let fromTZQuery = "SELECT TZNAME, FROM_TZ(TIMESTAMP '2000-01-01 00:00:00', TZNAME) FROM " + + "(SELECT DISTINCT TZNAME FROM \(view)) ORDER BY TZNAME" +if mapReadable { + for (form, sql) in [("AT TIME ZONE", OracleTimeZoneRegions.query), ("FROM_TZ", fromTZQuery)] { + let started = ContinuousClock.now + let rows: [[OracleCell]] + do { + rows = try await rawRows(sql) + } catch { + check(false, "\(form): the map query runs: \(error)") + continue + } + info("\(form): \(rows.count) rows in \(ContinuousClock.now - started)") + let pairs = rows.compactMap { cells -> (name: String, wire: [UInt8])? in + guard cells.count == 2, let name = try? cells[0].decode(String.self) else { return nil } + return (name, bytes(of: cells[1])) + } + let unflagged = pairs.filter { OracleTimeZoneRegions.regionID(wire: $0.wire) == nil }.map { "\($0.name) \(hex($0.wire))" } + check(unflagged.isEmpty, "\(form): every row carries a region id\(listed(unflagged))") + var namesByID: [Int: [String]] = [:] + for pair in pairs { + guard let id = OracleTimeZoneRegions.regionID(wire: pair.wire) else { continue } + namesByID[id, default: []].append(pair.name) + } + let shared = namesByID.filter { $0.value.count > 1 }.map { "\($0.key): \($0.value.joined(separator: ", "))" } + check(shared.isEmpty, "\(form): no two names share an id\(listed(shared))") + let lowBits = pairs.filter { $0.wire.count >= 13 && $0.wire[12] & 0x03 != 0 }.map { "\($0.name) \(hex($0.wire))" } + check(lowBits.isEmpty, "\(form): the low two bits of byte 12 are clear\(listed(lowBits))") + var ids: [String: Int] = [:] + for pair in pairs { + ids[pair.name] = OracleTimeZoneRegions.regionID(wire: pair.wire) + } + idsByForm[form] = ids + if form == "AT TIME ZONE" { + mapRows = pairs + let utcFields: [UInt8] = [0x78, 0x64, 0x01, 0x01, 0x01, 0x01, 0x01] + let moved = pairs.filter { Array($0.wire.prefix(7)) != utcFields }.map { "\($0.name) \(hex($0.wire))" } + check(moved.isEmpty, "AT TIME ZONE: bytes 1 to 7 of every row are 2000-01-01 00:00:00 UTC\(listed(moved))") + } + } + if idsByForm.count == 2 { + check(idsByForm["AT TIME ZONE"] == idsByForm["FROM_TZ"], "AT TIME ZONE and FROM_TZ give each name the same id") + } + info("OracleTimeZoneRegions keeps \(OracleTimeZoneRegions(rows: mapRows).count) of \(mapRows.count) rows") +} else { + info("skip: \(view) is not readable, so every region value reads as UTC") +} + +// The first name per id, as the map keeps it: the query orders by name. +var nameByID: [Int: String] = [:] +for row in mapRows { + guard let id = OracleTimeZoneRegions.regionID(wire: row.wire), nameByID[id] == nil else { continue } + nameByID[id] = row.name +} + +print("\n3. Names Foundation does not know") +let unknownNames = mapRows.map { $0.name }.filter { TimeZone(identifier: $0) == nil } +info("\(unknownNames.count) of \(mapRows.count) names read as UTC instead") +for name in unknownNames { + info("Foundation does not know \(name)") +} + +print("\n4. Stored region values") +let samples: [(id: Int, literal: String)] = [ + (1, "TIMESTAMP '2024-01-02 03:04:05.5 Europe/Paris'"), + (2, "TIMESTAMP '2024-07-01 14:00:00 Europe/Paris'"), + (3, "TIMESTAMP '2024-03-10 01:30:00 America/Los_Angeles'"), + (4, "TIMESTAMP '2024-11-03 12:00:00 US/Pacific'"), + (5, "TIMESTAMP '2024-06-15 08:00:00 UTC'"), + (6, "TIMESTAMP '2024-06-15 08:00:00 Etc/GMT+5'"), + (7, "TIMESTAMP '1880-06-01 12:00:00 Europe/Paris'"), + (8, "TIMESTAMP '2024-10-27 02:30:00 Europe/Paris CEST'"), +] +let overlapRow = 8 +await run("DROP TABLE TP_TZR_T PURGE") +if let failure = await attempt("CREATE TABLE TP_TZR_T (ID NUMBER PRIMARY KEY, TZ TIMESTAMP(6) WITH TIME ZONE)").failure { + print("cannot create TP_TZR_T: \(failure)") + exit(3) +} +var literalByID: [Int: String] = [:] +for sample in samples { + if let failure = await attempt("INSERT INTO TP_TZR_T VALUES (\(sample.id), \(sample.literal))").failure { + info("skip \(sample.literal): \(failure.prefix(100))") + } else { + literalByID[sample.id] = sample.literal + } +} +_ = await result("COMMIT") + +var wireByID: [Int: [UInt8]] = [:] +do { + for cells in try await rawRows("SELECT TO_CHAR(ID), TZ, SYS_EXTRACT_UTC(TZ), DUMP(TZ) FROM TP_TZR_T ORDER BY ID") { + guard cells.count == 4, let row = (try? cells[0].decode(String.self)).flatMap({ Int($0) }) else { continue } + let wire = bytes(of: cells[1]) + let utc = bytes(of: cells[2]) + let dump = (try? cells[3].decode(String.self)) ?? "" + let dumped = dump.split(separator: ":", maxSplits: 1).dropFirst().first.map { + $0.split(separator: ",").compactMap { UInt8($0.trimmingCharacters(in: .whitespaces)) } + } ?? [] + let id = OracleTimeZoneRegions.regionID(wire: wire) + let lowBits = wire.count >= 13 ? String(wire[12] & 0x03) : "?" + info("row \(row): \(hex(wire)), region id \(id.map { String($0) } ?? "none"), byte 12 low bits \(lowBits)") + check(id != nil, "row \(row): the stored value carries a region id") + check(Array(wire.prefix(7)) == Array(utc.prefix(7)), "row \(row): wire bytes 1 to 7 are SYS_EXTRACT_UTC's \(hex(Array(utc.prefix(7))))") + check(dump.hasPrefix("Typ=181") && dumped == wire, "row \(row): DUMP of the column matches the wire bytes: \(dump)") + wireByID[row] = wire + } +} catch { + check(false, "the stored values read raw: \(error)") +} + +let textQuery = """ + SELECT TO_CHAR(ID), TZ, TO_CHAR(TZ, 'YYYY-MM-DD HH24:MI:SS.FF6') || ' ' || EXTRACT(TIMEZONE_REGION FROM TZ), + TO_CHAR(SYS_EXTRACT_UTC(TZ), 'YYYY-MM-DD HH24:MI:SS.FF6') || '+00:00', + EXTRACT(TIMEZONE_REGION FROM TZ), TO_CHAR(TZ, 'TZR') + FROM TP_TZR_T ORDER BY ID + """ +var cellTextByID: [Int: String] = [:] +var regionByID: [Int: String] = [:] +for values in await result(textQuery)?.rows ?? [] { + guard values.count == 6, let row = values[0].stringValue.flatMap({ Int($0) }) else { continue } + let cell = values[1].stringValue ?? "NULL" + let server = values[2].stringValue ?? "" + let utcText = values[3].stringValue ?? "" + let region = values[4].stringValue ?? "" + let tzr = values[5].stringValue ?? "" + let mapName = wireByID[row].flatMap { OracleTimeZoneRegions.regionID(wire: $0) }.flatMap { nameByID[$0] } + if let mapName, mapName != region { + check(false, "row \(row): the map names its id \(mapName), the server \(region)") + } + let expected = mapName.map { TimeZone(identifier: $0) != nil } == true ? server : utcText + check(cell == expected, "row \(row) \(literalByID[row] ?? ""): reads \(cell)\(cell == expected ? "" : ", expected \(expected)")") + if tzr != region { + info("row \(row): TO_CHAR TZR spells \(tzr), EXTRACT spells \(region)") + } + cellTextByID[row] = cell + regionByID[row] = region +} + +print("\n5. A session zone set to a region") +_ = await result("ALTER SESSION SET TIME_ZONE = 'Europe/Paris'") +let sessionSuffix = nameByID.values.contains("Europe/Paris") ? " Europe/Paris" : "+00:00" +let regionForms = [ + "CURRENT_TIMESTAMP", + "FROM_TZ(TIMESTAMP '2024-01-02 03:04:05', 'Europe/Paris')", + "TIMESTAMP '2024-01-02 02:04:05 UTC' AT TIME ZONE 'Europe/Paris'", + "TIMESTAMP '2024-01-02 02:04:05 UTC' AT LOCAL", + "CAST(CAST(TIMESTAMP '2024-01-02 02:04:05 UTC' AS TIMESTAMP WITH LOCAL TIME ZONE) AS TIMESTAMP WITH TIME ZONE)", +] +if let values = await result("SELECT \(regionForms.joined(separator: ", ")) FROM DUAL")?.rows.first { + for (form, value) in zip(regionForms, values) { + let text = value.stringValue ?? "NULL" + check(text.hasSuffix(sessionSuffix), "\(form) reads \(text)") + } +} +if let first = cellTextByID.keys.min(), let before = cellTextByID[first] { + let after = await scalar("SELECT TZ FROM TP_TZR_T WHERE ID = \(first)") + check(after == before, "after the ALTER, row \(first) still reads \(after ?? "NULL")") +} + +print("\n6. Write-back through the writer's masks") +for (row, text) in cellTextByID.sorted(by: { $0.key < $1.key }) { + let matched = await scalar("SELECT COUNT(*) FROM TP_TZR_T WHERE ID = \(row) AND TZ = \(parsed(text))") + if row == overlapRow { + let resolved = await scalar("SELECT TO_CHAR(\(parsed(text)), 'YYYY-MM-DD HH24:MI:SS TZR TZD') FROM DUAL") + info("the repeated hour: \(text) matches \(matched ?? "no") stored row and resolves to \(resolved ?? "an error")") + continue + } + check(matched == "1", "\(text) parses back to the stored instant") + guard !text.hasSuffix("+00:00") else { continue } + let kept = await scalar("SELECT EXTRACT(TIMEZONE_REGION FROM \(parsed(text))) FROM DUAL") + check(kept == regionByID[row], "\(text) keeps its region: \(kept ?? "NULL")") +} + +let spellings: [(text: String, literal: String)] = [ + ("2024-01-02 03:04:05.123456-05:30", "TIMESTAMP '2024-01-02 03:04:05.123456 -05:30'"), + ("2000-01-01 00:00:00.000000+00:00", "TIMESTAMP '2000-01-01 00:00:00 +00:00'"), + ("2000-01-01 00:00:00", "FROM_TZ(TIMESTAMP '2000-01-01 00:00:00', SESSIONTIMEZONE)"), + ("2024-01-02 03:04:05.500000 Europe/Paris", "TIMESTAMP '2024-01-02 03:04:05.5 Europe/Paris'"), +] +await run("DROP TABLE TP_TZR_L PURGE") +_ = await result("CREATE TABLE TP_TZR_L (ID NUMBER PRIMARY KEY, L TIMESTAMP(6) WITH LOCAL TIME ZONE)") +for (index, spelling) in spellings.enumerated() { + let compared = await attempt("SELECT CASE WHEN \(parsed(spelling.text)) = \(spelling.literal) THEN 1 ELSE 0 END FROM DUAL") + let equal = compared.result?.rows.first?.first?.stringValue + check(equal == "1", "TIMESTAMP WITH TIME ZONE: '\(spelling.text)' parses to \(spelling.literal)\(compared.failure.map { ": \($0.prefix(100))" } ?? "")") + let inserted = await attempt("INSERT INTO TP_TZR_L VALUES (\(index), \(parsed(spelling.text)))") + var stored: String? + if inserted.failure == nil { + stored = await scalar("SELECT COUNT(*) FROM TP_TZR_L WHERE ID = \(index) AND L = \(spelling.literal)") + } + check(stored == "1", "TIMESTAMP WITH LOCAL TIME ZONE: '\(spelling.text)' writes \(spelling.literal)\(inserted.failure.map { ": \($0.prefix(100))" } ?? "")") +} + +await run("DROP TABLE TP_TZR_T PURGE") +await run("DROP TABLE TP_TZR_L PURGE") +try? await raw.close() +print(disagreements == 0 ? "\nall checks hold" : "\n\(disagreements) disagreement(s)") +exit(disagreements == 0 ? 0 : 1) +EOF + +echo "Building the probe against Packages/TableProOracle (the first build takes a minute)" +(cd "$WORK" && swift build -c debug > "$WORK/build.log" 2>&1) || { + grep -E "error:" "$WORK/build.log" | head -20 >&2 + exit 3 +} +"$WORK/.build/debug/Probe" "$HOST" "$PORT" "$SERVICE" "$USER_NAME" "$PASSWORD" "$MASK" "$OFFSET_MASK" 2> /dev/null diff --git a/apple/scripts/probes/check-parquet-export-values.sh b/apple/scripts/probes/check-parquet-export-values.sh index 966b46b5bd..8036d34fe2 100755 --- a/apple/scripts/probes/check-parquet-export-values.sh +++ b/apple/scripts/probes/check-parquet-export-values.sh @@ -5,8 +5,8 @@ # The plugin links libduckdb.a, so no test bundle can hold its staging database. This compiles # DuckDBStagingDatabase.swift against the shipped library, stages a table holding the values the # export used to change (binary, a zero-length blob, bit(8), BIGINT UNSIGNED above 2^63, a TIME -# past 24 hours, an Oracle DATE with a time, text in an integer column), writes the file and reads -# it back. +# past 24 hours, an Oracle DATE with a time, text in an integer column, timestamps and a time +# with a UTC offset), writes the file and reads it back. # # Usage: # scripts/probes/check-parquet-export-values.sh @@ -74,7 +74,12 @@ enum ParquetValuesProbe { ("seen", "DATE", "Oracle"), ("count", "int", "SQLite"), ("day", "date", "PostgreSQL"), - ("note", "text", "PostgreSQL") + ("note", "text", "PostgreSQL"), + ("zoned", "timestamp with time zone", "PostgreSQL"), + ("instant", "timestamptz", "PostgreSQL"), + ("clock", "time with time zone", "PostgreSQL"), + ("oracle_tz", "TIMESTAMP(6) WITH TIME ZONE", "Oracle"), + ("oracle_ltz", "TIMESTAMP(6) WITH LOCAL TIME ZONE", "Oracle") ] let types = declared.map { ParquetTypeMapper.duckDBType(forColumnType: $0.type, databaseTypeId: $0.engine) } @@ -83,15 +88,21 @@ enum ParquetValuesProbe { try staging.insert(rows: [ [ .text("1"), .bytes(Data([0x41, 0x42, 0x43])), .text("10101010"), .text("18446744073709551615"), - .text("100:00:00"), .text("2024-01-01 12:30:00"), .text("7"), .text("2024-03-01"), .text("it's") + .text("100:00:00"), .text("2024-01-01 12:30:00"), .text("7"), .text("2024-03-01"), .text("it's"), + .text("2024-01-01 12:00:00+07"), .text("2024-10-27 02:30:00+02"), .text("12:00:00+07"), + .text("2026-10-06 15:22:20.000001-05:30"), .text("2026-10-06 15:22:20.000001+07:00") ], [ .text("2"), .bytes(Data()), .text("1"), .text("5"), - .text("10:00:00"), .text("2024-01-02 00:00:00"), .text("seven"), .text("2024-03-02"), .text("") + .text("10:00:00"), .text("2024-01-02 00:00:00"), .text("seven"), .text("2024-03-02"), .text(""), + .null, .text("2024-10-27 02:30:00+01"), .null, .null, .null ] ]) try staging.insert(rows: [ - [.text("3"), .bytes(Data([0x00, 0xFF, 0x5C, 0x78])), .null, .null, .null, .null, .null, .null, .null] + [ + .text("3"), .bytes(Data([0x00, 0xFF, 0x5C, 0x78])), .null, .null, .null, .null, .null, .null, .null, + .null, .null, .null, .null, .null + ] ]) let fallbacks = try staging.copyToParquet(fileURL: file, compression: "SNAPPY", rowGroupSize: 122_880) @@ -106,7 +117,14 @@ enum ParquetValuesProbe { ("bit", "PostgreSQL", "BOOLEAN"), ("bit(1)", "MySQL", "BOOLEAN"), ("bit", "SQL Server", "BOOLEAN"), ("bit(8)", "MySQL", "VARCHAR"), ("bit varying(8)", "PostgreSQL", "VARCHAR"), ("int unsigned", "MySQL", "BIGINT"), ("BIGINT(20) UNSIGNED", "MySQL", "UBIGINT"), - ("bigint", "MySQL", "BIGINT"), ("date", "MySQL", "DATE"), ("DATE", "Oracle", "TIMESTAMP") + ("bigint", "MySQL", "BIGINT"), ("date", "MySQL", "DATE"), ("DATE", "Oracle", "TIMESTAMP"), + ("timestamp with time zone", "PostgreSQL", "TIMESTAMPTZ"), ("timestamptz", "PostgreSQL", "TIMESTAMPTZ"), + ("timestamp(3) with time zone", "PostgreSQL", "TIMESTAMPTZ"), + ("TIMESTAMP(6) WITH TIME ZONE", "Oracle", "TIMESTAMPTZ"), ("TIMESTAMP WITH TIME ZONE", "DuckDB", "TIMESTAMPTZ"), + ("time with time zone", "PostgreSQL", "TIMETZ"), ("timetz", "PostgreSQL", "TIMETZ"), + ("timestamp", "PostgreSQL", "TIMESTAMP"), ("timestamp without time zone", "PostgreSQL", "TIMESTAMP"), + ("time", "PostgreSQL", "TIME"), ("TIMESTAMP(6) WITH LOCAL TIME ZONE", "Oracle", "TIMESTAMPTZ"), + ("timestamp", "MySQL", "TIMESTAMP"), ("datetimeoffset", "SQL Server", "VARCHAR") ] for entry in mapped { expect( @@ -126,7 +144,9 @@ enum ParquetValuesProbe { let typeIds = try reader.columnTypes("SELECT * FROM \(source)") let expectedTypes: [duckdb_type] = [ DUCKDB_TYPE_BIGINT, DUCKDB_TYPE_BLOB, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_UBIGINT, DUCKDB_TYPE_VARCHAR, - DUCKDB_TYPE_TIMESTAMP, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_DATE, DUCKDB_TYPE_VARCHAR + DUCKDB_TYPE_TIMESTAMP, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_DATE, DUCKDB_TYPE_VARCHAR, + DUCKDB_TYPE_TIMESTAMP_TZ, DUCKDB_TYPE_TIMESTAMP_TZ, DUCKDB_TYPE_TIME_TZ, DUCKDB_TYPE_TIMESTAMP_TZ, + DUCKDB_TYPE_TIMESTAMP_TZ ] for (index, column) in declared.enumerated() { expect("type of \(column.name)", "\(typeIds[index].rawValue)", "\(expectedTypes[index].rawValue)") @@ -149,6 +169,12 @@ enum ParquetValuesProbe { expect("quoted text", try cell("note", id: 1), "it's") expect("empty text", try cell("note = ''", id: 2), "true") expect("null", try cell("note IS NULL AND big IS NULL AND seen IS NULL", id: 3), "true") + expect("timestamp with time zone in UTC", try cell("timezone('UTC', zoned)", id: 1), "2024-01-01 05:00:00") + expect("first 02:30 of the fall-back hour", try cell("timezone('UTC', instant)", id: 1), "2024-10-27 00:30:00") + expect("second 02:30 of the fall-back hour", try cell("timezone('UTC', instant)", id: 2), "2024-10-27 01:30:00") + expect("time with time zone in UTC", try cell("clock", id: 1), "05:00:00+00") + expect("Oracle timestamp with time zone", try cell("timezone('UTC', oracle_tz)", id: 1), "2026-10-06 20:52:20.000001") + expect("Oracle timestamp with local time zone", try cell("timezone('UTC', oracle_ltz)", id: 1), "2026-10-06 08:22:20.000001") if failures > 0 { exit(1) } print("PASS: the Parquet file holds every staged value") diff --git a/apple/scripts/probes/check-postgresql-copy-in.sh b/apple/scripts/probes/check-postgresql-copy-in.sh new file mode 100755 index 0000000000..233983bc9d --- /dev/null +++ b/apple/scripts/probes/check-postgresql-copy-in.sh @@ -0,0 +1,424 @@ +#!/usr/bin/env bash +# +# Checks what the PostgreSQL driver's copy-in (LibPQCopyIn) relies on, with the libpq the app links +# (apple/Libs/libpq.a) against a live server: rows go as CopyData without psql's `\.` line and end with +# CopyDone; pieces may split a row or a UTF-8 sequence; CRLF rows and a last row with no line break load; +# a bad row and a CopyFail both fail the COPY and abort an open block; a binary COPY shows on its +# PGRES_COPY_IN result; a refused COPY never enters copy-in; a read-only transaction refuses it with +# 25006; a cancel request mid-stream fails it with 57014. It also prints what a sent `\.` line does, +# which PostgreSQL 18 reads as data in CSV. +# +# Stop, sent the way LibPQCopyIn sends rows (nonblocking, waiting for write-ready in 100 ms slices): a Stop +# while the server takes rows ends the COPY with its CopyFail at once; a Stop while the server waits on a row +# lock takes no data, so CopyFail alone waits for the lock, and one cancel request ends it at once. Either way +# the session runs the next statement, an open block is left aborted and no row stays. +# +# Usage: +# apple/scripts/probes/check-postgresql-copy-in.sh "host=127.0.0.1 port=5432 user=postgres dbname=postgres" +# +# Needs apple/scripts/download-libs.sh to have run. Creates and drops the table copy_probe. Measured on +# PostgreSQL 17.11 and 18.6 with libpq 17.4. Exits non-zero on a failure. + +set -euo pipefail + +CONNINFO="${1:?usage: check-postgresql-copy-in.sh }" +APPLE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LIBS="$APPLE/Libs" +[ -f "$LIBS/libpq.a" ] || { echo "apple/Libs/libpq.a not found; run apple/scripts/download-libs.sh" >&2; exit 3; } + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +cat > "$WORK_DIR/probe.c" << 'C_SOURCE' +#include +#include +#include +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(const char *label, int ok, const char *detail) { + printf("%s %s%s%s\n", ok ? "ok " : "FAIL", label, detail ? ": " : "", detail ? detail : ""); + if (!ok) failures++; +} + +static void run(PGconn *c, const char *sql) { + PGresult *r = PQexec(c, sql); + ExecStatusType s = PQresultStatus(r); + if (s != PGRES_COMMAND_OK && s != PGRES_TUPLES_OK) { + fprintf(stderr, "setup failed: %s: %s", sql, PQerrorMessage(c)); + exit(3); + } + PQclear(r); +} + +static char *scalar(PGconn *c, const char *sql) { + static char buf[4096]; + PGresult *r = PQexec(c, sql); + snprintf(buf, sizeof buf, "%s", PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0) + ? PQgetvalue(r, 0, 0) : ""); + PQclear(r); + return buf; +} + +typedef struct { + ExecStatusType status; + char tuples[32]; + char tag[64]; + char state[8]; + char message[512]; + int extra; +} Outcome; + +// Sends `data` in pieces of `piece` bytes after a header that answered COPY_IN, ends it, reads every result. +static Outcome copy_in(PGconn *c, const char *header, const char *data, size_t length, size_t piece, + const char *fail_reason, int *binary) { + Outcome o; + memset(&o, 0, sizeof o); + PGresult *start = PQexec(c, header); + o.status = PQresultStatus(start); + if (binary) *binary = PQbinaryTuples(start); + if (o.status != PGRES_COPY_IN) { + const char *st = PQresultErrorField(start, PG_DIAG_SQLSTATE); + snprintf(o.state, sizeof o.state, "%s", st ? st : ""); + snprintf(o.message, sizeof o.message, "%s", PQresultErrorField(start, PG_DIAG_MESSAGE_PRIMARY) ?: ""); + PQclear(start); + while ((start = PQgetResult(c))) { o.extra++; PQclear(start); } + return o; + } + PQclear(start); + for (size_t at = 0; at < length && !fail_reason; at += piece) { + size_t n = length - at < piece ? length - at : piece; + if (PQputCopyData(c, data + at, (int)n) != 1) { snprintf(o.message, sizeof o.message, "put failed"); break; } + } + if (PQputCopyEnd(c, fail_reason) != 1) snprintf(o.message, sizeof o.message, "end failed"); + PGresult *r = PQgetResult(c); + o.status = PQresultStatus(r); + snprintf(o.tuples, sizeof o.tuples, "%s", PQcmdTuples(r)); + snprintf(o.tag, sizeof o.tag, "%s", PQcmdStatus(r)); + const char *st = PQresultErrorField(r, PG_DIAG_SQLSTATE); + snprintf(o.state, sizeof o.state, "%s", st ? st : ""); + const char *msg = PQresultErrorField(r, PG_DIAG_MESSAGE_PRIMARY); + if (msg) snprintf(o.message, sizeof o.message, "%s", msg); + PQclear(r); + while ((r = PQgetResult(c))) { o.extra++; PQclear(r); } + return o; +} + +static int is_ok(Outcome o, const char *tuples) { + return o.status == PGRES_COMMAND_OK && strcmp(o.tuples, tuples) == 0 && o.extra == 0; +} + +static void describe(char *out, size_t size, Outcome o) { + snprintf(out, size, "status=%s tuples=%s tag='%s' state=%s msg='%s' extra=%d", PQresStatus(o.status), o.tuples, + o.tag, o.state, o.message, o.extra); +} + +typedef struct { PGconn *conn; } CancelArgs; + +static void *cancel_after(void *arg) { + usleep(300000); + PGcancel *cancel = PQgetCancel(((CancelArgs *)arg)->conn); + char err[256]; + PQcancel(cancel, err, sizeof err); + PQfreeCancel(cancel); + return NULL; +} + +static atomic_int stop_requested; +static double stop_time; + +static double now_ms(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return t.tv_sec * 1e3 + t.tv_nsec / 1e6; +} + +static void *stop_after(void *arg) { + usleep((useconds_t)(*(int *)arg) * 1000); + stop_time = now_ms(); + atomic_store(&stop_requested, 1); + return NULL; +} + +typedef struct { + Outcome outcome; + double stop_to_end_ms; + int sent_cancel; + int gave_up; +} Stopped; + +// One 100 ms slice of waiting for the queued bytes to go out: 0 once they all went, -1 on a lost connection. +static int flush_slice(PGconn *c) { + int flushed = PQflush(c); + if (flushed <= 0) return flushed; + if (PQsocketPoll(PQsocket(c), 1, 1, PQgetCurrentTimeUSec() + 100000) > 0) PQconsumeInput(c); + return 1; +} + +// One slice of waiting for the COPY's result once it ended: 0 once the result is in, -1 on a lost connection. A +// proxy can take every byte while the server reads none, so a flushed CopyFail is not an answered one. +static int answer_slice(PGconn *c) { + int flushed = PQflush(c); + if (flushed < 0) return -1; + if (flushed == 0 && !PQisBusy(c)) return 0; + if (PQsocketPoll(PQsocket(c), 1, flushed, PQgetCurrentTimeUSec() + 100000) > 0 && !PQconsumeInput(c)) return -1; + return flushed == 0 && !PQisBusy(c) ? 0 : 1; +} + +// LibPQCopyIn's loop: pieces queued in nonblocking mode and flushed in slices, so a Stop is seen while the server +// takes no data. A Stop while rows go out ends the copy with CopyFail, and with `cancels` one cancel request follows +// once a slice passes with no answer; a Stop after CopyDone sends the cancel request at once. Without `cancels` the +// wait gives up after `give_up_ms` and the copy stays open for the caller to finish. +static Stopped copy_in_stoppable(PGconn *c, const char *header, const char *data, size_t length, size_t piece, + int stop_delay_ms, int cancels, double give_up_ms) { + Stopped s; + memset(&s, 0, sizeof s); + atomic_store(&stop_requested, 0); + PGresult *start = PQexec(c, header); + if (PQresultStatus(start) != PGRES_COPY_IN) { + s.outcome.status = PQresultStatus(start); + PQclear(start); + while ((start = PQgetResult(c))) PQclear(start); + return s; + } + PQclear(start); + pthread_t timer; + pthread_create(&timer, NULL, stop_after, &stop_delay_ms); + PQsetnonblocking(c, 1); + int failed_copy = 0; + for (size_t at = 0; at < length && !failed_copy; at += piece) { + size_t n = length - at < piece ? length - at : piece; + if (atomic_load(&stop_requested)) { failed_copy = 1; break; } + if (PQputCopyData(c, data + at, (int)n) != 1) { snprintf(s.outcome.message, sizeof s.outcome.message, "put failed"); break; } + int flushing; + while ((flushing = flush_slice(c)) == 1) { + if (atomic_load(&stop_requested)) { failed_copy = 1; break; } + } + if (flushing < 0) break; + } + PQputCopyEnd(c, failed_copy ? "canceled by user" : NULL); + int waited = 0; + while (answer_slice(c) == 1) { + if (!atomic_load(&stop_requested)) continue; + if (cancels && !s.sent_cancel && (!failed_copy || waited >= 1)) { + PGcancel *cancel = PQgetCancel(c); + char err[256]; + PQcancel(cancel, err, sizeof err); + PQfreeCancel(cancel); + s.sent_cancel = 1; + } + waited++; + if (!cancels && now_ms() - stop_time > give_up_ms) { s.gave_up = 1; break; } + } + pthread_join(timer, NULL); + if (s.gave_up) return s; + PQsetnonblocking(c, 0); + PGresult *r = PQgetResult(c); + s.stop_to_end_ms = now_ms() - stop_time; + s.outcome.status = PQresultStatus(r); + snprintf(s.outcome.tuples, sizeof s.outcome.tuples, "%s", PQcmdTuples(r)); + const char *st = PQresultErrorField(r, PG_DIAG_SQLSTATE); + snprintf(s.outcome.state, sizeof s.outcome.state, "%s", st ? st : ""); + const char *msg = PQresultErrorField(r, PG_DIAG_MESSAGE_PRIMARY); + if (msg) snprintf(s.outcome.message, sizeof s.outcome.message, "%s", msg); + PQclear(r); + while ((r = PQgetResult(c))) { s.outcome.extra++; PQclear(r); } + return s; +} + +static void describe_stop(char *out, size_t size, Stopped s) { + char d[700]; + describe(d, sizeof d, s.outcome); + snprintf(out, size, "%s, %.0f ms from Stop to the result, cancel request %s", d, s.stop_to_end_ms, + s.sent_cancel ? "sent" : "not sent"); +} + +// The rest of the block after a Stop: what a session that takes the next statement answers. +static int runs_next_statement(PGconn *c) { + return strcmp(scalar(c, "SELECT 1"), "1") == 0; +} + +int main(int argc, char **argv) { + setvbuf(stdout, NULL, _IONBF, 0); + PGconn *c = PQconnectdb(argv[1]); + if (PQstatus(c) != CONNECTION_OK) { fprintf(stderr, "no server: %s", PQerrorMessage(c)); return 3; } + printf("libpq %d, server %d\n", PQlibVersion(), PQserverVersion(c)); + char d[1024]; + Outcome o; + + run(c, "DROP TABLE IF EXISTS copy_probe; CREATE TABLE copy_probe (a int, b text)"); + + const char *rows = "1\tone\n2\ttwo\n3\t\\N\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("text rows without \\. end with CopyDone as COPY 3", is_ok(o, "3") && strcmp(o.tag, "COPY 3") == 0, d); + check("a NULL marker loads as NULL", strcmp(scalar(c, "SELECT count(*) FROM copy_probe WHERE b IS NULL"), "1") == 0, NULL); + check("an autocommit COPY leaves the session idle", PQtransactionStatus(c) == PQTRANS_IDLE, NULL); + + const char *with_marker = "4\tfour\n\\.\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", with_marker, strlen(with_marker), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + printf("info text rows with the \\. line sent: %s\n", d); + + run(c, "TRUNCATE copy_probe"); + const char *csv = "a,b\n5,\"five, quoted\"\n6,\"multi\nline\"\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN WITH (FORMAT csv, HEADER true)", csv, strlen(csv), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("CSV with a header line and a quoted line break is COPY 2", is_ok(o, "2"), d); + check("the quoted line break is kept", strcmp(scalar(c, "SELECT b FROM copy_probe WHERE a = 6"), "multi\nline") == 0, NULL); + + const char *csv_marker = "7,seven\n\\.\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN WITH (FORMAT csv)", csv_marker, strlen(csv_marker), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + printf("info CSV rows with the \\. line sent: %s\n", d); + + run(c, "TRUNCATE copy_probe"); + const char *utf8 = "8\tcaf\xc3\xa9 \xf0\x9f\x98\x80\n9\tlast row without a line break"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", utf8, strlen(utf8), 3, NULL, NULL); + describe(d, sizeof d, o); + check("3-byte pieces that split a UTF-8 sequence and a last row with no line break load", is_ok(o, "2"), d); + check("the split sequence arrives whole", strcmp(scalar(c, "SELECT b FROM copy_probe WHERE a = 8"), "caf\xc3\xa9 \xf0\x9f\x98\x80") == 0, NULL); + + run(c, "TRUNCATE copy_probe"); + const char *crlf = "10\tten\r\n11\televen\r\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", crlf, strlen(crlf), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("CRLF text rows load", is_ok(o, "2"), d); + check("a CRLF row keeps no carriage return", strcmp(scalar(c, "SELECT b FROM copy_probe WHERE a = 10"), "ten") == 0, NULL); + + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", "", 0, 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("no rows is COPY 0", is_ok(o, "0"), d); + + run(c, "BEGIN"); + const char *bad = "12\ttwelve\nx\tbad\n13\tthirteen\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", bad, strlen(bad), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("a bad row fails the COPY after CopyDone with 22P02", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "22P02") == 0 && o.extra == 0, d); + check("inside a block the failed COPY leaves it aborted", PQtransactionStatus(c) == PQTRANS_INERROR, NULL); + run(c, "ROLLBACK"); + + run(c, "BEGIN"); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, "canceled by user", NULL); + describe(d, sizeof d, o); + check("CopyFail answers 57014 'COPY from stdin failed: canceled by user'", + o.status == PGRES_FATAL_ERROR && strcmp(o.state, "57014") == 0 && strstr(o.message, "canceled by user") != NULL, d); + check("CopyFail aborts the block", PQtransactionStatus(c) == PQTRANS_INERROR, NULL); + run(c, "ROLLBACK"); + + run(c, "BEGIN"); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + check("a COPY inside a block leaves it open", is_ok(o, "3") && PQtransactionStatus(c) == PQTRANS_INTRANS, NULL); + run(c, "ROLLBACK"); + + int binary = 0; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN WITH (FORMAT binary)", "", 0, 1, "binary COPY data cannot come from a script", &binary); + describe(d, sizeof d, o); + check("a binary COPY reports binary tuples on its COPY_IN result", binary == 1, NULL); + check("ending a binary COPY with CopyFail fails it with 57014", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "57014") == 0, d); + + o = copy_in(c, "COPY no_such_table FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("a COPY the server refuses never enters COPY_IN (42P01)", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "42P01") == 0, d); + + run(c, "START TRANSACTION READ ONLY; SELECT 1"); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("a read-only transaction refuses the COPY before COPY_IN (25006)", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "25006") == 0, d); + run(c, "ROLLBACK"); + + // Cancel while rows stream: the server keeps reading pieces and answers 57014 after CopyDone. + size_t big = 64u * 1024u * 1024u; + char *lots = malloc(big); + size_t usable = big - (big % 7); + for (size_t i = 0; i + 7 <= usable; i += 7) memcpy(lots + i, "1\tabcd\n", 7); + pthread_t thread; + CancelArgs args = { c }; + pthread_create(&thread, NULL, cancel_after, &args); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", lots, usable, 1 << 16, NULL, NULL); + pthread_join(thread, NULL); + describe(d, sizeof d, o); + check("a cancel request mid-stream fails the COPY with 57014", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "57014") == 0, d); + free(lots); + + // Stop while the server takes rows: CopyFail goes out behind the piece being sent. + size_t streamed = 128u * 1024u * 1024u; + char *stream = malloc(streamed); + size_t streamable = streamed - (streamed % 7); + for (size_t i = 0; i + 7 <= streamable; i += 7) memcpy(stream + i, "1\tabcd\n", 7); + run(c, "TRUNCATE copy_probe"); + Stopped s = copy_in_stoppable(c, "COPY copy_probe (a, b) FROM STDIN", stream, streamable, 1 << 20, 300, 1, 0); + describe_stop(d, sizeof d, s); + check("a Stop mid-stream ends the COPY within a second", + s.outcome.status == PGRES_FATAL_ERROR && strcmp(s.outcome.state, "57014") == 0 && s.stop_to_end_ms < 1000, d); + check("after the Stop an autocommit session is idle and runs the next statement", + PQtransactionStatus(c) == PQTRANS_IDLE && runs_next_statement(c), NULL); + check("no row of the stopped COPY stays", strcmp(scalar(c, "SELECT count(*) FROM copy_probe"), "0") == 0, NULL); + s = copy_in_stoppable(c, "COPY copy_probe (a, b) FROM STDIN", stream, streamable, 1 << 20, 300, 0, 10000); + describe_stop(d, sizeof d, s); + printf("info the same Stop with CopyFail alone: %s\n", s.gave_up ? "no answer in 10 s" : d); + + run(c, "BEGIN"); + s = copy_in_stoppable(c, "COPY copy_probe (a, b) FROM STDIN", stream, streamable, 1 << 20, 300, 1, 0); + describe_stop(d, sizeof d, s); + check("a Stop mid-stream inside a block fails the COPY", s.outcome.status == PGRES_FATAL_ERROR && strcmp(s.outcome.state, "57014") == 0, d); + check("the stopped COPY leaves the block aborted", PQtransactionStatus(c) == PQTRANS_INERROR, NULL); + run(c, "ROLLBACK"); + check("after ROLLBACK the session runs the next statement", runs_next_statement(c), NULL); + free(stream); + + // Stop while the server waits on a row lock another session holds: it takes no data until the lock goes. + PGconn *holder = PQconnectdb(argv[1]); + if (PQstatus(holder) != CONNECTION_OK) { fprintf(stderr, "no second session: %s", PQerrorMessage(holder)); return 3; } + run(c, "DROP TABLE IF EXISTS copy_lock; CREATE TABLE copy_lock (a int PRIMARY KEY)"); + size_t keys = 20000000; + char *numbers = malloc(keys * 10); + size_t numbers_length = 0; + for (size_t k = 1; k <= keys; k++) numbers_length += (size_t)sprintf(numbers + numbers_length, "%zu\n", k); + + run(holder, "BEGIN; INSERT INTO copy_lock VALUES (400000)"); + s = copy_in_stoppable(c, "COPY copy_lock FROM STDIN", numbers, numbers_length, 1 << 20, 1500, 0, 2000); + check("on a server waiting for a row lock, CopyFail alone is still unsent 2 s after the Stop", s.gave_up, NULL); + run(holder, "ROLLBACK"); + while (PQflush(c) == 1) PQsocketPoll(PQsocket(c), 1, 1, PQgetCurrentTimeUSec() + 100000); + PQsetnonblocking(c, 0); + PGresult *late = PQgetResult(c); + const char *late_state = PQresultErrorField(late, PG_DIAG_SQLSTATE); + check("once the lock goes, the CopyFail fails the COPY", PQresultStatus(late) == PGRES_FATAL_ERROR && late_state && strcmp(late_state, "57014") == 0, + PQresultErrorField(late, PG_DIAG_MESSAGE_PRIMARY)); + PQclear(late); + while ((late = PQgetResult(c))) PQclear(late); + + run(holder, "BEGIN; INSERT INTO copy_lock VALUES (400000)"); + run(c, "BEGIN"); + s = copy_in_stoppable(c, "COPY copy_lock FROM STDIN", numbers, numbers_length, 1 << 20, 1500, 1, 0); + describe_stop(d, sizeof d, s); + check("on a server waiting for a row lock, a Stop with one cancel request ends the COPY within a second", + s.sent_cancel && s.outcome.status == PGRES_FATAL_ERROR && strcmp(s.outcome.state, "57014") == 0 && s.stop_to_end_ms < 1000, d); + check("the COPY stopped in a lock wait leaves the block aborted", PQtransactionStatus(c) == PQTRANS_INERROR, NULL); + run(c, "ROLLBACK"); + check("after the cancel request the session runs the next statement", runs_next_statement(c), NULL); + check("no row of the COPY stopped in a lock wait stays", strcmp(scalar(c, "SELECT count(*) FROM copy_lock"), "0") == 0, NULL); + run(holder, "ROLLBACK"); + PQfinish(holder); + free(numbers); + run(c, "DROP TABLE copy_lock"); + + run(c, "DROP TABLE copy_probe"); + PQfinish(c); + printf("%d failure(s)\n", failures); + return failures ? 1 : 0; +} +C_SOURCE + +clang -O1 -o "$WORK_DIR/probe" "$WORK_DIR/probe.c" \ + -I"$APPLE/Plugins/PostgreSQLDriverPlugin/CLibPQ/include" \ + "$LIBS/libpq.a" "$LIBS/libpgcommon.a" "$LIBS/libpgport.a" \ + -L"$LIBS/dylibs" -lssl.3 -lcrypto.3 -lz -framework GSS -Wl,-rpath,"$LIBS/dylibs" + +"$WORK_DIR/probe" "$CONNINFO" diff --git a/apple/scripts/probes/check-postgresql-read-only-functions.sh b/apple/scripts/probes/check-postgresql-read-only-functions.sh new file mode 100755 index 0000000000..bd391a3539 --- /dev/null +++ b/apple/scripts/probes/check-postgresql-read-only-functions.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Checks the server fact Read-Only Safe Mode rests on for the PostgreSQL family: a read the driver runs the way it runs +# one at Read-Only (`START TRANSACTION READ ONLY; SELECT 1`, the statement, `ROLLBACK`) cannot write through a function +# it calls. A SQL function that deletes rows, `nextval`, and `set_config` at session level are each sent that way; the +# delete and the sequence must be refused with SQLSTATE 25006 and leave the data as it was, and the setting must be +# gone after the ROLLBACK. The same delete sent without the wrap must write, so the probe proves it can see a write. +# +# Usage: scripts/probes/check-postgresql-read-only-functions.sh [psql connection arguments…] +# scripts/probes/check-postgresql-read-only-functions.sh -h 127.0.0.1 -p 5432 -U postgres -d postgres +# The login needs CREATE on the database: the probe makes and drops its own schema. +# Prints OK, or BUG with what the server let through. Exits 2 when the server cannot be reached. +set -uo pipefail + +PSQL="${PSQL:-$(command -v psql || true)}" +[[ -n "$PSQL" ]] || { echo "FAIL: psql not found; set PSQL" >&2; exit 2; } +SCHEMA="tablepro_read_only_probe_$$" +BEGIN_READ_ONLY="START TRANSACTION READ ONLY; SELECT 1" + +run() { "$PSQL" -X -q -At -v VERBOSITY=verbose "$@" 2>&1; } + +run "$@" -c 'SELECT 1' > /dev/null || { echo "FAIL: cannot connect" >&2; exit 2; } +cleanup() { run "$@" -c "DROP SCHEMA IF EXISTS $SCHEMA CASCADE" > /dev/null; } +trap 'cleanup "$@"' EXIT + +run "$@" -v ON_ERROR_STOP=1 \ + -c "CREATE SCHEMA $SCHEMA" \ + -c "CREATE TABLE $SCHEMA.orders (id integer)" \ + -c "INSERT INTO $SCHEMA.orders SELECT generate_series(1, 5)" \ + -c "CREATE SEQUENCE $SCHEMA.ticket" \ + -c "CREATE FUNCTION $SCHEMA.purge_orders() RETURNS void LANGUAGE sql AS 'DELETE FROM $SCHEMA.orders'" \ + > /dev/null || { echo "FAIL: cannot create the probe schema" >&2; exit 2; } +echo "PostgreSQL $(run "$@" -c 'SHOW server_version')" + +failures=() +wrapped() { run "$@" -c "$BEGIN_READ_ONLY" -c "$STATEMENT" -c "ROLLBACK"; } + +STATEMENT="SELECT $SCHEMA.purge_orders()" +answer="$(wrapped "$@")" +[[ "$answer" == *25006* ]] || failures+=("purge_orders() was not refused: $answer") +[[ "$(run "$@" -c "SELECT count(*) FROM $SCHEMA.orders")" == 5 ]] || failures+=("purge_orders() deleted rows") + +STATEMENT="SELECT nextval('$SCHEMA.ticket')" +answer="$(wrapped "$@")" +[[ "$answer" == *25006* ]] || failures+=("nextval() was not refused: $answer") + +answer="$(run "$@" -c "$BEGIN_READ_ONLY" -c "SELECT set_config('application_name', 'probe_leak', false)" \ + -c "ROLLBACK" -c "SHOW application_name")" +[[ "$answer" != *$'\n'probe_leak ]] || failures+=("set_config() outlived the ROLLBACK") + +run "$@" -c "SELECT $SCHEMA.purge_orders()" > /dev/null +[[ "$(run "$@" -c "SELECT count(*) FROM $SCHEMA.orders")" == 0 ]] || failures+=("control: the unwrapped purge did not write") + +if ((${#failures[@]} == 0)); then + echo "OK: the read-only transaction refused every write through a function" + exit 0 +fi +printf 'BUG: %s\n' "${failures[@]}" +exit 1 diff --git a/apple/scripts/probes/check-qdrant.sh b/apple/scripts/probes/check-qdrant.sh index 1a4b551f33..a4056a0dba 100755 --- a/apple/scripts/probes/check-qdrant.sh +++ b/apple/scripts/probes/check-qdrant.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # -# Run TableProQdrantCore against a live Qdrant node: cursor paging, every filter kind, sorting with -# keyless points, typed payload and vector writes, the insert id check, facet, export, truncate and -# drop. The probe creates and drops a collection named tablepro_probe. +# Run TableProQdrantCore and TableProQdrantClient against a live Qdrant node: cursor paging, every +# filter kind, sorting with keyless points, typed payload and vector writes, the insert id check, +# facet, export, truncate and drop. The probe creates and drops a collection named tablepro_probe. # # Usage: # scripts/probes/check-qdrant.sh [host] [port] @@ -35,7 +35,10 @@ let package = Package( name: "probe", platforms: [.macOS(.v14)], dependencies: [.package(path: "$CORE")], - targets: [.executableTarget(name: "probe", dependencies: [.product(name: "TableProQdrantCore", package: "TableProCore")])] + targets: [.executableTarget(name: "probe", dependencies: [ + .product(name: "TableProQdrantCore", package: "TableProCore"), + .product(name: "TableProQdrantClient", package: "TableProCore") + ])] ) MANIFEST diff --git a/apple/scripts/probes/check-rds-cname-chain.sh b/apple/scripts/probes/check-rds-cname-chain.sh new file mode 100755 index 0000000000..1c6bda8e5e --- /dev/null +++ b/apple/scripts/probes/check-rds-cname-chain.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# Check how AWSEndpointHost finds the RDS endpoint behind a custom DNS name, against the system resolver: the +# hop-by-hop CNAME walk TablePro signs and dials, beside what getaddrinfo's AI_CANONNAME reports, which is the end +# of the chain (an ec2, vpce or replica name) and never the name a token is signed for. +# +# Usage: +# apple/scripts/probes/check-rds-cname-chain.sh [name...] +# +# With no names it uses public CNAMEs that point at RDS instances. Needs network access. +# Exits non-zero when a default name no longer leads to an RDS endpoint, 3 when a prerequisite is missing. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +AWS="$ROOT/Plugins/TableProPluginKit/AWS" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +if [ -z "${DEVELOPER_DIR:-}" ]; then + DEVELOPER_DIR="$(xcode-select -p)" + case "$DEVELOPER_DIR" in + *CommandLineTools*) DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer ;; + esac +fi +export DEVELOPER_DIR +command -v xcrun > /dev/null || { echo "xcrun not found" >&2; exit 3; } + +cat > "$WORK/main.swift" << 'SWIFT' +import Darwin +import Foundation + +struct Hop: CNAMEResolving { + let inner = SystemCNAMEResolver() + + func cnameTarget(of name: String, within timeout: Duration?) async -> CNAMELookup { + let started = ContinuousClock.now + let lookup = await inner.cnameTarget(of: name, within: timeout) + print(" CNAME \(name) -> \(lookup) (\(started.duration(to: .now)))") + return lookup + } +} + +func canonicalName(_ name: String) -> String { + var hints = addrinfo() + hints.ai_flags = AI_CANONNAME + hints.ai_socktype = SOCK_STREAM + var result: UnsafeMutablePointer? + guard getaddrinfo(name, nil, &hints, &result) == 0, let result else { return "lookup failed" } + defer { freeaddrinfo(result) } + return result.pointee.ai_canonname.map { String(cString: $0) } ?? "none" +} + +@main +enum Probe { + static func main() async { + let checksDefaults = CommandLine.arguments.contains("--defaults") + var failures = 0 + for name in CommandLine.arguments.dropFirst() where name != "--defaults" { + print("== \(name)") + print(" AI_CANONNAME: \(canonicalName(name))") + switch await AWSEndpointHost.rdsEndpoint(behind: name, resolver: Hop(), within: .seconds(10)) { + case .found(let endpoint): + print(" signs and dials: \(endpoint), region \(AWSEndpointHost.rdsRegion(forHost: endpoint) ?? "none")") + case .notFound: + print(" signs and dials: the name as typed") + if checksDefaults { failures += 1 } + case .failed: + print(" the lookup failed") + failures += 1 + } + } + exit(failures == 0 ? 0 : 1) + } +} +SWIFT + +xcrun swiftc -swift-version 6 -parse-as-library -module-name CNAMEProbe \ + "$AWS/AWSEndpointHost.swift" "$AWS/CNAMEResolver.swift" "$AWS/AWSPartition.swift" "$AWS/AWSAuthError.swift" \ + "$WORK/main.swift" -o "$WORK/probe" 2> "$WORK/build.log" || { + cat "$WORK/build.log" >&2 + exit 3 +} + +if [ "$#" -eq 0 ]; then + "$WORK/probe" --defaults \ + users-mysql-1.machines.pythonanywhere-services.com \ + users-mysql-3.machines.pythonanywhere-services.com +else + "$WORK/probe" "$@" +fi diff --git a/apple/scripts/probes/check-sftp-read-ahead.sh b/apple/scripts/probes/check-sftp-read-ahead.sh new file mode 100755 index 0000000000..ba322b0560 --- /dev/null +++ b/apple/scripts/probes/check-sftp-read-ahead.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Measures how the buffer passed to libssh2_sftp_read sets SFTP download speed over a link with +# latency, against the vendored libssh2 and a real OpenSSH server. +# +# libssh2 keeps buffer_size * 4 bytes of FXP_READ requests in flight (sftp_read in src/sftp.c, +# capped at four channel windows), so a download moves about four buffers per round trip. On +# loopback that never shows. LibSSH2SFTPSession sizes its download buffer on this; rerun after a +# libssh2 bump. +# +# Usage: +# scripts/probes/check-sftp-read-ahead.sh [delay-ms] [size-mb] # default 50 ms, 32 MB +# +# Needs Docker. Starts a throwaway OpenSSH container with NET_ADMIN, delays every packet the server +# sends with tc netem, which adds that much to each round trip, and removes the container on exit. Exits non-zero if a 2 MB buffer is not at least three times +# faster than a 32 KB one. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../lib/openssh-fixture.sh +source "$ROOT/scripts/lib/openssh-fixture.sh" + +DELAY_MS="${1:-50}" +SIZE_MB="${2:-32}" +CONTAINER="tp-sftp-read-ahead" +PORT="${TABLEPRO_SFTP_READ_AHEAD_PORT:-22023}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"; docker rm -f -v "$CONTAINER" > /dev/null 2>&1 || true' EXIT + +docker rm -f -v "$CONTAINER" > /dev/null 2>&1 || true +docker run -d --name "$CONTAINER" --cap-add NET_ADMIN \ + -p "127.0.0.1:$PORT:2222" \ + -e USER_NAME=tp -e USER_PASSWORD=tppass -e PASSWORD_ACCESS=true -e SUDO_ACCESS=false \ + "$OPENSSH_FIXTURE_IMAGE_10" > /dev/null +openssh_wait_for_banner 127.0.0.1 "$PORT" 60 || { echo "no SSH banner within 60s" >&2; exit 2; } + +docker exec "$CONTAINER" sh -c "apk add --no-cache iproute2 > /dev/null && tc qdisc add dev eth0 root netem delay ${DELAY_MS}ms" +docker exec "$CONTAINER" sh -c "head -c $((SIZE_MB * 1024 * 1024)) /dev/urandom > /config/read-ahead.bin && chmod 644 /config/read-ahead.bin" + +cat > "$WORK/probe.c" << 'PROBE' +#include +#include +#include +#include +#include +#include +#include +#include +#include "libssh2.h" +#include "libssh2_sftp.h" + +static double download(int port, size_t buffer_size, size_t *bytes) { + int sock = socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_in a; + memset(&a, 0, sizeof(a)); + a.sin_family = AF_INET; + a.sin_port = htons((unsigned short)port); + a.sin_addr.s_addr = inet_addr("127.0.0.1"); + if (connect(sock, (struct sockaddr *)&a, sizeof(a))) { puts("tcp connect failed"); exit(2); } + LIBSSH2_SESSION *ses = libssh2_session_init(); + if (libssh2_session_handshake(ses, sock)) { puts("handshake failed"); exit(2); } + if (libssh2_userauth_password(ses, "tp", "tppass")) { puts("auth failed"); exit(2); } + LIBSSH2_SFTP *sftp = libssh2_sftp_init(ses); + if (!sftp) { puts("sftp refused"); exit(2); } + const char *path = "/config/read-ahead.bin"; + LIBSSH2_SFTP_HANDLE *h = libssh2_sftp_open_ex(sftp, path, (unsigned)strlen(path), LIBSSH2_FXF_READ, 0, LIBSSH2_SFTP_OPENFILE); + if (!h) { puts("open failed"); exit(2); } + char *buffer = malloc(buffer_size); + struct timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); + *bytes = 0; + for (;;) { + ssize_t n = libssh2_sftp_read(h, buffer, buffer_size); + if (n == 0) break; + if (n < 0) { printf("read failed %zd\n", n); exit(2); } + *bytes += (size_t)n; + } + clock_gettime(CLOCK_MONOTONIC, &end); + free(buffer); + libssh2_sftp_close_handle(h); + libssh2_sftp_shutdown(sftp); + libssh2_session_disconnect(ses, "done"); + libssh2_session_free(ses); + close(sock); + return (double)(end.tv_sec - start.tv_sec) + (double)(end.tv_nsec - start.tv_nsec) / 1e9; +} + +int main(int argc, char **argv) { + int port = atoi(argv[1]); + libssh2_init(0); + size_t sizes[] = { 32 * 1024, 256 * 1024, 1024 * 1024, 2 * 1024 * 1024 }; + double rates[4]; + for (int i = 0; i < 4; i++) { + size_t bytes = 0; + double seconds = download(port, sizes[i], &bytes); + rates[i] = (double)bytes / seconds / (1024.0 * 1024.0); + printf("buffer %5zu KB: %zu bytes in %.2f s, %.1f MB/s\n", sizes[i] / 1024, bytes, seconds, rates[i]); + } + libssh2_exit(); + return rates[3] >= rates[0] * 3 ? 0 : 1; +} +PROBE + +clang -w -arch arm64 -o "$WORK/probe" "$WORK/probe.c" \ + -I "$ROOT/TablePro/Core/SSH/CLibSSH2/include" \ + "$ROOT/Libs/libssh2_arm64.a" "$ROOT/Libs/libssl_arm64.a" "$ROOT/Libs/libcrypto_arm64.a" \ + -lz -framework Security -framework CoreFoundation + +echo "libssh2 SFTP download, ${DELAY_MS} ms added to each round trip, ${SIZE_MB} MB file:" +"$WORK/probe" "$PORT" diff --git a/apple/scripts/probes/check-sql-lexical-grammar.sh b/apple/scripts/probes/check-sql-lexical-grammar.sh index 081d63ec2a..0feda2d90f 100755 --- a/apple/scripts/probes/check-sql-lexical-grammar.sh +++ b/apple/scripts/probes/check-sql-lexical-grammar.sh @@ -108,6 +108,8 @@ CASE_NAMES=( "dollars glued to an identifier" "dollars glued to a non-ASCII identifier" "canonically equivalent dollar tag" + "E'' after a zero width space" + "dollar quote after a zero width space" ) CASE_SQL=( "SELECT 'C:\\' AS p; DROP TABLE lexer_canary" @@ -125,6 +127,8 @@ CASE_SQL=( "SELECT 1 AS x\$\$; DROP TABLE lexer_canary; --\$\$" "SELECT 1 AS é\$\$; DROP TABLE lexer_canary; --\$\$" "SELECT \$K\$ x \$K\$ , \$K\$; DROP TABLE lexer_canary; -- \$K\$" # the second tag is U+212A KELVIN SIGN + $'SELECT varchar\xe2\x80\x8bE\'\\\'\'; DROP TABLE lexer_canary; SELECT \'x\'' # DuckDB blanks the U+200B, PostgreSQL keeps it in the name + $'SELECT 1 AS a,\xe2\x80\x8b$$\'$$; DROP TABLE lexer_canary; SELECT \'x\'' ) SQLITE_DB="$WORK/lexer.sqlite" diff --git a/apple/scripts/probes/check-sqlite-agent-app-nap.sh b/apple/scripts/probes/check-sqlite-agent-app-nap.sh new file mode 100755 index 0000000000..86bfbfd51e --- /dev/null +++ b/apple/scripts/probes/check-sqlite-agent-app-nap.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# +# Shows whether App Nap delays the Remote SQLite heartbeat past the agent's 60 s watchdog. +# +# Builds the SQLite driver's sources into a small app, connects it to the shipped agent (local python3, the real +# watchdog) over the loopback relay the driver tests use, launches it in the background so it is never frontmost, and +# leaves it hidden and idle. It records each heartbeat as it reaches the relay, then runs one statement. A second +# copy with the heartbeat off is the control: the watchdog ends its session after about 60 s. +# +# The script prints the main thread's scheduling priority once the apps are hidden: 4 means App Nap took them. +# Exit 1 when the session with heartbeats did not survive, 3 when the probe cannot run. +# +# Usage: +# scripts/probes/check-sqlite-agent-app-nap.sh [--minutes N] +# +# --minutes how long the app stays hidden, default 7 + +set -uo pipefail + +MINUTES=7 +while [ $# -gt 0 ]; do + case "$1" in + --minutes) MINUTES="${2:?}"; shift 2 ;; + *) echo "unknown argument $1" >&2; exit 3 ;; + esac +done +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +[ -f "$ROOT/Libs/libsqlite3_vendored.a" ] || { echo "run scripts/download-libs.sh first" >&2; exit 3; } +command -v python3 > /dev/null || { echo "python3 not found" >&2; exit 3; } + +WORK="$(mktemp -d)" || exit 3 +[ -n "$WORK" ] || exit 3 +cleanup() { + pkill -f "$WORK/" 2> /dev/null + rm -rf "$WORK" +} +trap cleanup EXIT + +SOURCES="$WORK/Sources/NapProbe" +mkdir -p "$SOURCES" +for source in "$ROOT"/Plugins/SQLiteDriverPlugin/*.swift "$ROOT"/Plugins/SharedSpill/*.swift; do + ln -s "$source" "$SOURCES/driver_$(basename "$source")" +done +for shared in PluginConnectTimeout PluginQueryTimeout PluginURLSessionTaskBox; do + [ -f "$ROOT/Plugins/Shared/$shared.swift" ] && ln -s "$ROOT/Plugins/Shared/$shared.swift" "$SOURCES/shared_$shared.swift" +done +for helper in TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift \ + Plugins/SQLiteDriverTests/SQLiteAgentLoopback.swift; do + ln -s "$ROOT/$helper" "$SOURCES/agent_$(basename "$helper")" +done + +cat > "$WORK/Package.swift" << MANIFEST +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "NapProbe", + platforms: [.macOS(.v14)], + dependencies: [.package(path: "$ROOT/Packages/TableProCore")], + targets: [ + .executableTarget( + name: "NapProbe", + dependencies: [ + .product(name: "TableProPluginKit", package: "TableProCore"), + .product(name: "TableProSQLGrammar", package: "TableProCore"), + .product(name: "TableProProcess", package: "TableProCore"), + .product(name: "CSQLite", package: "TableProCore"), + .product(name: "TableProSQLiteCore", package: "TableProCore"), + .product(name: "TableProSQLQuoting", package: "TableProCore"), + .product(name: "TableProSQLVocabulary", package: "TableProCore"), + .product(name: "TableProWireIO", package: "TableProCore"), + ], + path: "Sources/NapProbe", + swiftSettings: [.swiftLanguageMode(.v6), .enableUpcomingFeature("MemberImportVisibility")], + linkerSettings: [.unsafeFlags(["$ROOT/Libs/libsqlite3_vendored.a"])] + ), + ] +) +MANIFEST + +cat > "$SOURCES/main.swift" << 'SWIFT' +import AppKit +import Foundation +import TableProPluginKit + +// Arguments: